Add basic unit test that evlauates the module

flake.lock

@@ -16,6 +16,21 @@
         "type": "github"
       }
     },
+    "my-flake": {
+      "locked": {
+        "lastModified": 1670879933,
+        "narHash": "sha256-V45PH0cnFLilx66x4td5qQnWNn/V/6/6b7FQDIHvdyI=",
+        "owner": "Jaculabilis",
+        "repo": "my-flake",
+        "rev": "2b2cd07a6d971b15fc5f65d6d963d0da551a5892",
+        "type": "github"
+      },
+      "original": {
+        "owner": "Jaculabilis",
+        "repo": "my-flake",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1669833724,
@@ -35,6 +50,7 @@
     "root": {
       "inputs": {
         "flake-compat": "flake-compat",
+        "my-flake": "my-flake",
         "nixpkgs": "nixpkgs"
       }
     }

flake.nix

@@ -1,5 +1,6 @@
 {
   inputs = {
+    my-flake.url = "github:Jaculabilis/my-flake";
     nixpkgs.url = "github:NixOS/nixpkgs?ref=refs/tags/22.11";
     flake-compat = {
       url = "github:edolstra/flake-compat";
@@ -7,21 +8,68 @@
     };
   };
 
-  outputs = { self, nixpkgs, flake-compat }:
+  outputs = { self, my-flake, nixpkgs, flake-compat }:
   let
-    system = "x86_64-linux";
+    systems = [ "aarch64-linux" "x86_64-linux" ];
-    pkgs = nixpkgs.legacyPackages.${system};
+    each = system:
-  in
+    let
-  {
+      pkgs = (import nixpkgs {
-    packages.${system}.default =
+        inherit system;
-      (pkgs.poetry2nix.mkPoetryApplication {
+        overlays = [ self.overlays.default ];
-        projectDir = ./.;
+      });
-      }).dependencyEnv;
+    in {
+      packages.${system} = {
+        default = self.packages.${system}.inquisitor;
+        inquisitor = pkgs.inquisitor;
+        env = pkgs.inquisitor.dependencyEnv;
+      };
 
-    defaultPackage.${system} = self.packages.${system}.default;
+      devShells.${system} = {
+        default = self.devShells.${system}.inquisitor;
+        inquisitor = pkgs.mkShell {
+          buildInputs = [ (pkgs.python3.withPackages (p: [p.poetry])) ];
+          shellHook = ''
+            PS1="(inquisitor) $PS1"
+          '';
+        };
+        sources = pkgs.mkShell {
+          buildInputs = [ self.packages.${system}.env ];
+          shellHook = ''
+            PS1="(sources) $PS1"
+          '';
+        };
+      };
 
-    devShell.${system} = pkgs.mkShell {
+      checks.${system}.test-module = let
-      buildInputs = [ (pkgs.python3.withPackages (p: [p.poetry])) ];
+        test-lib = import "${nixpkgs}/nixos/lib/testing-python.nix" {
+          inherit system;
+        };
+      in
+        test-lib.makeTest {
+          name = "inquisitor-test-module";
+          nodes = {
+            host = { ... }: {
+              imports = [ self.nixosModules.default ];
+              services.inquisitor.enable = true;
+            };
+          };
+          testScript = ''
+            start_all()
+            host.wait_for_unit("multi-user.target")
+            # Check that the setup script ran
+            host.succeed("[ -e /var/lib/inquisitor ]")
+            # Check that the service came up
+            host.succeed("systemctl is-enabled inquisitor")
+            host.succeed("systemctl is-active inquisitor")
+          '';
+        };
     };
+  in (my-flake.outputs-for each systems) // {
+    overlays.default = final: prev: {
+      inquisitor = final.poetry2nix.mkPoetryApplication {
+        projectDir = ./.;
+      };
+    };
+    nixosModules.default = import ./module.nix self;
   };
 }

inquisitor/cli.py

@@ -236,6 +236,11 @@ def command_help(args):
 	return 0
 
 
+def nocommand(args):
+	print("command required")
+	return 0
+
+
 def main():
 	"""CLI entry point"""
 	# Enable piping
@@ -283,8 +288,11 @@ def main():
 	add_logging_handler(verbose=args.verbose, log_filename=None)
 
 	# Execute command
-	if args.command:
+	try:
-		sys.exit(commands[args.command](args.args))
+		command = commands.get(args.command, nocommand)
-	else:
+		sys.exit(command(args.args))
-		print("command required")
+	except BrokenPipeError:
-		sys.exit(0)
+		# See https://docs.python.org/3.10/library/signal.html#note-on-sigpipe
+		devnull = os.open(os.devnull, os.O_WRONLY)
+		os.dup2(devnull, sys.stdout.fileno())
+		sys.exit(1)

inquisitor/configs/resolver.py

@@ -42,7 +42,8 @@ def read_config_file(config_path):
 	"""
 	# Parse the config file into key-value pairs
 	if not os.path.isfile(config_path):
-		raise FileNotFoundError(f'No config file found at {config_path}')
+
+		raise FileNotFoundError(f'No config file found at {config_path}, try setting {CONFIG_ENVVAR}')
 	accumulated_configs = {}
 	current_key = None
 	with open(config_path, 'r', encoding='utf8') as cfg:

module.nix

@@ -0,0 +1,154 @@
+flake: { config, lib, pkgs, ... }:
+
+let
+  inherit (lib) mkIf mkOption types;
+
+  cfg = config.services.inquisitor;
+in
+{
+  options = {
+    services.inquisitor = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Enable the Inquisitor aggregator.";
+      };
+
+      listen.addr = mkOption {
+        type = types.str;
+        default = "0.0.0.0";
+        description = "Listen address passed to nginx.";
+      };
+
+      listen.port = mkOption {
+        type = types.port;
+        default = 80;
+        description = "Listen port passed to nginx.";
+      };
+    };
+  };
+
+  config =
+  let
+    # Get the inquisitor package from the flake.
+    inquisitor = flake.packages.${pkgs.stdenv.hostPlatform.system}.env;
+
+    # Define the inquisitor state directory.
+    stateDir = "/var/lib/inquisitor";
+
+    # Define an scp helper for item callbacks to use.
+    scp-helper = pkgs.writeShellScriptBin "scp-helper" ''
+      ${pkgs.openssh}/bin/scp -i ${stateDir}/.ssh/inquisitor.key -oStrictHostKeyChecking=no "$@"
+    '';
+
+    # Define the inquisitor service user.
+    svcUser = {
+      name = "inquisitor";
+      group = "inquisitor";
+      description = "Inquisitor service user";
+      isSystemUser = true;
+      shell = pkgs.bashInteractive;
+      packages = [ inquisitor pkgs.cron ];
+    };
+
+    # Create a config file pointing to the state directory.
+    inqConfig = pkgs.writeTextFile {
+      name = "inquisitor.conf";
+      text = ''
+        DataPath = ${stateDir}/data/
+        SourcePath = ${stateDir}/sources/
+        CachePath = ${stateDir}/cache/
+        Verbose = false
+        LogFile = ${stateDir}/inquisitor.log
+      '';
+    };
+
+    # Create a setup script to ensure the service directory state.
+    inqSetup = pkgs.writeShellScript "inquisitor-setup.sh" ''
+      # Ensure the required directories exist.
+      ${pkgs.coreutils}/bin/mkdir -p ${stateDir}/data/inquisitor/
+      ${pkgs.coreutils}/bin/mkdir -p ${stateDir}/sources/
+      ${pkgs.coreutils}/bin/mkdir -p ${stateDir}/cache/
+      if [ ! -f ${stateDir}/data/inquisitor/state ]; then
+        ${pkgs.coreutils}/bin/echo "{}" > ${stateDir}/data/inquisitor/state
+      fi
+
+      # Ensure the service owns the folders.
+      ${pkgs.coreutils}/bin/chown -R ${svcUser.name} ${stateDir}
+
+      # Ensure the scp helper is present
+      if [ -f ${stateDir}/scp-helper ]; then
+        ${pkgs.coreutils}/bin/rm ${stateDir}/scp-helper
+      fi
+      ln -s -t ${stateDir} ${scp-helper}/bin/scp-helper
+    '';
+
+    # Create a run script for the service.
+    inqRun = pkgs.writeShellScript "inquisitor-run.sh" ''
+      cd ${stateDir}
+      ${inquisitor}/bin/gunicorn \
+        --bind=localhost:24133 \
+        --workers=4 \
+        --timeout 120 \
+        --log-level debug \
+        "inquisitor.app:wsgi()"
+    '';
+
+    # Create a wrapper to execute the cli as the service user.
+    # (needed to avoid creating files in the state dir the service can't read)
+    inqWrapper = pkgs.writeShellScriptBin "inq" ''
+      sudo --user=${svcUser.name} ${inquisitor}/bin/inquisitor "$@"
+    '';
+  in mkIf cfg.enable
+  {
+    users.users.inquisitor = svcUser;
+    users.groups.inquisitor = {};
+
+    # Link the config in /etc to avoid envvar shenanigans
+    environment.etc."inquisitor.conf".source = inqConfig;
+
+    # Give all users the wrapper program.
+    environment.systemPackages = [ inqWrapper ];
+    # Allow the sudo in the cli wrapper without password.
+    security.sudo.extraRules = [{
+      commands = [{
+        command = "${inquisitor}/bin/inquisitor";
+        options = [ "NOPASSWD" ];
+      }];
+      runAs = svcUser.name;
+      groups = [ "users" ];
+    }];
+
+    # Run the setup script on activation.
+    system.activationScripts.inquisitorSetup = "${inqSetup}";
+
+    # Set up the inquisitor service.
+    systemd.services.inquisitor = {
+      description = "Inquisitor server";
+      script = "${inqRun}";
+      serviceConfig = {
+        User = svcUser.name;
+        Type = "simple";
+      };
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      enable = true;
+    };
+
+    # Set up the nginx reverse proxy to the server.
+    services.nginx.enable = true;
+    services.nginx.virtualHosts.inquisitorHost = {
+      listen = [ cfg.listen ];
+      locations."/".extraConfig = ''
+        access_log /var/log/nginx/access.inquisitor.log;
+        proxy_buffering off;
+        proxy_pass http://localhost:24133/;
+      '';
+    };
+    networking.firewall.allowedTCPPorts = [ cfg.listen.port ];
+
+    # Enable cron so the service can use it to schedule fetches.
+    services.cron.enable = true;
+  };
+}
+

poetry.lock

@@ -172,14 +172,6 @@ dev = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.4.0,<0.5)", "betamax-seria
 lint = ["black", "flake8", "flynt", "pre-commit", "pydocstyle"]
 test = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.4.0,<0.5)", "betamax-serializers (>=0.2.0,<0.3)", "mock (>=0.8)", "pytest", "testfixtures (>4.13.2,<7)"]
 
-[[package]]
-name = "protobuf"
-version = "4.21.12"
-description = ""
-category = "main"
-optional = false
-python-versions = ">=3.7"
-
 [[package]]
 name = "requests"
 version = "2.28.1"
@@ -289,7 +281,7 @@ dev = ["praw", "gunicorn", "feedparser"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.10"
-content-hash = "f7202e31b8297ce5c72738178aa28256ba6a10d03dc741804a10f1a8ba952d2b"
+content-hash = "2d6e1c8843f9821ef246fdeeeef572bcdaa2db452bbaa74531e8af91f8d89bb5"
 
 [metadata.files]
 beautifulsoup4 = [
@@ -386,22 +378,6 @@ prawcore = [
     {file = "prawcore-2.3.0-py3-none-any.whl", hash = "sha256:48c17db447fa06a13ca3e722201f443031437708daa736c05a1df895fbcceff5"},
     {file = "prawcore-2.3.0.tar.gz", hash = "sha256:daf1ccd4b7a80dc4e6567d48336d782e94a9a6dad83770fc2edf76dc9a84f56d"},
 ]
-protobuf = [
-    {file = "protobuf-4.21.12-cp310-abi3-win32.whl", hash = "sha256:b135410244ebe777db80298297a97fbb4c862c881b4403b71bac9d4107d61fd1"},
-    {file = "protobuf-4.21.12-cp310-abi3-win_amd64.whl", hash = "sha256:89f9149e4a0169cddfc44c74f230d7743002e3aa0b9472d8c28f0388102fc4c2"},
-    {file = "protobuf-4.21.12-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:299ea899484ee6f44604deb71f424234f654606b983cb496ea2a53e3c63ab791"},
-    {file = "protobuf-4.21.12-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:d1736130bce8cf131ac7957fa26880ca19227d4ad68b4888b3be0dea1f95df97"},
-    {file = "protobuf-4.21.12-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:78a28c9fa223998472886c77042e9b9afb6fe4242bd2a2a5aced88e3f4422aa7"},
-    {file = "protobuf-4.21.12-cp37-cp37m-win32.whl", hash = "sha256:3d164928ff0727d97022957c2b849250ca0e64777ee31efd7d6de2e07c494717"},
-    {file = "protobuf-4.21.12-cp37-cp37m-win_amd64.whl", hash = "sha256:f45460f9ee70a0ec1b6694c6e4e348ad2019275680bd68a1d9314b8c7e01e574"},
-    {file = "protobuf-4.21.12-cp38-cp38-win32.whl", hash = "sha256:6ab80df09e3208f742c98443b6166bcb70d65f52cfeb67357d52032ea1ae9bec"},
-    {file = "protobuf-4.21.12-cp38-cp38-win_amd64.whl", hash = "sha256:1f22ac0ca65bb70a876060d96d914dae09ac98d114294f77584b0d2644fa9c30"},
-    {file = "protobuf-4.21.12-cp39-cp39-win32.whl", hash = "sha256:27f4d15021da6d2b706ddc3860fac0a5ddaba34ab679dc182b60a8bb4e1121cc"},
-    {file = "protobuf-4.21.12-cp39-cp39-win_amd64.whl", hash = "sha256:237216c3326d46808a9f7c26fd1bd4b20015fb6867dc5d263a493ef9a539293b"},
-    {file = "protobuf-4.21.12-py2.py3-none-any.whl", hash = "sha256:a53fd3f03e578553623272dc46ac2f189de23862e68565e83dde203d41b76fc5"},
-    {file = "protobuf-4.21.12-py3-none-any.whl", hash = "sha256:b98d0148f84e3a3c569e19f52103ca1feacdac0d2df8d6533cf983d1fda28462"},
-    {file = "protobuf-4.21.12.tar.gz", hash = "sha256:7cd532c4566d0e6feafecc1059d04c7915aec8e182d1cf7adee8b24ef1e2e6ab"},
-]
 requests = [
     {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"},
     {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"},

pyproject.toml

@@ -14,7 +14,6 @@ beautifulsoup4 = "^4.11.1"
 praw = {version = "^7.6.1", optional = true}
 gunicorn = {version = "^20.1.0", optional = true}
 feedparser = {version = "^6.0.10", optional = true}
-protobuf = "^4.21.12"
 
 [tool.poetry.extras]
 dev = ["praw", "gunicorn", "feedparser"]