Add NixOS module and vm demo

I can try it again after I know I can get the module and vm working again

.gitignore

@@ -1,2 +1,3 @@
 .direnv
 tmp/
+nixos.qcow2

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help serve test-data
+.PHONY: help serve test-data demo
 
 help: ## display this help
 	@awk 'BEGIN{FS = ":.*##"; printf "\033[1m\nUsage\n \033[1;92m make\033[0;36m <target>\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } ' $(MAKEFILE_LIST)
@@ -7,4 +7,7 @@ serve: ## Run "intake serve" with live reload
 	@air -build.cmd "go build -o tmp/intake" -build.bin tmp/intake -build.args_bin serve,--data-dir,tmp -build.include_ext "go,html,css"
 
 test-data: ## Recreate test data in tmp/
-	@test/test_items.sh
+	@test/test_items.sh
+
+demo: ## Run the demo vm
+	@nix run ".#nixosConfigurations.demo.config.system.build.nixos-shell"

README.md

@@ -4,6 +4,8 @@ Intake is an arbitrary feed aggregator that generalizes the concept of a feed.
 Rather than being restricted to parsing items out of an RSS feed, Intake provides a middle layer of executing arbitrary commands that conform to a JSON-based specification.
 An Intake source can parse an RSS feed, but it can also scrape a website without a feed, provide additional logic to filter or annotate feed items, or integrate with an API.
 
+A demo running in a NixOS VM is available via `make demo` or using `nix run` on the `nixosConfigurations.demo.config.system.build.nixos-shell` flake attribute.
+
 ## Overview
 
 In Intake, a _source_ represents a single content feed of discrete _items_, such as a blog and its posts or a website and its pages.
@@ -103,32 +105,20 @@ Instead, the web interface can be locked behind a password set via `intake passw
 
 Parity features
 
-* [x] web feed supports item TTS
-* [x] item punt
-* [x] web feed paging
-* [x] web fetch
-* [x] crontab integration
 * [ ] source batching
-* [x] add item from web
+* [x] NixOS module
-* [x] web edit channels
+* [x] NixOS vm demo
-* web edit sources
-  * [x] edit action argv
-  * [x] edit source envs
-  * [x] edit source crontab
-* [x] Nix build
-* [ ] NixOS module
-* [ ] NixOS vm demo
 
 Future features
 
+* [ ] CLI simplification?
 * [ ] on_delete triggers
 * [ ] manual item edits, CLI
 * [ ] manual item edits, web
-* [x] source-level TTS
 * [ ] metric reporting
-* [x] on action failure, create an error item with logs
 * [ ] items gracefully add new fields and `action` keys
 * [ ] arbitrary date punt
-* [x] sort crontab entries
 * [ ] TUI feed view
-* [ ] Nix flake templates
+* Nix flake templates
+  * [ ] parsing a news feed
+  * [ ] following a webcomic

cmd/sourceEnv.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"log"
+	"strings"
 
 	"github.com/Jaculabilis/intake/core"
 	"github.com/spf13/cobra"
@@ -50,4 +51,16 @@ func sourceEnv(source string, env []string) {
 	if err := core.SetEnvs(db, source, env); err != nil {
 		log.Fatalf("failed to set envs: %v", err)
 	}
+
+	for _, envval := range env {
+		if strings.HasPrefix(envval, "INTAKE_CRON=") {
+			specs, err := core.GetCronSources(db)
+			if err != nil {
+				log.Fatalf("failed to get cron specs: %v", err)
+			}
+			if err = core.UpdateCrontab(db, specs); err != nil {
+				log.Fatalf("failed to update crontab: %v", err)
+			}
+		}
+	}
 }

demo/alice.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+if [ -f /home/alice/.intake-setup-done ]; then
+    echo "already done"
+    exit 0
+fi
+
+# intake service gets a crontab wrapper, cheat here
+export PATH="/run/wrappers/bin:$PATH"
+
+mkdir -p $INTAKE_DATA_DIR
+
+intake source add -s echo
+intake action add -s echo -a fetch -- jq -cn '{id: env.ID, title: env.MESSAGE}'
+intake source env -s echo --set "ID=hello"
+intake source env -s echo --set "MESSAGE=Hello, world!"
+intake channel add -s echo -c home
+
+intake source add -s currenttime
+intake action add -s currenttime -a fetch -- sh -c "date +%Y-%m-%d-%H-%M | jq -cR '{id: .}'"
+intake source env -s currenttime --set "INTAKE_CRON=* * * * *"
+intake channel add -s currenttime -c home
+
+touch /home/alice/.intake-setup-done
+echo "done"

demo/bob.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+if [ -f /home/bob/.intake-setup-done ]; then
+    echo "already done"
+    exit 0
+fi
+
+# intake service gets a crontab wrapper, cheat here
+export PATH="/run/wrappers/bin:$PATH"
+
+mkdir -p $INTAKE_DATA_DIR
+
+intake source add -s echo
+intake action add -s echo -a fetch -- jq -cn '{id: env.ID, title: env.MESSAGE}'
+intake source env -s echo --set "ID=goodbye"
+intake source env -s echo --set "MESSAGE=Goodbye, world!"
+intake channel add -s echo -c home
+
+touch /home/bob/.intake-setup-done
+echo "done"

demo/default.nix

@@ -0,0 +1,106 @@
+{ pkgs, lib, ... }:
+
+{
+  system.stateVersion = "25.05";
+
+  environment.systemPackages = with pkgs; [
+    jq
+  ];
+
+  # Set up two users to demonstrate the user separation
+  users.users.alice = {
+    isNormalUser = true;
+    password = "a";
+    uid = 1000;
+  };
+
+  users.users.bob = {
+    isNormalUser = true;
+    password = "b";
+    uid = 1001;
+  };
+
+  # Set up intake for both users
+  services.intake.extraPackages = with pkgs; [
+    jq
+  ];
+  services.intake.users = {
+    alice = {
+      enable = true;
+      listen.addr = "0.0.0.0";
+      listen.port = 6001;
+    };
+    bob = {
+      enable = true;
+      listen.addr = "0.0.0.0";
+      listen.port = 6002;
+    };
+  };
+
+  # Forward both ports
+  virtualisation.forwardPorts = [
+    {
+      from = "host";
+      host.port = 6001;
+      guest.port = 6001;
+    }
+    {
+      from = "host";
+      host.port = 6002;
+      guest.port = 6002;
+    }
+  ];
+
+  # Disable nixos-shell autologin
+  services.getty.autologinUser = lib.mkForce null;
+
+  # Disable default mounts
+  nixos-shell.mounts = {
+    mountHome = false;
+    mountNixProfile = false;
+    cache = "none";
+  };
+
+  # Define a setup service to create some demo content
+  systemd.services =
+  let
+    setupFor = userName: script: {
+      description = "Intake demo setup for ${userName}";
+      serviceConfig = {
+        User = userName;
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+      path = [ pkgs.intake ];
+      environment.INTAKE_DATA_DIR = "/home/${userName}/.local/share/intake";
+      wantedBy = [ "intake-${userName}.service" ];
+      before = [ "intake-${userName}.service" ];
+      after = [ "network.target" ];
+      script = builtins.readFile ./${userName}.sh;
+    };
+  in
+  {
+    intake-alice-setup = setupFor "alice";
+    intake-bob-setup = setupFor "bob";
+  };
+
+  # Include some demo instructions
+  environment.etc.issue.text = ''
+    ###
+    # Welcome to the intake demo! Log in as `alice` with password `a` to begin.
+    #
+    # Exit the VM with ctrl+a x, or switch to the qemu console with ctrl+a c and `quit`.
+    ###
+
+  '';
+
+  users.motd = ''
+
+    ###
+    # The web interfaces are exposed at http://localhost:6001 and http://localhost:6002
+    #
+    # Within this demo VM, you can run `intake` CLI commands.
+    ###
+
+  '';
+}

flake.lock

@@ -1,20 +1,22 @@
 {
   "nodes": {
-    "flake-parts": {
+    "nixos-shell": {
       "inputs": {
-        "nixpkgs-lib": "nixpkgs-lib"
+        "nixpkgs": [
+          "nixpkgs"
+        ]
       },
       "locked": {
-        "lastModified": 1736143030,
+        "lastModified": 1732727306,
-        "narHash": "sha256-+hu54pAoLDEZT9pjHlqL9DNzWz0NbUn8NEAHP7PQPzU=",
+        "narHash": "sha256-4R+OVEmJ8yR7/gsxMQtC39b9f61SvELYQwKeXGAyFfo=",
-        "owner": "hercules-ci",
+        "owner": "Mic92",
-        "repo": "flake-parts",
+        "repo": "nixos-shell",
-        "rev": "b905f6fc23a9051a6e1b741e1438dbfc0634c6de",
+        "rev": "c61dce7cf5dc263d237ba8a7fc175b09642f96eb",
         "type": "github"
       },
       "original": {
-        "owner": "hercules-ci",
+        "owner": "Mic92",
-        "repo": "flake-parts",
+        "repo": "nixos-shell",
         "type": "github"
       }
     },
@@ -34,21 +36,9 @@
         "type": "github"
       }
     },
-    "nixpkgs-lib": {
-      "locked": {
-        "lastModified": 1735774519,
-        "narHash": "sha256-CewEm1o2eVAnoqb6Ml+Qi9Gg/EfNAxbRx1lANGVyoLI=",
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/e9b51731911566bbf7e4895475a87fe06961de0b.tar.gz"
-      },
-      "original": {
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/e9b51731911566bbf7e4895475a87fe06961de0b.tar.gz"
-      }
-    },
     "root": {
       "inputs": {
-        "flake-parts": "flake-parts",
+        "nixos-shell": "nixos-shell",
         "nixpkgs": "nixpkgs"
       }
     }

flake.nix

@@ -2,45 +2,64 @@
   description = "Universal and extensible feed aggregator";
 
   inputs = {
-    flake-parts.url = "github:hercules-ci/flake-parts";
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    nixos-shell.url = "github:Mic92/nixos-shell";
+    nixos-shell.inputs.nixpkgs.follows = "nixpkgs";
   };
 
   outputs =
-    inputs@{ flake-parts, ... }:
+    {
-    flake-parts.lib.mkFlake { inherit inputs; } {
+      self,
-      systems = [
+      nixpkgs,
-        "x86_64-linux"
+      nixos-shell,
-        "aarch64-linux"
+    }:
-        "aarch64-darwin"
+    let
-        "x86_64-darwin"
+      inherit (nixpkgs.lib) nixosSystem;
-      ];
+      system = "x86_64-linux";
-      perSystem =
+      pkgs = nixpkgs.legacyPackages.${system};
-        {
+    in
-          pkgs,
+    {
-          self',
+      formatter = nixpkgs.legacyPackages.${system}.nixfmt-rfc-style;
-          ...
-        }:
-        {
-          formatter = pkgs.nixfmt-rfc-style;
 
-          packages = {
+      packages.${system} =
-            intake = pkgs.callPackage ./package.nix { };
+        let
-            default = self'.packages.intake;
+          pkgs = (
-          };
+            import nixpkgs {
-
+              inherit system;
-          devShells.default = pkgs.mkShell {
+              overlays = [ self.overlays.default ];
-            packages = [
+            }
-              pkgs.go
+          );
-              pkgs.gopls
+        in 
-              pkgs.go-tools
+        {
-              pkgs.gotools
+          default = self.packages.${system}.intake;
-              pkgs.cobra-cli
+          inherit (pkgs) intake;
-              pkgs.air
-            ];
-          };
         };
-      flake = {
+
+      devShells.${system}.default = pkgs.mkShell {
+        packages = [
+          pkgs.go
+          pkgs.gopls
+          pkgs.go-tools
+          pkgs.gotools
+          pkgs.cobra-cli
+          pkgs.air
+          pkgs.nixos-shell
+        ];
+      };
+
+      overlays.default = final: prev: {
+        intake = final.callPackage ./package.nix { };
+      };
+    
+      nixosModules.default = import ./module.nix self;
+
+      nixosConfigurations."demo" = nixosSystem {
+        inherit system;
+        modules = [
+          nixos-shell.nixosModules.nixos-shell
+          self.nixosModules.default
+          ./demo
+        ];
       };
     };
 }

module.nix

@@ -0,0 +1,130 @@
+flake:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  inherit (lib)
+    filterAttrs
+    foldl
+    imap1
+    mapAttrsToList
+    mkEnableOption
+    mkIf
+    mkMerge
+    mkOption
+    mkPackageOption
+    types
+    ;
+in
+{
+  options = {
+    services.intake = {
+      package = mkPackageOption pkgs "intake" { };
+
+      extraPackages = mkOption {
+        type = types.listOf types.package;
+        default = [ ];
+        description = "Extra packages available to all enabled users and their intake services.";
+      };
+
+      users = mkOption {
+        description = "User intake service definitions.";
+        default = { };
+        type = types.attrsOf (
+          types.submodule {
+            options = {
+              enable = mkEnableOption "intake, a universal and extensible feed aggregator.";
+
+              dataDir = mkOption {
+                type = types.str;
+                default = "/home/$USER/.local/share/intake";
+                description = "The data directory for this user's intake service.";
+              };
+
+              listen.addr = mkOption {
+                type = types.str;
+                default = "127.0.0.1";
+                description = "The listen address for this user's intake service.";
+              };
+
+              listen.port = mkOption {
+                type = types.port;
+                default = 80;
+                description = "The listen port for this user's intake service.";
+              };
+
+              extraPackages = mkOption {
+                type = types.listOf types.package;
+                default = [ ];
+                description = "Extra packages available to this user and their intake service.";
+              };
+            };
+          }
+        );
+      };
+    };
+  };
+
+  config =
+    let
+      intakeCfg = config.services.intake;
+      enabledUsers = filterAttrs (userName: userCfg: userCfg.enable) intakeCfg.users;
+      enabledUserNames = mapAttrsToList (userName: userCfg: userName) enabledUsers;
+      userPackages = userName: [ intakeCfg.package ] ++ intakeCfg.extraPackages ++ intakeCfg.users.${userName}.extraPackages;
+      crontabWrapper = pkgs.writeShellScriptBin "crontab" ''
+        exec ${config.security.wrapperDir}/crontab "$@"
+      '';
+    in
+    {
+      # Apply the overlay so intake is included in pkgs.
+      nixpkgs.overlays = [ flake.overlays.default ];
+
+      # Give every intake user the shared packages and their user-specific packages.
+      users.users =
+        let
+          addPackagesToUser = userName: {
+            ${userName}.packages = userPackages userName;
+          };
+        in
+        mkMerge (map addPackagesToUser enabledUserNames);
+
+      # Enable cron
+      services.cron.enable = true;
+
+      # Define a user service for each configured user
+      systemd.services =
+        let
+          runScript =
+            userName:
+            pkgs.writeShellScript "intake-run.sh" ''
+              mkdir -p $INTAKE_DATA_DIR
+              # Add the setuid wrapper directory so `crontab` is accessible
+              export PATH="${config.security.wrapperDir}:$PATH"
+              ${intakeCfg.package}/bin/intake serve --addr ${enabledUsers.${userName}.listen.addr} --port ${toString enabledUsers.${userName}.listen.port}
+            '';
+          # systemd service definition for a single user, given `services.intake.users.userName` = `userCfg`
+          userServiceConfig = userName: userCfg: {
+            "intake-${userName}" = {
+              description = "Intake service for user ${userName}";
+              script = "${runScript userName}";
+              path = [ crontabWrapper ] ++ intakeCfg.extraPackages ++ userCfg.extraPackages;
+              environment = {
+                INTAKE_DATA_DIR = "/home/${userName}/.local/share/intake";
+              };
+              serviceConfig = {
+                User = userName;
+                Type = "simple";
+              };
+              wantedBy = [ "multi-user.target" ];
+              after = [ "network.target" ];
+              enable = userCfg.enable;
+            };
+          };
+        in
+        mkMerge (mapAttrsToList userServiceConfig enabledUsers);
+    };
+}

web/source.go

@@ -128,6 +128,17 @@ func (env *Env) editSource(writer http.ResponseWriter, req *http.Request) {
 			http.Error(writer, err.Error(), 500)
 			return
 		}
+		if envName == "INTAKE_CRON" {
+			specs, err := core.GetCronSources(env.db)
+			if err != nil {
+				log.Printf("error: failed to get cron specs: %v", err)
+			} else {
+				err = core.UpdateCrontab(env.db, specs)
+				if err != nil {
+					log.Printf("error: failed to update crontab: %v", err)
+				}
+			}
+		}
 	}
 
 	actionName := req.PostForm.Get("actionName")