Add crontab integration

README.md

@@ -55,6 +55,10 @@ A minimally functional source requires a `fetch` action that returns items.
 TTL, TTD, and TTS can be configured at the source level by setting the environment variables `INTAKE_TTL`, `INTAKE_TTS`, or `INTAKE_TTS` to an integer value.
 These values override any `ttl`, `ttd`, or `tts` value returned by a fetch or action.
 
+Intake provides integration with `cron`.
+To create a cron job for a source, set the `INTAKE_CRON` environment variable to a five-element crontab spec (e.g. `0 0 * * *`).
+The `intake crontab` command will synchronize source cron jobs to your crontab.
+
 ### Action API
 
 The Intake action API defines how programs should behave to be used with Intake sources.
@@ -103,7 +107,7 @@ Parity features
 * [x] item punt
 * [x] web feed paging
 * [x] web fetch
-* [ ] crontab integration
+* [x] crontab integration
 * [ ] source batching
 * [x] add item from web
 * [x] Nix build

cmd/crontab.go

@@ -0,0 +1,51 @@
+package cmd
+
+import (
+	"fmt"
+	"log"
+	"sort"
+
+	"github.com/Jaculabilis/intake/core"
+	"github.com/spf13/cobra"
+)
+
+var crontabCmd = &cobra.Command{
+	Use:   "crontab",
+	Short: "Update crontab entries",
+	Long: `Update crontab entries.
+
+A source's cron job is defined by its INTAKE_CRON environment variable.
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		crontab(boolArg(cmd, "list"))
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(crontabCmd)
+
+	crontabCmd.Flags().BoolP("list", "l", false, "List crontab entries")
+}
+
+func crontab(list bool) {
+	db := openAndMigrateDb()
+
+	specs, err := core.GetCronSources(db)
+	if err != nil {
+		log.Fatalf("error: failed to get crontab sources: %v", err)
+	}
+	if list {
+		var sources []string
+		for source := range specs {
+			sources = append(sources, source)
+		}
+		sort.Strings(sources)
+		for _, source := range sources {
+			fmt.Println(specs[source])
+		}
+	} else {
+		if err := core.UpdateCrontab(db, specs); err != nil {
+			log.Fatalf("error: failed to update crontab: %v", err)
+		}
+	}
+}

core/crontab.go

@@ -0,0 +1,121 @@
+package core
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"sort"
+	"strings"
+)
+
+var IntakeCronBegin = "### begin intake-managed crontab entries"
+var IntakeCronEnd = "### end intake-managed crontab entries"
+
+func makeCrontabEntry(source string, spec string) string {
+	// TODO the /etc/profile setup is NixOS-specific, maybe there's another way to do this
+	return fmt.Sprintf("%-20s . /etc/profile; intake source fetch -s %s", spec, source)
+}
+
+func GetCronSources(db DB) (specs map[string]string, err error) {
+	res, err := db.Query(`
+		select source, value
+		from envs
+		where name = 'INTAKE_CRON'
+	`)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get source crontabs: %v", err)
+	}
+	specs = make(map[string]string)
+	for res.Next() {
+		var source string
+		var value string
+		if err = res.Scan(&source, &value); err != nil {
+			return nil, fmt.Errorf("failed to scan source crontab: %v", err)
+		}
+		specs[source] = makeCrontabEntry(source, value)
+	}
+	return
+}
+
+// Update the intake-managed section of the user's crontab.
+func UpdateCrontab(db DB, specs map[string]string) (err error) {
+	// If there is no crontab command available, quit early.
+	crontabPath, err := exec.LookPath("crontab")
+	if err != nil {
+		return fmt.Errorf("no crontab found")
+	}
+
+	// Get the current crontab without extra header lines via `EDITOR=cat crontab -e`
+	cmdLoad := exec.Command(crontabPath, "-e")
+	cmdLoad.Env = append(os.Environ(), "EDITOR=cat")
+	output, err := cmdLoad.Output()
+	if err != nil {
+		return fmt.Errorf("error: failed to get current crontab: %v", err)
+	}
+	lines := strings.Split(string(output), "\n")
+
+	// Sort the new intake crons
+	var sources []string
+	for source := range specs {
+		sources = append(sources, source)
+	}
+	sort.Strings(sources)
+
+	// Splice the intake crons into the crontab
+	var newCrontab []string
+	headerFound := false
+	inSection := false
+	for i := range lines {
+		switch {
+		case !headerFound && lines[i] == IntakeCronBegin:
+			headerFound = true
+			inSection = true
+			newCrontab = append(newCrontab, IntakeCronBegin)
+			for _, source := range sources {
+				newCrontab = append(newCrontab, specs[source])
+			}
+
+		case lines[i] == IntakeCronEnd:
+			newCrontab = append(newCrontab, IntakeCronEnd)
+			inSection = false
+
+		case !inSection:
+			newCrontab = append(newCrontab, lines[i])
+		}
+	}
+
+	// If the splice mark was never found, append the whole section to the end
+	if !headerFound {
+		newCrontab = append(newCrontab, IntakeCronBegin)
+		for _, source := range sources {
+			newCrontab = append(newCrontab, specs[source])
+		}
+		newCrontab = append(newCrontab, IntakeCronEnd)
+	}
+
+	log.Printf("Updating %d crontab entries", len(specs))
+
+	// Save the updated crontab
+	cmdSave := exec.Command(crontabPath, "-")
+	stdin, err := cmdSave.StdinPipe()
+	if err != nil {
+		return fmt.Errorf("failed to open stdin: %v", err)
+	}
+	if _, err := io.WriteString(stdin, strings.Join(newCrontab, "\n")); err != nil {
+		return fmt.Errorf("failed to write to crontab: %v", err)
+	}
+	if err := stdin.Close(); err != nil {
+		return fmt.Errorf("failed to close stdin: %v", err)
+	}
+	output, err = cmdSave.CombinedOutput()
+	if err != nil {
+		log.Printf("failed to read crontab output: %v", err)
+	}
+	if len(output) > 0 {
+		log.Printf("crontab output: %s", string(output))
+	}
+
+	return nil
+}

test/test_items.sh

@@ -57,3 +57,8 @@ tmp/intake item add -s page --id 212 --title "Item 212" --body "This is the body
 # default password, comment out to test no password
 echo "hello" | tmp/intake passwd --stdin
 echo "hello" | tmp/intake passwd --stdin --verify
+
+# crontab integration
+tmp/intake source env -s page --set "INTAKE_CRON=0 0 * * *"
+tmp/intake source env -s spook --set "INTAKE_CRON=0 0 * * *"
+tmp/intake source env -s feedtest --set "INTAKE_CRON=0 0 * * *"