Web channel editing

README.md

@@ -55,6 +55,10 @@ A minimally functional source requires a `fetch` action that returns items.
 TTL, TTD, and TTS can be configured at the source level by setting the environment variables `INTAKE_TTL`, `INTAKE_TTS`, or `INTAKE_TTS` to an integer value.
 These values override any `ttl`, `ttd`, or `tts` value returned by a fetch or action.
 
+Intake provides integration with `cron`.
+To create a cron job for a source, set the `INTAKE_CRON` environment variable to a five-element crontab spec (e.g. `0 0 * * *`).
+The `intake crontab` command will synchronize source cron jobs to your crontab.
+
 ### Action API
 
 The Intake action API defines how programs should behave to be used with Intake sources.
@@ -103,23 +107,28 @@ Parity features
 * [x] item punt
 * [x] web feed paging
 * [x] web fetch
-* [ ] crontab integration
+* [x] crontab integration
 * [ ] source batching
 * [x] add item from web
+* [ ] web edit channels
+* web edit sources
+  * [ ] edit action argv
+  * [ ] edit source envs
+  * [ ] edit source crontab
 * [x] Nix build
 * [ ] NixOS module
 * [ ] NixOS vm demo
-* [ ] Nix flake templates
 
 Future features
 
 * [ ] on_delete triggers
 * [ ] manual item edits, CLI
 * [ ] manual item edits, web
-* [ ] source-level TTS
+* [x] source-level TTS
 * [ ] metric reporting
 * [x] on action failure, create an error item with logs
 * [ ] items gracefully add new fields and `action` keys
 * [ ] arbitrary date punt
-* [ ] sort crontab entries
+* [x] sort crontab entries
 * [ ] TUI feed view
+* [ ] Nix flake templates

cmd/crontab.go

@@ -0,0 +1,51 @@
+package cmd
+
+import (
+	"fmt"
+	"log"
+	"sort"
+
+	"github.com/Jaculabilis/intake/core"
+	"github.com/spf13/cobra"
+)
+
+var crontabCmd = &cobra.Command{
+	Use:   "crontab",
+	Short: "Update crontab entries",
+	Long: `Update crontab entries.
+
+A source's cron job is defined by its INTAKE_CRON environment variable.
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		crontab(boolArg(cmd, "list"))
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(crontabCmd)
+
+	crontabCmd.Flags().BoolP("list", "l", false, "List crontab entries")
+}
+
+func crontab(list bool) {
+	db := openAndMigrateDb()
+
+	specs, err := core.GetCronSources(db)
+	if err != nil {
+		log.Fatalf("error: failed to get crontab sources: %v", err)
+	}
+	if list {
+		var sources []string
+		for source := range specs {
+			sources = append(sources, source)
+		}
+		sort.Strings(sources)
+		for _, source := range sources {
+			fmt.Println(specs[source])
+		}
+	} else {
+		if err := core.UpdateCrontab(db, specs); err != nil {
+			log.Fatalf("error: failed to update crontab: %v", err)
+		}
+	}
+}

core/channel.go

@@ -1,5 +1,7 @@
 package core
 
+import "time"
+
 func AddSourceToChannel(db DB, channel string, source string) error {
 	_, err := db.Exec(`
 		insert into channels (name, source)
@@ -41,12 +43,16 @@ func GetSourcesInChannel(db DB) (map[string][]string, error) {
 }
 
 func GetChannelsAndActiveCounts(db DB) (map[string]int, error) {
+	now := int(time.Now().Unix()) // TODO pass this value in
 	rows, err := db.Query(`
 		select c.name, count(i.id) as count
 		from channels c
-		left outer join items i on c.source = i.source and i.active = 1
+		left outer join items i
+			on c.source = i.source
+			and i.active <> 0
+			and (i.tts = 0 or i.created + i.tts < ?)
 		group by c.name
-	`)
+	`, now)
 	if err != nil {
 		return nil, err
 	}

core/crontab.go

@@ -0,0 +1,121 @@
+package core
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"sort"
+	"strings"
+)
+
+var IntakeCronBegin = "### begin intake-managed crontab entries"
+var IntakeCronEnd = "### end intake-managed crontab entries"
+
+func makeCrontabEntry(source string, spec string) string {
+	// TODO the /etc/profile setup is NixOS-specific, maybe there's another way to do this
+	return fmt.Sprintf("%-20s . /etc/profile; intake source fetch -s %s", spec, source)
+}
+
+func GetCronSources(db DB) (specs map[string]string, err error) {
+	res, err := db.Query(`
+		select source, value
+		from envs
+		where name = 'INTAKE_CRON'
+	`)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get source crontabs: %v", err)
+	}
+	specs = make(map[string]string)
+	for res.Next() {
+		var source string
+		var value string
+		if err = res.Scan(&source, &value); err != nil {
+			return nil, fmt.Errorf("failed to scan source crontab: %v", err)
+		}
+		specs[source] = makeCrontabEntry(source, value)
+	}
+	return
+}
+
+// Update the intake-managed section of the user's crontab.
+func UpdateCrontab(db DB, specs map[string]string) (err error) {
+	// If there is no crontab command available, quit early.
+	crontabPath, err := exec.LookPath("crontab")
+	if err != nil {
+		return fmt.Errorf("no crontab found")
+	}
+
+	// Get the current crontab without extra header lines via `EDITOR=cat crontab -e`
+	cmdLoad := exec.Command(crontabPath, "-e")
+	cmdLoad.Env = append(os.Environ(), "EDITOR=cat")
+	output, err := cmdLoad.Output()
+	if err != nil {
+		return fmt.Errorf("error: failed to get current crontab: %v", err)
+	}
+	lines := strings.Split(string(output), "\n")
+
+	// Sort the new intake crons
+	var sources []string
+	for source := range specs {
+		sources = append(sources, source)
+	}
+	sort.Strings(sources)
+
+	// Splice the intake crons into the crontab
+	var newCrontab []string
+	headerFound := false
+	inSection := false
+	for i := range lines {
+		switch {
+		case !headerFound && lines[i] == IntakeCronBegin:
+			headerFound = true
+			inSection = true
+			newCrontab = append(newCrontab, IntakeCronBegin)
+			for _, source := range sources {
+				newCrontab = append(newCrontab, specs[source])
+			}
+
+		case lines[i] == IntakeCronEnd:
+			newCrontab = append(newCrontab, IntakeCronEnd)
+			inSection = false
+
+		case !inSection:
+			newCrontab = append(newCrontab, lines[i])
+		}
+	}
+
+	// If the splice mark was never found, append the whole section to the end
+	if !headerFound {
+		newCrontab = append(newCrontab, IntakeCronBegin)
+		for _, source := range sources {
+			newCrontab = append(newCrontab, specs[source])
+		}
+		newCrontab = append(newCrontab, IntakeCronEnd)
+	}
+
+	log.Printf("Updating %d crontab entries", len(specs))
+
+	// Save the updated crontab
+	cmdSave := exec.Command(crontabPath, "-")
+	stdin, err := cmdSave.StdinPipe()
+	if err != nil {
+		return fmt.Errorf("failed to open stdin: %v", err)
+	}
+	if _, err := io.WriteString(stdin, strings.Join(newCrontab, "\n")); err != nil {
+		return fmt.Errorf("failed to write to crontab: %v", err)
+	}
+	if err := stdin.Close(); err != nil {
+		return fmt.Errorf("failed to close stdin: %v", err)
+	}
+	output, err = cmdSave.CombinedOutput()
+	if err != nil {
+		log.Printf("failed to read crontab output: %v", err)
+	}
+	if len(output) > 0 {
+		log.Printf("crontab output: %s", string(output))
+	}
+
+	return nil
+}

test/test_items.sh

@@ -57,3 +57,8 @@ tmp/intake item add -s page --id 212 --title "Item 212" --body "This is the body
 # default password, comment out to test no password
 echo "hello" | tmp/intake passwd --stdin
 echo "hello" | tmp/intake passwd --stdin --verify
+
+# crontab integration
+tmp/intake source env -s page --set "INTAKE_CRON=0 0 * * *"
+tmp/intake source env -s spook --set "INTAKE_CRON=0 0 * * *"
+tmp/intake source env -s feedtest --set "INTAKE_CRON=0 0 * * *"

web/channel.go

@@ -1,12 +1,76 @@
 package web
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/Jaculabilis/intake/core"
 	"github.com/Jaculabilis/intake/web/html"
 )
 
+func (env *Env) getChannels(writer http.ResponseWriter, req *http.Request) {
+	allSources, err := core.GetSources(env.db)
+	if err != nil {
+		http.Error(writer, err.Error(), 400)
+		return
+	}
+
+	channelSources, err := core.GetSourcesInChannel(env.db)
+	if err != nil {
+		http.Error(writer, err.Error(), 400)
+		return
+	}
+
+	data := html.EditChannelsData{
+		Sources:        allSources,
+		ChannelSources: channelSources,
+	}
+	html.EditChannels(writer, data)
+}
+
+func (env *Env) editChannel(writer http.ResponseWriter, req *http.Request) {
+	if err := req.ParseForm(); err != nil {
+		http.Error(writer, err.Error(), 400)
+		return
+	}
+
+	channel := req.Form.Get("channel")
+	source := req.Form.Get("source")
+
+	if channel == "" {
+		http.Error(writer, "missing channel", 400)
+		return
+	}
+	if source == "" {
+		http.Error(writer, "missing source", 400)
+		return
+	}
+	if exists, err := core.SourceExists(env.db, source); err != nil || !exists {
+		http.Error(writer, fmt.Sprintf("could not find source %s: %v", source, err), 500)
+		return
+	}
+
+	if req.Method == http.MethodPost {
+		if err := core.AddSourceToChannel(env.db, channel, source); err != nil {
+			http.Error(writer, err.Error(), 500)
+			return
+		}
+		writer.Header()["HX-Refresh"] = []string{"true"}
+		writer.WriteHeader(http.StatusNoContent)
+		return
+	}
+
+	if req.Method == http.MethodDelete {
+		if err := core.DeleteSourceFromChannel(env.db, channel, source); err != nil {
+			http.Error(writer, err.Error(), 500)
+			return
+		}
+		writer.Header()["HX-Refresh"] = []string{"true"}
+		writer.WriteHeader(http.StatusNoContent)
+		return
+	}
+}
+
 func (env *Env) getChannel(writer http.ResponseWriter, req *http.Request) {
 	channel := req.PathValue("channel")
 

web/html/editChannels.html

@@ -0,0 +1,49 @@
+{{ define "title" }}Channels - Intake{{ end }}
+
+{{ define "content" -}}
+<nav class="center">
+    <span class="feed-controls">
+    <a href="/">Home</a>
+    </span>
+</nav>
+
+<nav>
+<span class="feed-controls">Edit channels</span>
+
+<p>
+<form>
+    <label for="source">Add</label>
+    <select name="source">{{ range .Sources }}
+        <option value="{{ . }}">{{ . }}</option>{{ end }}
+    </select>
+    <label for="channel">to</label>
+    <input type="text" name="channel" list="channel-options">
+    <button
+    hx-post="/channel/"
+    >Submit</button>
+</form>
+</p>
+
+<datalist id="channel-options">{{ range $channel, $_ := .ChannelSources }}
+    <option value="{{ $channel }}"></option>{{ end }}
+</datalist>
+
+{{ range $channel, $sources := .ChannelSources }}
+<p><b><a href="/channel/{{ $channel }}">{{ $channel }}</a></b></p>
+<table>
+{{- range $sources }}
+<tr>
+<td>
+    <button
+        hx-delete="/channel/?channel={{ $channel }}&source={{ . }}"
+    >&#10005;</button>
+</td>
+    </form>
+<td><a href="/source/{{ . }}">{{ . }}</a></td>
+</tr>
+{{ end }}
+
+</table>
+{{ end }}
+</nav>
+{{- end }}

web/html/feed.html

@@ -1,7 +1,7 @@
 {{ define "title" }}{{ if .Items }}({{ len .Items }}) {{ end }}Intake{{ end }}
 
 {{ define "content" -}}
-<article class="center">
+<nav class="center">
 <span class="feed-controls flex-between">
 <a href="?hidden={{ .ShowHidden }}&page={{ page .Page -1 }}&count={{ .Count }}">&lt;--</a>
 <a href="/">Home</a>
@@ -14,20 +14,20 @@
 </span>
 <a href="?hidden={{ .ShowHidden }}&page={{ page .Page 1 }}&count={{ .Count }}">--&gt;</a>
 </span>
-</article>
+</nav>
 
 {{ if .Items }}
 {{ range .Items }}
 	{{ template "item" . }}
 {{ end }}
 
-<article class="center">
+<nav class="center">
 <button
     hx-post="/mass-deactivate"
     hx-vals='{{ massDeacVars .Items }}'
     hx-confirm="Deactivate {{ len .Items }} items?"
 >Deactivate All</button>
-</article>
+</nav>
 
 {{ else }}
 <article class="center">

web/html/fetch.html

@@ -1,9 +1,9 @@
 {{ define "title" }}Intake - fetch result for {{ .Source }}{{ end }}
 
 {{ define "content" -}}
-<article class="center">
+<nav class="center">
 <span class="feed-controls">Fetch results for <a href="/source/{{ .Source }}">{{ .Source }}</a></span>
-</article>
+</nav>
 
 <article>
 <p>{{ .Added }} new items, {{ .Updated }} updated items, {{ .Deleted }} deleted items</p>

web/html/home.html

@@ -1,14 +1,14 @@
 {{ define "title" }}Intake{{ end }}
 
 {{ define "content" -}}
-<article>
+<nav>
 <details open>
 <summary><span class="feed-controls">Channels</span></summary>
 {{ if .Channels }}
 {{ range .Channels }}
 <p><a href="/channel/{{ .Name }}">
 {{ if .Active }}
-{{ .Name }} ({{ .Active }})
+({{ .Active }}) {{ .Name }}
 {{ else }}
 {{ .Name }}
 {{ end }}
@@ -17,10 +17,12 @@
 {{ else }}
 <p>No channels found.</p>
 {{ end }}
-</details>
-</article>
 
-<article>
+<p><a href="/channel/">(Edit channels)</a></p>
+</details>
+</nav>
+
+<nav>
 <details>
 <summary><span class="feed-controls">Sources</span></summary>
 {{ if .Sources }}
@@ -41,9 +43,9 @@
 <p>No sources found.</p>
 {{ end }}
 </details>
-</article>
+</nav>
 
-<article>
+<nav>
 <details open>
 <summary><span class="feed-controls">Add item</span></summary>
 <form action="/item" method="post">
@@ -73,5 +75,5 @@
 </p>
 </form>
 </details>
-</article>
+</nav>
 {{- end }}

web/html/html.go

@@ -169,3 +169,16 @@ func Fetch(writer io.Writer, data FetchData) {
 		log.Printf("error: failed to render fetch: %v", err)
 	}
 }
+
+var editChannels = load("editChannels.html")
+
+type EditChannelsData struct {
+	Sources        []string
+	ChannelSources map[string][]string
+}
+
+func EditChannels(writer io.Writer, data EditChannelsData) {
+	if err := editChannels.Execute(writer, data); err != nil {
+		log.Printf("error: failed to render edit channels: %v", err)
+	}
+}

web/html/intake.css

@@ -3,7 +3,7 @@ main {
 	max-width: 700px;
 	margin: 0 auto;
 }
-article {
+article, nav {
 	border: 1px solid black; border-radius: 6px;
 	padding: 5px;
 	margin-bottom: 20px;
@@ -88,7 +88,7 @@ table.feed-control td {
 .intake-sources form {
 	margin: 0
 }
-article.center {
+.center {
 	text-align: center;
 }
 article textarea {

web/html/itemPage.html

@@ -1,11 +1,11 @@
 {{ define "title" }}{{ if .Item.Title }}{{ .Item.Title }}{{ else }}{{ .Item.Source }}/{{ .Item.Id }}{{ end }} - Intake [{{ .Item.Source }}]{{ end }}
 
 {{ define "content" -}}
-<article class="center">
+<nav class="center">
     <span class="feed-controls">
     <a href="/">Home</a>
     </span>
-</article>
+</nav>
 
 {{ template "item" .Item }}
 {{- end }}

web/html/login.html

@@ -1,7 +1,7 @@
 {{ define "title" }}Intake - Login{{ end }}
 
 {{ define "content" -}}
-<article class="center">
+<nav class="center">
 <form method="post" action="/login">
     <p>
         <input name="password" type="password"/>
@@ -13,5 +13,5 @@
 >Submit</button>
 </form>
 <p id="errors">{{ .Error }}</p>
-</article>
+</nav>
 {{ end }}

web/main.go

@@ -41,6 +41,9 @@ func RunServer(db core.DB, addr string, port string) {
 	handleFunc("POST   /login", env.login, logged)
 	handleFunc("GET    /source/{source}", env.getSource, env.authed, logged)
 	handleFunc("POST   /source/{source}/fetch", env.fetchSource, env.authed, logged)
+	handleFunc("GET    /channel/", env.getChannels, env.authed, logged)
+	handleFunc("POST   /channel/", env.editChannel, env.authed, logged)
+	handleFunc("DELETE /channel/", env.editChannel, env.authed, logged)
 	handleFunc("GET    /channel/{channel}", env.getChannel, env.authed, logged)
 	handleFunc("POST   /item", env.addItem, env.authed, logged)
 	handleFunc("GET    /item/{source}/{id}", env.getItem, env.authed, logged)