Implement web fetch

README.md

@@ -99,7 +99,7 @@ Parity features
 * [x] web feed supports item TTS
 * [x] item punt
 * [x] web feed paging
-* [ ] web fetch
+* [x] web fetch
 * [ ] set a working directory for item actions
 * [ ] crontab integration
 * [ ] source batching

test/test_items.sh

@@ -26,11 +26,14 @@ tmp/intake item add -s feedtest --id o --title "TTS 30s" --tts 30
 tmp/intake item add -s feedtest --id p --title "TTS 10d" --tts 864000
 tmp/intake item add -s feedtest --id q --title "TTS -10d" --tts "-864000"
 
+tmp/intake action add -s feedtest -a fetch -- jq -cn '{id: "0", title: "Returned by fetch"}'
+
 tmp/intake source add -s spook
 tmp/intake action add -s spook -a spookier -- jq -c '.title = .title + "o"'
 tmp/intake item add -s spook --id boo --title "Boo" --action '{"spookier": true}'
 
 tmp/intake source add -s nothing
+tmp/intake action add -s nothing -a fetch -- true
 
 tmp/intake channel add -c all -s feedtest
 tmp/intake channel add -c all -s spook

web/html/feed.html

@@ -2,7 +2,7 @@
 
 {{ define "content" -}}
 <article class="center">
-<span class="feed-controls">
+<span class="feed-controls flex-between">
 <a href="?hidden={{ .ShowHidden }}&page={{ page .Page -1 }}&count={{ .Count }}">&lt;--</a>
 <a href="/">Home</a>
 <span>

web/html/fetch.html

@@ -0,0 +1,16 @@
+{{ define "title" }}Intake - fetch result for {{ .Source }}{{ end }}
+
+{{ define "content" -}}
+<article class="center">
+<span class="feed-controls">Fetch results for <a href="/source/{{ .Source }}">{{ .Source }}</a></span>
+</article>
+
+<article>
+<p>{{ .Added }} new items, {{ .Updated }} updated items, {{ .Deleted }} deleted items</p>
+<ul>
+{{ range .Items -}}
+<li><a href="/item/{{ .Source }}/{{ .Id }}">{{ or .Title .Id | raw }}</a></li>
+{{ end -}}
+</ul>
+</article>
+{{- end }}

web/html/home.html

@@ -27,6 +27,12 @@
 <table class="intake-sources">
 {{ range .Sources }}
 <tr>
+<td>
+    <form method="post" action="/source/{{ .Name }}/fetch">
+        <button type="submit">fetch</button>
+    </form>
+</td>
+</td>
 <td><a href="/source/{{ .Name }}">{{ .Name }}</a></td>
 </tr>
 {{ end }}

web/html/html.go

@@ -154,3 +154,19 @@ func Login(writer io.Writer, data LoginData) {
 		log.Printf("error: failed to render login: %v", err)
 	}
 }
+
+var fetch = load("fetch.html")
+
+type FetchData struct {
+	Source  string
+	Added   int
+	Updated int
+	Deleted int
+	Items   []core.Item
+}
+
+func Fetch(writer io.Writer, data FetchData) {
+	if err := fetch.Execute(writer, data); err != nil {
+		log.Printf("error: failed to render fetch: %v", err)
+	}
+}

web/html/intake.css

@@ -12,6 +12,8 @@ article {
 }
 .feed-controls {
 	font-size: 1.2em;
+}
+.flex-between {
 	display: flex;
 	justify-content: space-between;
 }

web/html/login.html

@@ -13,4 +13,5 @@
 >Submit</button>
 </form>
 <p id="errors">{{ .Error }}</p>
+</article>
 {{ end }}

web/main.go

@@ -40,6 +40,7 @@ func RunServer(db core.DB, addr string, port string) {
 	handleFunc("GET    /htmx.org@2.0.4.js", env.getScript, logged)
 	handleFunc("POST   /login", env.login, logged)
 	handleFunc("GET    /source/{source}", env.getSource, env.authed, logged)
+	handleFunc("POST   /source/{source}/fetch", env.fetchSource, env.authed, logged)
 	handleFunc("GET    /channel/{channel}", env.getChannel, env.authed, logged)
 	handleFunc("GET    /item/{source}/{id}", env.getItem, env.authed, logged)
 	handleFunc("DELETE /item/{source}/{id}", env.deleteItem, env.authed, logged)

web/source.go

@@ -1,7 +1,10 @@
 package web
 
 import (
+	"fmt"
+	"log"
 	"net/http"
+	"time"
 
 	"github.com/Jaculabilis/intake/core"
 	"github.com/Jaculabilis/intake/web/html"
@@ -38,3 +41,57 @@ func (env *Env) getSource(writer http.ResponseWriter, req *http.Request) {
 	}
 	html.Feed(writer, data)
 }
+
+func (env *Env) fetchSource(writer http.ResponseWriter, req *http.Request) {
+	source := req.PathValue("source")
+	if exists, err := core.SourceExists(env.db, source); !exists || err != nil {
+		http.NotFound(writer, req)
+		return
+	}
+
+	state, err := core.GetState(env.db, source)
+	if err != nil {
+		http.Error(writer, fmt.Sprintf("error: failed to get state: %v", err.Error()), 500)
+		return
+	}
+
+	envs, err := core.GetEnvs(env.db, source)
+	if err != nil {
+		http.Error(writer, fmt.Sprintf("error: failed to get envs: %v", err.Error()), 500)
+		return
+	}
+
+	argv, err := core.GetArgvForAction(env.db, source, "fetch")
+	if err != nil {
+		http.Error(writer, fmt.Sprintf("error: failed to get fetch argv: %v", err.Error()), 500)
+		return
+	}
+
+	postProcess, err := core.GetSourcePostProcessor(env.db, source)
+	if err != nil {
+		http.Error(writer, fmt.Sprintf("error: failed to get post-processor: %v", err.Error()), 500)
+		return
+	}
+
+	items, newState, err := core.Execute(source, argv, envs, state, "", time.Minute, postProcess)
+	if err != nil {
+		http.Error(writer, fmt.Sprintf("error: failed to execute fetch: %v", err.Error()), 500)
+		return
+	}
+
+	added, deleted, err := core.UpdateWithFetchedItems(env.db, source, newState, items, time.Now())
+	if err != nil {
+		http.Error(writer, fmt.Sprintf("error: failed to update: %v", err.Error()), 500)
+		return
+	}
+	log.Printf("%s added %d items, updated %d items, and deleted %d items", source, added, len(items)-added, deleted)
+
+	data := html.FetchData{
+		Source:  source,
+		Added:   added,
+		Updated: len(items) - added,
+		Deleted: deleted,
+		Items:   items,
+	}
+	html.Fetch(writer, data)
+}