From 08dbeda80abfe0bf3494ee25f8a94836b090acbb Mon Sep 17 00:00:00 2001 From: Tim Van Baak Date: Fri, 7 Feb 2025 14:37:36 -0800 Subject: [PATCH] Implement web fetch --- README.md | 2 +- test/test_items.sh | 3 +++ web/html/feed.html | 2 +- web/html/fetch.html | 16 +++++++++++++ web/html/home.html | 6 +++++ web/html/html.go | 16 +++++++++++++ web/html/intake.css | 2 ++ web/html/login.html | 1 + web/main.go | 1 + web/source.go | 57 +++++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 web/html/fetch.html diff --git a/README.md b/README.md index c560336..46c22d0 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ Parity features * [x] web feed supports item TTS * [x] item punt * [x] web feed paging -* [ ] web fetch +* [x] web fetch * [ ] set a working directory for item actions * [ ] crontab integration * [ ] source batching diff --git a/test/test_items.sh b/test/test_items.sh index cafcf5b..06d0850 100755 --- a/test/test_items.sh +++ b/test/test_items.sh @@ -26,11 +26,14 @@ tmp/intake item add -s feedtest --id o --title "TTS 30s" --tts 30 tmp/intake item add -s feedtest --id p --title "TTS 10d" --tts 864000 tmp/intake item add -s feedtest --id q --title "TTS -10d" --tts "-864000" +tmp/intake action add -s feedtest -a fetch -- jq -cn '{id: "0", title: "Returned by fetch"}' + tmp/intake source add -s spook tmp/intake action add -s spook -a spookier -- jq -c '.title = .title + "o"' tmp/intake item add -s spook --id boo --title "Boo" --action '{"spookier": true}' tmp/intake source add -s nothing +tmp/intake action add -s nothing -a fetch -- true tmp/intake channel add -c all -s feedtest tmp/intake channel add -c all -s spook diff --git a/web/html/feed.html b/web/html/feed.html index 361befc..cc3ea82 100644 --- a/web/html/feed.html +++ b/web/html/feed.html @@ -2,7 +2,7 @@ {{ define "content" -}}
- + <-- Home diff --git a/web/html/fetch.html b/web/html/fetch.html new file mode 100644 index 0000000..c083a28 --- /dev/null +++ b/web/html/fetch.html @@ -0,0 +1,16 @@ +{{ define "title" }}Intake - fetch result for {{ .Source }}{{ end }} + +{{ define "content" -}} + + +
+

{{ .Added }} new items, {{ .Updated }} updated items, {{ .Deleted }} deleted items

+ +
+{{- end }} diff --git a/web/html/home.html b/web/html/home.html index 7328f8d..6d270d6 100644 --- a/web/html/home.html +++ b/web/html/home.html @@ -27,6 +27,12 @@ {{ range .Sources }} + + {{ end }} diff --git a/web/html/html.go b/web/html/html.go index 31aa7a6..23555e0 100644 --- a/web/html/html.go +++ b/web/html/html.go @@ -154,3 +154,19 @@ func Login(writer io.Writer, data LoginData) { log.Printf("error: failed to render login: %v", err) } } + +var fetch = load("fetch.html") + +type FetchData struct { + Source string + Added int + Updated int + Deleted int + Items []core.Item +} + +func Fetch(writer io.Writer, data FetchData) { + if err := fetch.Execute(writer, data); err != nil { + log.Printf("error: failed to render fetch: %v", err) + } +} diff --git a/web/html/intake.css b/web/html/intake.css index ca8ad7b..858f6fc 100644 --- a/web/html/intake.css +++ b/web/html/intake.css @@ -12,6 +12,8 @@ article { } .feed-controls { font-size: 1.2em; +} +.flex-between { display: flex; justify-content: space-between; } diff --git a/web/html/login.html b/web/html/login.html index e41c5a6..3a65cb9 100644 --- a/web/html/login.html +++ b/web/html/login.html @@ -13,4 +13,5 @@ >Submit

{{ .Error }}

+ {{ end }} diff --git a/web/main.go b/web/main.go index 5b4bf27..6d2c563 100644 --- a/web/main.go +++ b/web/main.go @@ -40,6 +40,7 @@ func RunServer(db core.DB, addr string, port string) { handleFunc("GET /htmx.org@2.0.4.js", env.getScript, logged) handleFunc("POST /login", env.login, logged) handleFunc("GET /source/{source}", env.getSource, env.authed, logged) + handleFunc("POST /source/{source}/fetch", env.fetchSource, env.authed, logged) handleFunc("GET /channel/{channel}", env.getChannel, env.authed, logged) handleFunc("GET /item/{source}/{id}", env.getItem, env.authed, logged) handleFunc("DELETE /item/{source}/{id}", env.deleteItem, env.authed, logged) diff --git a/web/source.go b/web/source.go index c7dfa9e..baf9d51 100644 --- a/web/source.go +++ b/web/source.go @@ -1,7 +1,10 @@ package web import ( + "fmt" + "log" "net/http" + "time" "github.com/Jaculabilis/intake/core" "github.com/Jaculabilis/intake/web/html" @@ -38,3 +41,57 @@ func (env *Env) getSource(writer http.ResponseWriter, req *http.Request) { } html.Feed(writer, data) } + +func (env *Env) fetchSource(writer http.ResponseWriter, req *http.Request) { + source := req.PathValue("source") + if exists, err := core.SourceExists(env.db, source); !exists || err != nil { + http.NotFound(writer, req) + return + } + + state, err := core.GetState(env.db, source) + if err != nil { + http.Error(writer, fmt.Sprintf("error: failed to get state: %v", err.Error()), 500) + return + } + + envs, err := core.GetEnvs(env.db, source) + if err != nil { + http.Error(writer, fmt.Sprintf("error: failed to get envs: %v", err.Error()), 500) + return + } + + argv, err := core.GetArgvForAction(env.db, source, "fetch") + if err != nil { + http.Error(writer, fmt.Sprintf("error: failed to get fetch argv: %v", err.Error()), 500) + return + } + + postProcess, err := core.GetSourcePostProcessor(env.db, source) + if err != nil { + http.Error(writer, fmt.Sprintf("error: failed to get post-processor: %v", err.Error()), 500) + return + } + + items, newState, err := core.Execute(source, argv, envs, state, "", time.Minute, postProcess) + if err != nil { + http.Error(writer, fmt.Sprintf("error: failed to execute fetch: %v", err.Error()), 500) + return + } + + added, deleted, err := core.UpdateWithFetchedItems(env.db, source, newState, items, time.Now()) + if err != nil { + http.Error(writer, fmt.Sprintf("error: failed to update: %v", err.Error()), 500) + return + } + log.Printf("%s added %d items, updated %d items, and deleted %d items", source, added, len(items)-added, deleted) + + data := html.FetchData{ + Source: source, + Added: added, + Updated: len(items) - added, + Deleted: deleted, + Items: items, + } + html.Fetch(writer, data) +}
+
+ +
+		 {{ .Name }}