Jaculabilis/intake
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add actions to items

Browse Source
This commit is contained in:
Tim Van Baak 2025-01-29 08:48:12 -08:00
parent d23efdf00b
commit f804299180
7 changed files with 93 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
core/action_test.go
View File

@ -147,4 +147,36 @@ func TestExecute(t *testing.T) {
res, err = execute([]string{"jq", "-cn", `["a", "a"] | .[] | {id: .}`})
assertNil(err)
assertLen(res, 2)
// Action keys are detected even with empty values
res, err = execute([]string{"jq", "-cn", `{id: "test", action: {"hello": null}}`})
assertNil(err)
assertLen(res, 1)
if res[0].Action["hello"] == nil {
t.Fatal("missing hello action")
}
if res[0].Action["goodbye"] != nil {
t.Fatal("nonexistent action should key to nil in Action")
}
res, err = execute([]string{"jq", "-cn", `{id: "test", action: {"hello": ""}}`})
assertNil(err)
assertLen(res, 1)
if res[0].Action["hello"] == nil {
t.Fatal("missing hello action")
}
res, err = execute([]string{"jq", "-cn", `{id: "test", action: {"hello": []}}`})
assertNil(err)
assertLen(res, 1)
if res[0].Action["hello"] == nil {
t.Fatal("missing hello action")
}
res, err = execute([]string{"jq", "-cn", `{id: "test", action: {"hello": {}}}`})
assertNil(err)
assertLen(res, 1)
if res[0].Action["hello"] == nil {
t.Fatal("missing hello action")
}
}

6
core/db_test.go
View File

@ -17,8 +17,8 @@ func TestDeleteSourceCascade(t *testing.T) {
t.Fatalf("failed to add source2: %v", err)
}
if err := AddItems(db, []Item{
{"source1", "item1", 0, true, "", "", "", "", 0},
{"source2", "item2", 0, true, "", "", "", "", 0},
{"source1", "item1", 0, true, "", "", "", "", 0, nil},
{"source2", "item2", 0, true, "", "", "", "", 0, nil},
}); err != nil {
t.Fatalf("failed to add items: %v", err)
}
@ -42,7 +42,7 @@ func TestDeleteSourceCascade(t *testing.T) {
t.Fatalf("Expected only 1 item after source delete, got %d", len(items))
}
err = AddItems(db, []Item{{"source1", "item3", 0, true, "", "", "", "", 0}})
err = AddItems(db, []Item{{"source1", "item3", 0, true, "", "", "", "", 0, nil}})
if err == nil {
t.Fatal("Unexpected success adding item for nonexistent source")
}

12
core/item.go
View File

@ -1,11 +1,22 @@
package core
import (
"database/sql/driver"
"encoding/json"
"fmt"
"log"
)
type Actions map[string]json.RawMessage
func (a Actions) Value() (driver.Value, error) {
return json.Marshal(a)
}
func (a *Actions) Scan(value interface{}) error {
return json.Unmarshal([]byte(value.(string)), a)
}
type Item struct {
Source string `json:"source"`
Id string `json:"id"`
@ -16,6 +27,7 @@ type Item struct {
Body string `json:"body"`
Link string `json:"link"`
Time int `json:"time"`
Action Actions `json:"action"`
}
// Whether an item that no longer appears in a fetch can be deleted.

8
core/item_test.go
View File

@ -1,6 +1,9 @@
package core
import "testing"
import (
"encoding/json"
"testing"
)
func TestItemFormatsExist(t *testing.T) {
for name := range AvailableFormats {
@ -30,6 +33,9 @@ func TestItemRoundTrip(t *testing.T) {
Body: "body",
Link: "link",
Time: 123456,
Action: map[string]json.RawMessage{
"hello": json.RawMessage(`"world"`),
},
}
if err := AddItems(db, []Item{item1}); err != nil {
t.Fatalf("update failed: %v", err)

37
core/source.go
View File

@ -51,16 +51,20 @@ func DeleteSource(db *DB, name string) error {
func AddItems(db *DB, items []Item) error {
return db.Transact(func(tx *sql.Tx) error {
stmt, err := tx.Prepare(`
insert into items (source, id, active, title, author, body, link, time)
values (?, ?, ?, ?, ?, ?, ?, ?)
insert into items (source, id, active, title, author, body, link, time, action)
values (?, ?, ?, ?, ?, ?, ?, ?, jsonb(?))
`)
if err != nil {
return err
return fmt.Errorf("failed to prepare insert: %v", err)
}
for _, item := range items {
_, err = stmt.Exec(item.Source, item.Id, true, item.Title, item.Author, item.Body, item.Link, item.Time)
actions, err := json.Marshal(item.Action)
if err != nil {
return err
return fmt.Errorf("failed to marshal actions for %s/%s: %v", item.Source, item.Id, err)
}
_, err = stmt.Exec(item.Source, item.Id, true, item.Title, item.Author, item.Body, item.Link, item.Time, actions)
if err != nil {
return fmt.Errorf("failed to insert %s/%s: %v", item.Source, item.Id, err)
}
}
return nil
@ -100,7 +104,8 @@ func UpdateItems(db *DB, items []Item) error {
author = ?,
body = ?,
link = ?,
time = ?
time = ?,
action = jsonb(?)
where source = ?
and id = ?
`)
@ -108,7 +113,11 @@ func UpdateItems(db *DB, items []Item) error {
return err
}
for _, item := range items {
_, err = stmt.Exec(item.Title, item.Author, item.Body, item.Link, item.Time, item.Source, item.Id)
actions, err := json.Marshal(item.Action)
if err != nil {
return fmt.Errorf("failed to marshal actions for %s/%s: %v", item.Source, item.Id, err)
}
_, err = stmt.Exec(item.Title, item.Author, item.Body, item.Link, item.Time, actions, item.Source, item.Id)
if err != nil {
return err
}
@ -161,7 +170,7 @@ func getItems(db *DB, query string, args ...any) ([]Item, error) {
var items []Item
for rows.Next() {
var item Item
err = rows.Scan(&item.Source, &item.Id, &item.Created, &item.Active, &item.Title, &item.Author, &item.Body, &item.Link, &item.Time)
err = rows.Scan(&item.Source, &item.Id, &item.Created, &item.Active, &item.Title, &item.Author, &item.Body, &item.Link, &item.Time, &item.Action)
if err != nil {
return nil, err
}
@ -172,7 +181,7 @@ func getItems(db *DB, query string, args ...any) ([]Item, error) {
func GetItem(db *DB, source string, id string) (Item, error) {
items, err := getItems(db, `
select source, id, created, active, title, author, body, link, time
select source, id, created, active, title, author, body, link, time, json(action)
from items
where source = ?
and id = ?
@ -189,7 +198,7 @@ func GetItem(db *DB, source string, id string) (Item, error) {
func GetAllActiveItems(db *DB) ([]Item, error) {
return getItems(db, `
select
source, id, created, active, title, author, body, link, time
source, id, created, active, title, author, body, link, time, json(action)
from items
where active <> 0
`)
@ -198,7 +207,7 @@ func GetAllActiveItems(db *DB) ([]Item, error) {
func GetAllItems(db *DB) ([]Item, error) {
return getItems(db, `
select
source, id, created, active, title, author, body, link, time
source, id, created, active, title, author, body, link, time, json(action)
from items
`)
}
@ -206,7 +215,7 @@ func GetAllItems(db *DB) ([]Item, error) {
func GetActiveItemsForSource(db *DB, source string) ([]Item, error) {
return getItems(db, `
select
source, id, created, active, title, author, body, link, time
source, id, created, active, title, author, body, link, time, json(action)
from items
where
source = ?
@ -217,7 +226,7 @@ func GetActiveItemsForSource(db *DB, source string) ([]Item, error) {
func GetAllItemsForSource(db *DB, source string) ([]Item, error) {
return getItems(db, `
select
source, id, created, active, title, author, body, link, time
source, id, created, active, title, author, body, link, time, json(action)
from items
where
source = ?
@ -225,6 +234,8 @@ func GetAllItemsForSource(db *DB, source string) ([]Item, error) {
}
// Given the results of a fetch, add new items, update existing items, and delete expired items.
//
// Returns the number of new and deleted items on success.
func UpdateWithFetchedItems(db *DB, source string, items []Item) (int, int, error) {
// Get the existing items
existingItems, err := GetAllItemsForSource(db, source)

9
core/source_test.go
View File

@ -61,8 +61,8 @@ func TestAddItem(t *testing.T) {
}
if err := AddItems(db, []Item{
{"test", "one", 0, true, "", "", "", "", 0},
{"test", "two", 0, true, "title", "author", "body", "link", 123456},
{"test", "one", 0, true, "", "", "", "", 0, nil},
{"test", "two", 0, true, "title", "author", "body", "link", 123456, nil},
}); err != nil {
t.Fatalf("failed to add items: %v", err)
}
@ -219,8 +219,9 @@ func TestOnCreateAction(t *testing.T) {
if add != 1 || err != nil {
t.Fatal("failed update with alter oncreate")
}
if getItem("two").Title != "Goodbye, World" {
t.Fatal("title not updated")
two := getItem("two")
if two.Title != "Goodbye, World" {
t.Fatalf("title not updated, is: %s", two.Title)
}
// on_create can add a field

1
core/sql/0001_initial_schema.sql
View File

@ -19,6 +19,7 @@ create table items(
body text,
link text,
time int,
action blob,
primary key (source, id),
foreign key (source) references sources (name) on delete cascade
) strict;