86 lines
2.6 KiB
Go
86 lines
2.6 KiB
Go
package cmd
|
|
|
|
import (
|
|
"fmt"
|
|
"log"
|
|
"slices"
|
|
"time"
|
|
|
|
"github.com/Jaculabilis/intake/core"
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
var sourceFetchCmd = &cobra.Command{
|
|
Use: "fetch",
|
|
Short: "Fetch items for a source and update the feed",
|
|
Long: fmt.Sprintf(`Fetch items from a feed source using the configured "fetch" action.
|
|
Items returned by a successful fetch will be used to update the source.
|
|
A fetch is successful if all items output by the fetch are parsed successfully
|
|
and the exit code is 0. No changes will be made to the source if the fetch
|
|
does not succeed.
|
|
|
|
In a dry run, the items will be printed according to the chosen format and
|
|
the source will not be updated with the fetch result.
|
|
|
|
%s`, makeFormatHelpText()),
|
|
Run: func(cmd *cobra.Command, args []string) {
|
|
sourceFetch(
|
|
stringArg(cmd, "source"),
|
|
stringArg(cmd, "format"),
|
|
stringArg(cmd, "timeout"),
|
|
boolArg(cmd, "dry-run"),
|
|
)
|
|
},
|
|
}
|
|
|
|
func init() {
|
|
sourceCmd.AddCommand(sourceFetchCmd)
|
|
|
|
sourceFetchCmd.Flags().StringP("source", "s", "", "Source name to fetch (required)")
|
|
sourceFetchCmd.MarkFlagRequired("source")
|
|
|
|
sourceFetchCmd.Flags().StringP("format", "f", "headlines", "Feed format for returned items.")
|
|
sourceFetchCmd.Flags().Bool("dry-run", false, "Instead of updating the source, print the fetched items")
|
|
sourceFetchCmd.Flags().StringP("timeout", "t", core.DefaultTimeout.String(),
|
|
fmt.Sprintf("Timeout duration (default: %s)", core.DefaultTimeout.String()))
|
|
}
|
|
|
|
func sourceFetch(source string, format string, timeout string, dryRun bool) {
|
|
formatter := formatAs(format)
|
|
|
|
duration, err := time.ParseDuration(timeout)
|
|
if err != nil {
|
|
log.Fatalf("error: invalid duration: %v", err)
|
|
}
|
|
|
|
db := openAndMigrateDb()
|
|
|
|
state, envs, argv, postProcess, err := core.GetSourceActionInputs(db, source, "fetch")
|
|
if err != nil {
|
|
log.Fatalf("error: failed to load data for %s: %v", source, err)
|
|
}
|
|
|
|
items, newState, errItem, err := core.Execute(source, argv, envs, state, "", duration, postProcess)
|
|
if err != nil {
|
|
core.AddErrorItem(db, errItem)
|
|
log.Fatalf("error: failed to execute fetch: %v", err)
|
|
}
|
|
|
|
if dryRun {
|
|
log.Printf("Fetch returned %d items", len(items))
|
|
if !slices.Equal(state, newState) {
|
|
log.Printf("State update (%d => %d bytes)", len(state), len(newState))
|
|
}
|
|
for _, item := range items {
|
|
fmt.Println(formatter(item))
|
|
}
|
|
return
|
|
}
|
|
|
|
added, deleted, err := core.UpdateWithFetchedItems(db, source, newState, items, time.Now())
|
|
if err != nil {
|
|
log.Fatalf("error: failed to update: %v", err)
|
|
}
|
|
log.Printf("%s added %d items, updated %d items, and deleted %d items", source, added, len(items)-added, deleted)
|
|
}
|