Jaculabilis
/
intake
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fixup add update logic

This commit is contained in:
Tim Van Baak 2023-05-29 17:17:51 -07:00
parent 1df9ef8734
commit 893c48175e
1 changed files with 18 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
intake/source.py
View File

@ -14,6 +14,7 @@ class LocalSource:
""" """
An intake source backed by a filesystem directory. An intake source backed by a filesystem directory.
""" """
def __init__(self, data_path: Path, source_name: str): def __init__(self, data_path: Path, source_name: str):
self.data_path: Path = data_path self.data_path: Path = data_path
self.source_name = source_name self.source_name = source_name
@ -34,7 +35,8 @@ class LocalSource:
return [ return [
filepath.name[:-5] filepath.name[:-5]
for filepath in self.source_path.iterdir() for filepath in self.source_path.iterdir()
if filepath.name.endswith(".item")] if filepath.name.endswith(".item")
]
def item_exists(self, item_id) -> bool: def item_exists(self, item_id) -> bool:
return self.get_item_path(item_id).exists() return self.get_item_path(item_id).exists()
@ -61,7 +63,7 @@ class LocalSource:
tmp_path = self.source_path / f"{item['id']}.item.tmp" tmp_path = self.source_path / f"{item['id']}.item.tmp"
with tmp_path.open("w") as f: with tmp_path.open("w") as f:
f.write(json.dumps(item, indent=2)) f.write(json.dumps(item, indent=2))
os.rename(tmp_path, self.get_item_path(item['id'])) os.rename(tmp_path, self.get_item_path(item["id"]))
def delete_item(self, item_id) -> None: def delete_item(self, item_id) -> None:
os.remove(self.get_item_path(item_id)) os.remove(self.get_item_path(item_id))
@ -166,7 +168,7 @@ def update_items(source: LocalSource, fetched_items):
""" """
# Get a list of item ids that already existed for this source. # Get a list of item ids that already existed for this source.
prior_ids = source.get_item_ids() prior_ids = source.get_item_ids()
print(f'Found {len(prior_ids)} prior items') print(f"Found {len(prior_ids)} prior items")
# Determine which items are new and which are updates. # Determine which items are new and which are updates.
new_items = [] new_items = []
@ -185,7 +187,17 @@ def update_items(source: LocalSource, fetched_items):
# Update the other items using the fetched items' values. # Update the other items using the fetched items' values.
for upd_item in upd_items: for upd_item in upd_items:
old_item = source.get_item(upd_item["id"]) old_item = source.get_item(upd_item["id"])
for field in ("title", "tags", "link", "time", "author", "body", "ttl", "ttd", "tts"): for field in (
"title",
"tags",
"link",
"time",
"author",
"body",
"ttl",
"ttd",
"tts",
):
if field in upd_item and old_item[field] != upd_item[field]: if field in upd_item and old_item[field] != upd_item[field]:
old_item[field] = upd_item[field] old_item[field] = upd_item[field]
if "callback" in upd_item: if "callback" in upd_item:
@ -193,16 +205,14 @@ def update_items(source: LocalSource, fetched_items):
# in the callback when the item is new will keep their original # in the callback when the item is new will keep their original
# values, as those values reappear in new_item on subsequent # values, as those values reappear in new_item on subsequent
# updates. # updates.
old_item['callback'] = {**old_item['callback'], **upd_item['callback']} old_item["callback"] = {**old_item["callback"], **upd_item["callback"]}
# Items are removed when they are old (not in the latest fetch) and # Items are removed when they are old (not in the latest fetch) and
# inactive. Some item fields change this basic behavior. # inactive. Some item fields change this basic behavior.
del_count = 0 del_count = 0
now = int(time.time()) now = int(time.time())
upd_ids = [item["id"] for item in upd_items] upd_ids = [item["id"] for item in upd_items]
old_item_ids = [ old_item_ids = [item_id for item_id in prior_ids if item_id not in upd_ids]
item_id for item_id in prior_ids
if item_id not in upd_ids]
for item_id in old_item_ids: for item_id in old_item_ids:
item = source.get_item(item_id) item = source.get_item(item_id)