Add an Item wrapper object

intake/app.py

@@ -8,7 +8,7 @@ import time
 
 from flask import Flask, render_template, request, jsonify, abort, redirect, url_for
 
-from intake.source import LocalSource, execute_action
+from intake.source import LocalSource, execute_action, Item
 
 # Globals
 app = Flask(__name__)
@@ -24,18 +24,8 @@ def intake_data_dir() -> Path:
     raise Exception("No intake data directory defined")
 
 
-def item_sort_key(item):
+def item_sort_key(item: Item):
-    item_date = item.get("time", item.get("created", 0))
+    return item.sort_key
-    return (item_date, item["id"])
-
-
-def show_item(item):
-    """
-    Whether to show an item based on active and tts.
-    """
-    return item["active"] and (
-        "tts" not in item or item["created"] + item["tts"] < int(time.time())
-    )
 
 
 @app.template_filter("datetimeformat")
@@ -80,9 +70,7 @@ def source_feed(name):
     if not source.source_path.exists():
         abort(404)
 
-    return _sources_feed(
+    return _sources_feed(name, [source], show_hidden=request.args.get("hidden", True))
-        name, [source], show_hidden=request.args.get("hidden", True)
-    )
 
 
 @app.get("/channel/<string:name>")
@@ -98,9 +86,7 @@ def channel_feed(name):
         abort(404)
     sources = [LocalSource(intake_data_dir(), name) for name in channels[name]]
 
-    return _sources_feed(
+    return _sources_feed(name, sources, show_hidden=request.args.get("hidden", False))
-        name, sources, show_hidden=request.args.get("hidden", False)
-    )
 
 
 def _sources_feed(name: str, sources: List[LocalSource], show_hidden: bool):
@@ -113,7 +99,7 @@ def _sources_feed(name: str, sources: List[LocalSource], show_hidden: bool):
             item
             for source in sources
             for item in source.get_all_items()
-            if show_item(item) or show_hidden
+            if not item.is_hidden or show_hidden
         ],
         key=item_sort_key,
     )
@@ -308,21 +294,17 @@ def add_item():
         source_path.mkdir()
     config_path = source_path / "intake.json"
     if not config_path.exists():
-        config_path.write_text(json.dumps({
+        config_path.write_text(
-            "action": {
+            json.dumps({"action": {"fetch": {"exe": "true"}}}, indent=2)
-                "fetch": {
+        )
-                    "exe": "true"
-                }
-            }
-        }, indent=2))
     source = LocalSource(source_path.parent, source_path.name)
 
     # Clean up the fields
-    item = {key: value for key, value in request.form.items() if value}
+    fields = {key: value for key, value in request.form.items() if value}
-    item["id"] = '{:x}'.format(getrandbits(16 * 4))
+    fields["id"] = "{:x}".format(getrandbits(16 * 4))
     # TODO: this doesn't support tags or ttX fields correctly
-
+    item = Item.create(source, **fields)
-    source.new_item(item)
+    source.save_item(item)
 
     return redirect(url_for("source_feed", name="default"))
 

intake/cli.py

@@ -224,7 +224,10 @@ def cmd_feed(cmd_args):
         for name in args.sources
         if (data / name / "intake.json").exists()
     ]
-    items = [item for source in sources for item in source.get_all_items()]
+    items = sorted(
+        [item for source in sources for item in source.get_all_items()],
+        key=lambda item: item.sort_key,
+    )
 
     if not items:
         print("Feed is empty")
@@ -234,7 +237,7 @@ def cmd_feed(cmd_args):
     width = min(80, size.columns)
 
     for item in items:
-        title = item["title"] if "title" in item else ""
+        title = item.display_title
         titles = [title]
         while len(titles[-1]) > width - 4:
             i = titles[-1][: width - 4].rfind(" ")

intake/source.py

@@ -2,15 +2,137 @@ from datetime import timedelta
 from pathlib import Path
 from subprocess import Popen, PIPE, TimeoutExpired
 from threading import Thread
+from time import time as current_time
 from typing import List
 import json
 import os
 import os.path
-import time
 
 from intake.types import InvalidConfigException, SourceUpdateException
 
 
+class Item:
+    """
+    A wrapper for an item object.
+    """
+
+    def __init__(self, source: "LocalSource", item: dict):
+        self.source = source
+        self._item = item
+
+    # Methods to allow Item as a drop-in replacement for the item dict itself
+    def __contains__(self, key):
+        return self._item.__contains__(key)
+
+    def __iter__(self):
+        return self._item.__iter__
+
+    def __getitem__(self, key):
+        return self._item.__getitem__(key)
+
+    def __setitem__(self, key, value):
+        return self._item.__setitem__(key, value)
+
+    def get(self, key, default=None):
+        return self._item.get(key, default)
+
+    @staticmethod
+    def create(source: "LocalSource", **fields) -> "Item":
+        if "id" not in fields:
+            raise KeyError("id")
+        item = {
+            "id": fields["id"],
+            "source": source.source_name,
+            "created": int(current_time()),
+            "active": True,
+        }
+        for field_name in (
+            "title",
+            "author",
+            "body",
+            "link",
+            "time",
+            "tags",
+            "tts",
+            "ttl",
+            "ttd",
+            "action",
+        ):
+            if val := fields.get(field_name):
+                item[field_name] = val
+        return Item(source, item)
+
+    @property
+    def display_title(self):
+        return self._item.get("title", self._item["id"])
+
+    @property
+    def abs_tts(self):
+        if "tts" not in self._item:
+            return None
+        return self._item["created"] + self._item["tts"]
+
+    @property
+    def can_remove(self):
+        # The time-to-live fields protects an item from removal until expiry.
+        # This is mainly used to avoid old items resurfacing when their source
+        # cannot guarantee monotonocity.
+        if "ttl" in self._item:
+            ttl_date = self._item["created"] + self._item["ttl"]
+            if ttl_date > current_time():
+                return False
+
+        # The time-to-die field puts a maximum lifespan on an item, removing it
+        # even if it is active.
+        if "ttd" in self._item:
+            ttd_date = self._item["created"] + self._item["ttd"]
+            if ttd_date < current_time():
+                return True
+
+        return not self._item["active"]
+
+    @property
+    def before_tts(self):
+        return (
+            "tts" in self._item
+            and self._item["created"] + self._item["tts"] < current_time()
+        )
+
+    @property
+    def is_hidden(self):
+        return not self._item["active"] or self.before_tts
+
+    @property
+    def sort_key(self):
+        item_date = self._item.get(
+            "time",
+            self._item.get(
+                "created",
+            ),
+        )
+        return (item_date, self._item["id"])
+
+    def serialize(self, indent=True):
+        return json.dumps(self._item, indent=2 if indent else None)
+
+    def update_from(self, updated: "Item") -> None:
+        for field in (
+            "title",
+            "author",
+            "body",
+            "link",
+            "time",
+            "tags",
+            "tts",
+            "ttl",
+            "ttd",
+        ):
+            if field in updated and self[field] != updated[field]:
+                self[field] = updated[field]
+        # Actions are not updated since the available actions and associated
+        # content is left to the action executor to manage.
+
+
 class LocalSource:
     """
     An intake source backed by a filesystem directory.
@@ -49,38 +171,25 @@ class LocalSource:
     def item_exists(self, item_id) -> bool:
         return self.get_item_path(item_id).exists()
 
-    def new_item(self, item: dict) -> dict:
+    def get_item(self, item_id: str) -> Item:
-        # Ensure required fields
-        if "id" not in item:
-            raise KeyError("id")
-        item["source"] = self.source_name
-        item["active"] = True
-        item["created"] = int(time.time())
-        item["title"] = item.get("title", item["id"])
-        item["tags"] = item.get("tags", [self.source_name])
-        # All other fields are optiona
-        self.save_item(item)
-        return item
-
-    def get_item(self, item_id: str) -> dict:
         with self.get_item_path(item_id).open() as f:
-            return json.load(f)
+            return Item(self, json.load(f))
 
-    def save_item(self, item: dict) -> None:
+    def save_item(self, item: Item) -> None:
         # Write to a tempfile first to avoid losing the item on write failure
         item_path = self.get_item_path(item["id"])
         tmp_path = item_path.with_name(f"{item_path.name}.tmp")
         with tmp_path.open("w") as f:
-            f.write(json.dumps(item, indent=2))
+            f.write(item.serialize())
         os.rename(tmp_path, item_path)
 
     def delete_item(self, item_id) -> None:
         os.remove(self.get_item_path(item_id))
 
-    def get_all_items(self) -> List[dict]:
+    def get_all_items(self) -> List[Item]:
         for filepath in self.source_path.iterdir():
             if filepath.name.endswith(".item"):
-                yield json.loads(filepath.read_text(encoding="utf8"))
+                yield Item(self, json.loads(filepath.read_text(encoding="utf8")))
 
 
 def _read_stdout(process: Popen, output: list) -> None:
@@ -188,7 +297,7 @@ def fetch_items(source: LocalSource, timeout: int = 60) -> List[dict]:
 
     for line in output:
         try:
-            item = json.loads(line)
+            item = Item.create(source, **json.loads(line))
             items.append(item)
         except json.JSONDecodeError:
             raise SourceUpdateException("invalid json")
@@ -202,23 +311,23 @@ def execute_action(
     """
     Execute the action for a feed source.
     """
-    item = source.get_item(item_id)
+    item: Item = source.get_item(item_id)
 
     output = _execute_source_action(
-        source, action, json.dumps(item), timedelta(timeout)
+        source, action, item.serialize(indent=False), timedelta(timeout)
     )
     if not output:
         raise SourceUpdateException("no item")
 
     try:
-        item = json.loads(output[0])
+        item = Item(source, json.loads(output[0]))
         source.save_item(item)
         return item
     except json.JSONDecodeError:
         raise SourceUpdateException("invalid json")
 
 
-def update_items(source: LocalSource, fetched_items):
+def update_items(source: LocalSource, fetched_items: List[Item]):
     """
     Update the source with a batch of new items, doing creations, updates, and
     deletions as necessary.
@@ -228,8 +337,8 @@ def update_items(source: LocalSource, fetched_items):
     print(f"Found {len(prior_ids)} prior items")
 
     # Determine which items are new and which are updates.
-    new_items = []
+    new_items: List[Item] = []
-    upd_items = []
+    upd_items: List[Item] = []
     for item in fetched_items:
         if source.item_exists(item["id"]):
             upd_items.append(item)
@@ -239,60 +348,24 @@ def update_items(source: LocalSource, fetched_items):
     # Write all the new items to the source directory.
     for item in new_items:
         # TODO: support on-create trigger
-        source.new_item(item)
+        source.save_item(item)
 
     # Update the other items using the fetched items' values.
     for upd_item in upd_items:
         old_item = source.get_item(upd_item["id"])
-        for field in (
+        old_item.update_from(upd_item)
-            "title",
+        source.save_item(old_item)
-            "tags",
-            "link",
-            "time",
-            "author",
-            "body",
-            "ttl",
-            "ttd",
-            "tts",
-        ):
-            if field in upd_item and old_item[field] != upd_item[field]:
-                old_item[field] = upd_item[field]
-        if "callback" in upd_item:
-            # Because of the way this update happens, any fields that are set
-            # in the callback when the item is new will keep their original
-            # values, as those values reappear in new_item on subsequent
-            # updates.
-            old_item["callback"] = {**old_item["callback"], **upd_item["callback"]}
 
     # Items are removed when they are old (not in the latest fetch) and
     # inactive. Some item fields change this basic behavior.
     del_count = 0
-    now = int(time.time())
+    # now = int(current_time())
     upd_ids = [item["id"] for item in upd_items]
     old_item_ids = [item_id for item_id in prior_ids if item_id not in upd_ids]
 
     for item_id in old_item_ids:
-        item = source.get_item(item_id)
+        if source.get_item(item_id).can_remove:
-        remove = not item["active"]
+            source.delete_item(item_id)
-
-        # The time-to-live field protects an item from removal until expiry.
-        # This is mainly used to avoid old items resurfacing when their source
-        # cannot guarantee monotonicity.
-        if "ttl" in item:
-            ttl_date = item["created"] + item["ttl"]
-            if ttl_date > now:
-                continue
-
-        # The time-to-die field puts a maximum lifespan on an item, removing it
-        # even if it is active.
-        if "ttd" in item:
-            ttd_date = item["created"] + item["ttd"]
-            if ttd_date < now:
-                remove = True
-
-        # Items to be removed are deleted.
-        if remove:
-            source.delete_item(item["id"])
             del_count += 1
 
     print(len(new_items), "new,", del_count, "deleted")

intake/templates/feed.jinja2

@@ -127,8 +127,8 @@ var doAction = function (source, itemid, action) {
 {% if items %}
 {% for item in items %}
 	<div class="readable-item
-	{%- if not item.active %} strikethru fade{% endif %}
+	{%- if not item.active %} strikethru{% endif %}
-	{%- if item.active and item.tts and item.created + item.tts > now %} fade{% endif -%}
+	{%- if item.is_hidden %} fade{% endif -%}
 	" id="{{item.source}}-{{item.id}}">
 	{% if item.id %}
 	<button class="item-button" onclick="javascript:deactivate('{{item.source}}', '{{item.id}}')" title="Deactivate">&#10005;</button>
@@ -143,7 +143,7 @@ var doAction = function (source, itemid, action) {
 	{# The item title is a clickable <summary> if there is body content #}
 	{% if item.body or item.action %}
 	<details>
-	<summary><span class="item-title">{{item.title}}</span></summary>
+	<summary><span class="item-title">{{item.display_title}}</span></summary>
 	{% if item.body %}
 	<p>{{item.body|safe}}</p>
 	{% endif %}
@@ -152,7 +152,7 @@ var doAction = function (source, itemid, action) {
 	{% endfor %}
 	</details>
 	{% else %}
-	<span class="item-title">{{item.title}}</span><br>
+	<span class="item-title">{{item.display_title}}</span><br>
 	{% endif %}
 
 	{# author/time footer line #}