diff --git a/inquisitor/__init__.py b/inquisitor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/inquisitor/__main__.py b/inquisitor/__main__.py new file mode 100644 index 0000000..f3c8323 --- /dev/null +++ b/inquisitor/__main__.py @@ -0,0 +1,5 @@ +from cli import run + +if __name__ == "__main__": + run() + diff --git a/inquisitor/app.py b/inquisitor/app.py new file mode 100644 index 0000000..88b69a6 --- /dev/null +++ b/inquisitor/app.py @@ -0,0 +1,47 @@ +# Standard library imports +from datetime import datetime +import logging + +# Third party imports +from flask import Flask, render_template, request, jsonify + +# Application imports +from inquisitor import dungeon, core + +# Globals +logger = logging.getLogger("inquisitor.app") +logger.setLevel(logging.INFO) +console = logging.StreamHandler() +console.setLevel(logging.INFO) +formatter = logging.Formatter('[%(asctime)s %(levelname)s:%(filename)s:%(lineno)d] %(message)s') +console.setFormatter(formatter) +logger.addHandler(console) + +app = Flask(__name__) +dungeon = dungeon.Dungeon("dungeon") +itemsources = core.load_all_sources("sources") + + +@app.route("/feed/") +def root(): + active_items = dungeon.get_active_items() + logger.info("Found {} active items".format(len(active_items))) + for item in active_items: + item['time_readable'] = str(datetime.fromtimestamp(item['time'])) + active_items.sort(key=lambda i: i['time']) + return render_template("feed.html", items=active_items[:100]) + + +@app.route("/deactivate/", methods=['POST']) +def deactivate(): + params = request.get_json() + if 'source' not in params and 'itemid' not in params: + logger.error("Bad request params: {}".format(params)) + item = dungeon.deactivate_item(params['source'], params['itemid']) + return jsonify({'active': item['active']}) + + +@app.route("/feed.css") +def css(): + with open("feed.css") as f: + return f.read() diff --git a/inquisitor/cli.py b/inquisitor/cli.py new file mode 100644 index 0000000..4660d53 --- /dev/null +++ b/inquisitor/cli.py @@ -0,0 +1,55 @@ +# Standard library imports +import argparse +import logging +import os + +# Application imports +from core import load_all_sources +from dungeon import Dungeon + +# Globals +logger = logging.getLogger("inquisitor.cli") + + +def run(): + parser = argparse.ArgumentParser() + parser.add_argument("--log", default="INFO", help="Set the log level (default: INFO)") + subparsers = parser.add_subparsers(help="Command to execute", dest="command") + subparsers.required = True + + update_parser = subparsers.add_parser("update", help="Fetch new items") + update_parser.add_argument("--srcdir", help="Path to sources folder (default ./sources)", + default="./sources") + update_parser.add_argument("--dungeon", help="Path to item cache folder (default ./dungeon)", + default="./dungeon") + update_parser.add_argument("--sources", help="Sources to update, by name", + nargs="*") + update_parser.set_defaults(func=update) + + args = parser.parse_args() + + # Configure logging + loglevel = getattr(logging, args.log.upper()) + if not isinstance(loglevel, int): + raise ValueError("Invalid log level: {}".format(args.log)) + logging.basicConfig(format='[%(levelname)s:%(filename)s:%(lineno)d] %(message)s', level=loglevel) + + args.func(args) + + +def update(args): + """Fetches new items from sources and stores them in the dungeon.""" + if not os.path.isdir(args.srcdir): + logger.error("srcdir must be a directory") + exit(-1) + if not os.path.isdir(args.dungeon): + logger.error("dungeon must be a directory") + exit(-1) + sources = load_all_sources(args.srcdir) + names = args.sources or [s.SOURCE for s in sources] + dungeon = Dungeon(args.dungeon) + for itemsource in sources: + if itemsource.SOURCE in names: + new_items = dungeon.update(itemsource) + items = dungeon.get_active_items_for_folder(itemsource.SOURCE) + logger.info("{} new item{}".format(new_items, "s" if new_items != 1 else "")) diff --git a/inquisitor/core.py b/inquisitor/core.py new file mode 100644 index 0000000..5c4ee92 --- /dev/null +++ b/inquisitor/core.py @@ -0,0 +1,42 @@ +# Standard library imports +import importlib.util +import os +import logging + +# Globals +logger = logging.getLogger("inquisitor.core") + + +def load_source_module(source_path): + """Loads a source module and checks for necessary members.""" + logger.debug("load_source_module('{}')".format(source_path)) + spec = importlib.util.spec_from_file_location("itemsource", source_path) + itemsource = importlib.util.module_from_spec(spec) + spec.loader.exec_module(itemsource) + if not hasattr(itemsource, 'SOURCE'): + raise ImportError("SOURCE missing") + if not hasattr(itemsource, 'fetch_new'): + raise ImportError("fetch_new missing") + return itemsource + + +def load_all_sources(source_folder): + """Loads all source modules in the given folder.""" + # Navigate to the sources folder + cwd = os.getcwd() + os.chdir(source_folder) + # Load all sources + source_names = [ + filename + for filename in os.listdir() + if filename.endswith(".py")] + sources = [] + for source_name in source_names: + try: + itemsource = load_source_module(source_name) + sources.append(itemsource) + except ImportError as e: + logger.error("Error importing {}: {}".format(source_name, e)) + # Return to cwd + os.chdir(cwd) + return sources diff --git a/inquisitor/dungeon.py b/inquisitor/dungeon.py new file mode 100644 index 0000000..ddef37d --- /dev/null +++ b/inquisitor/dungeon.py @@ -0,0 +1,111 @@ +# Standard library imports +import os +import logging +import ast + + +class Dungeon(): + def __init__(self, path): + """ + Serves as an interface between Inquisitor and a folder of + serialized readable items. + """ + self.path = path + self.log = logging.getLogger("inquisitor.dungeon") + + def load_path(self, path): + self.log.debug("Loading item from {}".format(path)) + with open(path, 'r', encoding='utf-8') as f: + item = ast.literal_eval(f.read()) + return item + + def load_item(self, source, itemid): + item_path = os.path.join(self.path, source, itemid + ".item") + return self.load_path(item_path) + + def save_item(self, item): + path = os.path.join(self.path, item['source'], item['id'] + ".item") + self.log.debug("Saving item {} to {}".format(item['id'], path)) + with open(path, 'w', encoding='utf-8') as f: + f.write(str(item)) + + def update(self, itemsource): + """ + Fetches items from the given source, saves new active items, + and clears out old inactive items. + """ + new_items = 0 + self.log.info("Updating source {}".format(itemsource.SOURCE)) + # Check if the source has a folder. + source_folder = os.path.join(self.path, itemsource.SOURCE) + source_state = os.path.join(source_folder, "state") + if not os.path.isdir(source_folder): + self.log.info("Creating folder {}".format(source_folder)) + os.mkdir(source_folder) + # Initialize persistent state. + with open(source_state, 'w') as f: + f.write("{}") + # Load source persistent state. + state = self.load_path(source_state) + # Any inactive items that no longer show up as new should be + # removed. Track which items to check for inactivity. + extant_items_to_check = [ + filename + for filename in os.listdir(source_folder) + if filename.endswith(".item")] + # Get the new items from the source. + source_items = itemsource.fetch_new(state) + with open(source_state, 'w', encoding='utf-8') as f: + f.write(str(state)) + for source_item in source_items: + file_path = os.path.join(source_folder, source_item['id'] + ".item") + if os.path.isfile(file_path): + # Still-new items are exempt from activity checks. + extant_items_to_check.remove(source_item['id'] + ".item") + item = self.load_path(file_path) + if not item['active']: + # Don't reactivate inactive items. + continue + else: + new_items += 1 + # Add new items and update active ones. + self.save_item(source_item) + # Check old items for inactivity. + for extant_item_filename in extant_items_to_check: + file_path = os.path.join(source_folder, extant_item_filename) + item = self.load_path(file_path) + if not item['active']: + # Remove old inactive items. + self.log.info("Deleting {}".format(file_path)) + os.remove(file_path) + return new_items + + def get_active_items(self): + source_folders = os.listdir(self.path) + items = [] + for source_folder_name in source_folders: + items.extend(self.get_active_items_for_folder(source_folder_name)) + return items + + def get_active_items_for_folder(self, source): + source_folder = os.path.join(self.path, source) + item_filenames = os.listdir(source_folder) + items = [] + for item_filename in item_filenames: + if not item_filename.endswith(".item"): + continue + file_path = os.path.join(source_folder, item_filename) + item = self.load_path(file_path) + if item['active']: + items.append(item) + return items + + def deactivate_item(self, source, itemid): + item_path = os.path.join(self.path, source, itemid + ".item") + if not os.path.isfile(item_path): + self.log.error("No item found: {}".format(item_path)) + return + item = self.load_path(item_path) + item['active'] = False + self.save_item(item) + return item diff --git a/inquisitor/resources/feed.css b/inquisitor/resources/feed.css new file mode 100644 index 0000000..d76f799 --- /dev/null +++ b/inquisitor/resources/feed.css @@ -0,0 +1,39 @@ +div#wrapper { + max-width: 700px; + margin: 0 auto; +} +.readable-item { + border: 1px solid black; + border-radius: 6px; + padding: 5px; + margin-bottom: 20px; +} +.item-title { + font-size: 1.4em; +} +.readable-item button { + font-size: 1em; + float:right; +} +.item-link { + text-decoration: none; + float:right; + margin: 0px 2px; + font-size: 1em; + padding: 2px 7px; + border: 1px solid; + border-radius: 2px; +} +.item-info { + color: rgba(0, 0, 0, 0.7); +} +button, summary { + cursor: pointer; +} +summary:focus { + outline: 1px dotted gray; +} +.strikethru span, .strikethru p { + text-decoration: line-through; + color: rgba(0, 0, 0, 0.2); +} \ No newline at end of file diff --git a/inquisitor/templates/feed.html b/inquisitor/templates/feed.html new file mode 100644 index 0000000..0c8f8d0 --- /dev/null +++ b/inquisitor/templates/feed.html @@ -0,0 +1,51 @@ + + + + + + +
+ {% if items %} + {% for item in items %} +
+ + {% if item.link %} + ⇗ + {% endif %} + {% if item.body %} +
+ {{item.title}} +

{{item.body|safe}}

+
+ {% else %} + {{item.title}}
+ {% endif %} + {{item.time_readable}} {{item.author}}
+ {{item.source}} {{item.id}} {{item.time_readable}} +
+ {% endfor %} + {% else %} +
+ Feed is empty +
+ {% endif %} +
+ + \ No newline at end of file