Remove extension dependencies

flake.lock

@@ -16,6 +16,21 @@
         "type": "github"
       }
     },
+    "my-flake": {
+      "locked": {
+        "lastModified": 1670879933,
+        "narHash": "sha256-V45PH0cnFLilx66x4td5qQnWNn/V/6/6b7FQDIHvdyI=",
+        "owner": "Jaculabilis",
+        "repo": "my-flake",
+        "rev": "2b2cd07a6d971b15fc5f65d6d963d0da551a5892",
+        "type": "github"
+      },
+      "original": {
+        "owner": "Jaculabilis",
+        "repo": "my-flake",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1669833724,
@@ -35,6 +50,7 @@
     "root": {
       "inputs": {
         "flake-compat": "flake-compat",
+        "my-flake": "my-flake",
         "nixpkgs": "nixpkgs"
       }
     }

flake.nix

@@ -1,5 +1,6 @@
 {
   inputs = {
+    my-flake.url = "github:Jaculabilis/my-flake";
     nixpkgs.url = "github:NixOS/nixpkgs?ref=refs/tags/22.11";
     flake-compat = {
       url = "github:edolstra/flake-compat";
@@ -7,21 +8,64 @@
     };
   };
 
-  outputs = { self, nixpkgs, flake-compat }:
+  outputs = { self, my-flake, nixpkgs, flake-compat }:
   let
-    system = "x86_64-linux";
+    systems = [ "aarch64-linux" "x86_64-linux" ];
-    pkgs = nixpkgs.legacyPackages.${system};
+    each = system:
-  in
+    let
-  {
+      pkgs = (import nixpkgs {
-    packages.${system}.default =
+        inherit system;
-      (pkgs.poetry2nix.mkPoetryApplication {
+        overlays = [ self.overlays.default ];
-        projectDir = ./.;
+      });
-      }).dependencyEnv;
+    in {
+      packages.${system} = {
+        default = self.packages.${system}.inquisitor;
+        inquisitor = pkgs.inquisitor;
+        env = pkgs.inquisitor.dependencyEnv;
+      };
 
-    defaultPackage.${system} = self.packages.${system}.default;
+      devShells.${system} = {
+        default = self.devShells.${system}.inquisitor;
+        inquisitor = pkgs.mkShell {
+          buildInputs = [ (pkgs.python3.withPackages (p: [p.poetry])) ];
+          shellHook = ''
+            PS1="(inquisitor) $PS1"
+          '';
+        };
+        sources = pkgs.mkShell {
+          buildInputs = [ self.packages.${system}.env ];
+          shellHook = ''
+            PS1="(sources) $PS1"
+          '';
+        };
+      };
 
-    devShell.${system} = pkgs.mkShell {
+      checks.${system}.test-module = let
-      buildInputs = [ (pkgs.python3.withPackages (p: [p.poetry])) ];
+        test-lib = import "${nixpkgs}/nixos/lib/testing-python.nix" {
+          inherit system;
+        };
+      in
+        test-lib.makeTest {
+          name = "inquisitor-test-module";
+          nodes = {
+            host = { ... }: {
+              imports = [ self.nixosModules.default ];
+              services.inquisitor.enable = true;
+            };
+          };
+          testScript = ''
+            start_all()
+            host.wait_for_unit("multi-user.target")
+            host.succeed("[ -e /var/lib/inquisitor ]")
+          '';
+        };
     };
+  in (my-flake.outputs-for each systems) // {
+    overlays.default = final: prev: {
+      inquisitor = final.poetry2nix.mkPoetryApplication {
+        projectDir = ./.;
+      };
+    };
+    nixosModules.default = import ./module.nix self;
   };
 }

inquisitor/__main__.py

@@ -1,2 +1,3 @@
 from cli import main
+
 main()

inquisitor/app.py

@@ -8,13 +8,14 @@ from flask import Flask, render_template, request, jsonify, abort, redirect, url
 
 # Application imports
 from inquisitor.configs import (
-	DUNGEON_PATH,
+    DUNGEON_PATH,
-	SOURCES_PATH,
+    SOURCES_PATH,
-	CACHE_PATH,
+    CACHE_PATH,
-	subfeeds,
+    subfeeds,
-	get_subfeed_overrides,
+    get_subfeed_overrides,
-	logger,
+    logger,
-	init_default_logging)
+    init_default_logging,
+)
 from inquisitor import sources, loader, timestamp
 
 # Globals
@@ -22,173 +23,202 @@ app = Flask(__name__)
 
 
 def make_query_link(text, wl, bl):
-	wlp = "only=" + ",".join(wl)
+    wlp = "only=" + ",".join(wl)
-	blp = "not="  + ",".join(bl)
+    blp = "not=" + ",".join(bl)
-	params = [p for p in (wlp, blp) if not p.endswith("=")]
+    params = [p for p in (wlp, blp) if not p.endswith("=")]
-	query = "?{}".format("&".join(params))
+    query = "?{}".format("&".join(params))
-	return '<a href="{1}">{0}</a>'.format(text, query)
+    return '<a href="{1}">{0}</a>'.format(text, query)
+
 
 @app.template_filter("datetimeformat")
 def datetimeformat(value):
-	return timestamp.stamp_to_readable(value) if value is not None else ""
+    return timestamp.stamp_to_readable(value) if value is not None else ""
+
 
 @app.route("/")
 def root():
-	return redirect(url_for('feed'))
+    return redirect(url_for("feed"))
+
 
 @app.route("/feed/")
 def feed():
-	return feed_for_sources(source_names=None)
+    return feed_for_sources(source_names=None)
+
 
 @app.route("/feed/<string:feed_name>/")
 def subfeed(feed_name):
-	# Check for and apply subfeed overrides
+    # Check for and apply subfeed overrides
-	subfeed_overrides = get_subfeed_overrides()
+    subfeed_overrides = get_subfeed_overrides()
-	subfeed_config = subfeed_overrides or subfeeds or {}
+    subfeed_config = subfeed_overrides or subfeeds or {}
 
-	# The built-in inquisitor subfeed contains sources not in another subfeed
+    # The built-in inquisitor subfeed contains sources not in another subfeed
-	if feed_name == 'inquisitor':
+    if feed_name == "inquisitor":
-		all_sources = os.listdir(DUNGEON_PATH)
+        all_sources = os.listdir(DUNGEON_PATH)
-		for subfeed, sources in subfeed_config.items():
+        for subfeed, sources in subfeed_config.items():
-			for source_name in sources:
+            for source_name in sources:
-				if source_name in all_sources:
+                if source_name in all_sources:
-					all_sources.remove(source_name)
+                    all_sources.remove(source_name)
-		return feed_for_sources(all_sources)
+        return feed_for_sources(all_sources)
+
+    if feed_name not in subfeed_config:
+        return abort(404)
+    return feed_for_sources(subfeed_config[feed_name])
 
-	if feed_name not in subfeed_config:
-		return abort(404)
-	return feed_for_sources(subfeed_config[feed_name])
 
 def feed_for_sources(source_names):
-	# Determine exclusion filters
+    # Determine exclusion filters
-	filters = []
+    filters = []
-	wl_param = request.args.get('only')
+    wl_param = request.args.get("only")
-	wl = wl_param.split(",") if wl_param else []
+    wl = wl_param.split(",") if wl_param else []
-	bl_param = request.args.get('not')
+    bl_param = request.args.get("not")
-	bl = bl_param.split(",") if bl_param else []
+    bl = bl_param.split(",") if bl_param else []
-	if wl:
+    if wl:
-		filters.append(lambda item: not any([tag in wl for tag in item['tags']]))
+        filters.append(lambda item: not any([tag in wl for tag in item["tags"]]))
-	if bl:
+    if bl:
-		filters.append(lambda item: any([tag in bl for tag in item['tags']]))
+        filters.append(lambda item: any([tag in bl for tag in item["tags"]]))
 
-	# Get all active+filtered items and all active tags
+    # Get all active+filtered items and all active tags
-	total = 0
+    total = 0
-	items, errors = loader.load_active_items(source_names)
+    items, errors = loader.load_active_items(source_names)
-	active_items = []
+    active_items = []
-	active_tags = {}
+    active_tags = {}
-	for item in items:
+    for item in items:
-		if item['active']:
+        if item["active"]:
-			for tag in item['tags']:
+            for tag in item["tags"]:
-				if tag not in active_tags: active_tags[tag] = 0
+                if tag not in active_tags:
-				active_tags[tag] += 1
+                    active_tags[tag] = 0
-			# active_tags |= set(item['tags'])
+                active_tags[tag] += 1
-			total += 1
+            # active_tags |= set(item['tags'])
-			if not any(map(lambda f: f(item), filters)):
+            total += 1
-				active_items.append(item)
+            if not any(map(lambda f: f(item), filters)):
-	# Sort items by time
+                active_items.append(item)
-	active_items.sort(key=lambda i: i['time'] if 'time' in i and i['time'] else i['created'] if 'created' in i and i['created'] else 0)
+    # Sort items by time
+    active_items.sort(
+        key=lambda i: i["time"]
+        if "time" in i and i["time"]
+        else i["created"]
+        if "created" in i and i["created"]
+        else 0
+    )
 
-	logger.info("Returning {} of {} items".format(len(active_items), total))
+    logger.info("Returning {} of {} items".format(len(active_items), total))
-	if errors:
+    if errors:
-		read_ex = {
+        read_ex = {
-			'title': 'Read errors',
+            "title": "Read errors",
-			'body': "<pre>{}</pre>".format("\n\n".join(errors)),
+            "body": "<pre>{}</pre>".format("\n\n".join(errors)),
-			'created': None,
+            "created": None,
-		}
+        }
-		active_items.insert(0, read_ex)
+        active_items.insert(0, read_ex)
 
-	if total > 0:
+    if total > 0:
-		# Create the feed control item
+        # Create the feed control item
-		link_table = ["<tr><td>{0}</td><td>{1}</td><td></td><td></td></tr>".format(
+        link_table = [
-			total, make_query_link("all", [], []))]
+            "<tr><td>{0}</td><td>{1}</td><td></td><td></td></tr>".format(
-		for tag, count in sorted(active_tags.items(), key=lambda i: i[0].lower()):
+                total, make_query_link("all", [], [])
-			links = [count]
+            )
-			links.append(make_query_link(tag, [tag], []))
+        ]
-			if tag in wl:
+        for tag, count in sorted(active_tags.items(), key=lambda i: i[0].lower()):
-				new_wl = [t for t in wl if t != tag]
+            links = [count]
-				links.append(make_query_link("-only", new_wl, bl))
+            links.append(make_query_link(tag, [tag], []))
-			else:
+            if tag in wl:
-				new_bl = [t for t in bl if t != tag]
+                new_wl = [t for t in wl if t != tag]
-				links.append(make_query_link("+only", wl + [tag], new_bl))
+                links.append(make_query_link("-only", new_wl, bl))
-			if tag in bl:
+            else:
-				new_bl = [t for t in bl if t != tag]
+                new_bl = [t for t in bl if t != tag]
-				links.append(make_query_link("-not", wl, new_bl))
+                links.append(make_query_link("+only", wl + [tag], new_bl))
-			else:
+            if tag in bl:
-				new_wl = [t for t in wl if t != tag]
+                new_bl = [t for t in bl if t != tag]
-				links.append(make_query_link("+not", new_wl, bl + [tag]))
+                links.append(make_query_link("-not", wl, new_bl))
-			link_table.append("<tr><td>{0}</td><td>{1}</td><td>{2}</td><td>{3}</td></tr>".format(*links))
+            else:
-		body = '<table class="feed-control">{}</table>'.format("\n".join(link_table))
+                new_wl = [t for t in wl if t != tag]
+                links.append(make_query_link("+not", new_wl, bl + [tag]))
+            link_table.append(
+                "<tr><td>{0}</td><td>{1}</td><td>{2}</td><td>{3}</td></tr>".format(
+                    *links
+                )
+            )
+        body = '<table class="feed-control">{}</table>'.format("\n".join(link_table))
 
-		feed_control = {
+        feed_control = {
-			'title': 'Feed Control [{}/{}]'.format(len(active_items), total),
+            "title": "Feed Control [{}/{}]".format(len(active_items), total),
-			'body': body,
+            "body": body,
-		}
+        }
-		active_items.insert(0, feed_control)
+        active_items.insert(0, feed_control)
 
-	selection = active_items[:100]
+    selection = active_items[:100]
 
-	return render_template("feed.jinja2",
+    return render_template(
-		items=selection,
+        "feed.jinja2",
-		mdeac=[
+        items=selection,
-			{'source': item['source'], 'itemid': item['id']}
+        mdeac=[
-			for item in selection
+            {"source": item["source"], "itemid": item["id"]}
-			if 'id' in item])
+            for item in selection
+            if "id" in item
+        ],
+    )
 
-@app.route("/deactivate/", methods=['POST'])
+
+@app.route("/deactivate/", methods=["POST"])
 def deactivate():
-	params = request.get_json()
+    params = request.get_json()
-	if 'source' not in params and 'itemid' not in params:
+    if "source" not in params and "itemid" not in params:
-		logger.error("Bad request params: {}".format(params))
+        logger.error("Bad request params: {}".format(params))
-	item = loader.load_item(params['source'], params['itemid'])
+    item = loader.load_item(params["source"], params["itemid"])
-	if item['active']:
+    if item["active"]:
-		logger.debug(f"Deactivating {params['source']}/{params['itemid']}")
+        logger.debug(f"Deactivating {params['source']}/{params['itemid']}")
-	item['active'] = False
+    item["active"] = False
-	return jsonify({'active': item['active']})
+    return jsonify({"active": item["active"]})
 
-@app.route("/punt/", methods=['POST'])
+
+@app.route("/punt/", methods=["POST"])
 def punt():
-	params = request.get_json()
+    params = request.get_json()
-	if 'source' not in params and 'itemid' not in params:
+    if "source" not in params and "itemid" not in params:
-		logger.error("Bad request params: {}".format(params))
+        logger.error("Bad request params: {}".format(params))
-	item = loader.load_item(params['source'], params['itemid'])
+    item = loader.load_item(params["source"], params["itemid"])
-	tomorrow = datetime.now() + timedelta(days=1)
+    tomorrow = datetime.now() + timedelta(days=1)
-	morning = datetime(tomorrow.year, tomorrow.month, tomorrow.day, 6, 0, 0)
+    morning = datetime(tomorrow.year, tomorrow.month, tomorrow.day, 6, 0, 0)
-	til_then = morning.timestamp() - item['created']
+    til_then = morning.timestamp() - item["created"]
-	item['tts'] = til_then
+    item["tts"] = til_then
-	return jsonify(item.item)
+    return jsonify(item.item)
 
-@app.route("/mass-deactivate/", methods=['POST'])
+
+@app.route("/mass-deactivate/", methods=["POST"])
 def mass_deactivate():
-	params = request.get_json()
+    params = request.get_json()
-	if 'items' not in params:
+    if "items" not in params:
-		logger.error("Bad request params: {}".format(params))
+        logger.error("Bad request params: {}".format(params))
-	for info in params.get('items', []):
+    for info in params.get("items", []):
-		source = info['source']
+        source = info["source"]
-		itemid = info['itemid']
+        itemid = info["itemid"]
-		item = loader.load_item(source, itemid)
+        item = loader.load_item(source, itemid)
-		if item['active']:
+        if item["active"]:
-			logger.debug(f"Deactivating {info['source']}/{info['itemid']}")
+            logger.debug(f"Deactivating {info['source']}/{info['itemid']}")
-		item['active'] = False
+        item["active"] = False
-	return jsonify({})
+    return jsonify({})
 
-@app.route("/callback/", methods=['POST'])
+
+@app.route("/callback/", methods=["POST"])
 def callback():
-	params = request.get_json()
+    params = request.get_json()
-	if 'source' not in params and 'itemid' not in params:
+    if "source" not in params and "itemid" not in params:
-		logger.error("Bad request params: {}".format(params))
+        logger.error("Bad request params: {}".format(params))
-	logger.info('Executing callback for {}/{}'.format(params['source'], params['itemid']))
+    logger.info(
-	sources.item_callback(params['source'], params['itemid'])
+        "Executing callback for {}/{}".format(params["source"], params["itemid"])
-	return jsonify({})
+    )
+    sources.item_callback(params["source"], params["itemid"])
+    return jsonify({})
 
-@app.route('/cache/<path:cache_path>')
+
+@app.route("/cache/<path:cache_path>")
 def cache(cache_path):
-	path = os.path.join(CACHE_PATH, cache_path)
+    path = os.path.join(CACHE_PATH, cache_path)
-	if not os.path.isfile(path):
+    if not os.path.isfile(path):
-		return abort(404)
+        return abort(404)
-	with open(path, 'rb') as f:
+    with open(path, "rb") as f:
-		return f.read()
+        return f.read()
 
 
 def wsgi():
-	init_default_logging()
+    init_default_logging()
-	return app
+    return app

inquisitor/cli.py

@@ -11,280 +11,324 @@ from inquisitor.configs import logger, DUNGEON_PATH, SOURCES_PATH, add_logging_h
 
 
 def command_test(args):
-	"""Echo config file values."""
+    """Echo config file values."""
-	from inquisitor.configs.resolver import (
+    from inquisitor.configs.resolver import (
-		config_path,
+        config_path,
-		CONFIG_DATA, data_path,
+        CONFIG_DATA,
-		CONFIG_SOURCES, source_path,
+        data_path,
-		CONFIG_CACHE, cache_path,
+        CONFIG_SOURCES,
-		CONFIG_LOGFILE, log_file,
+        source_path,
-		CONFIG_VERBOSE, is_verbose,
+        CONFIG_CACHE,
-		CONFIG_SUBFEEDS, subfeeds,
+        cache_path,
-	)
+        CONFIG_LOGFILE,
-	subfeeds = '; '.join(
+        log_file,
-		'{0}: {1}'.format(
+        CONFIG_VERBOSE,
-			sf_name,
+        is_verbose,
-			' '.join(sf_sources)
+        CONFIG_SUBFEEDS,
-		)
+        subfeeds,
-		for sf_name, sf_sources
+    )
-		in subfeeds.items()
+
-	) if subfeeds else ''
+    subfeeds = (
-	print(f'Inquisitor configured from {config_path}')
+        "; ".join(
-	print(f'    {CONFIG_DATA} = {data_path}')
+            "{0}: {1}".format(sf_name, " ".join(sf_sources))
-	print(f'    {CONFIG_SOURCES} = {source_path}')
+            for sf_name, sf_sources in subfeeds.items()
-	print(f'    {CONFIG_CACHE} = {cache_path}')
+        )
-	print(f'    {CONFIG_LOGFILE} = {log_file}')
+        if subfeeds
-	print(f'    {CONFIG_VERBOSE} = {is_verbose}')
+        else ""
-	print(f'    {CONFIG_SUBFEEDS} = {subfeeds}')
+    )
-	return 0
+    print(f"Inquisitor configured from {config_path}")
+    print(f"    {CONFIG_DATA} = {data_path}")
+    print(f"    {CONFIG_SOURCES} = {source_path}")
+    print(f"    {CONFIG_CACHE} = {cache_path}")
+    print(f"    {CONFIG_LOGFILE} = {log_file}")
+    print(f"    {CONFIG_VERBOSE} = {is_verbose}")
+    print(f"    {CONFIG_SUBFEEDS} = {subfeeds}")
+    return 0
 
 
 def command_update(args):
-	"""Fetch and store new items from the specified sources."""
+    """Fetch and store new items from the specified sources."""
-	parser = argparse.ArgumentParser(
+    parser = argparse.ArgumentParser(
-		prog="inquisitor update",
+        prog="inquisitor update", description=command_update.__doc__, add_help=False
-		description=command_update.__doc__,
+    )
-		add_help=False)
+    parser.add_argument("source", nargs="*", help="Sources to update.")
-	parser.add_argument("source",
+    args = parser.parse_args(args)
-		nargs="*",
-		help="Sources to update.")
-	args = parser.parse_args(args)
 
-	if len(args.source) == 0:
+    if len(args.source) == 0:
-		parser.print_help()
+        parser.print_help()
-		return 0
+        return 0
-	if not os.path.isdir(DUNGEON_PATH):
+    if not os.path.isdir(DUNGEON_PATH):
-		logger.error("Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon")
+        logger.error(
-		return -1
+            "Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon"
-	if not os.path.isdir(SOURCES_PATH):
+        )
-		logger.error("Couldn't find sources. Set INQUISITOR_SOURCES or cd to parent folder of ./sources")
+        return -1
+    if not os.path.isdir(SOURCES_PATH):
+        logger.error(
+            "Couldn't find sources. Set INQUISITOR_SOURCES or cd to parent folder of ./sources"
+        )
 
-	# Update sources
+    # Update sources
-	from inquisitor.sources import update_sources
+    from inquisitor.sources import update_sources
-	update_sources(*args.source)
+
-	return 0
+    update_sources(*args.source)
+    return 0
 
 
 def command_deactivate(args):
-	"""Deactivate all items in the specified dungeon cells."""
+    """Deactivate all items in the specified dungeon cells."""
-	parser = argparse.ArgumentParser(
+    parser = argparse.ArgumentParser(
-		prog="inquisitor deactivate",
+        prog="inquisitor deactivate",
-		description=command_deactivate.__doc__,
+        description=command_deactivate.__doc__,
-		add_help=False)
+        add_help=False,
-	parser.add_argument("source",
+    )
-		nargs="*",
+    parser.add_argument("source", nargs="*", help="Cells to deactivate.")
-		help="Cells to deactivate.")
+    parser.add_argument("--tag", help="Only deactivate items with this tag")
-	parser.add_argument("--tag",
+    parser.add_argument(
-		help="Only deactivate items with this tag")
+        "--title", help="Only deactivate items with titles containing this substring"
-	parser.add_argument("--title",
+    )
-		help="Only deactivate items with titles containing this substring")
+    args = parser.parse_args(args)
-	args = parser.parse_args(args)
 
-	if len(args.source) == 0:
+    if len(args.source) == 0:
-		parser.print_help()
+        parser.print_help()
-		return 0
+        return 0
-	if not os.path.isdir(DUNGEON_PATH):
+    if not os.path.isdir(DUNGEON_PATH):
-		logger.error("Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon")
+        logger.error(
-		return -1
+            "Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon"
+        )
+        return -1
 
-	# Deactivate all items in each source.
+    # Deactivate all items in each source.
-	from inquisitor.loader import load_items
+    from inquisitor.loader import load_items
-	for source_name in args.source:
-		path = os.path.join(DUNGEON_PATH, source_name)
-		if not os.path.isdir(path):
-			logger.warning("'{}' is not an extant source".format(source_name))
-		count = 0
-		items, _ = load_items(source_name)
-		for item in items.values():
-			if args.tag and args.tag not in item['tags']:
-				continue
-			if args.title and args.title not in item['title']:
-				continue
-			if item['active']:
-				item['active'] = False
-				count += 1
-		logger.info("Deactivated {} items in '{}'".format(count, source_name))
 
-	return 0
+    for source_name in args.source:
+        path = os.path.join(DUNGEON_PATH, source_name)
+        if not os.path.isdir(path):
+            logger.warning("'{}' is not an extant source".format(source_name))
+        count = 0
+        items, _ = load_items(source_name)
+        for item in items.values():
+            if args.tag and args.tag not in item["tags"]:
+                continue
+            if args.title and args.title not in item["title"]:
+                continue
+            if item["active"]:
+                item["active"] = False
+                count += 1
+        logger.info("Deactivated {} items in '{}'".format(count, source_name))
+
+    return 0
 
 
 def command_add(args):
-	"""Creates an item."""
+    """Creates an item."""
-	parser = argparse.ArgumentParser(
+    parser = argparse.ArgumentParser(
-		prog="inquisitor add",
+        prog="inquisitor add", description=command_add.__doc__, add_help=False
-		description=command_add.__doc__,
+    )
-		add_help=False)
+    parser.add_argument("--id", help="String")
-	parser.add_argument("--id", help="String")
+    parser.add_argument("--source", help="String")
-	parser.add_argument("--source", help="String")
+    parser.add_argument("--title", help="String")
-	parser.add_argument("--title", help="String")
+    parser.add_argument("--link", help="URL")
-	parser.add_argument("--link", help="URL")
+    parser.add_argument("--time", type=int, help="Unix timestmap")
-	parser.add_argument("--time", type=int, help="Unix timestmap")
+    parser.add_argument("--author", help="String")
-	parser.add_argument("--author", help="String")
+    parser.add_argument("--body", help="HTML")
-	parser.add_argument("--body", help="HTML")
+    parser.add_argument("--tags", help="Comma-separated list")
-	parser.add_argument("--tags", help="Comma-separated list")
+    parser.add_argument("--ttl", type=int, help="Cleanup protection in seconds")
-	parser.add_argument("--ttl", type=int, help="Cleanup protection in seconds")
+    parser.add_argument("--ttd", type=int, help="Cleanup force in seconds")
-	parser.add_argument("--ttd", type=int, help="Cleanup force in seconds")
+    parser.add_argument("--tts", type=int, help="Display delay in seconds")
-	parser.add_argument("--tts", type=int, help="Display delay in seconds")
+    parser.add_argument(
-	parser.add_argument("--create", action="store_true", help="Create source if it doesn't exist")
+        "--create", action="store_true", help="Create source if it doesn't exist"
-	args = parser.parse_args(args)
+    )
+    args = parser.parse_args(args)
 
-	if not args.title:
+    if not args.title:
-		parser.print_help()
+        parser.print_help()
-		return 0
+        return 0
-	if not os.path.isdir(DUNGEON_PATH):
+    if not os.path.isdir(DUNGEON_PATH):
-		logger.error("Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon")
+        logger.error(
-		return -1
+            "Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon"
+        )
+        return -1
 
-	source = args.source or 'inquisitor'
+    source = args.source or "inquisitor"
-	cell_path = os.path.join(DUNGEON_PATH, source)
+    cell_path = os.path.join(DUNGEON_PATH, source)
-	if args.create:
+    if args.create:
-		from inquisitor.sources import ensure_cell
+        from inquisitor.sources import ensure_cell
-		ensure_cell(source)
-	elif not os.path.isdir(cell_path):
-		logger.error("Source '{}' does not exist".format(source))
-		return -1
 
-	item = {
+        ensure_cell(source)
-		'id': args.id or '{:x}'.format(random.getrandbits(16 * 4)),
+    elif not os.path.isdir(cell_path):
-		'source': source,
+        logger.error("Source '{}' does not exist".format(source))
-	}
+        return -1
-	if args.title: item['title'] = str(args.title)
-	if args.link: item['link'] = str(args.link)
-	if args.time: item['time'] = int(args.time)
-	if args.author: item['author'] = str(args.author)
-	if args.body: item['body'] = str(args.body)
-	if args.tags: item['tags'] = [str(tag) for tag in args.tags.split(",")]
-	if args.ttl: item['ttl'] = int(args.ttl)
-	if args.ttd: item['ttd'] = int(args.ttd)
-	if args.tts: item['tts'] = int(args.tts)
 
-	from inquisitor.loader import new_item
+    item = {
-	saved_item = new_item(source, item)
+        "id": args.id or "{:x}".format(random.getrandbits(16 * 4)),
-	logger.info(saved_item)
+        "source": source,
+    }
+    if args.title:
+        item["title"] = str(args.title)
+    if args.link:
+        item["link"] = str(args.link)
+    if args.time:
+        item["time"] = int(args.time)
+    if args.author:
+        item["author"] = str(args.author)
+    if args.body:
+        item["body"] = str(args.body)
+    if args.tags:
+        item["tags"] = [str(tag) for tag in args.tags.split(",")]
+    if args.ttl:
+        item["ttl"] = int(args.ttl)
+    if args.ttd:
+        item["ttd"] = int(args.ttd)
+    if args.tts:
+        item["tts"] = int(args.tts)
+
+    from inquisitor.loader import new_item
+
+    saved_item = new_item(source, item)
+    logger.info(saved_item)
 
 
 def command_feed(args):
-	"""Print the current feed."""
+    """Print the current feed."""
-	if not os.path.isdir(DUNGEON_PATH):
+    if not os.path.isdir(DUNGEON_PATH):
-		logger.error("Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon")
+        logger.error(
-		return -1
+            "Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon"
+        )
+        return -1
 
-	import shutil
+    import shutil
-	from inquisitor import loader
+    from inquisitor import loader
-	from inquisitor import timestamp
+    from inquisitor import timestamp
 
-	items, errors = loader.load_active_items(source_names=None)
+    items, errors = loader.load_active_items(source_names=None)
-	if not items and not errors:
+    if not items and not errors:
-		print("Feed is empty")
+        print("Feed is empty")
-		return 0
+        return 0
 
-	if errors:
+    if errors:
-		items.insert(0, {
+        items.insert(
-			'title': '{} read errors: {}'.format(len(errors), ' '.join(errors)),
+            0,
-			'body': "\n".join(errors)
+            {
-		})
+                "title": "{} read errors: {}".format(len(errors), " ".join(errors)),
+                "body": "\n".join(errors),
+            },
+        )
 
-	size = shutil.get_terminal_size((80, 20))
+    size = shutil.get_terminal_size((80, 20))
-	width = min(80, size.columns)
+    width = min(80, size.columns)
 
-	for item in items:
+    for item in items:
-		title = item['title'] if 'title' in item else ""
+        title = item["title"] if "title" in item else ""
-		titles = [title]
+        titles = [title]
-		while len(titles[-1]) > width - 4:
+        while len(titles[-1]) > width - 4:
-			i = titles[-1][:width - 4].rfind(' ')
+            i = titles[-1][: width - 4].rfind(" ")
-			titles = titles[:-1] + [titles[-1][:i].strip(), titles[-1][i:].strip()]
+            titles = titles[:-1] + [titles[-1][:i].strip(), titles[-1][i:].strip()]
-		print('+' + (width - 2) * '-' + '+')
+        print("+" + (width - 2) * "-" + "+")
-		for title in titles:
+        for title in titles:
-			print("| {0:<{1}} |".format(title, width - 4))
+            print("| {0:<{1}} |".format(title, width - 4))
-		print("|{0:<{1}}|".format("", width - 2))
+        print("|{0:<{1}}|".format("", width - 2))
-		info1 = ""
+        info1 = ""
-		if 'author' in title and item['author']:
+        if "author" in title and item["author"]:
-			info1 += item['author'] + "  "
+            info1 += item["author"] + "  "
-		if 'time' in item and item['time']:
+        if "time" in item and item["time"]:
-			info1 += timestamp.stamp_to_readable(item['time'])
+            info1 += timestamp.stamp_to_readable(item["time"])
-		print("| {0:<{1}} |".format(info1, width - 4))
+        print("| {0:<{1}} |".format(info1, width - 4))
-		created = timestamp.stamp_to_readable(item['created']) if 'created' in item else ""
+        created = (
-		info2 = "{0}  {1}  {2}".format(
+            timestamp.stamp_to_readable(item["created"]) if "created" in item else ""
-			item.get('source', ''), item.get('id', ''), created)
+        )
-		print("| {0:<{1}} |".format(info2, width - 4))
+        info2 = "{0}  {1}  {2}".format(
-		print('+' + (width - 2) * '-' + '+')
+            item.get("source", ""), item.get("id", ""), created
-		print()
+        )
+        print("| {0:<{1}} |".format(info2, width - 4))
+        print("+" + (width - 2) * "-" + "+")
+        print()
 
 
 def command_run(args):
-	"""Run the default Flask server."""
+    """Run the default Flask server."""
-	parser = argparse.ArgumentParser(
+    parser = argparse.ArgumentParser(
-		prog="inquisitor run",
+        prog="inquisitor run", description=command_run.__doc__, add_help=False
-		description=command_run.__doc__,
+    )
-		add_help=False)
+    parser.add_argument("--debug", action="store_true")
-	parser.add_argument("--debug", action="store_true")
+    parser.add_argument("--port", type=int, default=5000)
-	parser.add_argument("--port", type=int, default=5000)
+    args = parser.parse_args(args)
-	args = parser.parse_args(args)
 
-	try:
+    try:
-		from inquisitor.app import app
+        from inquisitor.app import app
-		app.run(port=args.port, debug=args.debug)
+
-		return 0
+        app.run(port=args.port, debug=args.debug)
-	except Exception as e:
+        return 0
-		logger.error(e)
+    except Exception as e:
-		return -1
+        logger.error(e)
+        return -1
 
 
 def command_help(args):
-	"""Print this help message and exit."""
+    """Print this help message and exit."""
-	print_usage()
+    print_usage()
-	return 0
+    return 0
+
+
+def nocommand(args):
+    print("command required")
+    return 0
 
 
 def main():
-	"""CLI entry point"""
+    """CLI entry point"""
-	# Enable piping
+    # Enable piping
-	from signal import signal, SIGPIPE, SIG_DFL
+    from signal import signal, SIGPIPE, SIG_DFL
-	signal(SIGPIPE, SIG_DFL)
 
-	# Collect the commands from this module
+    signal(SIGPIPE, SIG_DFL)
-	import inquisitor.cli
-	commands = {
-		name[8:] : func
-		for name, func in vars(inquisitor.cli).items()
-		if name.startswith('command_')
-	}
-	descriptions = "\n".join([
-		"- {0}: {1}".format(name, func.__doc__)
-		for name, func in commands.items()])
 
-	# Set up the parser
+    # Collect the commands from this module
-	parser = argparse.ArgumentParser(
+    import inquisitor.cli
-		description="Available commands:\n{}\n".format(descriptions),
-		formatter_class=argparse.RawDescriptionHelpFormatter,
-		add_help=False)
-	parser.add_argument("command",
-		nargs="?",
-		default="help",
-		help="The command to execute",
-		choices=commands,
-		metavar="command")
-	parser.add_argument("args",
-		nargs=argparse.REMAINDER,
-		help="Command arguments",
-		metavar="args")
-	parser.add_argument("-v",
-		action="store_true",
-		dest="verbose",
-		help="Enable debug logging")
 
-	# Extract the usage print for command_help
+    commands = {
-	global print_usage
+        name[8:]: func
-	print_usage = parser.print_help
+        for name, func in vars(inquisitor.cli).items()
+        if name.startswith("command_")
+    }
+    descriptions = "\n".join(
+        ["- {0}: {1}".format(name, func.__doc__) for name, func in commands.items()]
+    )
 
-	args = parser.parse_args()
+    # Set up the parser
+    parser = argparse.ArgumentParser(
+        description="Available commands:\n{}\n".format(descriptions),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        add_help=False,
+    )
+    parser.add_argument(
+        "command",
+        nargs="?",
+        default="help",
+        help="The command to execute",
+        choices=commands,
+        metavar="command",
+    )
+    parser.add_argument(
+        "args", nargs=argparse.REMAINDER, help="Command arguments", metavar="args"
+    )
+    parser.add_argument(
+        "-v", action="store_true", dest="verbose", help="Enable debug logging"
+    )
 
-	# Initialize a console logger
+    # Extract the usage print for command_help
-	add_logging_handler(verbose=args.verbose, log_filename=None)
+    global print_usage
+    print_usage = parser.print_help
 
-	# Execute command
+    args = parser.parse_args()
-	if args.command:
+
-		sys.exit(commands[args.command](args.args))
+    # Initialize a console logger
-	else:
+    add_logging_handler(verbose=args.verbose, log_filename=None)
-		print("command required")
+
-		sys.exit(0)
+    # Execute command
+    try:
+        command = commands.get(args.command, nocommand)
+        sys.exit(command(args.args))
+    except BrokenPipeError:
+        # See https://docs.python.org/3.10/library/signal.html#note-on-sigpipe
+        devnull = os.open(os.devnull, os.O_WRONLY)
+        os.dup2(devnull, sys.stdout.fileno())
+        sys.exit(1)

inquisitor/configs/__init__.py

@@ -1,10 +1,5 @@
 from .resolver import data_path as DUNGEON_PATH
 from .resolver import source_path as SOURCES_PATH
 from .resolver import cache_path as CACHE_PATH
-from .resolver import (
+from .resolver import logger, subfeeds
-	logger,
+from .resolver import add_logging_handler, init_default_logging, get_subfeed_overrides
-	subfeeds)
-from .resolver import (
-	add_logging_handler,
-	init_default_logging,
-	get_subfeed_overrides)

inquisitor/configs/resolver.py

@@ -4,173 +4,172 @@ import logging
 
 # Constants governing config resolution:
 # Path to the config file, containing key-value pairs of the other settings
-CONFIG_ENVVAR = 'INQUISITOR_CONFIG'
+CONFIG_ENVVAR = "INQUISITOR_CONFIG"
-DEFAULT_CONFIG_PATH = '/etc/inquisitor.conf'
+DEFAULT_CONFIG_PATH = "/etc/inquisitor.conf"
 
 # Path to the folder where items are stored
-CONFIG_DATA = 'DataPath'
+CONFIG_DATA = "DataPath"
-DEFAULT_DATA_PATH = '/var/inquisitor/data/'
+DEFAULT_DATA_PATH = "/var/inquisitor/data/"
 
 # Path to the folder where source modules are stored
-CONFIG_SOURCES = 'SourcePath'
+CONFIG_SOURCES = "SourcePath"
-DEFAULT_SOURCES_PATH = '/var/inquisitor/sources/'
+DEFAULT_SOURCES_PATH = "/var/inquisitor/sources/"
 
 # Path to the folder where cached files are stored
-CONFIG_CACHE = 'CachePath'
+CONFIG_CACHE = "CachePath"
-DEFAULT_CACHE_PATH = '/var/inquisitor/cache/'
+DEFAULT_CACHE_PATH = "/var/inquisitor/cache/"
 
 # Path to a log file where logging will be redirected
-CONFIG_LOGFILE = 'LogFile'
+CONFIG_LOGFILE = "LogFile"
 DEFAULT_LOG_FILE = None
 
 # Whether logging is verbose
-CONFIG_VERBOSE = 'Verbose'
+CONFIG_VERBOSE = "Verbose"
-DEFAULT_VERBOSITY = 'false'
+DEFAULT_VERBOSITY = "false"
 
 # Subfeed source lists, with each subfeed config separated by lines and
 # sources within a subfeed separated by spaces
-CONFIG_SUBFEEDS = 'Subfeeds'
+CONFIG_SUBFEEDS = "Subfeeds"
 DEFAULT_SUBFEEDS = None
-SUBFEED_CONFIG_FILE = 'subfeeds.conf'
+SUBFEED_CONFIG_FILE = "subfeeds.conf"
 
 
 def read_config_file(config_path):
-	"""
+    """
-	Reads a config file of key-value pairs, where non-blank lines are
+    Reads a config file of key-value pairs, where non-blank lines are
-	either comments beginning with the character '#' or keys and values
+    either comments beginning with the character '#' or keys and values
-	separated by the character '='.
+    separated by the character '='.
-	"""
+    """
-	# Parse the config file into key-value pairs
+    # Parse the config file into key-value pairs
-	if not os.path.isfile(config_path):
+    if not os.path.isfile(config_path):
-		raise FileNotFoundError(f'No config file found at {config_path}')
-	accumulated_configs = {}
-	current_key = None
-	with open(config_path, 'r', encoding='utf8') as cfg:
-		line_no = 0
-		for line in cfg:
-			line_no += 1
-			# Skip blank lines and comments
-			if not line.strip() or line.lstrip().startswith('#'):
-				continue
-			# Accumulate config keyvalue pairs
-			if '=' in line:
-				# "key = value" begins a new keyvalue pair
-				current_key, value = line.split('=', maxsplit=1)
-				current_key = current_key.strip()
-				accumulated_configs[current_key] = value.strip()
-			else:
-				# If there's no '=' and no previous key, throw
-				if not current_key:
-					raise ValueError(f'Invalid config format on line {line_no}')
-				else:
-					accumulated_configs[current_key] += '\n' + line.strip()
 
-	return accumulated_configs
+        raise FileNotFoundError(
+            f"No config file found at {config_path}, try setting {CONFIG_ENVVAR}"
+        )
+    accumulated_configs = {}
+    current_key = None
+    with open(config_path, "r", encoding="utf8") as cfg:
+        line_no = 0
+        for line in cfg:
+            line_no += 1
+            # Skip blank lines and comments
+            if not line.strip() or line.lstrip().startswith("#"):
+                continue
+            # Accumulate config keyvalue pairs
+            if "=" in line:
+                # "key = value" begins a new keyvalue pair
+                current_key, value = line.split("=", maxsplit=1)
+                current_key = current_key.strip()
+                accumulated_configs[current_key] = value.strip()
+            else:
+                # If there's no '=' and no previous key, throw
+                if not current_key:
+                    raise ValueError(f"Invalid config format on line {line_no}")
+                else:
+                    accumulated_configs[current_key] += "\n" + line.strip()
+
+    return accumulated_configs
 
 
 def parse_subfeed_value(value):
-	sf_defs = [sf.strip() for sf in value.split('\n') if sf.strip()]
+    sf_defs = [sf.strip() for sf in value.split("\n") if sf.strip()]
-	subfeeds = {}
+    subfeeds = {}
-	for sf_def in sf_defs:
+    for sf_def in sf_defs:
-		if ':' not in sf_def:
+        if ":" not in sf_def:
-			raise ValueError(f'Invalid subfeed definition: {sf_def}')
+            raise ValueError(f"Invalid subfeed definition: {sf_def}")
-		sf_name, sf_sources = sf_def.split(':', maxsplit=1)
+        sf_name, sf_sources = sf_def.split(":", maxsplit=1)
-		sf_sources = sf_sources.split()
+        sf_sources = sf_sources.split()
-		subfeeds[sf_name.strip()] = [source.strip() for source in sf_sources]
+        subfeeds[sf_name.strip()] = [source.strip() for source in sf_sources]
-	return subfeeds
+    return subfeeds
 
 
 # Read envvar for config file location, with fallback to default
-config_path = os.path.abspath(
+config_path = os.path.abspath(os.environ.get(CONFIG_ENVVAR) or DEFAULT_CONFIG_PATH)
-	os.environ.get(CONFIG_ENVVAR) or
-	DEFAULT_CONFIG_PATH
-)
 
 configs = read_config_file(config_path)
 
 # Extract and validate config values
 data_path = configs.get(CONFIG_DATA) or DEFAULT_DATA_PATH
 if not os.path.isabs(data_path):
-	raise ValueError(f'Non-absolute data path: {data_path}')
+    raise ValueError(f"Non-absolute data path: {data_path}")
 if not os.path.isdir(data_path):
-	raise FileNotFoundError(f'Cannot find directory {data_path}')
+    raise FileNotFoundError(f"Cannot find directory {data_path}")
 
 source_path = configs.get(CONFIG_SOURCES) or DEFAULT_SOURCES_PATH
 if not os.path.isabs(source_path):
-	raise ValueError(f'Non-absolute source path: {source_path}')
+    raise ValueError(f"Non-absolute source path: {source_path}")
 if not os.path.isdir(source_path):
-	raise FileNotFoundError(f'Cannot find directory {source_path}')
+    raise FileNotFoundError(f"Cannot find directory {source_path}")
 
 cache_path = configs.get(CONFIG_CACHE) or DEFAULT_CACHE_PATH
 if not os.path.isabs(cache_path):
-	raise ValueError(f'Non-absolute cache path: {cache_path}')
+    raise ValueError(f"Non-absolute cache path: {cache_path}")
 if not os.path.isdir(cache_path):
-	raise FileNotFoundError(f'Cannot find directory {cache_path}')
+    raise FileNotFoundError(f"Cannot find directory {cache_path}")
 
 log_file = configs.get(CONFIG_LOGFILE) or DEFAULT_LOG_FILE
 if log_file and not os.path.isabs(log_file):
-	raise ValueError(f'Non-absolute log file path: {log_file}')
+    raise ValueError(f"Non-absolute log file path: {log_file}")
 
 is_verbose = configs.get(CONFIG_VERBOSE) or DEFAULT_VERBOSITY
-if is_verbose != 'true' and is_verbose != 'false':
+if is_verbose != "true" and is_verbose != "false":
-	raise ValueError(f'Invalid verbose value (must be "true" or "false"): {is_verbose}')
+    raise ValueError(f'Invalid verbose value (must be "true" or "false"): {is_verbose}')
-is_verbose = (is_verbose == 'true')
+is_verbose = is_verbose == "true"
 
 subfeeds = configs.get(CONFIG_SUBFEEDS) or DEFAULT_SUBFEEDS
 if subfeeds:
-	subfeeds = parse_subfeed_value(subfeeds)
+    subfeeds = parse_subfeed_value(subfeeds)
 
 
 def get_subfeed_overrides():
-	"""
+    """
-	Check for and parse the secondary subfeed configuration file
+    Check for and parse the secondary subfeed configuration file
-	"""
+    """
-	path = os.path.join(source_path, SUBFEED_CONFIG_FILE)
+    path = os.path.join(source_path, SUBFEED_CONFIG_FILE)
-	if not os.path.isfile(path):
+    if not os.path.isfile(path):
-		return None
+        return None
-	overrides = read_config_file(path)
+    overrides = read_config_file(path)
-	if CONFIG_SUBFEEDS not in overrides:
+    if CONFIG_SUBFEEDS not in overrides:
-		return None
+        return None
-	value = overrides[CONFIG_SUBFEEDS]
+    value = overrides[CONFIG_SUBFEEDS]
-	if not value:
+    if not value:
-		return None
+        return None
-	parsed_value = parse_subfeed_value(value)
+    parsed_value = parse_subfeed_value(value)
-	return parsed_value
+    return parsed_value
 
 
 # Set up logging
 logger = logging.getLogger("inquisitor")
 logger.setLevel(logging.DEBUG)
 
+
 def add_logging_handler(verbose, log_filename):
-	"""
+    """
-	Adds a logging handler according to the given settings
+    Adds a logging handler according to the given settings
-	"""
+    """
-	log_format = (
+    log_format = (
-		'[{asctime}] [{levelname}:{filename}:{lineno}] {message}'
+        "[{asctime}] [{levelname}:{filename}:{lineno}] {message}"
-		if verbose else
+        if verbose
-		'[{levelname}] {message}'
+        else "[{levelname}] {message}"
-	)
+    )
-	formatter = logging.Formatter(log_format, style='{')
+    formatter = logging.Formatter(log_format, style="{")
 
-	log_level = (
+    log_level = logging.DEBUG if verbose else logging.INFO
-		logging.DEBUG
+    handler = (
-		if verbose else
+        logging.handlers.RotatingFileHandler(
-		logging.INFO
+            log_filename,
-	)
+            encoding="utf8",
-	handler = (
+            maxBytes=2**22,  # 4 MB per log file
-		logging.handlers.RotatingFileHandler(
+            backupCount=4,
-			log_filename,
+        )  # 16 MB total
-			encoding='utf8',
+        if log_filename
-			maxBytes=2**22, # 4 MB per log file
+        else logging.StreamHandler()
-			backupCount=4)  # 16 MB total
+    )
-		if log_filename else
+    handler.setFormatter(formatter)
-		logging.StreamHandler()
+    handler.setLevel(log_level)
-	)
+
-	handler.setFormatter(formatter)
+    logger.addHandler(handler)
-	handler.setLevel(log_level)
 
-	logger.addHandler(handler)
 
 def init_default_logging():
-	add_logging_handler(is_verbose, log_file)
+    add_logging_handler(is_verbose, log_file)

inquisitor/error.py

@@ -8,19 +8,20 @@ from inquisitor.configs import DUNGEON_PATH, logger
 
 logger = logging.getLogger("inquisitor")
 
+
 def as_item(title, body=None):
-	iid = '{:x}'.format(random.getrandbits(16 * 4))
+    iid = "{:x}".format(random.getrandbits(16 * 4))
-	item = {
+    item = {
-		'id': iid,
+        "id": iid,
-		'source': 'inquisitor',
+        "source": "inquisitor",
-		'title': title,
+        "title": title,
-		'active': True,
+        "active": True,
-		'created': timestamp.now(),
+        "created": timestamp.now(),
-		'tags': ['inquisitor', 'error'],
+        "tags": ["inquisitor", "error"],
-	}
+    }
-	if body is not None:
+    if body is not None:
-		item['body'] = '<pre>{}</pre>'.format(body)
+        item["body"] = "<pre>{}</pre>".format(body)
-	path = os.path.join(DUNGEON_PATH, 'inquisitor', iid + ".item")
+    path = os.path.join(DUNGEON_PATH, "inquisitor", iid + ".item")
-	logger.error(json.dumps(item))
+    logger.error(json.dumps(item))
-	with open(path, 'w') as f:
+    with open(path, "w") as f:
-		f.write(json.dumps(item, indent=2))
+        f.write(json.dumps(item, indent=2))

inquisitor/loader.py

@@ -7,180 +7,180 @@ from inquisitor import error
 from inquisitor import timestamp
 
 
-class WritethroughDict():
+class WritethroughDict:
-	"""A wrapper for a dictionary saved to the file system."""
+    """A wrapper for a dictionary saved to the file system."""
 
-	@staticmethod
+    @staticmethod
-	def create(path, item):
+    def create(path, item):
-		"""
+        """
-		Creates a writethrough dictionary from a dictionary in memory and
+        Creates a writethrough dictionary from a dictionary in memory and
-		initializes a file to save it.
+        initializes a file to save it.
-		"""
+        """
-		if os.path.isfile(path):
+        if os.path.isfile(path):
-			raise FileExistsError(path)
+            raise FileExistsError(path)
-		wd = WritethroughDict(path, item)
+        wd = WritethroughDict(path, item)
-		wd.flush()
+        wd.flush()
-		return wd
+        return wd
 
-	@staticmethod
+    @staticmethod
-	def load(path):
+    def load(path):
-		"""
+        """
-		Creates a writethrough dictionary from an existing file in the
+        Creates a writethrough dictionary from an existing file in the
-		file system.
+        file system.
-		"""
+        """
-		if not os.path.isfile(path):
+        if not os.path.isfile(path):
-			raise FileNotFoundError(path)
+            raise FileNotFoundError(path)
-		with open(path) as f:
+        with open(path) as f:
-			item = json.load(f)
+            item = json.load(f)
-		return WritethroughDict(path, item)
+        return WritethroughDict(path, item)
 
-	def __init__(self, path, item):
+    def __init__(self, path, item):
-		self.path = path
+        self.path = path
-		self.item = item
+        self.item = item
 
-	def __getitem__(self, key):
+    def __getitem__(self, key):
-		return self.item[key]
+        return self.item[key]
 
-	def get(self, *args, **kwargs):
+    def get(self, *args, **kwargs):
-		return self.item.get(*args, **kwargs)
+        return self.item.get(*args, **kwargs)
 
-	def __setitem__(self, key, value):
+    def __setitem__(self, key, value):
-		self.item[key] = value
+        self.item[key] = value
-		self.flush()
+        self.flush()
 
-	def __contains__(self, key):
+    def __contains__(self, key):
-		return key in self.item
+        return key in self.item
 
-	def __repr__(self):
+    def __repr__(self):
-		return repr(self.item)
+        return repr(self.item)
 
-	def __str__(self):
+    def __str__(self):
-		return str(self.item)
+        return str(self.item)
 
-	def flush(self):
+    def flush(self):
-		s = json.dumps(self.item, indent=2)
+        s = json.dumps(self.item, indent=2)
-		with open(self.path, 'w', encoding="utf8") as f:
+        with open(self.path, "w", encoding="utf8") as f:
-			f.write(s)
+            f.write(s)
 
 
 def load_state(source_name):
-	"""Loads the state dictionary for a source."""
+    """Loads the state dictionary for a source."""
-	state_path = os.path.join(DUNGEON_PATH, source_name, "state")
+    state_path = os.path.join(DUNGEON_PATH, source_name, "state")
-	return WritethroughDict.load(state_path)
+    return WritethroughDict.load(state_path)
 
 
 def load_item(source_name, item_id):
-	"""Loads an item from a source."""
+    """Loads an item from a source."""
-	item_path = os.path.join(DUNGEON_PATH, source_name, f'{item_id}.item')
+    item_path = os.path.join(DUNGEON_PATH, source_name, f"{item_id}.item")
-	return WritethroughDict.load(item_path)
+    return WritethroughDict.load(item_path)
 
 
 def item_exists(source_name, item_id):
-	"""
+    """
-	Checks for the existence of an item.
+    Checks for the existence of an item.
-	"""
+    """
-	item_path = os.path.join(DUNGEON_PATH, source_name, f'{item_id}.item')
+    item_path = os.path.join(DUNGEON_PATH, source_name, f"{item_id}.item")
-	return os.path.isfile(item_path)
+    return os.path.isfile(item_path)
 
 
 def get_item_ids(cell_name):
-	"""
+    """
-	Returns a list of item ids in the given cell.
+    Returns a list of item ids in the given cell.
-	"""
+    """
-	cell_path = os.path.join(DUNGEON_PATH, cell_name)
+    cell_path = os.path.join(DUNGEON_PATH, cell_name)
-	return [
+    return [
-		filename[:-5]
+        filename[:-5]
-		for filename in os.listdir(cell_path)
+        for filename in os.listdir(cell_path)
-		if filename.endswith('.item')
+        if filename.endswith(".item")
-	]
+    ]
 
 
 def new_item(source_name, item):
-	"""
+    """
-	Creates a new item with the fields in the provided dictionary.
+    Creates a new item with the fields in the provided dictionary.
-	Initializes other fields to their default values.
+    Initializes other fields to their default values.
-	"""
+    """
-	# id is required
+    # id is required
-	if 'id' not in item:
+    if "id" not in item:
-		raise Exception(f'Cannot create item with no id. Value = {item}')
+        raise Exception(f"Cannot create item with no id. Value = {item}")
 
-	# source must be filled in, so if it is absent it is auto-populated with
+    # source must be filled in, so if it is absent it is auto-populated with
-	# source_name. Note: this allows sources to fill in a different source.
+    # source_name. Note: this allows sources to fill in a different source.
-	if 'source' not in item:
+    if "source" not in item:
-		item['source'] = source_name
+        item["source"] = source_name
 
-	# active is forced to True for new items
+    # active is forced to True for new items
-	item['active'] = True
+    item["active"] = True
 
-	# created is forced to the current timestamp
+    # created is forced to the current timestamp
-	item['created'] = timestamp.now()
+    item["created"] = timestamp.now()
 
-	# title is auto-populated with the id if missing
+    # title is auto-populated with the id if missing
-	if 'title' not in item:
+    if "title" not in item:
-		item['title'] = item['id']
+        item["title"] = item["id"]
 
-	# tags is auto-populated if missing (not if empty!)
+    # tags is auto-populated if missing (not if empty!)
-	if 'tags' not in item:
+    if "tags" not in item:
-		item['tags'] = [source_name]
+        item["tags"] = [source_name]
 
-	# All other fields are optional.
+    # All other fields are optional.
-	item_path = os.path.join(DUNGEON_PATH, item['source'], f'{item["id"]}.item')
+    item_path = os.path.join(DUNGEON_PATH, item["source"], f'{item["id"]}.item')
-	return WritethroughDict.create(item_path, item)
+    return WritethroughDict.create(item_path, item)
 
 
 def delete_item(source_name, item_id):
-	"""
+    """
-	Delete an item.
+    Delete an item.
-	"""
+    """
-	item_path = os.path.join(DUNGEON_PATH, source_name, f'{item_id}.item')
+    item_path = os.path.join(DUNGEON_PATH, source_name, f"{item_id}.item")
-	os.remove(item_path)
+    os.remove(item_path)
 
 
 def load_items(source_name):
-	"""
+    """
-	Returns a map of ids to items and a list of unreadable files.
+    Returns a map of ids to items and a list of unreadable files.
-	"""
+    """
-	cell_path = os.path.join(DUNGEON_PATH, source_name)
+    cell_path = os.path.join(DUNGEON_PATH, source_name)
-	items = {}
+    items = {}
-	errors = []
+    errors = []
-	for filename in os.listdir(cell_path):
+    for filename in os.listdir(cell_path):
-		if filename.endswith('.item'):
+        if filename.endswith(".item"):
-			try:
+            try:
-				item = load_item(source_name, filename[:-5])
+                item = load_item(source_name, filename[:-5])
-				items[item['id']] = item
+                items[item["id"]] = item
-			except Exception:
+            except Exception:
-				errors.append(filename)
+                errors.append(filename)
-	return items, errors
+    return items, errors
 
 
 def load_active_items(source_names):
-	"""
+    """
-	Returns a list of active items and a list of unreadable items. If
+    Returns a list of active items and a list of unreadable items. If
-	`source_names` is defined, load only from sources in that list.
+    `source_names` is defined, load only from sources in that list.
-	"""
+    """
-	items = []
+    items = []
-	errors = []
+    errors = []
-	now = timestamp.now()
+    now = timestamp.now()
-	check_list = source_names or os.listdir(DUNGEON_PATH)
+    check_list = source_names or os.listdir(DUNGEON_PATH)
-	for source_name in check_list:
+    for source_name in check_list:
-		source_path = os.path.join(DUNGEON_PATH, source_name)
+        source_path = os.path.join(DUNGEON_PATH, source_name)
-		if not os.path.isdir(source_path):
+        if not os.path.isdir(source_path):
-			logger.warning(f'Skipping nonexistent source {source_name}')
+            logger.warning(f"Skipping nonexistent source {source_name}")
-			continue
+            continue
-		for filename in os.listdir(source_path):
+        for filename in os.listdir(source_path):
-			if not filename.endswith('.item'):
+            if not filename.endswith(".item"):
-				continue
+                continue
-			try:
+            try:
-				item = load_item(source_name, filename[:-5])
+                item = load_item(source_name, filename[:-5])
-				# The time-to-show field hides items until an expiry date.
+                # The time-to-show field hides items until an expiry date.
-				if 'tts' in item:
+                if "tts" in item:
-					tts_date = item['created'] + item['tts']
+                    tts_date = item["created"] + item["tts"]
-					if now < tts_date:
+                    if now < tts_date:
-						continue
+                        continue
-				# Don't show inactive items
+                # Don't show inactive items
-				if not item['active']:
+                if not item["active"]:
-					continue
+                    continue
-				items.append(item)
+                items.append(item)
-			except Exception:
+            except Exception:
-				errors.append(filename)
+                errors.append(filename)
-	return items, errors
+    return items, errors

inquisitor/sources.py

@@ -10,229 +10,234 @@ from inquisitor.configs import SOURCES_PATH, DUNGEON_PATH, logger
 
 
 USE_NEWEST = (
-	'title',
+    "title",
-	'tags',
+    "tags",
-	'link',
+    "link",
-	'time'
+    "time" "author",
-	'author',
+    "body",
-	'body',
+    "ttl",
-	'ttl',
+    "ttd",
-	'ttd',
+    "tts",
-	'tts',
 )
 
 
 class InquisitorStubSource:
-	"""A dummy source-like object for clearing out ad-hoc inquisitor items"""
+    """A dummy source-like object for clearing out ad-hoc inquisitor items"""
-	def fetch_new(self, state):
+
-		return []
+    def fetch_new(self, state):
+        return []
 
 
 def ensure_cell(name):
-	"""
+    """
-	Creates a cell in the dungeon. Idempotent.
+    Creates a cell in the dungeon. Idempotent.
-	"""
+    """
-	cell_path = os.path.join(DUNGEON_PATH, name)
+    cell_path = os.path.join(DUNGEON_PATH, name)
-	if not os.path.isdir(cell_path):
+    if not os.path.isdir(cell_path):
-		logger.info(f'Creating cell for source "{name}"')
+        logger.info(f'Creating cell for source "{name}"')
-		os.mkdir(cell_path)
+        os.mkdir(cell_path)
-	state_path = os.path.join(cell_path, 'state')
+    state_path = os.path.join(cell_path, "state")
-	if not os.path.isfile(state_path):
+    if not os.path.isfile(state_path):
-		with open(state_path, 'w', encoding='utf8') as state:
+        with open(state_path, "w", encoding="utf8") as state:
-			json.dump({}, state)
+            json.dump({}, state)
 
 
 def update_sources(*source_names):
-	"""
+    """
-	Attempts to update each given source.
+    Attempts to update each given source.
-	"""
+    """
-	for source_name in source_names:
+    for source_name in source_names:
-		# Import the source
+        # Import the source
-		try:
+        try:
-			source_module = load_source(source_name)
+            source_module = load_source(source_name)
-		except Exception:
+        except Exception:
-			error.as_item(
+            error.as_item(
-				f'Error importing source "{source_name}"',
+                f'Error importing source "{source_name}"', traceback.format_exc()
-				traceback.format_exc())
+            )
-			continue
+            continue
 
-		# If it doesn't have a cell yet, create one
+        # If it doesn't have a cell yet, create one
-		try:
+        try:
-			ensure_cell(source_name)
+            ensure_cell(source_name)
-		except Exception:
+        except Exception:
-			error.as_item(
+            error.as_item(
-				f'Error initializing source "{source_name}"',
+                f'Error initializing source "{source_name}"', traceback.format_exc()
-				traceback.format_exc())
+            )
-			continue
+            continue
 
-		# Update the source
+        # Update the source
-		try:
+        try:
-			logger.info(f'Updating source "{source_name}"')
+            logger.info(f'Updating source "{source_name}"')
-			update_source(source_name, source_module)
+            update_source(source_name, source_module)
-		except Exception:
+        except Exception:
-			error.as_item(
+            error.as_item(
-				f'Error updating source "{source_name}"',
+                f'Error updating source "{source_name}"', traceback.format_exc()
-				traceback.format_exc())
+            )
 
 
 def load_source(source_name):
-	"""
+    """
-	Attempts to load the source module with the given name.
+    Attempts to load the source module with the given name.
-	Raises an exception on failure.
+    Raises an exception on failure.
-	"""
+    """
-	if source_name == 'inquisitor':
+    if source_name == "inquisitor":
-		return InquisitorStubSource()
+        return InquisitorStubSource()
 
-	cwd = os.getcwd()
+    cwd = os.getcwd()
-	try:
+    try:
-		# Push the sources directory.
+        # Push the sources directory.
-		os.chdir(SOURCES_PATH)
+        os.chdir(SOURCES_PATH)
-		# Make the sources directory importable while working with sources.
+        # Make the sources directory importable while working with sources.
-		if SOURCES_PATH not in sys.path:
+        if SOURCES_PATH not in sys.path:
-			sys.path.insert(0, SOURCES_PATH)
+            sys.path.insert(0, SOURCES_PATH)
 
-		# Check if the named source is present.
+        # Check if the named source is present.
-		source_file_name = source_name + '.py'
+        source_file_name = source_name + ".py"
-		if not os.path.isfile(source_file_name):
+        if not os.path.isfile(source_file_name):
-			raise FileNotFoundError(f'Missing "{source_name}" in "{SOURCES_PATH}"')
+            raise FileNotFoundError(f'Missing "{source_name}" in "{SOURCES_PATH}"')
 
-		# Import the source module by file path.
+        # Import the source module by file path.
-		logger.debug(f'Loading module "{source_file_name}"')
+        logger.debug(f'Loading module "{source_file_name}"')
-		spec = importlib.util.spec_from_file_location(source_name, source_file_name)
+        spec = importlib.util.spec_from_file_location(source_name, source_file_name)
-		itemsource = importlib.util.module_from_spec(spec)
+        itemsource = importlib.util.module_from_spec(spec)
-		spec.loader.exec_module(itemsource)
+        spec.loader.exec_module(itemsource)
-		itemsource = importlib.import_module(source_name)
+        itemsource = importlib.import_module(source_name)
 
-		# Require fetch_new().
+        # Require fetch_new().
-		if not hasattr(itemsource, 'fetch_new'):
+        if not hasattr(itemsource, "fetch_new"):
-			raise ImportError(f'Missing fetch_new in "{source_file_name}"')
+            raise ImportError(f'Missing fetch_new in "{source_file_name}"')
 
-		return itemsource
+        return itemsource
 
-	finally:
+    finally:
-		os.chdir(cwd)
+        os.chdir(cwd)
-		if SOURCES_PATH in sys.path:
+        if SOURCES_PATH in sys.path:
-			sys.path.remove(SOURCES_PATH)
+            sys.path.remove(SOURCES_PATH)
 
 
 def update_source(source_name, source):
-	"""
+    """
-	Attempts to update the given source. Raises an exception if the source does.
+    Attempts to update the given source. Raises an exception if the source does.
-	"""
+    """
-	# Get a list of item ids that already existed in this source's cell.
+    # Get a list of item ids that already existed in this source's cell.
-	prior_ids = loader.get_item_ids(source_name)
+    prior_ids = loader.get_item_ids(source_name)
-	logger.debug(f'Found {len(prior_ids)} prior items')
+    logger.debug(f"Found {len(prior_ids)} prior items")
 
-	# Get the feed items from the source's fetch method.
+    # Get the feed items from the source's fetch method.
-	state = loader.load_state(source_name)
+    state = loader.load_state(source_name)
-	fetched = source.fetch_new(state)
+    fetched = source.fetch_new(state)
-	state.flush()
+    state.flush()
-	logger.debug(f'Fetched {len(fetched)} items')
+    logger.debug(f"Fetched {len(fetched)} items")
-	fetched_items = {item['id']: item for item in fetched}
+    fetched_items = {item["id"]: item for item in fetched}
 
-	# Determine which items are new and which are updates.
+    # Determine which items are new and which are updates.
-	# We query the file system here instead of checking against this source's
+    # We query the file system here instead of checking against this source's
-	# item ids from above because sources are allowed to generate in other
+    # item ids from above because sources are allowed to generate in other
-	# sources' cells.
+    # sources' cells.
-	new_items = []
+    new_items = []
-	updated_items = []
+    updated_items = []
-	for item in fetched:
+    for item in fetched:
-		item_source = item.get('source', source_name)
+        item_source = item.get("source", source_name)
-		if loader.item_exists(item_source, item['id']):
+        if loader.item_exists(item_source, item["id"]):
-			updated_items.append(item)
+            updated_items.append(item)
-		else:
+        else:
-			new_items.append(item)
+            new_items.append(item)
 
-	# Write all the new items to the source's cell.
+    # Write all the new items to the source's cell.
-	has_create_handler = hasattr(source, 'on_create')
+    has_create_handler = hasattr(source, "on_create")
-	for item in new_items:
+    for item in new_items:
-		item_source = item.get('source', source_name)
+        item_source = item.get("source", source_name)
-		created_item = loader.new_item(item_source, item)
+        created_item = loader.new_item(item_source, item)
-		if has_create_handler:
+        if has_create_handler:
-			# Because some sources do not return items more than once,
+            # Because some sources do not return items more than once,
-			# exceptions in the on-create handler must be squashed.
+            # exceptions in the on-create handler must be squashed.
-			try:
+            try:
-				source.on_create(state, created_item)
+                source.on_create(state, created_item)
-			except:
+            except:
-				error.as_item(
+                error.as_item(
-					f'Exception in {source_name}.on_create',
+                    f"Exception in {source_name}.on_create", traceback.format_exc()
-					traceback.format_exc())
+                )
 
-	# Update the other items using the fetched items' values.
+    # Update the other items using the fetched items' values.
-	for new_item in updated_items:
+    for new_item in updated_items:
-		old_item = loader.load_item(new_item['source'], new_item['id'])
+        old_item = loader.load_item(new_item["source"], new_item["id"])
-		for field in USE_NEWEST:
+        for field in USE_NEWEST:
-			if field in new_item and old_item[field] != new_item[field]:
+            if field in new_item and old_item[field] != new_item[field]:
-				old_item[field] = new_item[field]
+                old_item[field] = new_item[field]
-		if 'callback' in new_item:
+        if "callback" in new_item:
-			old_callback = old_item.get('callback', {})
+            old_callback = old_item.get("callback", {})
-			# Because of the way this update happens, any fields that are set
+            # Because of the way this update happens, any fields that are set
-			# in the callback when the item is new will keep their original
+            # in the callback when the item is new will keep their original
-			# values, as those values reappear in new_item on subsequent
+            # values, as those values reappear in new_item on subsequent
-			# updates.
+            # updates.
-			old_item['callback'] = {**old_item['callback'], **new_item['callback']}
+            old_item["callback"] = {**old_item["callback"], **new_item["callback"]}
 
-	# In general, items are removed when they are old (not found in the last
+    # In general, items are removed when they are old (not found in the last
-	# fetch) and inactive. Some item fields can change this basic behavior.
+    # fetch) and inactive. Some item fields can change this basic behavior.
-	del_count = 0
+    del_count = 0
-	now = timestamp.now()
+    now = timestamp.now()
-	has_delete_handler = hasattr(source, 'on_delete')
+    has_delete_handler = hasattr(source, "on_delete")
-	fetched_ids = [item['id'] for item in updated_items]
+    fetched_ids = [item["id"] for item in updated_items]
-	old_item_ids = [
+    old_item_ids = [item_id for item_id in prior_ids if item_id not in fetched_ids]
-		item_id for item_id in prior_ids
+    for item_id in old_item_ids:
-		if item_id not in fetched_ids]
+        item = loader.load_item(source_name, item_id)
-	for item_id in old_item_ids:
+        remove = not item["active"]
-		item = loader.load_item(source_name, item_id)
+        # The time-to-live field protects an item from removal until expiry.
-		remove = not item['active']
+        # This is mainly used to avoid old items resurfacing when their source
-		# The time-to-live field protects an item from removal until expiry.
+        # cannot guarantee monotonicity.
-		# This is mainly used to avoid old items resurfacing when their source
+        if "ttl" in item:
-		# cannot guarantee monotonicity.
+            ttl_date = item["created"] + item["ttl"]
-		if 'ttl' in item:
+            if ttl_date > now:
-			ttl_date = item['created'] + item['ttl']
+                continue
-			if ttl_date > now:
+        # The time-to-die field can force an active item to be removed.
-				continue
+        if "ttd" in item:
-		# The time-to-die field can force an active item to be removed.
+            ttd_date = item["created"] + item["ttd"]
-		if 'ttd' in item:
+            if ttd_date < now:
-			ttd_date = item['created'] + item['ttd']
+                remove = True
-			if ttd_date < now:
+        # Items to be removed are deleted
-				remove = True
+        if remove:
-		# Items to be removed are deleted
+            try:
-		if remove:
+                if has_delete_handler:
-			try:
+                    # Run the delete handler so exceptions prevent deletions
-				if has_delete_handler:
+                    source.on_delete(state, item)
-					# Run the delete handler so exceptions prevent deletions
+                loader.delete_item(source_name, item["id"])
-					source.on_delete(state, item)
+                del_count += 1
-				loader.delete_item(source_name, item['id'])
+            except:
-				del_count += 1
+                error.as_item(
-			except:
+                    f'Failed to delete {source_name}/{item["id"]}',
-				error.as_item(
+                    traceback.format_exc(),
-					f'Failed to delete {source_name}/{item["id"]}',
+                )
-					traceback.format_exc())
 
-	# Note update timestamp in state
+    # Note update timestamp in state
-	state['last_updated'] = timestamp.now()
+    state["last_updated"] = timestamp.now()
 
-	# Log counts
+    # Log counts
-	logger.info("{} new item{}, {} deleted item{}".format(
+    logger.info(
-		len(new_items), "s" if len(new_items) != 1 else "",
+        "{} new item{}, {} deleted item{}".format(
-		del_count, "s" if del_count != 1 else ""))
+            len(new_items),
+            "s" if len(new_items) != 1 else "",
+            del_count,
+            "s" if del_count != 1 else "",
+        )
+    )
 
 
 def item_callback(source_name, itemid):
-	try:
+    try:
-		# Load the module with the callback function
+        # Load the module with the callback function
-		source_module = load_source(source_name)
+        source_module = load_source(source_name)
-		if not hasattr(source_module, 'callback'):
+        if not hasattr(source_module, "callback"):
-			raise ImportError(f"Missing callback in '{source_name}'")
+            raise ImportError(f"Missing callback in '{source_name}'")
-		# Load the source state and the origin item
+        # Load the source state and the origin item
-		state = loader.load_state(source_name)
+        state = loader.load_state(source_name)
-		item = loader.load_item(source_name, itemid)
+        item = loader.load_item(source_name, itemid)
-		# Execute callback
+        # Execute callback
-		source_module.callback(state, item)
+        source_module.callback(state, item)
-		# Save any changes
+        # Save any changes
-		item.flush()
+        item.flush()
-		state.flush()
+        state.flush()
-	except Exception:
+    except Exception:
-		error.as_item(
+        error.as_item(
-			f"Error executing callback for {source_name}/{itemid}",
+            f"Error executing callback for {source_name}/{itemid}",
-			traceback.format_exc())
+            traceback.format_exc(),
+        )

inquisitor/templates.py

@@ -17,212 +17,221 @@ import requests
 # Module imports
 from inquisitor import CACHE_PATH
 
-logger = logging.getLogger('inquisitor.templates')
+logger = logging.getLogger("inquisitor.templates")
 
 
 def cache_image(source, url, filename):
-	# Define some paths
+    # Define some paths
-	path = os.path.join(CACHE_PATH, source)
+    path = os.path.join(CACHE_PATH, source)
-	file_path = os.path.join(path, filename)
+    file_path = os.path.join(path, filename)
-	cached_url = f'/cache/{source}/{filename}'
+    cached_url = f"/cache/{source}/{filename}"
-	# Ensure cache folder
+    # Ensure cache folder
-	if not os.path.isdir(path):
+    if not os.path.isdir(path):
-		os.mkdir(path)
+        os.mkdir(path)
-	# Fetch url
+    # Fetch url
-	logger.info(f'Caching {url} to {file_path}')
+    logger.info(f"Caching {url} to {file_path}")
-	response = requests.get(url)
+    response = requests.get(url)
-	# Write file to disk
+    # Write file to disk
-	with open(file_path, 'wb') as f:
+    with open(file_path, "wb") as f:
-		f.write(response.content)
+        f.write(response.content)
-	# Return the inquisitor path to the file
+    # Return the inquisitor path to the file
-	return cached_url
+    return cached_url
 
 
 class LinearCrawler:
-	"""
+    """
-	An engine for generating items from web sources that link content
+    An engine for generating items from web sources that link content
-	together in a linear fashion, such as webcomics.
+    together in a linear fashion, such as webcomics.
-	"""
+    """
-	def fetch_new(self, state):
-		items = []
-		max_iter = self.max_iterations() - 1
-		new = self.try_fetch(state)
-		items.extend(new)
-		for iter in range(max_iter):
-			sleep(1)
-			# If we've already gotten some items out of this fetch, we don't
-			# want to lose them and have the state still be set to the next
-			# page, so we wrap further calls in a try block and force return
-			# if we hit an error.
-			try:
-				new = self.try_fetch(state)
-			except:
-				new = []
-			items.extend(new)
-			# Cut out early if there was nothing returned
-			if not new:
-				break
-		return items
 
-	def try_fetch(self, state):
+    def fetch_new(self, state):
-		# Check for whether a new page should be crawled
+        items = []
-		if 'current_page' not in state:
+        max_iter = self.max_iterations() - 1
-			next_page = self.get_start_url()
+        new = self.try_fetch(state)
-		else:
+        items.extend(new)
-			current = state['current_page']
+        for iter in range(max_iter):
-			response = requests.get(current)
+            sleep(1)
-			soup = BeautifulSoup(response.text, features='html.parser')
+            # If we've already gotten some items out of this fetch, we don't
-			next_page = self.get_next_page_url(current, soup)
+            # want to lose them and have the state still be set to the next
-		if not next_page:
+            # page, so we wrap further calls in a try block and force return
-			return []  # nothing new
+            # if we hit an error.
+            try:
+                new = self.try_fetch(state)
+            except:
+                new = []
+            items.extend(new)
+            # Cut out early if there was nothing returned
+            if not new:
+                break
+        return items
 
-		# Download the new page
+    def try_fetch(self, state):
-		logger.info('Fetching ' + next_page)
+        # Check for whether a new page should be crawled
-		response = requests.get(next_page)
+        if "current_page" not in state:
-		soup = BeautifulSoup(response.text, features="html.parser")
+            next_page = self.get_start_url()
+        else:
+            current = state["current_page"]
+            response = requests.get(current)
+            soup = BeautifulSoup(response.text, features="html.parser")
+            next_page = self.get_next_page_url(current, soup)
+        if not next_page:
+            return []  # nothing new
 
-		# Create an item from the page
+        # Download the new page
-		item = self.make_item(next_page, soup)
+        logger.info("Fetching " + next_page)
+        response = requests.get(next_page)
+        soup = BeautifulSoup(response.text, features="html.parser")
 
-		# Update the state and return the item
+        # Create an item from the page
-		state['current_page'] = next_page
+        item = self.make_item(next_page, soup)
-		return [item]
 
-	def max_iterations(self):
+        # Update the state and return the item
-		return 3
+        state["current_page"] = next_page
+        return [item]
 
-	def get_start_url(self):
+    def max_iterations(self):
-		raise NotImplementedError('get_start_url is required')
+        return 3
 
-	def get_next_page_url(self, url, soup):
+    def get_start_url(self):
-		raise NotImplementedError('get_next_page_url is required')
+        raise NotImplementedError("get_start_url is required")
 
-	def make_item(self, url, soup):
+    def get_next_page_url(self, url, soup):
-		raise NotImplementedError('make_item is required')
+        raise NotImplementedError("get_next_page_url is required")
+
+    def make_item(self, url, soup):
+        raise NotImplementedError("make_item is required")
 
 
 class RedditScraper:
-	"""
+    """
-	An engine for generating items from subreddits.
+    An engine for generating items from subreddits.
-	Requires defining source, subreddit_name
+    Requires defining source, subreddit_name
-	fetch new with RedditScraper.fetch_new(state, __name__, reddit)
+    fetch new with RedditScraper.fetch_new(state, __name__, reddit)
-	"""
+    """
-	@staticmethod
-	def fetch_new(state, name, reddit):
-		items = []
-		for name, obj in inspect.getmembers(sys.modules[name]):
-			if (inspect.isclass(obj)
-				and issubclass(obj, RedditScraper)
-				and obj is not RedditScraper
-			):
-				sub_items = obj(reddit).get_items()
-				items.extend(sub_items)
-		return items
 
-	def __init__(self, reddit):
+    @staticmethod
-		self.reddit = reddit
+    def fetch_new(state, name, reddit):
+        items = []
+        for name, obj in inspect.getmembers(sys.modules[name]):
+            if (
+                inspect.isclass(obj)
+                and issubclass(obj, RedditScraper)
+                and obj is not RedditScraper
+            ):
+                sub_items = obj(reddit).get_items()
+                items.extend(sub_items)
+        return items
 
-	def get_items(self):
+    def __init__(self, reddit):
-		sub_name = self.subreddit_name
+        self.reddit = reddit
-		logger.info(f'Fetching posts from r/{sub_name}')
-		subreddit = self.reddit.subreddit(sub_name)
-		posts = self.subreddit_page(subreddit)
-		items = []
-		for post in posts:
-			if self.filter_post(post):
-				items.append(self.item_from_post(post))
-		return items
 
-	def item_from_post(self, post):
+    def get_items(self):
-		item = {
+        sub_name = self.subreddit_name
-			'source': self.source,
+        logger.info(f"Fetching posts from r/{sub_name}")
-			'id': post.id,
+        subreddit = self.reddit.subreddit(sub_name)
-			'title': self.get_title(post),
+        posts = self.subreddit_page(subreddit)
-			'link': self.get_link(post),
+        items = []
-			'time': post.created_utc,
+        for post in posts:
-			'author': '/u/' + (post.author.name if post.author else "[deleted]"),
+            if self.filter_post(post):
-			'body': self.get_body(post),
+                items.append(self.item_from_post(post))
-			'tags': self.get_tags(post),
+        return items
-			'ttl': self.get_ttl(post),
-		}
-		ttl = self.get_ttl(post)
-		if ttl is not None: item['ttl'] = ttl
-		ttd = self.get_ttd(post)
-		if ttd is not None: item['ttd'] = ttd
-		tts = self.get_tts(post)
-		if tts is not None: item['tts'] = tts
-		callback = self.get_callback(post)
-		if callback is not None: item['callback'] = callback
-		return item
 
-	def subreddit_page(self, subreddit):
+    def item_from_post(self, post):
-		return subreddit.hot(limit=25)
+        item = {
+            "source": self.source,
+            "id": post.id,
+            "title": self.get_title(post),
+            "link": self.get_link(post),
+            "time": post.created_utc,
+            "author": "/u/" + (post.author.name if post.author else "[deleted]"),
+            "body": self.get_body(post),
+            "tags": self.get_tags(post),
+            "ttl": self.get_ttl(post),
+        }
+        ttl = self.get_ttl(post)
+        if ttl is not None:
+            item["ttl"] = ttl
+        ttd = self.get_ttd(post)
+        if ttd is not None:
+            item["ttd"] = ttd
+        tts = self.get_tts(post)
+        if tts is not None:
+            item["tts"] = tts
+        callback = self.get_callback(post)
+        if callback is not None:
+            item["callback"] = callback
+        return item
 
-	def filter_post(self, post):
+    def subreddit_page(self, subreddit):
-		return True
+        return subreddit.hot(limit=25)
 
-	def get_title(self, post):
+    def filter_post(self, post):
-		s = '[S] ' if post.spoiler else ''
+        return True
-		nsfw = '[NSFW] ' if post.over_18 else ''
-		return f'{s}{nsfw}/{post.subreddit_name_prefixed}: {post.title}'
 
-	def get_link(self, post):
+    def get_title(self, post):
-		return f'https://reddit.com{post.permalink}'
+        s = "[S] " if post.spoiler else ""
+        nsfw = "[NSFW] " if post.over_18 else ""
+        return f"{s}{nsfw}/{post.subreddit_name_prefixed}: {post.title}"
 
-	def get_body(self, post):
+    def get_link(self, post):
-		parts = []
+        return f"https://reddit.com{post.permalink}"
-		if not post.is_self:
-			parts.append(f'<i>link:</i> <a href="{post.url}">{post.url}</a>')
-		if hasattr(post, 'preview'):
-			try:
-				previews = post.preview['images'][0]['resolutions']
-				small_previews = [p for p in previews if p['width'] < 800]
-				preview = sorted(small_previews, key=lambda p:-p['width'])[0]
-				parts.append(f'<img src="{preview["url"]}">')
-			except:
-				pass
-		if getattr(post, 'is_gallery', False):
-			try:
-				for gallery_item in post.gallery_data['items']:
-					media_id = gallery_item['media_id']
-					metadata = post.media_metadata[media_id]
-					small_previews = [p for p in metadata['p'] if p['x'] < 800]
-					preview = sorted(small_previews, key=lambda p:-p['x'])[0]
-					parts.append(f'<i>link:</i> <a href="{metadata["s"]["u"]}">{metadata["s"]["u"]}</a>')
-					parts.append(f'<img src="{preview["u"]}">')
-			except:
-				pass
-		if post.selftext:
-			limit = post.selftext[1024:].find(' ')
-			preview_body = post.selftext[:1024 + limit]
-			if len(preview_body) < len(post.selftext):
-				preview_body += '[...]'
-			parts.append(f'<p>{preview_body}</p>')
-		return '<br><hr>'.join(parts)
 
-	def get_tags(self, post):
+    def get_body(self, post):
-		tags = ['reddit', post.subreddit_name_prefixed[2:]]
+        parts = []
-		if post.over_18:
+        if not post.is_self:
-			tags.append('nsfw')
+            parts.append(f'<i>link:</i> <a href="{post.url}">{post.url}</a>')
-		return tags
+        if hasattr(post, "preview"):
+            try:
+                previews = post.preview["images"][0]["resolutions"]
+                small_previews = [p for p in previews if p["width"] < 800]
+                preview = sorted(small_previews, key=lambda p: -p["width"])[0]
+                parts.append(f'<img src="{preview["url"]}">')
+            except:
+                pass
+        if getattr(post, "is_gallery", False):
+            try:
+                for gallery_item in post.gallery_data["items"]:
+                    media_id = gallery_item["media_id"]
+                    metadata = post.media_metadata[media_id]
+                    small_previews = [p for p in metadata["p"] if p["x"] < 800]
+                    preview = sorted(small_previews, key=lambda p: -p["x"])[0]
+                    parts.append(
+                        f'<i>link:</i> <a href="{metadata["s"]["u"]}">{metadata["s"]["u"]}</a>'
+                    )
+                    parts.append(f'<img src="{preview["u"]}">')
+            except:
+                pass
+        if post.selftext:
+            limit = post.selftext[1024:].find(" ")
+            preview_body = post.selftext[: 1024 + limit]
+            if len(preview_body) < len(post.selftext):
+                preview_body += "[...]"
+            parts.append(f"<p>{preview_body}</p>")
+        return "<br><hr>".join(parts)
 
-	def get_ttl(self, post):
+    def get_tags(self, post):
-		return 60 * 60 * 24 * 7  # 1 week
+        tags = ["reddit", post.subreddit_name_prefixed[2:]]
+        if post.over_18:
+            tags.append("nsfw")
+        return tags
 
-	def get_ttd(self, post):
+    def get_ttl(self, post):
-		return None
+        return 60 * 60 * 24 * 7  # 1 week
 
-	def get_tts(self, post):
+    def get_ttd(self, post):
-		return None
+        return None
 
-	def get_callback(self, post):
+    def get_tts(self, post):
-		return None
+        return None
 
-	def callback(self, state, item):
+    def get_callback(self, post):
-		raise NotImplementedError('callback')
+        return None
 
-	def on_create(self, state, item):
+    def callback(self, state, item):
-		raise NotImplementedError('on_create')
+        raise NotImplementedError("callback")
 
-	def on_delete(self, state, item):
+    def on_create(self, state, item):
-		raise NotImplementedError('on_delete')
+        raise NotImplementedError("on_create")
+
+    def on_delete(self, state, item):
+        raise NotImplementedError("on_delete")

inquisitor/timestamp.py

@@ -1,9 +1,11 @@
 import time
 import datetime
 
+
 def now():
-	return int(time.time())
+    return int(time.time())
+
 
 def stamp_to_readable(ts, formatstr="%Y-%m-%d %H:%M:%S"):
-	dt = datetime.datetime.fromtimestamp(ts)
+    dt = datetime.datetime.fromtimestamp(ts)
-	return dt.strftime(formatstr)
+    return dt.strftime(formatstr)

module.nix

@@ -0,0 +1,154 @@
+flake: { config, lib, pkgs, ... }:
+
+let
+  inherit (lib) mkIf mkOption types;
+
+  cfg = config.services.inquisitor;
+in
+{
+  options = {
+    services.inquisitor = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Enable the Inquisitor aggregator.";
+      };
+
+      listen.addr = mkOption {
+        type = types.str;
+        default = "0.0.0.0";
+        description = "Listen address passed to nginx.";
+      };
+
+      listen.port = mkOption {
+        type = types.port;
+        default = 80;
+        description = "Listen port passed to nginx.";
+      };
+    };
+  };
+
+  config =
+  let
+    # Get the inquisitor package from the flake.
+    inquisitor = flake.packages.${pkgs.stdenv.hostPlatform.system}.default;
+
+    # Define the inquisitor state directory.
+    stateDir = "/var/lib/inquisitor";
+
+    # Define an scp helper for item callbacks to use.
+    scp-helper = pkgs.writeShellScriptBin "scp-helper" ''
+      ${pkgs.openssh}/bin/scp -i ${stateDir}/.ssh/inquisitor.key -oStrictHostKeyChecking=no "$@"
+    '';
+
+    # Define the inquisitor service user.
+    svcUser = {
+      name = "inquisitor";
+      group = "inquisitor";
+      description = "Inquisitor service user";
+      isSystemUser = true;
+      shell = pkgs.bashInteractive;
+      packages = [ inquisitor pkgs.cron ];
+    };
+
+    # Create a config file pointing to the state directory.
+    inqConfig = pkgs.writeTextFile {
+      name = "inquisitor.conf";
+      text = ''
+        DataPath = ${stateDir}/data/
+        SourcePath = ${stateDir}/sources/
+        CachePath = ${stateDir}/cache/
+        Verbose = false
+        LogFile = ${stateDir}/inquisitor.log
+      '';
+    };
+
+    # Create a setup script to ensure the service directory state.
+    inqSetup = pkgs.writeShellScript "inquisitor-setup.sh" ''
+      # Ensure the required directories exist.
+      ${pkgs.coreutils}/bin/mkdir -p ${stateDir}/data/inquisitor/
+      ${pkgs.coreutils}/bin/mkdir -p ${stateDir}/sources/
+      ${pkgs.coreutils}/bin/mkdir -p ${stateDir}/cache/
+      if [ ! -f ${stateDir}/data/inquisitor/state ]; then
+        ${pkgs.coreutils}/bin/echo "{}" > ${stateDir}/data/inquisitor/state
+      fi
+
+      # Ensure the service owns the folders.
+      ${pkgs.coreutils}/bin/chown -R ${svcUser.name} ${stateDir}
+
+      # Ensure the scp helper is present
+      if [ -f ${stateDir}/scp-helper ]; then
+        ${pkgs.coreutils}/bin/rm ${stateDir}/scp-helper
+      fi
+      ln -s -t ${stateDir}/scp-helper ${scp-helper}/bin/scp-helper
+    '';
+
+    # Create a run script for the service.
+    inqRun = pkgs.writeShellScript "inquisitor-run.sh" ''
+      cd ${stateDir}
+      ${inquisitor}/bin/gunicorn \
+        --bind=localhost:24133 \
+        --workers=4 \
+        --timeout 120 \
+        --log-level debug \
+        "inquisitor.app:wsgi()"
+    '';
+
+    # Create a wrapper to execute the cli as the service user.
+    # (needed to avoid creating files in the state dir the service can't read)
+    inqWrapper = pkgs.writeShellScriptBin "inq" ''
+      sudo --user=${svcUser.name} ${inquisitor}/bin/inquisitor "$@"
+    '';
+  in mkIf cfg.enable
+  {
+    users.users.inquisitor = svcUser;
+    users.groups.inquisitor = {};
+
+    # Link the config in /etc to avoid envvar shenanigans
+    environment.etc."inquisitor.conf".source = inqConfig;
+
+    # Give all users the wrapper program.
+    environment.systemPackages = [ inqWrapper ];
+    # Allow the sudo in the cli wrapper without password.
+    security.sudo.extraRules = [{
+      commands = [{
+        command = "${inquisitor}/bin/inquisitor";
+        options = [ "NOPASSWD" ];
+      }];
+      runAs = svcUser.name;
+      groups = [ "users" ];
+    }];
+
+    # Run the setup script on activation.
+    system.activationScripts.inquisitorSetup = "${inqSetup}";
+
+    # Set up the inquisitor service.
+    systemd.services.inquisitor = {
+      description = "Inquisitor server";
+      script = "${inqRun}";
+      serviceConfig = {
+        User = svcUser.name;
+        Type = "simple";
+      };
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      enable = true;
+    };
+
+    # Set up the nginx reverse proxy to the server.
+    services.nginx.enable = true;
+    services.nginx.virtualHosts.inquisitorHost = {
+      listen = [ cfg.listen ];
+      locations."/".extraConfig = ''
+        access_log /var/log/nginx/access.inquisitor.log;
+        proxy_buffering off;
+        proxy_pass http://localhost:24133/;
+      '';
+    };
+    networking.firewall.allowedTCPPorts = [ cfg.listen.port ];
+
+    # Enable cron so the service can use it to schedule fetches.
+    services.cron.enable = true;
+  };
+}
+

poetry.lock

@@ -13,6 +13,27 @@ soupsieve = ">1.2"
 html5lib = ["html5lib"]
 lxml = ["lxml"]
 
+[[package]]
+name = "black"
+version = "22.12.0"
+description = "The uncompromising code formatter."
+category = "dev"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+click = ">=8.0.0"
+mypy-extensions = ">=0.4.3"
+pathspec = ">=0.9.0"
+platformdirs = ">=2"
+tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""}
+
+[package.extras]
+colorama = ["colorama (>=0.4.3)"]
+d = ["aiohttp (>=3.7.4)"]
+jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"]
+uvloop = ["uvloop (>=0.15.2)"]
+
 [[package]]
 name = "certifi"
 version = "2022.12.7"
@@ -51,17 +72,6 @@ category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
 
-[[package]]
-name = "feedparser"
-version = "6.0.10"
-description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds"
-category = "main"
-optional = true
-python-versions = ">=3.6"
-
-[package.dependencies]
-sgmllib3k = "*"
-
 [[package]]
 name = "flask"
 version = "2.2.2"
@@ -85,7 +95,7 @@ name = "gunicorn"
 version = "20.1.0"
 description = "WSGI HTTP Server for UNIX"
 category = "main"
-optional = true
+optional = false
 python-versions = ">=3.5"
 
 [package.dependencies]
@@ -136,41 +146,32 @@ optional = false
 python-versions = ">=3.7"
 
 [[package]]
-name = "praw"
+name = "mypy-extensions"
-version = "7.6.1"
+version = "0.4.3"
-description = "PRAW, an acronym for \"Python Reddit API Wrapper\", is a python package that allows for simple access to  Reddit's API."
+description = "Experimental type system extensions for programs checked with the mypy typechecker."
-category = "main"
+category = "dev"
-optional = true
+optional = false
-python-versions = "~=3.7"
+python-versions = "*"
-
-[package.dependencies]
-prawcore = ">=2.1,<3"
-update-checker = ">=0.18"
-websocket-client = ">=0.54.0"
-
-[package.extras]
-ci = ["coveralls"]
-dev = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.3.0,<0.5)", "packaging", "pre-commit", "pytest (>=2.7.3)", "requests (>=2.20.1,<3)", "sphinx", "sphinx-rtd-dark-mode", "sphinx-rtd-theme"]
-lint = ["pre-commit", "sphinx", "sphinx-rtd-dark-mode", "sphinx-rtd-theme"]
-readthedocs = ["sphinx", "sphinx-rtd-dark-mode", "sphinx-rtd-theme"]
-test = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.3.0,<0.5)", "pytest (>=2.7.3)", "requests (>=2.20.1,<3)"]
 
 [[package]]
-name = "prawcore"
+name = "pathspec"
-version = "2.3.0"
+version = "0.10.3"
-description = "Low-level communication layer for PRAW 4+."
+description = "Utility library for gitignore style pattern matching of file paths."
-category = "main"
+category = "dev"
-optional = true
+optional = false
-python-versions = "~=3.6"
+python-versions = ">=3.7"
 
-[package.dependencies]
+[[package]]
-requests = ">=2.6.0,<3.0"
+name = "platformdirs"
+version = "2.6.2"
+description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
+category = "dev"
+optional = false
+python-versions = ">=3.7"
 
 [package.extras]
-ci = ["coveralls"]
+docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"]
-dev = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.4.0,<0.5)", "betamax-serializers (>=0.2.0,<0.3)", "black", "flake8", "flynt", "mock (>=0.8)", "pre-commit", "pydocstyle", "pytest", "testfixtures (>4.13.2,<7)"]
+test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"]
-lint = ["black", "flake8", "flynt", "pre-commit", "pydocstyle"]
-test = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.4.0,<0.5)", "betamax-serializers (>=0.2.0,<0.3)", "mock (>=0.8)", "pytest", "testfixtures (>4.13.2,<7)"]
 
 [[package]]
 name = "requests"
@@ -195,7 +196,7 @@ name = "setuptools"
 version = "65.6.3"
 description = "Easily download, build, install, upgrade, and uninstall Python packages"
 category = "main"
-optional = true
+optional = false
 python-versions = ">=3.7"
 
 [package.extras]
@@ -203,14 +204,6 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-g
 testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
 testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"]
 
-[[package]]
-name = "sgmllib3k"
-version = "1.0.0"
-description = "Py3k port of sgmllib."
-category = "main"
-optional = true
-python-versions = "*"
-
 [[package]]
 name = "soupsieve"
 version = "2.3.2.post1"
@@ -220,20 +213,12 @@ optional = false
 python-versions = ">=3.6"
 
 [[package]]
-name = "update-checker"
+name = "tomli"
-version = "0.18.0"
+version = "2.0.1"
-description = "A python module that will check for package updates."
+description = "A lil' TOML parser"
-category = "main"
+category = "dev"
-optional = true
+optional = false
-python-versions = "*"
+python-versions = ">=3.7"
-
-[package.dependencies]
-requests = ">=2.3.0"
-
-[package.extras]
-dev = ["black", "flake8", "pytest (>=2.7.3)"]
-lint = ["black", "flake8"]
-test = ["pytest (>=2.7.3)"]
 
 [[package]]
 name = "urllib3"
@@ -248,19 +233,6 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
 secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
 socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
 
-[[package]]
-name = "websocket-client"
-version = "1.4.2"
-description = "WebSocket client for Python with low level API options"
-category = "main"
-optional = true
-python-versions = ">=3.7"
-
-[package.extras]
-docs = ["Sphinx (>=3.4)", "sphinx-rtd-theme (>=0.5)"]
-optional = ["python-socks", "wsaccel"]
-test = ["websockets"]
-
 [[package]]
 name = "werkzeug"
 version = "2.2.2"
@@ -275,19 +247,30 @@ MarkupSafe = ">=2.1.1"
 [package.extras]
 watchdog = ["watchdog"]
 
-[extras]
-dev = ["praw", "gunicorn", "feedparser"]
-
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.10"
-content-hash = "2d6e1c8843f9821ef246fdeeeef572bcdaa2db452bbaa74531e8af91f8d89bb5"
+content-hash = "fbb022ff050d55a4b5814e1ce7abcf002c5e32cc552d4171cd780dc99d0aa640"
 
 [metadata.files]
 beautifulsoup4 = [
     {file = "beautifulsoup4-4.11.1-py3-none-any.whl", hash = "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30"},
     {file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"},
 ]
+black = [
+    {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"},
+    {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"},
+    {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"},
+    {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"},
+    {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"},
+    {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"},
+    {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"},
+    {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"},
+    {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"},
+    {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"},
+    {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"},
+    {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"},
+]
 certifi = [
     {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"},
     {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"},
@@ -304,10 +287,6 @@ colorama = [
     {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
     {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
 ]
-feedparser = [
-    {file = "feedparser-6.0.10-py3-none-any.whl", hash = "sha256:79c257d526d13b944e965f6095700587f27388e50ea16fd245babe4dfae7024f"},
-    {file = "feedparser-6.0.10.tar.gz", hash = "sha256:27da485f4637ce7163cdeab13a80312b93b7d0c1b775bef4a47629a3110bca51"},
-]
 flask = [
     {file = "Flask-2.2.2-py3-none-any.whl", hash = "sha256:b9c46cc36662a7949f34b52d8ec7bb59c0d74ba08ba6cb9ce9adc1d8676d9526"},
     {file = "Flask-2.2.2.tar.gz", hash = "sha256:642c450d19c4ad482f96729bd2a8f6d32554aa1e231f4f6b4e7e5264b16cca2b"},
@@ -370,13 +349,17 @@ markupsafe = [
     {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"},
     {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"},
 ]
-praw = [
+mypy-extensions = [
-    {file = "praw-7.6.1-py3-none-any.whl", hash = "sha256:33ac091cd061d9bd607ae231d4ea40025060a4ecb5e11baa5ce9a25c6d2f5a6b"},
+    {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"},
-    {file = "praw-7.6.1.tar.gz", hash = "sha256:07fc95ffc52187351ab77b81fa6910c66e8fa084faf4b060b90864ad186dfb9e"},
+    {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"},
 ]
-prawcore = [
+pathspec = [
-    {file = "prawcore-2.3.0-py3-none-any.whl", hash = "sha256:48c17db447fa06a13ca3e722201f443031437708daa736c05a1df895fbcceff5"},
+    {file = "pathspec-0.10.3-py3-none-any.whl", hash = "sha256:3c95343af8b756205e2aba76e843ba9520a24dd84f68c22b9f93251507509dd6"},
-    {file = "prawcore-2.3.0.tar.gz", hash = "sha256:daf1ccd4b7a80dc4e6567d48336d782e94a9a6dad83770fc2edf76dc9a84f56d"},
+    {file = "pathspec-0.10.3.tar.gz", hash = "sha256:56200de4077d9d0791465aa9095a01d421861e405b5096955051deefd697d6f6"},
+]
+platformdirs = [
+    {file = "platformdirs-2.6.2-py3-none-any.whl", hash = "sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490"},
+    {file = "platformdirs-2.6.2.tar.gz", hash = "sha256:e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2"},
 ]
 requests = [
     {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"},
@@ -386,25 +369,18 @@ setuptools = [
     {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"},
     {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"},
 ]
-sgmllib3k = [
-    {file = "sgmllib3k-1.0.0.tar.gz", hash = "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9"},
-]
 soupsieve = [
     {file = "soupsieve-2.3.2.post1-py3-none-any.whl", hash = "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759"},
     {file = "soupsieve-2.3.2.post1.tar.gz", hash = "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"},
 ]
-update-checker = [
+tomli = [
-    {file = "update_checker-0.18.0-py3-none-any.whl", hash = "sha256:cbba64760a36fe2640d80d85306e8fe82b6816659190993b7bdabadee4d4bbfd"},
+    {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
-    {file = "update_checker-0.18.0.tar.gz", hash = "sha256:6a2d45bb4ac585884a6b03f9eade9161cedd9e8111545141e9aa9058932acb13"},
+    {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
 ]
 urllib3 = [
     {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"},
     {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"},
 ]
-websocket-client = [
-    {file = "websocket-client-1.4.2.tar.gz", hash = "sha256:d6e8f90ca8e2dd4e8027c4561adeb9456b54044312dba655e7cae652ceb9ae59"},
-    {file = "websocket_client-1.4.2-py3-none-any.whl", hash = "sha256:d6b06432f184438d99ac1f456eaf22fe1ade524c3dd16e661142dc54e9cba574"},
-]
 werkzeug = [
     {file = "Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5"},
     {file = "Werkzeug-2.2.2.tar.gz", hash = "sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f"},

pyproject.toml

@@ -11,16 +11,14 @@ python = "^3.10"
 flask = "^2.2.2"
 requests = "^2.28.1"
 beautifulsoup4 = "^4.11.1"
-praw = {version = "^7.6.1", optional = true}
+gunicorn = "^20.1.0"
-gunicorn = {version = "^20.1.0", optional = true}
-feedparser = {version = "^6.0.10", optional = true}
-
-[tool.poetry.extras]
-dev = ["praw", "gunicorn", "feedparser"]
 
 [tool.poetry.scripts]
 inquisitor = "inquisitor.cli:main"
 
+[tool.poetry.group.dev.dependencies]
+black = "^22.12.0"
+
 [build-system]
 requires = ["poetry>=0.12"]
 build-backend = "poetry.masonry.api"