Jaculabilis
/
inquisitor
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: Jaculabilis:cli-changes
Branches Tags
Jaculabilis:master
Jaculabilis:develop
Jaculabilis:cli-changes
..
compare: Jaculabilis:master
Branches Tags
Jaculabilis:develop
Jaculabilis:master
Jaculabilis:cli-changes

1 Commits
cli-change ... master

Author SHA1 Message Date
Tim Van Baak addcb4f1a5 Add protobuf to dependencies temporarily 
This fixes a private source easier than solving the problem of sources introducing dependencies
 2023-01-15 22:03:42 -08:00
16 changed files with 1164 additions and 1413 deletions
Show Stats Expand all files Collapse all files

16
flake.lock
View File

@ -16,21 +16,6 @@
"type": "github" "type": "github"
} }
}, },
"my-flake": {
"locked": {
"lastModified": 1670879933,
"narHash": "sha256-V45PH0cnFLilx66x4td5qQnWNn/V/6/6b7FQDIHvdyI=",
"owner": "Jaculabilis",
"repo": "my-flake",
"rev": "2b2cd07a6d971b15fc5f65d6d963d0da551a5892",
"type": "github"
},
"original": {
"owner": "Jaculabilis",
"repo": "my-flake",
"type": "github"
}
},
"nixpkgs": { "nixpkgs": {
"locked": { "locked": {
"lastModified": 1669833724, "lastModified": 1669833724,
@ -50,7 +35,6 @@
"root": { "root": {
"inputs": { "inputs": {
"flake-compat": "flake-compat", "flake-compat": "flake-compat",
"my-flake": "my-flake",
"nixpkgs": "nixpkgs" "nixpkgs": "nixpkgs"
} }
} }

70
flake.nix
View File

@ -1,6 +1,5 @@
{ {
inputs = { inputs = {
my-flake.url = "github:Jaculabilis/my-flake";
nixpkgs.url = "github:NixOS/nixpkgs?ref=refs/tags/22.11"; nixpkgs.url = "github:NixOS/nixpkgs?ref=refs/tags/22.11";
flake-compat = { flake-compat = {
url = "github:edolstra/flake-compat"; url = "github:edolstra/flake-compat";
@ -8,64 +7,21 @@
}; };
}; };
outputs = { self, my-flake, nixpkgs, flake-compat }: outputs = { self, nixpkgs, flake-compat }:
let let
systems = [ "aarch64-linux" "x86_64-linux" ]; system = "x86_64-linux";
each = system: pkgs = nixpkgs.legacyPackages.${system};
let in
pkgs = (import nixpkgs { {
inherit system; packages.${system}.default =
overlays = [ self.overlays.default ]; (pkgs.poetry2nix.mkPoetryApplication {
});
in {
packages.${system} = {
default = self.packages.${system}.inquisitor;
inquisitor = pkgs.inquisitor;
env = pkgs.inquisitor.dependencyEnv;
};
devShells.${system} = {
default = self.devShells.${system}.inquisitor;
inquisitor = pkgs.mkShell {
buildInputs = [ (pkgs.python3.withPackages (p: [p.poetry])) ];
shellHook = ''
PS1="(inquisitor) $PS1"
'';
};
sources = pkgs.mkShell {
buildInputs = [ self.packages.${system}.env ];
shellHook = ''
PS1="(sources) $PS1"
'';
};
};
checks.${system}.test-module = let
test-lib = import "${nixpkgs}/nixos/lib/testing-python.nix" {
inherit system;
};
in
test-lib.makeTest {
name = "inquisitor-test-module";
nodes = {
host = { ... }: {
imports = [ self.nixosModules.default ];
services.inquisitor.enable = true;
};
};
testScript = ''
start_all()
host.wait_for_unit("multi-user.target")
host.succeed("[ -e /var/lib/inquisitor ]")
'';
};
};
in (my-flake.outputs-for each systems) // {
overlays.default = final: prev: {
inquisitor = final.poetry2nix.mkPoetryApplication {
projectDir = ./.; projectDir = ./.;
}; }).dependencyEnv;
defaultPackage.${system} = self.packages.${system}.default;
devShell.${system} = pkgs.mkShell {
buildInputs = [ (pkgs.python3.withPackages (p: [p.poetry])) ];
}; };
nixosModules.default = import ./module.nix self;
}; };
} }

2
inquisitor/__init__.py
View File

@ -1,2 +1,2 @@
from inquisitor.configs import CACHE_PATH from inquisitor.configs import CACHE_PATH
from inquisitor.templates import cache_image, LinearCrawler, RedditScraper from inquisitor.templates import cache_image, LinearCrawler, RedditScraper

3
inquisitor/__main__.py
View File

@ -1,3 +1,2 @@
from cli import main from cli import main
main()
main()

314
inquisitor/app.py
View File

@ -8,14 +8,13 @@ from flask import Flask, render_template, request, jsonify, abort, redirect, url
# Application imports # Application imports
from inquisitor.configs import ( from inquisitor.configs import (
DUNGEON_PATH, DUNGEON_PATH,
SOURCES_PATH, SOURCES_PATH,
CACHE_PATH, CACHE_PATH,
subfeeds, subfeeds,
get_subfeed_overrides, get_subfeed_overrides,
logger, logger,
init_default_logging, init_default_logging)
)
from inquisitor import sources, loader, timestamp from inquisitor import sources, loader, timestamp
# Globals # Globals
@ -23,202 +22,173 @@ app = Flask(__name__)
def make_query_link(text, wl, bl): def make_query_link(text, wl, bl):
wlp = "only=" + ",".join(wl) wlp = "only=" + ",".join(wl)
blp = "not=" + ",".join(bl) blp = "not=" + ",".join(bl)
params = [p for p in (wlp, blp) if not p.endswith("=")] params = [p for p in (wlp, blp) if not p.endswith("=")]
query = "?{}".format("&".join(params)) query = "?{}".format("&".join(params))
return '<a href="{1}">{0}</a>'.format(text, query) return '<a href="{1}">{0}</a>'.format(text, query)
@app.template_filter("datetimeformat") @app.template_filter("datetimeformat")
def datetimeformat(value): def datetimeformat(value):
return timestamp.stamp_to_readable(value) if value is not None else "" return timestamp.stamp_to_readable(value) if value is not None else ""
@app.route("/") @app.route("/")
def root(): def root():
return redirect(url_for("feed")) return redirect(url_for('feed'))
@app.route("/feed/") @app.route("/feed/")
def feed(): def feed():
return feed_for_sources(source_names=None) return feed_for_sources(source_names=None)
@app.route("/feed/<string:feed_name>/") @app.route("/feed/<string:feed_name>/")
def subfeed(feed_name): def subfeed(feed_name):
# Check for and apply subfeed overrides # Check for and apply subfeed overrides
subfeed_overrides = get_subfeed_overrides() subfeed_overrides = get_subfeed_overrides()
subfeed_config = subfeed_overrides or subfeeds or {} subfeed_config = subfeed_overrides or subfeeds or {}
# The built-in inquisitor subfeed contains sources not in another subfeed # The built-in inquisitor subfeed contains sources not in another subfeed
if feed_name == "inquisitor": if feed_name == 'inquisitor':
all_sources = os.listdir(DUNGEON_PATH) all_sources = os.listdir(DUNGEON_PATH)
for subfeed, sources in subfeed_config.items(): for subfeed, sources in subfeed_config.items():
for source_name in sources: for source_name in sources:
if source_name in all_sources: if source_name in all_sources:
all_sources.remove(source_name) all_sources.remove(source_name)
return feed_for_sources(all_sources) return feed_for_sources(all_sources)
if feed_name not in subfeed_config:
return abort(404)
return feed_for_sources(subfeed_config[feed_name])
if feed_name not in subfeed_config:
return abort(404)
return feed_for_sources(subfeed_config[feed_name])
def feed_for_sources(source_names): def feed_for_sources(source_names):
# Determine exclusion filters # Determine exclusion filters
filters = [] filters = []
wl_param = request.args.get("only") wl_param = request.args.get('only')
wl = wl_param.split(",") if wl_param else [] wl = wl_param.split(",") if wl_param else []
bl_param = request.args.get("not") bl_param = request.args.get('not')
bl = bl_param.split(",") if bl_param else [] bl = bl_param.split(",") if bl_param else []
if wl: if wl:
filters.append(lambda item: not any([tag in wl for tag in item["tags"]])) filters.append(lambda item: not any([tag in wl for tag in item['tags']]))
if bl: if bl:
filters.append(lambda item: any([tag in bl for tag in item["tags"]])) filters.append(lambda item: any([tag in bl for tag in item['tags']]))
# Get all active+filtered items and all active tags # Get all active+filtered items and all active tags
total = 0 total = 0
items, errors = loader.load_active_items(source_names) items, errors = loader.load_active_items(source_names)
active_items = [] active_items = []
active_tags = {} active_tags = {}
for item in items: for item in items:
if item["active"]: if item['active']:
for tag in item["tags"]: for tag in item['tags']:
if tag not in active_tags: if tag not in active_tags: active_tags[tag] = 0
active_tags[tag] = 0 active_tags[tag] += 1
active_tags[tag] += 1 # active_tags |= set(item['tags'])
# active_tags |= set(item['tags']) total += 1
total += 1 if not any(map(lambda f: f(item), filters)):
if not any(map(lambda f: f(item), filters)): active_items.append(item)
active_items.append(item) # Sort items by time
# Sort items by time active_items.sort(key=lambda i: i['time'] if 'time' in i and i['time'] else i['created'] if 'created' in i and i['created'] else 0)
active_items.sort(
key=lambda i: i["time"]
if "time" in i and i["time"]
else i["created"]
if "created" in i and i["created"]
else 0
)
logger.info("Returning {} of {} items".format(len(active_items), total)) logger.info("Returning {} of {} items".format(len(active_items), total))
if errors: if errors:
read_ex = { read_ex = {
"title": "Read errors", 'title': 'Read errors',
"body": "<pre>{}</pre>".format("\n\n".join(errors)), 'body': "<pre>{}</pre>".format("\n\n".join(errors)),
"created": None, 'created': None,
} }
active_items.insert(0, read_ex) active_items.insert(0, read_ex)
if total > 0: if total > 0:
# Create the feed control item # Create the feed control item
link_table = [ link_table = ["<tr><td>{0}</td><td>{1}</td><td></td><td></td></tr>".format(
"<tr><td>{0}</td><td>{1}</td><td></td><td></td></tr>".format( total, make_query_link("all", [], []))]
total, make_query_link("all", [], []) for tag, count in sorted(active_tags.items(), key=lambda i: i[0].lower()):
) links = [count]
] links.append(make_query_link(tag, [tag], []))
for tag, count in sorted(active_tags.items(), key=lambda i: i[0].lower()): if tag in wl:
links = [count] new_wl = [t for t in wl if t != tag]
links.append(make_query_link(tag, [tag], [])) links.append(make_query_link("-only", new_wl, bl))
if tag in wl: else:
new_wl = [t for t in wl if t != tag] new_bl = [t for t in bl if t != tag]
links.append(make_query_link("-only", new_wl, bl)) links.append(make_query_link("+only", wl + [tag], new_bl))
else: if tag in bl:
new_bl = [t for t in bl if t != tag] new_bl = [t for t in bl if t != tag]
links.append(make_query_link("+only", wl + [tag], new_bl)) links.append(make_query_link("-not", wl, new_bl))
if tag in bl: else:
new_bl = [t for t in bl if t != tag] new_wl = [t for t in wl if t != tag]
links.append(make_query_link("-not", wl, new_bl)) links.append(make_query_link("+not", new_wl, bl + [tag]))
else: link_table.append("<tr><td>{0}</td><td>{1}</td><td>{2}</td><td>{3}</td></tr>".format(*links))
new_wl = [t for t in wl if t != tag] body = '<table class="feed-control">{}</table>'.format("\n".join(link_table))
links.append(make_query_link("+not", new_wl, bl + [tag]))
link_table.append(
"<tr><td>{0}</td><td>{1}</td><td>{2}</td><td>{3}</td></tr>".format(
*links
)
)
body = '<table class="feed-control">{}</table>'.format("\n".join(link_table))
feed_control = { feed_control = {
"title": "Feed Control [{}/{}]".format(len(active_items), total), 'title': 'Feed Control [{}/{}]'.format(len(active_items), total),
"body": body, 'body': body,
} }
active_items.insert(0, feed_control) active_items.insert(0, feed_control)
selection = active_items[:100] selection = active_items[:100]
return render_template( return render_template("feed.jinja2",
"feed.jinja2", items=selection,
items=selection, mdeac=[
mdeac=[ {'source': item['source'], 'itemid': item['id']}
{"source": item["source"], "itemid": item["id"]} for item in selection
for item in selection if 'id' in item])
if "id" in item
],
)
@app.route("/deactivate/", methods=['POST'])
@app.route("/deactivate/", methods=["POST"])
def deactivate(): def deactivate():
params = request.get_json() params = request.get_json()
if "source" not in params and "itemid" not in params: if 'source' not in params and 'itemid' not in params:
logger.error("Bad request params: {}".format(params)) logger.error("Bad request params: {}".format(params))
item = loader.load_item(params["source"], params["itemid"]) item = loader.load_item(params['source'], params['itemid'])
if item["active"]: if item['active']:
logger.debug(f"Deactivating {params['source']}/{params['itemid']}") logger.debug(f"Deactivating {params['source']}/{params['itemid']}")
item["active"] = False item['active'] = False
return jsonify({"active": item["active"]}) return jsonify({'active': item['active']})
@app.route("/punt/", methods=['POST'])
@app.route("/punt/", methods=["POST"])
def punt(): def punt():
params = request.get_json() params = request.get_json()
if "source" not in params and "itemid" not in params: if 'source' not in params and 'itemid' not in params:
logger.error("Bad request params: {}".format(params)) logger.error("Bad request params: {}".format(params))
item = loader.load_item(params["source"], params["itemid"]) item = loader.load_item(params['source'], params['itemid'])
tomorrow = datetime.now() + timedelta(days=1) tomorrow = datetime.now() + timedelta(days=1)
morning = datetime(tomorrow.year, tomorrow.month, tomorrow.day, 6, 0, 0) morning = datetime(tomorrow.year, tomorrow.month, tomorrow.day, 6, 0, 0)
til_then = morning.timestamp() - item["created"] til_then = morning.timestamp() - item['created']
item["tts"] = til_then item['tts'] = til_then
return jsonify(item.item) return jsonify(item.item)
@app.route("/mass-deactivate/", methods=['POST'])
@app.route("/mass-deactivate/", methods=["POST"])
def mass_deactivate(): def mass_deactivate():
params = request.get_json() params = request.get_json()
if "items" not in params: if 'items' not in params:
logger.error("Bad request params: {}".format(params)) logger.error("Bad request params: {}".format(params))
for info in params.get("items", []): for info in params.get('items', []):
source = info["source"] source = info['source']
itemid = info["itemid"] itemid = info['itemid']
item = loader.load_item(source, itemid) item = loader.load_item(source, itemid)
if item["active"]: if item['active']:
logger.debug(f"Deactivating {info['source']}/{info['itemid']}") logger.debug(f"Deactivating {info['source']}/{info['itemid']}")
item["active"] = False item['active'] = False
return jsonify({}) return jsonify({})
@app.route("/callback/", methods=['POST'])
@app.route("/callback/", methods=["POST"])
def callback(): def callback():
params = request.get_json() params = request.get_json()
if "source" not in params and "itemid" not in params: if 'source' not in params and 'itemid' not in params:
logger.error("Bad request params: {}".format(params)) logger.error("Bad request params: {}".format(params))
logger.info( logger.info('Executing callback for {}/{}'.format(params['source'], params['itemid']))
"Executing callback for {}/{}".format(params["source"], params["itemid"]) sources.item_callback(params['source'], params['itemid'])
) return jsonify({})
sources.item_callback(params["source"], params["itemid"])
return jsonify({})
@app.route('/cache/<path:cache_path>')
@app.route("/cache/<path:cache_path>")
def cache(cache_path): def cache(cache_path):
path = os.path.join(CACHE_PATH, cache_path) path = os.path.join(CACHE_PATH, cache_path)
if not os.path.isfile(path): if not os.path.isfile(path):
return abort(404) return abort(404)
with open(path, "rb") as f: with open(path, 'rb') as f:
return f.read() return f.read()
def wsgi(): def wsgi():
init_default_logging() init_default_logging()
return app return app

514
inquisitor/cli.py
View File

@ -11,324 +11,280 @@ from inquisitor.configs import logger, DUNGEON_PATH, SOURCES_PATH, add_logging_h
def command_test(args): def command_test(args):
"""Echo config file values.""" """Echo config file values."""
from inquisitor.configs.resolver import ( from inquisitor.configs.resolver import (
config_path, config_path,
CONFIG_DATA, CONFIG_DATA, data_path,
data_path, CONFIG_SOURCES, source_path,
CONFIG_SOURCES, CONFIG_CACHE, cache_path,
source_path, CONFIG_LOGFILE, log_file,
CONFIG_CACHE, CONFIG_VERBOSE, is_verbose,
cache_path, CONFIG_SUBFEEDS, subfeeds,
CONFIG_LOGFILE, )
log_file, subfeeds = '; '.join(
CONFIG_VERBOSE, '{0}: {1}'.format(
is_verbose, sf_name,
CONFIG_SUBFEEDS, ' '.join(sf_sources)
subfeeds, )
) for sf_name, sf_sources
in subfeeds.items()
subfeeds = ( ) if subfeeds else ''
"; ".join( print(f'Inquisitor configured from {config_path}')
"{0}: {1}".format(sf_name, " ".join(sf_sources)) print(f' {CONFIG_DATA} = {data_path}')
for sf_name, sf_sources in subfeeds.items() print(f' {CONFIG_SOURCES} = {source_path}')
) print(f' {CONFIG_CACHE} = {cache_path}')
if subfeeds print(f' {CONFIG_LOGFILE} = {log_file}')
else "" print(f' {CONFIG_VERBOSE} = {is_verbose}')
) print(f' {CONFIG_SUBFEEDS} = {subfeeds}')
print(f"Inquisitor configured from {config_path}") return 0
print(f" {CONFIG_DATA} = {data_path}")
print(f" {CONFIG_SOURCES} = {source_path}")
print(f" {CONFIG_CACHE} = {cache_path}")
print(f" {CONFIG_LOGFILE} = {log_file}")
print(f" {CONFIG_VERBOSE} = {is_verbose}")
print(f" {CONFIG_SUBFEEDS} = {subfeeds}")
return 0
def command_update(args): def command_update(args):
"""Fetch and store new items from the specified sources.""" """Fetch and store new items from the specified sources."""
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
prog="inquisitor update", description=command_update.__doc__, add_help=False prog="inquisitor update",
) description=command_update.__doc__,
parser.add_argument("source", nargs="*", help="Sources to update.") add_help=False)
args = parser.parse_args(args) parser.add_argument("source",
nargs="*",
help="Sources to update.")
args = parser.parse_args(args)
if len(args.source) == 0: if len(args.source) == 0:
parser.print_help() parser.print_help()
return 0 return 0
if not os.path.isdir(DUNGEON_PATH): if not os.path.isdir(DUNGEON_PATH):
logger.error( logger.error("Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon")
"Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon" return -1
) if not os.path.isdir(SOURCES_PATH):
return -1 logger.error("Couldn't find sources. Set INQUISITOR_SOURCES or cd to parent folder of ./sources")
if not os.path.isdir(SOURCES_PATH):
logger.error(
"Couldn't find sources. Set INQUISITOR_SOURCES or cd to parent folder of ./sources"
)
# Update sources # Update sources
from inquisitor.sources import update_sources from inquisitor.sources import update_sources
update_sources(*args.source)
update_sources(*args.source) return 0
return 0
def command_deactivate(args): def command_deactivate(args):
"""Deactivate all items in the specified dungeon cells.""" """Deactivate all items in the specified dungeon cells."""
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
prog="inquisitor deactivate", prog="inquisitor deactivate",
description=command_deactivate.__doc__, description=command_deactivate.__doc__,
add_help=False, add_help=False)
) parser.add_argument("source",
parser.add_argument("source", nargs="*", help="Cells to deactivate.") nargs="*",
parser.add_argument("--tag", help="Only deactivate items with this tag") help="Cells to deactivate.")
parser.add_argument( parser.add_argument("--tag",
"--title", help="Only deactivate items with titles containing this substring" help="Only deactivate items with this tag")
) parser.add_argument("--title",
args = parser.parse_args(args) help="Only deactivate items with titles containing this substring")
args = parser.parse_args(args)
if len(args.source) == 0: if len(args.source) == 0:
parser.print_help() parser.print_help()
return 0 return 0
if not os.path.isdir(DUNGEON_PATH): if not os.path.isdir(DUNGEON_PATH):
logger.error( logger.error("Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon")
"Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon" return -1
)
return -1
# Deactivate all items in each source. # Deactivate all items in each source.
from inquisitor.loader import load_items from inquisitor.loader import load_items
for source_name in args.source:
path = os.path.join(DUNGEON_PATH, source_name)
if not os.path.isdir(path):
logger.warning("'{}' is not an extant source".format(source_name))
count = 0
items, _ = load_items(source_name)
for item in items.values():
if args.tag and args.tag not in item['tags']:
continue
if args.title and args.title not in item['title']:
continue
if item['active']:
item['active'] = False
count += 1
logger.info("Deactivated {} items in '{}'".format(count, source_name))
for source_name in args.source: return 0
path = os.path.join(DUNGEON_PATH, source_name)
if not os.path.isdir(path):
logger.warning("'{}' is not an extant source".format(source_name))
count = 0
items, _ = load_items(source_name)
for item in items.values():
if args.tag and args.tag not in item["tags"]:
continue
if args.title and args.title not in item["title"]:
continue
if item["active"]:
item["active"] = False
count += 1
logger.info("Deactivated {} items in '{}'".format(count, source_name))
return 0
def command_add(args): def command_add(args):
"""Creates an item.""" """Creates an item."""
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
prog="inquisitor add", description=command_add.__doc__, add_help=False prog="inquisitor add",
) description=command_add.__doc__,
parser.add_argument("--id", help="String") add_help=False)
parser.add_argument("--source", help="String") parser.add_argument("--id", help="String")
parser.add_argument("--title", help="String") parser.add_argument("--source", help="String")
parser.add_argument("--link", help="URL") parser.add_argument("--title", help="String")
parser.add_argument("--time", type=int, help="Unix timestmap") parser.add_argument("--link", help="URL")
parser.add_argument("--author", help="String") parser.add_argument("--time", type=int, help="Unix timestmap")
parser.add_argument("--body", help="HTML") parser.add_argument("--author", help="String")
parser.add_argument("--tags", help="Comma-separated list") parser.add_argument("--body", help="HTML")
parser.add_argument("--ttl", type=int, help="Cleanup protection in seconds") parser.add_argument("--tags", help="Comma-separated list")
parser.add_argument("--ttd", type=int, help="Cleanup force in seconds") parser.add_argument("--ttl", type=int, help="Cleanup protection in seconds")
parser.add_argument("--tts", type=int, help="Display delay in seconds") parser.add_argument("--ttd", type=int, help="Cleanup force in seconds")
parser.add_argument( parser.add_argument("--tts", type=int, help="Display delay in seconds")
"--create", action="store_true", help="Create source if it doesn't exist" parser.add_argument("--create", action="store_true", help="Create source if it doesn't exist")
) args = parser.parse_args(args)
args = parser.parse_args(args)
if not args.title: if not args.title:
parser.print_help() parser.print_help()
return 0 return 0
if not os.path.isdir(DUNGEON_PATH): if not os.path.isdir(DUNGEON_PATH):
logger.error( logger.error("Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon")
"Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon" return -1
)
return -1
source = args.source or "inquisitor" source = args.source or 'inquisitor'
cell_path = os.path.join(DUNGEON_PATH, source) cell_path = os.path.join(DUNGEON_PATH, source)
if args.create: if args.create:
from inquisitor.sources import ensure_cell from inquisitor.sources import ensure_cell
ensure_cell(source)
elif not os.path.isdir(cell_path):
logger.error("Source '{}' does not exist".format(source))
return -1
ensure_cell(source) item = {
elif not os.path.isdir(cell_path): 'id': args.id or '{:x}'.format(random.getrandbits(16 * 4)),
logger.error("Source '{}' does not exist".format(source)) 'source': source,
return -1 }
if args.title: item['title'] = str(args.title)
if args.link: item['link'] = str(args.link)
if args.time: item['time'] = int(args.time)
if args.author: item['author'] = str(args.author)
if args.body: item['body'] = str(args.body)
if args.tags: item['tags'] = [str(tag) for tag in args.tags.split(",")]
if args.ttl: item['ttl'] = int(args.ttl)
if args.ttd: item['ttd'] = int(args.ttd)
if args.tts: item['tts'] = int(args.tts)
item = { from inquisitor.loader import new_item
"id": args.id or "{:x}".format(random.getrandbits(16 * 4)), saved_item = new_item(source, item)
"source": source, logger.info(saved_item)
}
if args.title:
item["title"] = str(args.title)
if args.link:
item["link"] = str(args.link)
if args.time:
item["time"] = int(args.time)
if args.author:
item["author"] = str(args.author)
if args.body:
item["body"] = str(args.body)
if args.tags:
item["tags"] = [str(tag) for tag in args.tags.split(",")]
if args.ttl:
item["ttl"] = int(args.ttl)
if args.ttd:
item["ttd"] = int(args.ttd)
if args.tts:
item["tts"] = int(args.tts)
from inquisitor.loader import new_item
saved_item = new_item(source, item)
logger.info(saved_item)
def command_feed(args): def command_feed(args):
"""Print the current feed.""" """Print the current feed."""
if not os.path.isdir(DUNGEON_PATH): if not os.path.isdir(DUNGEON_PATH):
logger.error( logger.error("Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon")
"Couldn't find dungeon. Set INQUISITOR_DUNGEON or cd to parent folder of ./dungeon" return -1
)
return -1
import shutil import shutil
from inquisitor import loader from inquisitor import loader
from inquisitor import timestamp from inquisitor import timestamp
items, errors = loader.load_active_items(source_names=None) items, errors = loader.load_active_items(source_names=None)
if not items and not errors: if not items and not errors:
print("Feed is empty") print("Feed is empty")
return 0 return 0
if errors: if errors:
items.insert( items.insert(0, {
0, 'title': '{} read errors: {}'.format(len(errors), ' '.join(errors)),
{ 'body': "\n".join(errors)
"title": "{} read errors: {}".format(len(errors), " ".join(errors)), })
"body": "\n".join(errors),
},
)
size = shutil.get_terminal_size((80, 20)) size = shutil.get_terminal_size((80, 20))
width = min(80, size.columns) width = min(80, size.columns)
for item in items: for item in items:
title = item["title"] if "title" in item else "" title = item['title'] if 'title' in item else ""
titles = [title] titles = [title]
while len(titles[-1]) > width - 4: while len(titles[-1]) > width - 4:
i = titles[-1][: width - 4].rfind(" ") i = titles[-1][:width - 4].rfind(' ')
titles = titles[:-1] + [titles[-1][:i].strip(), titles[-1][i:].strip()] titles = titles[:-1] + [titles[-1][:i].strip(), titles[-1][i:].strip()]
print("+" + (width - 2) * "-" + "+") print('+' + (width - 2) * '-' + '+')
for title in titles: for title in titles:
print("| {0:<{1}} |".format(title, width - 4)) print("| {0:<{1}} |".format(title, width - 4))
print("|{0:<{1}}|".format("", width - 2)) print("|{0:<{1}}|".format("", width - 2))
info1 = "" info1 = ""
if "author" in title and item["author"]: if 'author' in title and item['author']:
info1 += item["author"] + " " info1 += item['author'] + " "
if "time" in item and item["time"]: if 'time' in item and item['time']:
info1 += timestamp.stamp_to_readable(item["time"]) info1 += timestamp.stamp_to_readable(item['time'])
print("| {0:<{1}} |".format(info1, width - 4)) print("| {0:<{1}} |".format(info1, width - 4))
created = ( created = timestamp.stamp_to_readable(item['created']) if 'created' in item else ""
timestamp.stamp_to_readable(item["created"]) if "created" in item else "" info2 = "{0} {1} {2}".format(
) item.get('source', ''), item.get('id', ''), created)
info2 = "{0} {1} {2}".format( print("| {0:<{1}} |".format(info2, width - 4))
item.get("source", ""), item.get("id", ""), created print('+' + (width - 2) * '-' + '+')
) print()
print("| {0:<{1}} |".format(info2, width - 4))
print("+" + (width - 2) * "-" + "+")
print()
def command_run(args): def command_run(args):
"""Run the default Flask server.""" """Run the default Flask server."""
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
prog="inquisitor run", description=command_run.__doc__, add_help=False prog="inquisitor run",
) description=command_run.__doc__,
parser.add_argument("--debug", action="store_true") add_help=False)
parser.add_argument("--port", type=int, default=5000) parser.add_argument("--debug", action="store_true")
args = parser.parse_args(args) parser.add_argument("--port", type=int, default=5000)
args = parser.parse_args(args)
try: try:
from inquisitor.app import app from inquisitor.app import app
app.run(port=args.port, debug=args.debug)
app.run(port=args.port, debug=args.debug) return 0
return 0 except Exception as e:
except Exception as e: logger.error(e)
logger.error(e) return -1
return -1
def command_help(args): def command_help(args):
"""Print this help message and exit.""" """Print this help message and exit."""
print_usage() print_usage()
return 0 return 0
def nocommand(args):
print("command required")
return 0
def main(): def main():
"""CLI entry point""" """CLI entry point"""
# Enable piping # Enable piping
from signal import signal, SIGPIPE, SIG_DFL from signal import signal, SIGPIPE, SIG_DFL
signal(SIGPIPE, SIG_DFL)
signal(SIGPIPE, SIG_DFL) # Collect the commands from this module
import inquisitor.cli
commands = {
name[8:] : func
for name, func in vars(inquisitor.cli).items()
if name.startswith('command_')
}
descriptions = "\n".join([
"- {0}: {1}".format(name, func.__doc__)
for name, func in commands.items()])
# Collect the commands from this module # Set up the parser
import inquisitor.cli parser = argparse.ArgumentParser(
description="Available commands:\n{}\n".format(descriptions),
formatter_class=argparse.RawDescriptionHelpFormatter,
add_help=False)
parser.add_argument("command",
nargs="?",
default="help",
help="The command to execute",
choices=commands,
metavar="command")
parser.add_argument("args",
nargs=argparse.REMAINDER,
help="Command arguments",
metavar="args")
parser.add_argument("-v",
action="store_true",
dest="verbose",
help="Enable debug logging")
commands = { # Extract the usage print for command_help
name[8:]: func global print_usage
for name, func in vars(inquisitor.cli).items() print_usage = parser.print_help
if name.startswith("command_")
}
descriptions = "\n".join(
["- {0}: {1}".format(name, func.__doc__) for name, func in commands.items()]
)
# Set up the parser args = parser.parse_args()
parser = argparse.ArgumentParser(
description="Available commands:\n{}\n".format(descriptions),
formatter_class=argparse.RawDescriptionHelpFormatter,
add_help=False,
)
parser.add_argument(
"command",
nargs="?",
default="help",
help="The command to execute",
choices=commands,
metavar="command",
)
parser.add_argument(
"args", nargs=argparse.REMAINDER, help="Command arguments", metavar="args"
)
parser.add_argument(
"-v", action="store_true", dest="verbose", help="Enable debug logging"
)
# Extract the usage print for command_help # Initialize a console logger
global print_usage add_logging_handler(verbose=args.verbose, log_filename=None)
print_usage = parser.print_help
args = parser.parse_args() # Execute command
if args.command:
# Initialize a console logger sys.exit(commands[args.command](args.args))
add_logging_handler(verbose=args.verbose, log_filename=None) else:
print("command required")
# Execute command sys.exit(0)
try:
command = commands.get(args.command, nocommand)
sys.exit(command(args.args))
except BrokenPipeError:
# See https://docs.python.org/3.10/library/signal.html#note-on-sigpipe
devnull = os.open(os.devnull, os.O_WRONLY)
os.dup2(devnull, sys.stdout.fileno())
sys.exit(1)

9
inquisitor/configs/__init__.py
View File

@ -1,5 +1,10 @@
from .resolver import data_path as DUNGEON_PATH from .resolver import data_path as DUNGEON_PATH
from .resolver import source_path as SOURCES_PATH from .resolver import source_path as SOURCES_PATH
from .resolver import cache_path as CACHE_PATH from .resolver import cache_path as CACHE_PATH
from .resolver import logger, subfeeds from .resolver import (
from .resolver import add_logging_handler, init_default_logging, get_subfeed_overrides logger,
subfeeds)
from .resolver import (
add_logging_handler,
init_default_logging,
get_subfeed_overrides)

215
inquisitor/configs/resolver.py
View File

@ -4,172 +4,173 @@ import logging
# Constants governing config resolution: # Constants governing config resolution:
# Path to the config file, containing key-value pairs of the other settings # Path to the config file, containing key-value pairs of the other settings
CONFIG_ENVVAR = "INQUISITOR_CONFIG" CONFIG_ENVVAR = 'INQUISITOR_CONFIG'
DEFAULT_CONFIG_PATH = "/etc/inquisitor.conf" DEFAULT_CONFIG_PATH = '/etc/inquisitor.conf'
# Path to the folder where items are stored # Path to the folder where items are stored
CONFIG_DATA = "DataPath" CONFIG_DATA = 'DataPath'
DEFAULT_DATA_PATH = "/var/inquisitor/data/" DEFAULT_DATA_PATH = '/var/inquisitor/data/'
# Path to the folder where source modules are stored # Path to the folder where source modules are stored
CONFIG_SOURCES = "SourcePath" CONFIG_SOURCES = 'SourcePath'
DEFAULT_SOURCES_PATH = "/var/inquisitor/sources/" DEFAULT_SOURCES_PATH = '/var/inquisitor/sources/'
# Path to the folder where cached files are stored # Path to the folder where cached files are stored
CONFIG_CACHE = "CachePath" CONFIG_CACHE = 'CachePath'
DEFAULT_CACHE_PATH = "/var/inquisitor/cache/" DEFAULT_CACHE_PATH = '/var/inquisitor/cache/'
# Path to a log file where logging will be redirected # Path to a log file where logging will be redirected
CONFIG_LOGFILE = "LogFile" CONFIG_LOGFILE = 'LogFile'
DEFAULT_LOG_FILE = None DEFAULT_LOG_FILE = None
# Whether logging is verbose # Whether logging is verbose
CONFIG_VERBOSE = "Verbose" CONFIG_VERBOSE = 'Verbose'
DEFAULT_VERBOSITY = "false" DEFAULT_VERBOSITY = 'false'
# Subfeed source lists, with each subfeed config separated by lines and # Subfeed source lists, with each subfeed config separated by lines and
# sources within a subfeed separated by spaces # sources within a subfeed separated by spaces
CONFIG_SUBFEEDS = "Subfeeds" CONFIG_SUBFEEDS = 'Subfeeds'
DEFAULT_SUBFEEDS = None DEFAULT_SUBFEEDS = None
SUBFEED_CONFIG_FILE = "subfeeds.conf" SUBFEED_CONFIG_FILE = 'subfeeds.conf'
def read_config_file(config_path): def read_config_file(config_path):
""" """
Reads a config file of key-value pairs, where non-blank lines are Reads a config file of key-value pairs, where non-blank lines are
either comments beginning with the character '#' or keys and values either comments beginning with the character '#' or keys and values
separated by the character '='. separated by the character '='.
""" """
# Parse the config file into key-value pairs # Parse the config file into key-value pairs
if not os.path.isfile(config_path): if not os.path.isfile(config_path):
raise FileNotFoundError(f'No config file found at {config_path}')
accumulated_configs = {}
current_key = None
with open(config_path, 'r', encoding='utf8') as cfg:
line_no = 0
for line in cfg:
line_no += 1
# Skip blank lines and comments
if not line.strip() or line.lstrip().startswith('#'):
continue
# Accumulate config keyvalue pairs
if '=' in line:
# "key = value" begins a new keyvalue pair
current_key, value = line.split('=', maxsplit=1)
current_key = current_key.strip()
accumulated_configs[current_key] = value.strip()
else:
# If there's no '=' and no previous key, throw
if not current_key:
raise ValueError(f'Invalid config format on line {line_no}')
else:
accumulated_configs[current_key] += '\n' + line.strip()
raise FileNotFoundError( return accumulated_configs
f"No config file found at {config_path}, try setting {CONFIG_ENVVAR}"
)
accumulated_configs = {}
current_key = None
with open(config_path, "r", encoding="utf8") as cfg:
line_no = 0
for line in cfg:
line_no += 1
# Skip blank lines and comments
if not line.strip() or line.lstrip().startswith("#"):
continue
# Accumulate config keyvalue pairs
if "=" in line:
# "key = value" begins a new keyvalue pair
current_key, value = line.split("=", maxsplit=1)
current_key = current_key.strip()
accumulated_configs[current_key] = value.strip()
else:
# If there's no '=' and no previous key, throw
if not current_key:
raise ValueError(f"Invalid config format on line {line_no}")
else:
accumulated_configs[current_key] += "\n" + line.strip()
return accumulated_configs
def parse_subfeed_value(value): def parse_subfeed_value(value):
sf_defs = [sf.strip() for sf in value.split("\n") if sf.strip()] sf_defs = [sf.strip() for sf in value.split('\n') if sf.strip()]
subfeeds = {} subfeeds = {}
for sf_def in sf_defs: for sf_def in sf_defs:
if ":" not in sf_def: if ':' not in sf_def:
raise ValueError(f"Invalid subfeed definition: {sf_def}") raise ValueError(f'Invalid subfeed definition: {sf_def}')
sf_name, sf_sources = sf_def.split(":", maxsplit=1) sf_name, sf_sources = sf_def.split(':', maxsplit=1)
sf_sources = sf_sources.split() sf_sources = sf_sources.split()
subfeeds[sf_name.strip()] = [source.strip() for source in sf_sources] subfeeds[sf_name.strip()] = [source.strip() for source in sf_sources]
return subfeeds return subfeeds
# Read envvar for config file location, with fallback to default # Read envvar for config file location, with fallback to default
config_path = os.path.abspath(os.environ.get(CONFIG_ENVVAR) or DEFAULT_CONFIG_PATH) config_path = os.path.abspath(
os.environ.get(CONFIG_ENVVAR) or
DEFAULT_CONFIG_PATH
)
configs = read_config_file(config_path) configs = read_config_file(config_path)
# Extract and validate config values # Extract and validate config values
data_path = configs.get(CONFIG_DATA) or DEFAULT_DATA_PATH data_path = configs.get(CONFIG_DATA) or DEFAULT_DATA_PATH
if not os.path.isabs(data_path): if not os.path.isabs(data_path):
raise ValueError(f"Non-absolute data path: {data_path}") raise ValueError(f'Non-absolute data path: {data_path}')
if not os.path.isdir(data_path): if not os.path.isdir(data_path):
raise FileNotFoundError(f"Cannot find directory {data_path}") raise FileNotFoundError(f'Cannot find directory {data_path}')
source_path = configs.get(CONFIG_SOURCES) or DEFAULT_SOURCES_PATH source_path = configs.get(CONFIG_SOURCES) or DEFAULT_SOURCES_PATH
if not os.path.isabs(source_path): if not os.path.isabs(source_path):
raise ValueError(f"Non-absolute source path: {source_path}") raise ValueError(f'Non-absolute source path: {source_path}')
if not os.path.isdir(source_path): if not os.path.isdir(source_path):
raise FileNotFoundError(f"Cannot find directory {source_path}") raise FileNotFoundError(f'Cannot find directory {source_path}')
cache_path = configs.get(CONFIG_CACHE) or DEFAULT_CACHE_PATH cache_path = configs.get(CONFIG_CACHE) or DEFAULT_CACHE_PATH
if not os.path.isabs(cache_path): if not os.path.isabs(cache_path):
raise ValueError(f"Non-absolute cache path: {cache_path}") raise ValueError(f'Non-absolute cache path: {cache_path}')
if not os.path.isdir(cache_path): if not os.path.isdir(cache_path):
raise FileNotFoundError(f"Cannot find directory {cache_path}") raise FileNotFoundError(f'Cannot find directory {cache_path}')
log_file = configs.get(CONFIG_LOGFILE) or DEFAULT_LOG_FILE log_file = configs.get(CONFIG_LOGFILE) or DEFAULT_LOG_FILE
if log_file and not os.path.isabs(log_file): if log_file and not os.path.isabs(log_file):
raise ValueError(f"Non-absolute log file path: {log_file}") raise ValueError(f'Non-absolute log file path: {log_file}')
is_verbose = configs.get(CONFIG_VERBOSE) or DEFAULT_VERBOSITY is_verbose = configs.get(CONFIG_VERBOSE) or DEFAULT_VERBOSITY
if is_verbose != "true" and is_verbose != "false": if is_verbose != 'true' and is_verbose != 'false':
raise ValueError(f'Invalid verbose value (must be "true" or "false"): {is_verbose}') raise ValueError(f'Invalid verbose value (must be "true" or "false"): {is_verbose}')
is_verbose = is_verbose == "true" is_verbose = (is_verbose == 'true')
subfeeds = configs.get(CONFIG_SUBFEEDS) or DEFAULT_SUBFEEDS subfeeds = configs.get(CONFIG_SUBFEEDS) or DEFAULT_SUBFEEDS
if subfeeds: if subfeeds:
subfeeds = parse_subfeed_value(subfeeds) subfeeds = parse_subfeed_value(subfeeds)
def get_subfeed_overrides(): def get_subfeed_overrides():
""" """
Check for and parse the secondary subfeed configuration file Check for and parse the secondary subfeed configuration file
""" """
path = os.path.join(source_path, SUBFEED_CONFIG_FILE) path = os.path.join(source_path, SUBFEED_CONFIG_FILE)
if not os.path.isfile(path): if not os.path.isfile(path):
return None return None
overrides = read_config_file(path) overrides = read_config_file(path)
if CONFIG_SUBFEEDS not in overrides: if CONFIG_SUBFEEDS not in overrides:
return None return None
value = overrides[CONFIG_SUBFEEDS] value = overrides[CONFIG_SUBFEEDS]
if not value: if not value:
return None return None
parsed_value = parse_subfeed_value(value) parsed_value = parse_subfeed_value(value)
return parsed_value return parsed_value
# Set up logging # Set up logging
logger = logging.getLogger("inquisitor") logger = logging.getLogger("inquisitor")
logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG)
def add_logging_handler(verbose, log_filename): def add_logging_handler(verbose, log_filename):
""" """
Adds a logging handler according to the given settings Adds a logging handler according to the given settings
""" """
log_format = ( log_format = (
"[{asctime}] [{levelname}:{filename}:{lineno}] {message}" '[{asctime}] [{levelname}:{filename}:{lineno}] {message}'
if verbose if verbose else
else "[{levelname}] {message}" '[{levelname}] {message}'
) )
formatter = logging.Formatter(log_format, style="{") formatter = logging.Formatter(log_format, style='{')
log_level = logging.DEBUG if verbose else logging.INFO log_level = (
handler = ( logging.DEBUG
logging.handlers.RotatingFileHandler( if verbose else
log_filename, logging.INFO
encoding="utf8", )
maxBytes=2**22, # 4 MB per log file handler = (
backupCount=4, logging.handlers.RotatingFileHandler(
) # 16 MB total log_filename,
if log_filename encoding='utf8',
else logging.StreamHandler() maxBytes=2**22, # 4 MB per log file
) backupCount=4) # 16 MB total
handler.setFormatter(formatter) if log_filename else
handler.setLevel(log_level) logging.StreamHandler()
)
logger.addHandler(handler) handler.setFormatter(formatter)
handler.setLevel(log_level)
logger.addHandler(handler)
def init_default_logging(): def init_default_logging():
add_logging_handler(is_verbose, log_file) add_logging_handler(is_verbose, log_file)

31
inquisitor/error.py
View File

@ -8,20 +8,19 @@ from inquisitor.configs import DUNGEON_PATH, logger
logger = logging.getLogger("inquisitor") logger = logging.getLogger("inquisitor")
def as_item(title, body=None): def as_item(title, body=None):
iid = "{:x}".format(random.getrandbits(16 * 4)) iid = '{:x}'.format(random.getrandbits(16 * 4))
item = { item = {
"id": iid, 'id': iid,
"source": "inquisitor", 'source': 'inquisitor',
"title": title, 'title': title,
"active": True, 'active': True,
"created": timestamp.now(), 'created': timestamp.now(),
"tags": ["inquisitor", "error"], 'tags': ['inquisitor', 'error'],
} }
if body is not None: if body is not None:
item["body"] = "<pre>{}</pre>".format(body) item['body'] = '<pre>{}</pre>'.format(body)
path = os.path.join(DUNGEON_PATH, "inquisitor", iid + ".item") path = os.path.join(DUNGEON_PATH, 'inquisitor', iid + ".item")
logger.error(json.dumps(item)) logger.error(json.dumps(item))
with open(path, "w") as f: with open(path, 'w') as f:
f.write(json.dumps(item, indent=2)) f.write(json.dumps(item, indent=2))

274
inquisitor/loader.py
View File

@ -7,180 +7,180 @@ from inquisitor import error
from inquisitor import timestamp from inquisitor import timestamp
class WritethroughDict: class WritethroughDict():
"""A wrapper for a dictionary saved to the file system.""" """A wrapper for a dictionary saved to the file system."""
@staticmethod @staticmethod
def create(path, item): def create(path, item):
""" """
Creates a writethrough dictionary from a dictionary in memory and Creates a writethrough dictionary from a dictionary in memory and
initializes a file to save it. initializes a file to save it.
""" """
if os.path.isfile(path): if os.path.isfile(path):
raise FileExistsError(path) raise FileExistsError(path)
wd = WritethroughDict(path, item) wd = WritethroughDict(path, item)
wd.flush() wd.flush()
return wd return wd
@staticmethod @staticmethod
def load(path): def load(path):
""" """
Creates a writethrough dictionary from an existing file in the Creates a writethrough dictionary from an existing file in the
file system. file system.
""" """
if not os.path.isfile(path): if not os.path.isfile(path):
raise FileNotFoundError(path) raise FileNotFoundError(path)
with open(path) as f: with open(path) as f:
item = json.load(f) item = json.load(f)
return WritethroughDict(path, item) return WritethroughDict(path, item)
def __init__(self, path, item): def __init__(self, path, item):
self.path = path self.path = path
self.item = item self.item = item
def __getitem__(self, key): def __getitem__(self, key):
return self.item[key] return self.item[key]
def get(self, *args, **kwargs): def get(self, *args, **kwargs):
return self.item.get(*args, **kwargs) return self.item.get(*args, **kwargs)
def __setitem__(self, key, value): def __setitem__(self, key, value):
self.item[key] = value self.item[key] = value
self.flush() self.flush()
def __contains__(self, key): def __contains__(self, key):
return key in self.item return key in self.item
def __repr__(self): def __repr__(self):
return repr(self.item) return repr(self.item)
def __str__(self): def __str__(self):
return str(self.item) return str(self.item)
def flush(self): def flush(self):
s = json.dumps(self.item, indent=2) s = json.dumps(self.item, indent=2)
with open(self.path, "w", encoding="utf8") as f: with open(self.path, 'w', encoding="utf8") as f:
f.write(s) f.write(s)
def load_state(source_name): def load_state(source_name):
"""Loads the state dictionary for a source.""" """Loads the state dictionary for a source."""
state_path = os.path.join(DUNGEON_PATH, source_name, "state") state_path = os.path.join(DUNGEON_PATH, source_name, "state")
return WritethroughDict.load(state_path) return WritethroughDict.load(state_path)
def load_item(source_name, item_id): def load_item(source_name, item_id):
"""Loads an item from a source.""" """Loads an item from a source."""
item_path = os.path.join(DUNGEON_PATH, source_name, f"{item_id}.item") item_path = os.path.join(DUNGEON_PATH, source_name, f'{item_id}.item')
return WritethroughDict.load(item_path) return WritethroughDict.load(item_path)
def item_exists(source_name, item_id): def item_exists(source_name, item_id):
""" """
Checks for the existence of an item. Checks for the existence of an item.
""" """
item_path = os.path.join(DUNGEON_PATH, source_name, f"{item_id}.item") item_path = os.path.join(DUNGEON_PATH, source_name, f'{item_id}.item')
return os.path.isfile(item_path) return os.path.isfile(item_path)
def get_item_ids(cell_name): def get_item_ids(cell_name):
""" """
Returns a list of item ids in the given cell. Returns a list of item ids in the given cell.
""" """
cell_path = os.path.join(DUNGEON_PATH, cell_name) cell_path = os.path.join(DUNGEON_PATH, cell_name)
return [ return [
filename[:-5] filename[:-5]
for filename in os.listdir(cell_path) for filename in os.listdir(cell_path)
if filename.endswith(".item") if filename.endswith('.item')
] ]
def new_item(source_name, item): def new_item(source_name, item):
""" """
Creates a new item with the fields in the provided dictionary. Creates a new item with the fields in the provided dictionary.
Initializes other fields to their default values. Initializes other fields to their default values.
""" """
# id is required # id is required
if "id" not in item: if 'id' not in item:
raise Exception(f"Cannot create item with no id. Value = {item}") raise Exception(f'Cannot create item with no id. Value = {item}')
# source must be filled in, so if it is absent it is auto-populated with # source must be filled in, so if it is absent it is auto-populated with
# source_name. Note: this allows sources to fill in a different source. # source_name. Note: this allows sources to fill in a different source.
if "source" not in item: if 'source' not in item:
item["source"] = source_name item['source'] = source_name
# active is forced to True for new items # active is forced to True for new items
item["active"] = True item['active'] = True
# created is forced to the current timestamp # created is forced to the current timestamp
item["created"] = timestamp.now() item['created'] = timestamp.now()
# title is auto-populated with the id if missing # title is auto-populated with the id if missing
if "title" not in item: if 'title' not in item:
item["title"] = item["id"] item['title'] = item['id']
# tags is auto-populated if missing (not if empty!) # tags is auto-populated if missing (not if empty!)
if "tags" not in item: if 'tags' not in item:
item["tags"] = [source_name] item['tags'] = [source_name]
# All other fields are optional. # All other fields are optional.
item_path = os.path.join(DUNGEON_PATH, item["source"], f'{item["id"]}.item') item_path = os.path.join(DUNGEON_PATH, item['source'], f'{item["id"]}.item')
return WritethroughDict.create(item_path, item) return WritethroughDict.create(item_path, item)
def delete_item(source_name, item_id): def delete_item(source_name, item_id):
""" """
Delete an item. Delete an item.
""" """
item_path = os.path.join(DUNGEON_PATH, source_name, f"{item_id}.item") item_path = os.path.join(DUNGEON_PATH, source_name, f'{item_id}.item')
os.remove(item_path) os.remove(item_path)
def load_items(source_name): def load_items(source_name):
""" """
Returns a map of ids to items and a list of unreadable files. Returns a map of ids to items and a list of unreadable files.
""" """
cell_path = os.path.join(DUNGEON_PATH, source_name) cell_path = os.path.join(DUNGEON_PATH, source_name)
items = {} items = {}
errors = [] errors = []
for filename in os.listdir(cell_path): for filename in os.listdir(cell_path):
if filename.endswith(".item"): if filename.endswith('.item'):
try: try:
item = load_item(source_name, filename[:-5]) item = load_item(source_name, filename[:-5])
items[item["id"]] = item items[item['id']] = item
except Exception: except Exception:
errors.append(filename) errors.append(filename)
return items, errors return items, errors
def load_active_items(source_names): def load_active_items(source_names):
""" """
Returns a list of active items and a list of unreadable items. If Returns a list of active items and a list of unreadable items. If
`source_names` is defined, load only from sources in that list. `source_names` is defined, load only from sources in that list.
""" """
items = [] items = []
errors = [] errors = []
now = timestamp.now() now = timestamp.now()
check_list = source_names or os.listdir(DUNGEON_PATH) check_list = source_names or os.listdir(DUNGEON_PATH)
for source_name in check_list: for source_name in check_list:
source_path = os.path.join(DUNGEON_PATH, source_name) source_path = os.path.join(DUNGEON_PATH, source_name)
if not os.path.isdir(source_path): if not os.path.isdir(source_path):
logger.warning(f"Skipping nonexistent source {source_name}") logger.warning(f'Skipping nonexistent source {source_name}')
continue continue
for filename in os.listdir(source_path): for filename in os.listdir(source_path):
if not filename.endswith(".item"): if not filename.endswith('.item'):
continue continue
try: try:
item = load_item(source_name, filename[:-5]) item = load_item(source_name, filename[:-5])
# The time-to-show field hides items until an expiry date. # The time-to-show field hides items until an expiry date.
if "tts" in item: if 'tts' in item:
tts_date = item["created"] + item["tts"] tts_date = item['created'] + item['tts']
if now < tts_date: if now < tts_date:
continue continue
# Don't show inactive items # Don't show inactive items
if not item["active"]: if not item['active']:
continue continue
items.append(item) items.append(item)
except Exception: except Exception:
errors.append(filename) errors.append(filename)
return items, errors return items, errors

389
inquisitor/sources.py
View File

@ -10,234 +10,229 @@ from inquisitor.configs import SOURCES_PATH, DUNGEON_PATH, logger
USE_NEWEST = ( USE_NEWEST = (
"title", 'title',
"tags", 'tags',
"link", 'link',
"time" "author", 'time'
"body", 'author',
"ttl", 'body',
"ttd", 'ttl',
"tts", 'ttd',
'tts',
) )
class InquisitorStubSource: class InquisitorStubSource:
"""A dummy source-like object for clearing out ad-hoc inquisitor items""" """A dummy source-like object for clearing out ad-hoc inquisitor items"""
def fetch_new(self, state):
def fetch_new(self, state): return []
return []
def ensure_cell(name): def ensure_cell(name):
""" """
Creates a cell in the dungeon. Idempotent. Creates a cell in the dungeon. Idempotent.
""" """
cell_path = os.path.join(DUNGEON_PATH, name) cell_path = os.path.join(DUNGEON_PATH, name)
if not os.path.isdir(cell_path): if not os.path.isdir(cell_path):
logger.info(f'Creating cell for source "{name}"') logger.info(f'Creating cell for source "{name}"')
os.mkdir(cell_path) os.mkdir(cell_path)
state_path = os.path.join(cell_path, "state") state_path = os.path.join(cell_path, 'state')
if not os.path.isfile(state_path): if not os.path.isfile(state_path):
with open(state_path, "w", encoding="utf8") as state: with open(state_path, 'w', encoding='utf8') as state:
json.dump({}, state) json.dump({}, state)
def update_sources(*source_names): def update_sources(*source_names):
""" """
Attempts to update each given source. Attempts to update each given source.
""" """
for source_name in source_names: for source_name in source_names:
# Import the source # Import the source
try: try:
source_module = load_source(source_name) source_module = load_source(source_name)
except Exception: except Exception:
error.as_item( error.as_item(
f'Error importing source "{source_name}"', traceback.format_exc() f'Error importing source "{source_name}"',
) traceback.format_exc())
continue continue
# If it doesn't have a cell yet, create one # If it doesn't have a cell yet, create one
try: try:
ensure_cell(source_name) ensure_cell(source_name)
except Exception: except Exception:
error.as_item( error.as_item(
f'Error initializing source "{source_name}"', traceback.format_exc() f'Error initializing source "{source_name}"',
) traceback.format_exc())
continue continue
# Update the source # Update the source
try: try:
logger.info(f'Updating source "{source_name}"') logger.info(f'Updating source "{source_name}"')
update_source(source_name, source_module) update_source(source_name, source_module)
except Exception: except Exception:
error.as_item( error.as_item(
f'Error updating source "{source_name}"', traceback.format_exc() f'Error updating source "{source_name}"',
) traceback.format_exc())
def load_source(source_name): def load_source(source_name):
""" """
Attempts to load the source module with the given name. Attempts to load the source module with the given name.
Raises an exception on failure. Raises an exception on failure.
""" """
if source_name == "inquisitor": if source_name == 'inquisitor':
return InquisitorStubSource() return InquisitorStubSource()
cwd = os.getcwd() cwd = os.getcwd()
try: try:
# Push the sources directory. # Push the sources directory.
os.chdir(SOURCES_PATH) os.chdir(SOURCES_PATH)
# Make the sources directory importable while working with sources. # Make the sources directory importable while working with sources.
if SOURCES_PATH not in sys.path: if SOURCES_PATH not in sys.path:
sys.path.insert(0, SOURCES_PATH) sys.path.insert(0, SOURCES_PATH)
# Check if the named source is present. # Check if the named source is present.
source_file_name = source_name + ".py" source_file_name = source_name + '.py'
if not os.path.isfile(source_file_name): if not os.path.isfile(source_file_name):
raise FileNotFoundError(f'Missing "{source_name}" in "{SOURCES_PATH}"') raise FileNotFoundError(f'Missing "{source_name}" in "{SOURCES_PATH}"')
# Import the source module by file path. # Import the source module by file path.
logger.debug(f'Loading module "{source_file_name}"') logger.debug(f'Loading module "{source_file_name}"')
spec = importlib.util.spec_from_file_location(source_name, source_file_name) spec = importlib.util.spec_from_file_location(source_name, source_file_name)
itemsource = importlib.util.module_from_spec(spec) itemsource = importlib.util.module_from_spec(spec)
spec.loader.exec_module(itemsource) spec.loader.exec_module(itemsource)
itemsource = importlib.import_module(source_name) itemsource = importlib.import_module(source_name)
# Require fetch_new(). # Require fetch_new().
if not hasattr(itemsource, "fetch_new"): if not hasattr(itemsource, 'fetch_new'):
raise ImportError(f'Missing fetch_new in "{source_file_name}"') raise ImportError(f'Missing fetch_new in "{source_file_name}"')
return itemsource return itemsource
finally: finally:
os.chdir(cwd) os.chdir(cwd)
if SOURCES_PATH in sys.path: if SOURCES_PATH in sys.path:
sys.path.remove(SOURCES_PATH) sys.path.remove(SOURCES_PATH)
def update_source(source_name, source): def update_source(source_name, source):
""" """
Attempts to update the given source. Raises an exception if the source does. Attempts to update the given source. Raises an exception if the source does.
""" """
# Get a list of item ids that already existed in this source's cell. # Get a list of item ids that already existed in this source's cell.
prior_ids = loader.get_item_ids(source_name) prior_ids = loader.get_item_ids(source_name)
logger.debug(f"Found {len(prior_ids)} prior items") logger.debug(f'Found {len(prior_ids)} prior items')
# Get the feed items from the source's fetch method. # Get the feed items from the source's fetch method.
state = loader.load_state(source_name) state = loader.load_state(source_name)
fetched = source.fetch_new(state) fetched = source.fetch_new(state)
state.flush() state.flush()
logger.debug(f"Fetched {len(fetched)} items") logger.debug(f'Fetched {len(fetched)} items')
fetched_items = {item["id"]: item for item in fetched} fetched_items = {item['id']: item for item in fetched}
# Determine which items are new and which are updates. # Determine which items are new and which are updates.
# We query the file system here instead of checking against this source's # We query the file system here instead of checking against this source's
# item ids from above because sources are allowed to generate in other # item ids from above because sources are allowed to generate in other
# sources' cells. # sources' cells.
new_items = [] new_items = []
updated_items = [] updated_items = []
for item in fetched: for item in fetched:
item_source = item.get("source", source_name) item_source = item.get('source', source_name)
if loader.item_exists(item_source, item["id"]): if loader.item_exists(item_source, item['id']):
updated_items.append(item) updated_items.append(item)
else: else:
new_items.append(item) new_items.append(item)
# Write all the new items to the source's cell. # Write all the new items to the source's cell.
has_create_handler = hasattr(source, "on_create") has_create_handler = hasattr(source, 'on_create')
for item in new_items: for item in new_items:
item_source = item.get("source", source_name) item_source = item.get('source', source_name)
created_item = loader.new_item(item_source, item) created_item = loader.new_item(item_source, item)
if has_create_handler: if has_create_handler:
# Because some sources do not return items more than once, # Because some sources do not return items more than once,
# exceptions in the on-create handler must be squashed. # exceptions in the on-create handler must be squashed.
try: try:
source.on_create(state, created_item) source.on_create(state, created_item)
except: except:
error.as_item( error.as_item(
f"Exception in {source_name}.on_create", traceback.format_exc() f'Exception in {source_name}.on_create',
) traceback.format_exc())
# Update the other items using the fetched items' values. # Update the other items using the fetched items' values.
for new_item in updated_items: for new_item in updated_items:
old_item = loader.load_item(new_item["source"], new_item["id"]) old_item = loader.load_item(new_item['source'], new_item['id'])
for field in USE_NEWEST: for field in USE_NEWEST:
if field in new_item and old_item[field] != new_item[field]: if field in new_item and old_item[field] != new_item[field]:
old_item[field] = new_item[field] old_item[field] = new_item[field]
if "callback" in new_item: if 'callback' in new_item:
old_callback = old_item.get("callback", {}) old_callback = old_item.get('callback', {})
# Because of the way this update happens, any fields that are set # Because of the way this update happens, any fields that are set
# in the callback when the item is new will keep their original # in the callback when the item is new will keep their original
# values, as those values reappear in new_item on subsequent # values, as those values reappear in new_item on subsequent
# updates. # updates.
old_item["callback"] = {**old_item["callback"], **new_item["callback"]} old_item['callback'] = {**old_item['callback'], **new_item['callback']}
# In general, items are removed when they are old (not found in the last # In general, items are removed when they are old (not found in the last
# fetch) and inactive. Some item fields can change this basic behavior. # fetch) and inactive. Some item fields can change this basic behavior.
del_count = 0 del_count = 0
now = timestamp.now() now = timestamp.now()
has_delete_handler = hasattr(source, "on_delete") has_delete_handler = hasattr(source, 'on_delete')
fetched_ids = [item["id"] for item in updated_items] fetched_ids = [item['id'] for item in updated_items]
old_item_ids = [item_id for item_id in prior_ids if item_id not in fetched_ids] old_item_ids = [
for item_id in old_item_ids: item_id for item_id in prior_ids
item = loader.load_item(source_name, item_id) if item_id not in fetched_ids]
remove = not item["active"] for item_id in old_item_ids:
# The time-to-live field protects an item from removal until expiry. item = loader.load_item(source_name, item_id)
# This is mainly used to avoid old items resurfacing when their source remove = not item['active']
# cannot guarantee monotonicity. # The time-to-live field protects an item from removal until expiry.
if "ttl" in item: # This is mainly used to avoid old items resurfacing when their source
ttl_date = item["created"] + item["ttl"] # cannot guarantee monotonicity.
if ttl_date > now: if 'ttl' in item:
continue ttl_date = item['created'] + item['ttl']
# The time-to-die field can force an active item to be removed. if ttl_date > now:
if "ttd" in item: continue
ttd_date = item["created"] + item["ttd"] # The time-to-die field can force an active item to be removed.
if ttd_date < now: if 'ttd' in item:
remove = True ttd_date = item['created'] + item['ttd']
# Items to be removed are deleted if ttd_date < now:
if remove: remove = True
try: # Items to be removed are deleted
if has_delete_handler: if remove:
# Run the delete handler so exceptions prevent deletions try:
source.on_delete(state, item) if has_delete_handler:
loader.delete_item(source_name, item["id"]) # Run the delete handler so exceptions prevent deletions
del_count += 1 source.on_delete(state, item)
except: loader.delete_item(source_name, item['id'])
error.as_item( del_count += 1
f'Failed to delete {source_name}/{item["id"]}', except:
traceback.format_exc(), error.as_item(
) f'Failed to delete {source_name}/{item["id"]}',
traceback.format_exc())
# Note update timestamp in state # Note update timestamp in state
state["last_updated"] = timestamp.now() state['last_updated'] = timestamp.now()
# Log counts # Log counts
logger.info( logger.info("{} new item{}, {} deleted item{}".format(
"{} new item{}, {} deleted item{}".format( len(new_items), "s" if len(new_items) != 1 else "",
len(new_items), del_count, "s" if del_count != 1 else ""))
"s" if len(new_items) != 1 else "",
del_count,
"s" if del_count != 1 else "",
)
)
def item_callback(source_name, itemid): def item_callback(source_name, itemid):
try: try:
# Load the module with the callback function # Load the module with the callback function
source_module = load_source(source_name) source_module = load_source(source_name)
if not hasattr(source_module, "callback"): if not hasattr(source_module, 'callback'):
raise ImportError(f"Missing callback in '{source_name}'") raise ImportError(f"Missing callback in '{source_name}'")
# Load the source state and the origin item # Load the source state and the origin item
state = loader.load_state(source_name) state = loader.load_state(source_name)
item = loader.load_item(source_name, itemid) item = loader.load_item(source_name, itemid)
# Execute callback # Execute callback
source_module.callback(state, item) source_module.callback(state, item)
# Save any changes # Save any changes
item.flush() item.flush()
state.flush() state.flush()
except Exception: except Exception:
error.as_item( error.as_item(
f"Error executing callback for {source_name}/{itemid}", f"Error executing callback for {source_name}/{itemid}",
traceback.format_exc(), traceback.format_exc())
)

361
inquisitor/templates.py
View File

@ -17,221 +17,212 @@ import requests
# Module imports # Module imports
from inquisitor import CACHE_PATH from inquisitor import CACHE_PATH
logger = logging.getLogger("inquisitor.templates") logger = logging.getLogger('inquisitor.templates')
def cache_image(source, url, filename): def cache_image(source, url, filename):
# Define some paths # Define some paths
path = os.path.join(CACHE_PATH, source) path = os.path.join(CACHE_PATH, source)
file_path = os.path.join(path, filename) file_path = os.path.join(path, filename)
cached_url = f"/cache/{source}/{filename}" cached_url = f'/cache/{source}/{filename}'
# Ensure cache folder # Ensure cache folder
if not os.path.isdir(path): if not os.path.isdir(path):
os.mkdir(path) os.mkdir(path)
# Fetch url # Fetch url
logger.info(f"Caching {url} to {file_path}") logger.info(f'Caching {url} to {file_path}')
response = requests.get(url) response = requests.get(url)
# Write file to disk # Write file to disk
with open(file_path, "wb") as f: with open(file_path, 'wb') as f:
f.write(response.content) f.write(response.content)
# Return the inquisitor path to the file # Return the inquisitor path to the file
return cached_url return cached_url
class LinearCrawler: class LinearCrawler:
""" """
An engine for generating items from web sources that link content An engine for generating items from web sources that link content
together in a linear fashion, such as webcomics. together in a linear fashion, such as webcomics.
""" """
def fetch_new(self, state):
items = []
max_iter = self.max_iterations() - 1
new = self.try_fetch(state)
items.extend(new)
for iter in range(max_iter):
sleep(1)
# If we've already gotten some items out of this fetch, we don't
# want to lose them and have the state still be set to the next
# page, so we wrap further calls in a try block and force return
# if we hit an error.
try:
new = self.try_fetch(state)
except:
new = []
items.extend(new)
# Cut out early if there was nothing returned
if not new:
break
return items
def fetch_new(self, state): def try_fetch(self, state):
items = [] # Check for whether a new page should be crawled
max_iter = self.max_iterations() - 1 if 'current_page' not in state:
new = self.try_fetch(state) next_page = self.get_start_url()
items.extend(new) else:
for iter in range(max_iter): current = state['current_page']
sleep(1) response = requests.get(current)
# If we've already gotten some items out of this fetch, we don't soup = BeautifulSoup(response.text, features='html.parser')
# want to lose them and have the state still be set to the next next_page = self.get_next_page_url(current, soup)
# page, so we wrap further calls in a try block and force return if not next_page:
# if we hit an error. return [] # nothing new
try:
new = self.try_fetch(state)
except:
new = []
items.extend(new)
# Cut out early if there was nothing returned
if not new:
break
return items
def try_fetch(self, state): # Download the new page
# Check for whether a new page should be crawled logger.info('Fetching ' + next_page)
if "current_page" not in state: response = requests.get(next_page)
next_page = self.get_start_url() soup = BeautifulSoup(response.text, features="html.parser")
else:
current = state["current_page"]
response = requests.get(current)
soup = BeautifulSoup(response.text, features="html.parser")
next_page = self.get_next_page_url(current, soup)
if not next_page:
return [] # nothing new
# Download the new page # Create an item from the page
logger.info("Fetching " + next_page) item = self.make_item(next_page, soup)
response = requests.get(next_page)
soup = BeautifulSoup(response.text, features="html.parser")
# Create an item from the page # Update the state and return the item
item = self.make_item(next_page, soup) state['current_page'] = next_page
return [item]
# Update the state and return the item def max_iterations(self):
state["current_page"] = next_page return 3
return [item]
def max_iterations(self): def get_start_url(self):
return 3 raise NotImplementedError('get_start_url is required')
def get_start_url(self): def get_next_page_url(self, url, soup):
raise NotImplementedError("get_start_url is required") raise NotImplementedError('get_next_page_url is required')
def get_next_page_url(self, url, soup): def make_item(self, url, soup):
raise NotImplementedError("get_next_page_url is required") raise NotImplementedError('make_item is required')
def make_item(self, url, soup):
raise NotImplementedError("make_item is required")
class RedditScraper: class RedditScraper:
""" """
An engine for generating items from subreddits. An engine for generating items from subreddits.
Requires defining source, subreddit_name Requires defining source, subreddit_name
fetch new with RedditScraper.fetch_new(state, __name__, reddit) fetch new with RedditScraper.fetch_new(state, __name__, reddit)
""" """
@staticmethod
def fetch_new(state, name, reddit):
items = []
for name, obj in inspect.getmembers(sys.modules[name]):
if (inspect.isclass(obj)
and issubclass(obj, RedditScraper)
and obj is not RedditScraper
):
sub_items = obj(reddit).get_items()
items.extend(sub_items)
return items
@staticmethod def __init__(self, reddit):
def fetch_new(state, name, reddit): self.reddit = reddit
items = []
for name, obj in inspect.getmembers(sys.modules[name]):
if (
inspect.isclass(obj)
and issubclass(obj, RedditScraper)
and obj is not RedditScraper
):
sub_items = obj(reddit).get_items()
items.extend(sub_items)
return items
def __init__(self, reddit): def get_items(self):
self.reddit = reddit sub_name = self.subreddit_name
logger.info(f'Fetching posts from r/{sub_name}')
subreddit = self.reddit.subreddit(sub_name)
posts = self.subreddit_page(subreddit)
items = []
for post in posts:
if self.filter_post(post):
items.append(self.item_from_post(post))
return items
def get_items(self): def item_from_post(self, post):
sub_name = self.subreddit_name item = {
logger.info(f"Fetching posts from r/{sub_name}") 'source': self.source,
subreddit = self.reddit.subreddit(sub_name) 'id': post.id,
posts = self.subreddit_page(subreddit) 'title': self.get_title(post),
items = [] 'link': self.get_link(post),
for post in posts: 'time': post.created_utc,
if self.filter_post(post): 'author': '/u/' + (post.author.name if post.author else "[deleted]"),
items.append(self.item_from_post(post)) 'body': self.get_body(post),
return items 'tags': self.get_tags(post),
'ttl': self.get_ttl(post),
}
ttl = self.get_ttl(post)
if ttl is not None: item['ttl'] = ttl
ttd = self.get_ttd(post)
if ttd is not None: item['ttd'] = ttd
tts = self.get_tts(post)
if tts is not None: item['tts'] = tts
callback = self.get_callback(post)
if callback is not None: item['callback'] = callback
return item
def item_from_post(self, post): def subreddit_page(self, subreddit):
item = { return subreddit.hot(limit=25)
"source": self.source,
"id": post.id,
"title": self.get_title(post),
"link": self.get_link(post),
"time": post.created_utc,
"author": "/u/" + (post.author.name if post.author else "[deleted]"),
"body": self.get_body(post),
"tags": self.get_tags(post),
"ttl": self.get_ttl(post),
}
ttl = self.get_ttl(post)
if ttl is not None:
item["ttl"] = ttl
ttd = self.get_ttd(post)
if ttd is not None:
item["ttd"] = ttd
tts = self.get_tts(post)
if tts is not None:
item["tts"] = tts
callback = self.get_callback(post)
if callback is not None:
item["callback"] = callback
return item
def subreddit_page(self, subreddit): def filter_post(self, post):
return subreddit.hot(limit=25) return True
def filter_post(self, post): def get_title(self, post):
return True s = '[S] ' if post.spoiler else ''
nsfw = '[NSFW] ' if post.over_18 else ''
return f'{s}{nsfw}/{post.subreddit_name_prefixed}: {post.title}'
def get_title(self, post): def get_link(self, post):
s = "[S] " if post.spoiler else "" return f'https://reddit.com{post.permalink}'
nsfw = "[NSFW] " if post.over_18 else ""
return f"{s}{nsfw}/{post.subreddit_name_prefixed}: {post.title}"
def get_link(self, post): def get_body(self, post):
return f"https://reddit.com{post.permalink}" parts = []
if not post.is_self:
parts.append(f'<i>link:</i> <a href="{post.url}">{post.url}</a>')
if hasattr(post, 'preview'):
try:
previews = post.preview['images'][0]['resolutions']
small_previews = [p for p in previews if p['width'] < 800]
preview = sorted(small_previews, key=lambda p:-p['width'])[0]
parts.append(f'<img src="{preview["url"]}">')
except:
pass
if getattr(post, 'is_gallery', False):
try:
for gallery_item in post.gallery_data['items']:
media_id = gallery_item['media_id']
metadata = post.media_metadata[media_id]
small_previews = [p for p in metadata['p'] if p['x'] < 800]
preview = sorted(small_previews, key=lambda p:-p['x'])[0]
parts.append(f'<i>link:</i> <a href="{metadata["s"]["u"]}">{metadata["s"]["u"]}</a>')
parts.append(f'<img src="{preview["u"]}">')
except:
pass
if post.selftext:
limit = post.selftext[1024:].find(' ')
preview_body = post.selftext[:1024 + limit]
if len(preview_body) < len(post.selftext):
preview_body += '[...]'
parts.append(f'<p>{preview_body}</p>')
return '<br><hr>'.join(parts)
def get_body(self, post): def get_tags(self, post):
parts = [] tags = ['reddit', post.subreddit_name_prefixed[2:]]
if not post.is_self: if post.over_18:
parts.append(f'<i>link:</i> <a href="{post.url}">{post.url}</a>') tags.append('nsfw')
if hasattr(post, "preview"): return tags
try:
previews = post.preview["images"][0]["resolutions"]
small_previews = [p for p in previews if p["width"] < 800]
preview = sorted(small_previews, key=lambda p: -p["width"])[0]
parts.append(f'<img src="{preview["url"]}">')
except:
pass
if getattr(post, "is_gallery", False):
try:
for gallery_item in post.gallery_data["items"]:
media_id = gallery_item["media_id"]
metadata = post.media_metadata[media_id]
small_previews = [p for p in metadata["p"] if p["x"] < 800]
preview = sorted(small_previews, key=lambda p: -p["x"])[0]
parts.append(
f'<i>link:</i> <a href="{metadata["s"]["u"]}">{metadata["s"]["u"]}</a>'
)
parts.append(f'<img src="{preview["u"]}">')
except:
pass
if post.selftext:
limit = post.selftext[1024:].find(" ")
preview_body = post.selftext[: 1024 + limit]
if len(preview_body) < len(post.selftext):
preview_body += "[...]"
parts.append(f"<p>{preview_body}</p>")
return "<br><hr>".join(parts)
def get_tags(self, post): def get_ttl(self, post):
tags = ["reddit", post.subreddit_name_prefixed[2:]] return 60 * 60 * 24 * 7 # 1 week
if post.over_18:
tags.append("nsfw")
return tags
def get_ttl(self, post): def get_ttd(self, post):
return 60 * 60 * 24 * 7 # 1 week return None
def get_ttd(self, post): def get_tts(self, post):
return None return None
def get_tts(self, post): def get_callback(self, post):
return None return None
def get_callback(self, post): def callback(self, state, item):
return None raise NotImplementedError('callback')
def callback(self, state, item): def on_create(self, state, item):
raise NotImplementedError("callback") raise NotImplementedError('on_create')
def on_create(self, state, item): def on_delete(self, state, item):
raise NotImplementedError("on_create") raise NotImplementedError('on_delete')
def on_delete(self, state, item):
raise NotImplementedError("on_delete")

8
inquisitor/timestamp.py
View File

@ -1,11 +1,9 @@
import time import time
import datetime import datetime
def now(): def now():
return int(time.time()) return int(time.time())
def stamp_to_readable(ts, formatstr="%Y-%m-%d %H:%M:%S"): def stamp_to_readable(ts, formatstr="%Y-%m-%d %H:%M:%S"):
dt = datetime.datetime.fromtimestamp(ts) dt = datetime.datetime.fromtimestamp(ts)
return dt.strftime(formatstr) return dt.strftime(formatstr)

154
module.nix
View File

@ -1,154 +0,0 @@
flake: { config, lib, pkgs, ... }:
let
inherit (lib) mkIf mkOption types;
cfg = config.services.inquisitor;
in
{
options = {
services.inquisitor = {
enable = mkOption {
type = types.bool;
default = true;
description = "Enable the Inquisitor aggregator.";
};
listen.addr = mkOption {
type = types.str;
default = "0.0.0.0";
description = "Listen address passed to nginx.";
};
listen.port = mkOption {
type = types.port;
default = 80;
description = "Listen port passed to nginx.";
};
};
};
config =
let
# Get the inquisitor package from the flake.
inquisitor = flake.packages.${pkgs.stdenv.hostPlatform.system}.default;
# Define the inquisitor state directory.
stateDir = "/var/lib/inquisitor";
# Define an scp helper for item callbacks to use.
scp-helper = pkgs.writeShellScriptBin "scp-helper" ''
${pkgs.openssh}/bin/scp -i ${stateDir}/.ssh/inquisitor.key -oStrictHostKeyChecking=no "$@"
'';
# Define the inquisitor service user.
svcUser = {
name = "inquisitor";
group = "inquisitor";
description = "Inquisitor service user";
isSystemUser = true;
shell = pkgs.bashInteractive;
packages = [ inquisitor pkgs.cron ];
};
# Create a config file pointing to the state directory.
inqConfig = pkgs.writeTextFile {
name = "inquisitor.conf";
text = ''
DataPath = ${stateDir}/data/
SourcePath = ${stateDir}/sources/
CachePath = ${stateDir}/cache/
Verbose = false
LogFile = ${stateDir}/inquisitor.log
'';
};
# Create a setup script to ensure the service directory state.
inqSetup = pkgs.writeShellScript "inquisitor-setup.sh" ''
# Ensure the required directories exist.
${pkgs.coreutils}/bin/mkdir -p ${stateDir}/data/inquisitor/
${pkgs.coreutils}/bin/mkdir -p ${stateDir}/sources/
${pkgs.coreutils}/bin/mkdir -p ${stateDir}/cache/
if [ ! -f ${stateDir}/data/inquisitor/state ]; then
${pkgs.coreutils}/bin/echo "{}" > ${stateDir}/data/inquisitor/state
fi
# Ensure the service owns the folders.
${pkgs.coreutils}/bin/chown -R ${svcUser.name} ${stateDir}
# Ensure the scp helper is present
if [ -f ${stateDir}/scp-helper ]; then
${pkgs.coreutils}/bin/rm ${stateDir}/scp-helper
fi
ln -s -t ${stateDir}/scp-helper ${scp-helper}/bin/scp-helper
'';
# Create a run script for the service.
inqRun = pkgs.writeShellScript "inquisitor-run.sh" ''
cd ${stateDir}
${inquisitor}/bin/gunicorn \
--bind=localhost:24133 \
--workers=4 \
--timeout 120 \
--log-level debug \
"inquisitor.app:wsgi()"
'';
# Create a wrapper to execute the cli as the service user.
# (needed to avoid creating files in the state dir the service can't read)
inqWrapper = pkgs.writeShellScriptBin "inq" ''
sudo --user=${svcUser.name} ${inquisitor}/bin/inquisitor "$@"
'';
in mkIf cfg.enable
{
users.users.inquisitor = svcUser;
users.groups.inquisitor = {};
# Link the config in /etc to avoid envvar shenanigans
environment.etc."inquisitor.conf".source = inqConfig;
# Give all users the wrapper program.
environment.systemPackages = [ inqWrapper ];
# Allow the sudo in the cli wrapper without password.
security.sudo.extraRules = [{
commands = [{
command = "${inquisitor}/bin/inquisitor";
options = [ "NOPASSWD" ];
}];
runAs = svcUser.name;
groups = [ "users" ];
}];
# Run the setup script on activation.
system.activationScripts.inquisitorSetup = "${inqSetup}";
# Set up the inquisitor service.
systemd.services.inquisitor = {
description = "Inquisitor server";
script = "${inqRun}";
serviceConfig = {
User = svcUser.name;
Type = "simple";
};
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
enable = true;
};
# Set up the nginx reverse proxy to the server.
services.nginx.enable = true;
services.nginx.virtualHosts.inquisitorHost = {
listen = [ cfg.listen ];
locations."/".extraConfig = ''
access_log /var/log/nginx/access.inquisitor.log;
proxy_buffering off;
proxy_pass http://localhost:24133/;
'';
};
networking.firewall.allowedTCPPorts = [ cfg.listen.port ];
# Enable cron so the service can use it to schedule fetches.
services.cron.enable = true;
};
}

206
poetry.lock generated
View File

@ -13,27 +13,6 @@ soupsieve = ">1.2"
html5lib = ["html5lib"] html5lib = ["html5lib"]
lxml = ["lxml"] lxml = ["lxml"]
[[package]]
name = "black"
version = "22.12.0"
description = "The uncompromising code formatter."
category = "dev"
optional = false
python-versions = ">=3.7"
[package.dependencies]
click = ">=8.0.0"
mypy-extensions = ">=0.4.3"
pathspec = ">=0.9.0"
platformdirs = ">=2"
tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""}
[package.extras]
colorama = ["colorama (>=0.4.3)"]
d = ["aiohttp (>=3.7.4)"]
jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"]
uvloop = ["uvloop (>=0.15.2)"]
[[package]] [[package]]
name = "certifi" name = "certifi"
version = "2022.12.7" version = "2022.12.7"
@ -72,6 +51,17 @@ category = "main"
optional = false optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
[[package]]
name = "feedparser"
version = "6.0.10"
description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds"
category = "main"
optional = true
python-versions = ">=3.6"
[package.dependencies]
sgmllib3k = "*"
[[package]] [[package]]
name = "flask" name = "flask"
version = "2.2.2" version = "2.2.2"
@ -95,7 +85,7 @@ name = "gunicorn"
version = "20.1.0" version = "20.1.0"
description = "WSGI HTTP Server for UNIX" description = "WSGI HTTP Server for UNIX"
category = "main" category = "main"
optional = false optional = true
python-versions = ">=3.5" python-versions = ">=3.5"
[package.dependencies] [package.dependencies]
@ -146,32 +136,49 @@ optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
[[package]] [[package]]
name = "mypy-extensions" name = "praw"
version = "0.4.3" version = "7.6.1"
description = "Experimental type system extensions for programs checked with the mypy typechecker." description = "PRAW, an acronym for \"Python Reddit API Wrapper\", is a python package that allows for simple access to Reddit's API."
category = "dev" category = "main"
optional = false optional = true
python-versions = "*" python-versions = "~=3.7"
[[package]] [package.dependencies]
name = "pathspec" prawcore = ">=2.1,<3"
version = "0.10.3" update-checker = ">=0.18"
description = "Utility library for gitignore style pattern matching of file paths." websocket-client = ">=0.54.0"
category = "dev"
optional = false
python-versions = ">=3.7"
[[package]]
name = "platformdirs"
version = "2.6.2"
description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
category = "dev"
optional = false
python-versions = ">=3.7"
[package.extras] [package.extras]
docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] ci = ["coveralls"]
test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] dev = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.3.0,<0.5)", "packaging", "pre-commit", "pytest (>=2.7.3)", "requests (>=2.20.1,<3)", "sphinx", "sphinx-rtd-dark-mode", "sphinx-rtd-theme"]
lint = ["pre-commit", "sphinx", "sphinx-rtd-dark-mode", "sphinx-rtd-theme"]
readthedocs = ["sphinx", "sphinx-rtd-dark-mode", "sphinx-rtd-theme"]
test = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.3.0,<0.5)", "pytest (>=2.7.3)", "requests (>=2.20.1,<3)"]
[[package]]
name = "prawcore"
version = "2.3.0"
description = "Low-level communication layer for PRAW 4+."
category = "main"
optional = true
python-versions = "~=3.6"
[package.dependencies]
requests = ">=2.6.0,<3.0"
[package.extras]
ci = ["coveralls"]
dev = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.4.0,<0.5)", "betamax-serializers (>=0.2.0,<0.3)", "black", "flake8", "flynt", "mock (>=0.8)", "pre-commit", "pydocstyle", "pytest", "testfixtures (>4.13.2,<7)"]
lint = ["black", "flake8", "flynt", "pre-commit", "pydocstyle"]
test = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.4.0,<0.5)", "betamax-serializers (>=0.2.0,<0.3)", "mock (>=0.8)", "pytest", "testfixtures (>4.13.2,<7)"]
[[package]]
name = "protobuf"
version = "4.21.12"
description = ""
category = "main"
optional = false
python-versions = ">=3.7"
[[package]] [[package]]
name = "requests" name = "requests"
@ -196,7 +203,7 @@ name = "setuptools"
version = "65.6.3" version = "65.6.3"
description = "Easily download, build, install, upgrade, and uninstall Python packages" description = "Easily download, build, install, upgrade, and uninstall Python packages"
category = "main" category = "main"
optional = false optional = true
python-versions = ">=3.7" python-versions = ">=3.7"
[package.extras] [package.extras]
@ -204,6 +211,14 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-g
testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"]
[[package]]
name = "sgmllib3k"
version = "1.0.0"
description = "Py3k port of sgmllib."
category = "main"
optional = true
python-versions = "*"
[[package]] [[package]]
name = "soupsieve" name = "soupsieve"
version = "2.3.2.post1" version = "2.3.2.post1"
@ -213,12 +228,20 @@ optional = false
python-versions = ">=3.6" python-versions = ">=3.6"
[[package]] [[package]]
name = "tomli" name = "update-checker"
version = "2.0.1" version = "0.18.0"
description = "A lil' TOML parser" description = "A python module that will check for package updates."
category = "dev" category = "main"
optional = false optional = true
python-versions = ">=3.7" python-versions = "*"
[package.dependencies]
requests = ">=2.3.0"
[package.extras]
dev = ["black", "flake8", "pytest (>=2.7.3)"]
lint = ["black", "flake8"]
test = ["pytest (>=2.7.3)"]
[[package]] [[package]]
name = "urllib3" name = "urllib3"
@ -233,6 +256,19 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[[package]]
name = "websocket-client"
version = "1.4.2"
description = "WebSocket client for Python with low level API options"
category = "main"
optional = true
python-versions = ">=3.7"
[package.extras]
docs = ["Sphinx (>=3.4)", "sphinx-rtd-theme (>=0.5)"]
optional = ["python-socks", "wsaccel"]
test = ["websockets"]
[[package]] [[package]]
name = "werkzeug" name = "werkzeug"
version = "2.2.2" version = "2.2.2"
@ -247,30 +283,19 @@ MarkupSafe = ">=2.1.1"
[package.extras] [package.extras]
watchdog = ["watchdog"] watchdog = ["watchdog"]
[extras]
dev = ["praw", "gunicorn", "feedparser"]
[metadata] [metadata]
lock-version = "1.1" lock-version = "1.1"
python-versions = "^3.10" python-versions = "^3.10"
content-hash = "fbb022ff050d55a4b5814e1ce7abcf002c5e32cc552d4171cd780dc99d0aa640" content-hash = "f7202e31b8297ce5c72738178aa28256ba6a10d03dc741804a10f1a8ba952d2b"
[metadata.files] [metadata.files]
beautifulsoup4 = [ beautifulsoup4 = [
{file = "beautifulsoup4-4.11.1-py3-none-any.whl", hash = "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30"}, {file = "beautifulsoup4-4.11.1-py3-none-any.whl", hash = "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30"},
{file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"}, {file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"},
] ]
black = [
{file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"},
{file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"},
{file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"},
{file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"},
{file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"},
{file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"},
{file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"},
{file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"},
{file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"},
{file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"},
{file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"},
{file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"},
]
certifi = [ certifi = [
{file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"},
{file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"},
@ -287,6 +312,10 @@ colorama = [
{file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
{file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
] ]
feedparser = [
{file = "feedparser-6.0.10-py3-none-any.whl", hash = "sha256:79c257d526d13b944e965f6095700587f27388e50ea16fd245babe4dfae7024f"},
{file = "feedparser-6.0.10.tar.gz", hash = "sha256:27da485f4637ce7163cdeab13a80312b93b7d0c1b775bef4a47629a3110bca51"},
]
flask = [ flask = [
{file = "Flask-2.2.2-py3-none-any.whl", hash = "sha256:b9c46cc36662a7949f34b52d8ec7bb59c0d74ba08ba6cb9ce9adc1d8676d9526"}, {file = "Flask-2.2.2-py3-none-any.whl", hash = "sha256:b9c46cc36662a7949f34b52d8ec7bb59c0d74ba08ba6cb9ce9adc1d8676d9526"},
{file = "Flask-2.2.2.tar.gz", hash = "sha256:642c450d19c4ad482f96729bd2a8f6d32554aa1e231f4f6b4e7e5264b16cca2b"}, {file = "Flask-2.2.2.tar.gz", hash = "sha256:642c450d19c4ad482f96729bd2a8f6d32554aa1e231f4f6b4e7e5264b16cca2b"},
@ -349,17 +378,29 @@ markupsafe = [
{file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"}, {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"},
{file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"},
] ]
mypy-extensions = [ praw = [
{file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "praw-7.6.1-py3-none-any.whl", hash = "sha256:33ac091cd061d9bd607ae231d4ea40025060a4ecb5e11baa5ce9a25c6d2f5a6b"},
{file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, {file = "praw-7.6.1.tar.gz", hash = "sha256:07fc95ffc52187351ab77b81fa6910c66e8fa084faf4b060b90864ad186dfb9e"},
] ]
pathspec = [ prawcore = [
{file = "pathspec-0.10.3-py3-none-any.whl", hash = "sha256:3c95343af8b756205e2aba76e843ba9520a24dd84f68c22b9f93251507509dd6"}, {file = "prawcore-2.3.0-py3-none-any.whl", hash = "sha256:48c17db447fa06a13ca3e722201f443031437708daa736c05a1df895fbcceff5"},
{file = "pathspec-0.10.3.tar.gz", hash = "sha256:56200de4077d9d0791465aa9095a01d421861e405b5096955051deefd697d6f6"}, {file = "prawcore-2.3.0.tar.gz", hash = "sha256:daf1ccd4b7a80dc4e6567d48336d782e94a9a6dad83770fc2edf76dc9a84f56d"},
] ]
platformdirs = [ protobuf = [
{file = "platformdirs-2.6.2-py3-none-any.whl", hash = "sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490"}, {file = "protobuf-4.21.12-cp310-abi3-win32.whl", hash = "sha256:b135410244ebe777db80298297a97fbb4c862c881b4403b71bac9d4107d61fd1"},
{file = "platformdirs-2.6.2.tar.gz", hash = "sha256:e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2"}, {file = "protobuf-4.21.12-cp310-abi3-win_amd64.whl", hash = "sha256:89f9149e4a0169cddfc44c74f230d7743002e3aa0b9472d8c28f0388102fc4c2"},
{file = "protobuf-4.21.12-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:299ea899484ee6f44604deb71f424234f654606b983cb496ea2a53e3c63ab791"},
{file = "protobuf-4.21.12-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:d1736130bce8cf131ac7957fa26880ca19227d4ad68b4888b3be0dea1f95df97"},
{file = "protobuf-4.21.12-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:78a28c9fa223998472886c77042e9b9afb6fe4242bd2a2a5aced88e3f4422aa7"},
{file = "protobuf-4.21.12-cp37-cp37m-win32.whl", hash = "sha256:3d164928ff0727d97022957c2b849250ca0e64777ee31efd7d6de2e07c494717"},
{file = "protobuf-4.21.12-cp37-cp37m-win_amd64.whl", hash = "sha256:f45460f9ee70a0ec1b6694c6e4e348ad2019275680bd68a1d9314b8c7e01e574"},
{file = "protobuf-4.21.12-cp38-cp38-win32.whl", hash = "sha256:6ab80df09e3208f742c98443b6166bcb70d65f52cfeb67357d52032ea1ae9bec"},
{file = "protobuf-4.21.12-cp38-cp38-win_amd64.whl", hash = "sha256:1f22ac0ca65bb70a876060d96d914dae09ac98d114294f77584b0d2644fa9c30"},
{file = "protobuf-4.21.12-cp39-cp39-win32.whl", hash = "sha256:27f4d15021da6d2b706ddc3860fac0a5ddaba34ab679dc182b60a8bb4e1121cc"},
{file = "protobuf-4.21.12-cp39-cp39-win_amd64.whl", hash = "sha256:237216c3326d46808a9f7c26fd1bd4b20015fb6867dc5d263a493ef9a539293b"},
{file = "protobuf-4.21.12-py2.py3-none-any.whl", hash = "sha256:a53fd3f03e578553623272dc46ac2f189de23862e68565e83dde203d41b76fc5"},
{file = "protobuf-4.21.12-py3-none-any.whl", hash = "sha256:b98d0148f84e3a3c569e19f52103ca1feacdac0d2df8d6533cf983d1fda28462"},
{file = "protobuf-4.21.12.tar.gz", hash = "sha256:7cd532c4566d0e6feafecc1059d04c7915aec8e182d1cf7adee8b24ef1e2e6ab"},
] ]
requests = [ requests = [
{file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"},
@ -369,18 +410,25 @@ setuptools = [
{file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"},
{file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"},
] ]
sgmllib3k = [
{file = "sgmllib3k-1.0.0.tar.gz", hash = "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9"},
]
soupsieve = [ soupsieve = [
{file = "soupsieve-2.3.2.post1-py3-none-any.whl", hash = "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759"}, {file = "soupsieve-2.3.2.post1-py3-none-any.whl", hash = "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759"},
{file = "soupsieve-2.3.2.post1.tar.gz", hash = "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"}, {file = "soupsieve-2.3.2.post1.tar.gz", hash = "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"},
] ]
tomli = [ update-checker = [
{file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "update_checker-0.18.0-py3-none-any.whl", hash = "sha256:cbba64760a36fe2640d80d85306e8fe82b6816659190993b7bdabadee4d4bbfd"},
{file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, {file = "update_checker-0.18.0.tar.gz", hash = "sha256:6a2d45bb4ac585884a6b03f9eade9161cedd9e8111545141e9aa9058932acb13"},
] ]
urllib3 = [ urllib3 = [
{file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"}, {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"},
{file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"},
] ]
websocket-client = [
{file = "websocket-client-1.4.2.tar.gz", hash = "sha256:d6e8f90ca8e2dd4e8027c4561adeb9456b54044312dba655e7cae652ceb9ae59"},
{file = "websocket_client-1.4.2-py3-none-any.whl", hash = "sha256:d6b06432f184438d99ac1f456eaf22fe1ade524c3dd16e661142dc54e9cba574"},
]
werkzeug = [ werkzeug = [
{file = "Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5"}, {file = "Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5"},
{file = "Werkzeug-2.2.2.tar.gz", hash = "sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f"}, {file = "Werkzeug-2.2.2.tar.gz", hash = "sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f"},

11
pyproject.toml
View File

@ -11,14 +11,17 @@ python = "^3.10"
flask = "^2.2.2" flask = "^2.2.2"
requests = "^2.28.1" requests = "^2.28.1"
beautifulsoup4 = "^4.11.1" beautifulsoup4 = "^4.11.1"
gunicorn = "^20.1.0" praw = {version = "^7.6.1", optional = true}
gunicorn = {version = "^20.1.0", optional = true}
feedparser = {version = "^6.0.10", optional = true}
protobuf = "^4.21.12"
[tool.poetry.extras]
dev = ["praw", "gunicorn", "feedparser"]
[tool.poetry.scripts] [tool.poetry.scripts]
inquisitor = "inquisitor.cli:main" inquisitor = "inquisitor.cli:main"
[tool.poetry.group.dev.dependencies]
black = "^22.12.0"
[build-system] [build-system]
requires = ["poetry>=0.12"] requires = ["poetry>=0.12"]
build-backend = "poetry.masonry.api" build-backend = "poetry.masonry.api"