Add source to git

inquisitor/__main__.py

@@ -0,0 +1,5 @@
+from cli import run
+
+if __name__ == "__main__":
+	run()
+

inquisitor/app.py

@@ -0,0 +1,47 @@
+# Standard library imports
+from datetime import datetime
+import logging
+
+# Third party imports
+from flask import Flask, render_template, request, jsonify
+
+# Application imports
+from inquisitor import dungeon, core
+
+# Globals
+logger = logging.getLogger("inquisitor.app")
+logger.setLevel(logging.INFO)
+console = logging.StreamHandler()
+console.setLevel(logging.INFO)
+formatter = logging.Formatter('[%(asctime)s %(levelname)s:%(filename)s:%(lineno)d] %(message)s')
+console.setFormatter(formatter)
+logger.addHandler(console)
+
+app = Flask(__name__)
+dungeon = dungeon.Dungeon("dungeon")
+itemsources = core.load_all_sources("sources")
+
+
+@app.route("/feed/")
+def root():
+	active_items = dungeon.get_active_items()
+	logger.info("Found {} active items".format(len(active_items)))
+	for item in active_items:
+		item['time_readable'] = str(datetime.fromtimestamp(item['time']))
+	active_items.sort(key=lambda i: i['time'])
+	return render_template("feed.html", items=active_items[:100])
+
+
+@app.route("/deactivate/", methods=['POST'])
+def deactivate():
+	params = request.get_json()
+	if 'source' not in params and 'itemid' not in params:
+		logger.error("Bad request params: {}".format(params))
+	item = dungeon.deactivate_item(params['source'], params['itemid'])
+	return jsonify({'active': item['active']})
+
+
+@app.route("/feed.css")
+def css():
+	with open("feed.css") as f:
+		return f.read()

inquisitor/cli.py

@@ -0,0 +1,55 @@
+# Standard library imports
+import argparse
+import logging
+import os
+
+# Application imports
+from core import load_all_sources
+from dungeon import Dungeon
+
+# Globals
+logger = logging.getLogger("inquisitor.cli")
+
+
+def run():
+	parser = argparse.ArgumentParser()
+	parser.add_argument("--log", default="INFO", help="Set the log level (default: INFO)")
+	subparsers = parser.add_subparsers(help="Command to execute", dest="command")
+	subparsers.required = True
+
+	update_parser = subparsers.add_parser("update", help="Fetch new items")
+	update_parser.add_argument("--srcdir", help="Path to sources folder (default ./sources)",
+		default="./sources")
+	update_parser.add_argument("--dungeon", help="Path to item cache folder (default ./dungeon)",
+		default="./dungeon")
+	update_parser.add_argument("--sources", help="Sources to update, by name",
+		nargs="*")
+	update_parser.set_defaults(func=update)
+
+	args = parser.parse_args()
+
+	# Configure logging
+	loglevel = getattr(logging, args.log.upper())
+	if not isinstance(loglevel, int):
+		raise ValueError("Invalid log level: {}".format(args.log))
+	logging.basicConfig(format='[%(levelname)s:%(filename)s:%(lineno)d] %(message)s', level=loglevel)
+
+	args.func(args)
+
+
+def update(args):
+	"""Fetches new items from sources and stores them in the dungeon."""
+	if not os.path.isdir(args.srcdir):
+		logger.error("srcdir must be a directory")
+		exit(-1)
+	if not os.path.isdir(args.dungeon):
+		logger.error("dungeon must be a directory")
+		exit(-1)
+	sources = load_all_sources(args.srcdir)
+	names = args.sources or [s.SOURCE for s in sources]
+	dungeon = Dungeon(args.dungeon)
+	for itemsource in sources:
+		if itemsource.SOURCE in names:
+			new_items = dungeon.update(itemsource)
+			items = dungeon.get_active_items_for_folder(itemsource.SOURCE)
+			logger.info("{} new item{}".format(new_items, "s" if new_items != 1 else ""))

inquisitor/core.py

@@ -0,0 +1,42 @@
+# Standard library imports
+import importlib.util
+import os
+import logging
+
+# Globals
+logger = logging.getLogger("inquisitor.core")
+
+
+def load_source_module(source_path):
+	"""Loads a source module and checks for necessary members."""
+	logger.debug("load_source_module('{}')".format(source_path))
+	spec = importlib.util.spec_from_file_location("itemsource", source_path)
+	itemsource = importlib.util.module_from_spec(spec)
+	spec.loader.exec_module(itemsource)
+	if not hasattr(itemsource, 'SOURCE'):
+		raise ImportError("SOURCE missing")
+	if not hasattr(itemsource, 'fetch_new'):
+		raise ImportError("fetch_new missing")
+	return itemsource
+
+
+def load_all_sources(source_folder):
+	"""Loads all source modules in the given folder."""
+	# Navigate to the sources folder
+	cwd = os.getcwd()
+	os.chdir(source_folder)
+	# Load all sources
+	source_names = [
+		filename
+		for filename in os.listdir()
+		if filename.endswith(".py")]
+	sources = []
+	for source_name in source_names:
+		try:
+			itemsource = load_source_module(source_name)
+			sources.append(itemsource)
+		except ImportError as e:
+			logger.error("Error importing {}: {}".format(source_name, e))
+	# Return to cwd
+	os.chdir(cwd)
+	return sources

inquisitor/dungeon.py

@@ -0,0 +1,111 @@
+# Standard library imports
+import os
+import logging
+import ast
+
+
+class Dungeon():
+	def __init__(self, path):
+		"""
+		Serves as an interface between Inquisitor and a folder of
+		serialized readable items.
+		"""
+		self.path = path
+		self.log = logging.getLogger("inquisitor.dungeon")
+
+	def load_path(self, path):
+		self.log.debug("Loading item from {}".format(path))
+		with open(path, 'r', encoding='utf-8') as f:
+			item = ast.literal_eval(f.read())
+		return item
+
+	def load_item(self, source, itemid):
+		item_path = os.path.join(self.path, source, itemid + ".item")
+		return self.load_path(item_path)
+
+	def save_item(self, item):
+		path = os.path.join(self.path, item['source'], item['id'] + ".item")
+		self.log.debug("Saving item {} to {}".format(item['id'], path))
+		with open(path, 'w', encoding='utf-8') as f:
+			f.write(str(item))
+
+	def update(self, itemsource):
+		"""
+		Fetches items from the given source, saves new active items,
+		and clears out old inactive items.
+		"""
+		new_items = 0
+		self.log.info("Updating source {}".format(itemsource.SOURCE))
+		# Check if the source has a folder.
+		source_folder = os.path.join(self.path, itemsource.SOURCE)
+		source_state = os.path.join(source_folder, "state")
+		if not os.path.isdir(source_folder):
+			self.log.info("Creating folder {}".format(source_folder))
+			os.mkdir(source_folder)
+			# Initialize persistent state.
+			with open(source_state, 'w') as f:
+				f.write("{}")
+		# Load source persistent state.
+		state = self.load_path(source_state)
+		# Any inactive items that no longer show up as new should be
+		# removed. Track which items to check for inactivity.
+		extant_items_to_check = [
+			filename
+			for filename in os.listdir(source_folder)
+			if filename.endswith(".item")]
+		# Get the new items from the source.
+		source_items = itemsource.fetch_new(state)
+		with open(source_state, 'w', encoding='utf-8') as f:
+			f.write(str(state))
+		for source_item in source_items:
+			file_path = os.path.join(source_folder, source_item['id'] + ".item")
+			if os.path.isfile(file_path):
+				# Still-new items are exempt from activity checks.
+				extant_items_to_check.remove(source_item['id'] + ".item")
+				item = self.load_path(file_path)
+				if not item['active']:
+					# Don't reactivate inactive items.
+					continue
+			else:
+				new_items += 1
+			# Add new items and update active ones.
+			self.save_item(source_item)
+		# Check old items for inactivity.
+		for extant_item_filename in extant_items_to_check:
+			file_path = os.path.join(source_folder, extant_item_filename)
+			item = self.load_path(file_path)
+			if not item['active']:
+				# Remove old inactive items.
+				self.log.info("Deleting {}".format(file_path))
+				os.remove(file_path)
+		return new_items
+
+	def get_active_items(self):
+		source_folders = os.listdir(self.path)
+		items = []
+		for source_folder_name in source_folders:
+			items.extend(self.get_active_items_for_folder(source_folder_name))
+		return items
+
+	def get_active_items_for_folder(self, source):
+		source_folder = os.path.join(self.path, source)
+		item_filenames = os.listdir(source_folder)
+		items = []
+		for item_filename in item_filenames:
+			if not item_filename.endswith(".item"):
+				continue
+			file_path = os.path.join(source_folder, item_filename)
+			item = self.load_path(file_path)
+			if item['active']:
+				items.append(item)
+		return items
+
+	def deactivate_item(self, source, itemid):
+		item_path = os.path.join(self.path, source, itemid + ".item")
+		if not os.path.isfile(item_path):
+			self.log.error("No item found: {}".format(item_path))
+			return
+		item = self.load_path(item_path)
+		item['active'] = False
+		self.save_item(item)
+		return item

inquisitor/resources/feed.css

@@ -0,0 +1,39 @@
+div#wrapper {
+	max-width: 700px;
+	margin: 0 auto;
+}
+.readable-item {
+	border: 1px solid black;
+	border-radius: 6px;
+	padding: 5px;
+	margin-bottom: 20px;
+}
+.item-title {
+	font-size: 1.4em;
+}
+.readable-item button {
+	font-size: 1em;
+	float:right;
+}
+.item-link {
+	text-decoration: none;
+	float:right;
+	margin: 0px 2px;
+	font-size: 1em;
+	padding: 2px 7px;
+	border: 1px solid;
+	border-radius: 2px;
+}
+.item-info {
+	color: rgba(0, 0, 0, 0.7);
+}
+button, summary {
+	cursor: pointer;
+}
+summary:focus {
+	outline: 1px dotted gray;
+}
+.strikethru span, .strikethru p {
+	text-decoration: line-through;
+	color: rgba(0, 0, 0, 0.2);
+}

inquisitor/templates/feed.html

@@ -0,0 +1,51 @@
+<html>
+	<head>
+		<link rel="stylesheet" type="text/css" href="/feed.css" />
+		<script>
+			var deactivate = function (source, itemid) {
+				fetch('/deactivate/', {
+					method: 'POST',
+					headers: {
+						'Content-Type': 'application/json; charset=UTF-8',
+					},
+					body: JSON.stringify({source: source, itemid: itemid}),
+				})
+				.then(response => response.json())
+				.then(function (data) {
+					if (!data.active) {
+						document.getElementById(source + "-" + itemid)
+							.classList.add("strikethru")
+					}
+				});
+			}
+		</script>
+	</head>
+	<body>
+		<div id="wrapper">
+			{% if items %}
+			{% for item in items %}
+			<div class="readable-item" id="{{item.source}}-{{item.id}}">
+				<button onclick="javascript:deactivate('{{item.source}}', '{{item.id}}')">&#10005;</button>
+				{% if item.link %}
+				<a class="item-link" href="{{item.link}}" target="_blank">&#8663;</a>
+				{% endif %}
+				{% if item.body %}
+				<details>
+				<summary><span class="item-title">{{item.title}}</span></summary>
+				<p>{{item.body|safe}}</p>
+				</details>
+				{% else %}
+				<span class="item-title">{{item.title}}</span><br>
+				{% endif %}
+				<span class="item-info">{{item.time_readable}} {{item.author}}</span><br>
+				<span class="item-info">{{item.source}} {{item.id}} {{item.time_readable}}</span>
+			</div>
+			{% endfor %}
+			{% else %}
+			<div class="readable-item">
+				<span class="item-title">Feed is empty</span>
+			</div>
+			{% endif %}
+		</div>
+	</body>
+</html>