Add source to git
This commit is contained in:
parent
9c6975dc17
commit
0d48c773ef
|
|
@ -0,0 +1,5 @@
|
|||
from cli import run
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
# Standard library imports
|
||||
from datetime import datetime
|
||||
import logging
|
||||
|
||||
# Third party imports
|
||||
from flask import Flask, render_template, request, jsonify
|
||||
|
||||
# Application imports
|
||||
from inquisitor import dungeon, core
|
||||
|
||||
# Globals
|
||||
logger = logging.getLogger("inquisitor.app")
|
||||
logger.setLevel(logging.INFO)
|
||||
console = logging.StreamHandler()
|
||||
console.setLevel(logging.INFO)
|
||||
formatter = logging.Formatter('[%(asctime)s %(levelname)s:%(filename)s:%(lineno)d] %(message)s')
|
||||
console.setFormatter(formatter)
|
||||
logger.addHandler(console)
|
||||
|
||||
app = Flask(__name__)
|
||||
dungeon = dungeon.Dungeon("dungeon")
|
||||
itemsources = core.load_all_sources("sources")
|
||||
|
||||
|
||||
@app.route("/feed/")
|
||||
def root():
|
||||
active_items = dungeon.get_active_items()
|
||||
logger.info("Found {} active items".format(len(active_items)))
|
||||
for item in active_items:
|
||||
item['time_readable'] = str(datetime.fromtimestamp(item['time']))
|
||||
active_items.sort(key=lambda i: i['time'])
|
||||
return render_template("feed.html", items=active_items[:100])
|
||||
|
||||
|
||||
@app.route("/deactivate/", methods=['POST'])
|
||||
def deactivate():
|
||||
params = request.get_json()
|
||||
if 'source' not in params and 'itemid' not in params:
|
||||
logger.error("Bad request params: {}".format(params))
|
||||
item = dungeon.deactivate_item(params['source'], params['itemid'])
|
||||
return jsonify({'active': item['active']})
|
||||
|
||||
|
||||
@app.route("/feed.css")
|
||||
def css():
|
||||
with open("feed.css") as f:
|
||||
return f.read()
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
# Standard library imports
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
|
||||
# Application imports
|
||||
from core import load_all_sources
|
||||
from dungeon import Dungeon
|
||||
|
||||
# Globals
|
||||
logger = logging.getLogger("inquisitor.cli")
|
||||
|
||||
|
||||
def run():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--log", default="INFO", help="Set the log level (default: INFO)")
|
||||
subparsers = parser.add_subparsers(help="Command to execute", dest="command")
|
||||
subparsers.required = True
|
||||
|
||||
update_parser = subparsers.add_parser("update", help="Fetch new items")
|
||||
update_parser.add_argument("--srcdir", help="Path to sources folder (default ./sources)",
|
||||
default="./sources")
|
||||
update_parser.add_argument("--dungeon", help="Path to item cache folder (default ./dungeon)",
|
||||
default="./dungeon")
|
||||
update_parser.add_argument("--sources", help="Sources to update, by name",
|
||||
nargs="*")
|
||||
update_parser.set_defaults(func=update)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Configure logging
|
||||
loglevel = getattr(logging, args.log.upper())
|
||||
if not isinstance(loglevel, int):
|
||||
raise ValueError("Invalid log level: {}".format(args.log))
|
||||
logging.basicConfig(format='[%(levelname)s:%(filename)s:%(lineno)d] %(message)s', level=loglevel)
|
||||
|
||||
args.func(args)
|
||||
|
||||
|
||||
def update(args):
|
||||
"""Fetches new items from sources and stores them in the dungeon."""
|
||||
if not os.path.isdir(args.srcdir):
|
||||
logger.error("srcdir must be a directory")
|
||||
exit(-1)
|
||||
if not os.path.isdir(args.dungeon):
|
||||
logger.error("dungeon must be a directory")
|
||||
exit(-1)
|
||||
sources = load_all_sources(args.srcdir)
|
||||
names = args.sources or [s.SOURCE for s in sources]
|
||||
dungeon = Dungeon(args.dungeon)
|
||||
for itemsource in sources:
|
||||
if itemsource.SOURCE in names:
|
||||
new_items = dungeon.update(itemsource)
|
||||
items = dungeon.get_active_items_for_folder(itemsource.SOURCE)
|
||||
logger.info("{} new item{}".format(new_items, "s" if new_items != 1 else ""))
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
# Standard library imports
|
||||
import importlib.util
|
||||
import os
|
||||
import logging
|
||||
|
||||
# Globals
|
||||
logger = logging.getLogger("inquisitor.core")
|
||||
|
||||
|
||||
def load_source_module(source_path):
|
||||
"""Loads a source module and checks for necessary members."""
|
||||
logger.debug("load_source_module('{}')".format(source_path))
|
||||
spec = importlib.util.spec_from_file_location("itemsource", source_path)
|
||||
itemsource = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(itemsource)
|
||||
if not hasattr(itemsource, 'SOURCE'):
|
||||
raise ImportError("SOURCE missing")
|
||||
if not hasattr(itemsource, 'fetch_new'):
|
||||
raise ImportError("fetch_new missing")
|
||||
return itemsource
|
||||
|
||||
|
||||
def load_all_sources(source_folder):
|
||||
"""Loads all source modules in the given folder."""
|
||||
# Navigate to the sources folder
|
||||
cwd = os.getcwd()
|
||||
os.chdir(source_folder)
|
||||
# Load all sources
|
||||
source_names = [
|
||||
filename
|
||||
for filename in os.listdir()
|
||||
if filename.endswith(".py")]
|
||||
sources = []
|
||||
for source_name in source_names:
|
||||
try:
|
||||
itemsource = load_source_module(source_name)
|
||||
sources.append(itemsource)
|
||||
except ImportError as e:
|
||||
logger.error("Error importing {}: {}".format(source_name, e))
|
||||
# Return to cwd
|
||||
os.chdir(cwd)
|
||||
return sources
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
# Standard library imports
|
||||
import os
|
||||
import logging
|
||||
import ast
|
||||
|
||||
|
||||
class Dungeon():
|
||||
def __init__(self, path):
|
||||
"""
|
||||
Serves as an interface between Inquisitor and a folder of
|
||||
serialized readable items.
|
||||
"""
|
||||
self.path = path
|
||||
self.log = logging.getLogger("inquisitor.dungeon")
|
||||
|
||||
def load_path(self, path):
|
||||
self.log.debug("Loading item from {}".format(path))
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
item = ast.literal_eval(f.read())
|
||||
return item
|
||||
|
||||
def load_item(self, source, itemid):
|
||||
item_path = os.path.join(self.path, source, itemid + ".item")
|
||||
return self.load_path(item_path)
|
||||
|
||||
def save_item(self, item):
|
||||
path = os.path.join(self.path, item['source'], item['id'] + ".item")
|
||||
self.log.debug("Saving item {} to {}".format(item['id'], path))
|
||||
with open(path, 'w', encoding='utf-8') as f:
|
||||
f.write(str(item))
|
||||
|
||||
def update(self, itemsource):
|
||||
"""
|
||||
Fetches items from the given source, saves new active items,
|
||||
and clears out old inactive items.
|
||||
"""
|
||||
new_items = 0
|
||||
self.log.info("Updating source {}".format(itemsource.SOURCE))
|
||||
# Check if the source has a folder.
|
||||
source_folder = os.path.join(self.path, itemsource.SOURCE)
|
||||
source_state = os.path.join(source_folder, "state")
|
||||
if not os.path.isdir(source_folder):
|
||||
self.log.info("Creating folder {}".format(source_folder))
|
||||
os.mkdir(source_folder)
|
||||
# Initialize persistent state.
|
||||
with open(source_state, 'w') as f:
|
||||
f.write("{}")
|
||||
# Load source persistent state.
|
||||
state = self.load_path(source_state)
|
||||
# Any inactive items that no longer show up as new should be
|
||||
# removed. Track which items to check for inactivity.
|
||||
extant_items_to_check = [
|
||||
filename
|
||||
for filename in os.listdir(source_folder)
|
||||
if filename.endswith(".item")]
|
||||
# Get the new items from the source.
|
||||
source_items = itemsource.fetch_new(state)
|
||||
with open(source_state, 'w', encoding='utf-8') as f:
|
||||
f.write(str(state))
|
||||
for source_item in source_items:
|
||||
file_path = os.path.join(source_folder, source_item['id'] + ".item")
|
||||
if os.path.isfile(file_path):
|
||||
# Still-new items are exempt from activity checks.
|
||||
extant_items_to_check.remove(source_item['id'] + ".item")
|
||||
item = self.load_path(file_path)
|
||||
if not item['active']:
|
||||
# Don't reactivate inactive items.
|
||||
continue
|
||||
else:
|
||||
new_items += 1
|
||||
# Add new items and update active ones.
|
||||
self.save_item(source_item)
|
||||
# Check old items for inactivity.
|
||||
for extant_item_filename in extant_items_to_check:
|
||||
file_path = os.path.join(source_folder, extant_item_filename)
|
||||
item = self.load_path(file_path)
|
||||
if not item['active']:
|
||||
# Remove old inactive items.
|
||||
self.log.info("Deleting {}".format(file_path))
|
||||
os.remove(file_path)
|
||||
return new_items
|
||||
|
||||
def get_active_items(self):
|
||||
source_folders = os.listdir(self.path)
|
||||
items = []
|
||||
for source_folder_name in source_folders:
|
||||
items.extend(self.get_active_items_for_folder(source_folder_name))
|
||||
return items
|
||||
|
||||
def get_active_items_for_folder(self, source):
|
||||
source_folder = os.path.join(self.path, source)
|
||||
item_filenames = os.listdir(source_folder)
|
||||
items = []
|
||||
for item_filename in item_filenames:
|
||||
if not item_filename.endswith(".item"):
|
||||
continue
|
||||
file_path = os.path.join(source_folder, item_filename)
|
||||
item = self.load_path(file_path)
|
||||
if item['active']:
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
def deactivate_item(self, source, itemid):
|
||||
item_path = os.path.join(self.path, source, itemid + ".item")
|
||||
if not os.path.isfile(item_path):
|
||||
self.log.error("No item found: {}".format(item_path))
|
||||
return
|
||||
item = self.load_path(item_path)
|
||||
item['active'] = False
|
||||
self.save_item(item)
|
||||
return item
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
div#wrapper {
|
||||
max-width: 700px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
.readable-item {
|
||||
border: 1px solid black;
|
||||
border-radius: 6px;
|
||||
padding: 5px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.item-title {
|
||||
font-size: 1.4em;
|
||||
}
|
||||
.readable-item button {
|
||||
font-size: 1em;
|
||||
float:right;
|
||||
}
|
||||
.item-link {
|
||||
text-decoration: none;
|
||||
float:right;
|
||||
margin: 0px 2px;
|
||||
font-size: 1em;
|
||||
padding: 2px 7px;
|
||||
border: 1px solid;
|
||||
border-radius: 2px;
|
||||
}
|
||||
.item-info {
|
||||
color: rgba(0, 0, 0, 0.7);
|
||||
}
|
||||
button, summary {
|
||||
cursor: pointer;
|
||||
}
|
||||
summary:focus {
|
||||
outline: 1px dotted gray;
|
||||
}
|
||||
.strikethru span, .strikethru p {
|
||||
text-decoration: line-through;
|
||||
color: rgba(0, 0, 0, 0.2);
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
<html>
|
||||
<head>
|
||||
<link rel="stylesheet" type="text/css" href="/feed.css" />
|
||||
<script>
|
||||
var deactivate = function (source, itemid) {
|
||||
fetch('/deactivate/', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=UTF-8',
|
||||
},
|
||||
body: JSON.stringify({source: source, itemid: itemid}),
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(function (data) {
|
||||
if (!data.active) {
|
||||
document.getElementById(source + "-" + itemid)
|
||||
.classList.add("strikethru")
|
||||
}
|
||||
});
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<div id="wrapper">
|
||||
{% if items %}
|
||||
{% for item in items %}
|
||||
<div class="readable-item" id="{{item.source}}-{{item.id}}">
|
||||
<button onclick="javascript:deactivate('{{item.source}}', '{{item.id}}')">✕</button>
|
||||
{% if item.link %}
|
||||
<a class="item-link" href="{{item.link}}" target="_blank">⇗</a>
|
||||
{% endif %}
|
||||
{% if item.body %}
|
||||
<details>
|
||||
<summary><span class="item-title">{{item.title}}</span></summary>
|
||||
<p>{{item.body|safe}}</p>
|
||||
</details>
|
||||
{% else %}
|
||||
<span class="item-title">{{item.title}}</span><br>
|
||||
{% endif %}
|
||||
<span class="item-info">{{item.time_readable}} {{item.author}}</span><br>
|
||||
<span class="item-info">{{item.source}} {{item.id}} {{item.time_readable}}</span>
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
<div class="readable-item">
|
||||
<span class="item-title">Feed is empty</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
Loading…
Reference in New Issue