Rewrite dungeon to encapsulate filesystem access
This commit is contained in:
parent
f8faa29caa
commit
6b2f898d36
|
|
@ -2,110 +2,258 @@
|
||||||
import os
|
import os
|
||||||
import logging
|
import logging
|
||||||
import ast
|
import ast
|
||||||
|
import importlib.util
|
||||||
|
import time
|
||||||
|
import random
|
||||||
|
import traceback
|
||||||
|
|
||||||
|
|
||||||
|
# Application imports
|
||||||
|
from item import create_item
|
||||||
|
|
||||||
|
|
||||||
|
# Globals
|
||||||
|
logger = logging.getLogger("inquisitor.dungeon")
|
||||||
|
|
||||||
|
|
||||||
|
class ReadableItem():
|
||||||
|
"""
|
||||||
|
An abstraction layer around items saved in dungeon cell folders.
|
||||||
|
Provides read-only access with a deactivation function.
|
||||||
|
If the underlying item is changed, those changes may be overwritten by deactivation.
|
||||||
|
|
||||||
|
Members:
|
||||||
|
path: path to the wrapped item's file
|
||||||
|
cell: name of the cell the item is in
|
||||||
|
id: the item's id
|
||||||
|
item: the wrapped item
|
||||||
|
"""
|
||||||
|
def __init__(self, dungeon_path, cell_name, item_id):
|
||||||
|
self.path = os.path.join(dungeon_path, cell_name, item_id + ".item")
|
||||||
|
self.cell = cell_name
|
||||||
|
self.id = item_id
|
||||||
|
logger.debug("Loading '{0.id}' from '{0.cell}'".format(self))
|
||||||
|
with open(self.path, 'r', encoding='utf-8') as f:
|
||||||
|
self.item = ast.literal_eval(f.read())
|
||||||
|
|
||||||
|
def __getitem__(self, key):
|
||||||
|
return self.item[key]
|
||||||
|
|
||||||
|
def __setitem__(self, key, value):
|
||||||
|
raise TypeError("ReadableItem is not writable")
|
||||||
|
|
||||||
|
def __contains__(self, key):
|
||||||
|
return key in self.item
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return "ReadableItem({0.cell}/{0.id})".format(self)
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return repr(self)
|
||||||
|
|
||||||
|
def deactivate(self):
|
||||||
|
self.item['active'] = False
|
||||||
|
logger.debug("Deactivating item at {0.path}".format(self))
|
||||||
|
with open(self.path, 'w', encoding='utf-8') as f:
|
||||||
|
f.write(str(self.item))
|
||||||
|
|
||||||
|
|
||||||
|
class DungeonCell():
|
||||||
|
"""
|
||||||
|
An abstraction layer around a folder containing items generated by an item source.
|
||||||
|
|
||||||
|
Members:
|
||||||
|
name: the cell's source's name
|
||||||
|
dungeon_path: the path to the dungeon containing this cell
|
||||||
|
path: the path to this cell
|
||||||
|
state: the source's persistent state dictionary
|
||||||
|
"""
|
||||||
|
def __init__(self, dungeon_path, name):
|
||||||
|
self.name = name
|
||||||
|
self.dungeon_path = dungeon_path
|
||||||
|
self.path = os.path.join(dungeon_path, name)
|
||||||
|
state_path = os.path.join(self.path, "state")
|
||||||
|
if not os.path.isdir(self.path):
|
||||||
|
# Initialize cell state on the disk.
|
||||||
|
logger.info("Creating folder for cell {}".format(name))
|
||||||
|
os.mkdir(self.path)
|
||||||
|
self.state = {}
|
||||||
|
with open(state_path, 'w', encoding='utf-8') as f:
|
||||||
|
f.write(str(self.state))
|
||||||
|
else:
|
||||||
|
# Load cell state from the disk.
|
||||||
|
with open(state_path, 'r', encoding='utf-8') as f:
|
||||||
|
self.state = ast.literal_eval(f.read())
|
||||||
|
|
||||||
|
def _item_path(key):
|
||||||
|
return os.path.join(self.path, key + ".item")
|
||||||
|
|
||||||
|
def __getitem__(self, key):
|
||||||
|
filepath = self._item_path(key)
|
||||||
|
if not os.path.isfile(filepath):
|
||||||
|
raise KeyError("No item '{}' in cell '{}'".format(key, self.name))
|
||||||
|
return ReadableItem(self.dungeon_path, self.name, key)
|
||||||
|
|
||||||
|
def __setitem__(self, key, value):
|
||||||
|
logger.info("Setting item {} in cell {}".format(key, self.name))
|
||||||
|
if type(value) is ReadableItem:
|
||||||
|
value = value.item
|
||||||
|
if type(value) is not dict:
|
||||||
|
raise TypeError("Can't store a '{}' as '{}': not a dict".format(type(value), key))
|
||||||
|
filepath = self._item_path(key)
|
||||||
|
with open(filepath, 'w', encoding='utf-8') as f:
|
||||||
|
f.write(str(value))
|
||||||
|
|
||||||
|
def __delitem__(self, key):
|
||||||
|
logger.info("Deleting item '{}' in cell '{}'".format(key, self.name))
|
||||||
|
filepath = self._item_path(key)
|
||||||
|
if os.path.isfile(filepath):
|
||||||
|
os.remove(filepath)
|
||||||
|
|
||||||
|
def __contains__(self, key):
|
||||||
|
for item in self:
|
||||||
|
if item == key:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def __iter__(self):
|
||||||
|
for filename in os.listdir(self.path):
|
||||||
|
if filename.endswith(".item"):
|
||||||
|
yield filename[:-5]
|
||||||
|
|
||||||
|
def save_state(self):
|
||||||
|
filepath = os.path.join(self.path, 'state')
|
||||||
|
with open(filepath, 'w', encoding='utf-8') as f:
|
||||||
|
f.write(str(self.state))
|
||||||
|
|
||||||
|
def update_from_source(self, source, args):
|
||||||
|
logger.info("Updating source {}".format(self.name))
|
||||||
|
# Get the ids of the existing items.
|
||||||
|
prior_item_ids = [item_id for item_id in self]
|
||||||
|
# Get the new items.
|
||||||
|
new_items = itemsource.fetch_new(state, args)
|
||||||
|
self.save_state()
|
||||||
|
new_count = del_count = 0
|
||||||
|
for item in new_items:
|
||||||
|
# Store new items unconditionally.
|
||||||
|
if item['id'] not in prior_item_ids:
|
||||||
|
new_count += 1
|
||||||
|
self[item['id']] = item
|
||||||
|
# Update extant items if active, otherwise inactive items will be
|
||||||
|
# reactivated by the overwrite.
|
||||||
|
else:
|
||||||
|
prior_item = self[item['id']]
|
||||||
|
if prior_item['active']:
|
||||||
|
self[item['id']] = item
|
||||||
|
# Remove the id from the list to track its continued presence
|
||||||
|
# in the source's queue of new items.
|
||||||
|
prior_item_ids.remove(item['id'])
|
||||||
|
# Any extant item left in the list is old. Remove any items that are
|
||||||
|
# both old and inactive.
|
||||||
|
for prior_id in prior_item_ids:
|
||||||
|
if not self[prior_id]['active']:
|
||||||
|
del_count += 1
|
||||||
|
del self[prior_id]
|
||||||
|
# Return counts
|
||||||
|
return new_count, del_count
|
||||||
|
|
||||||
|
|
||||||
class Dungeon():
|
class Dungeon():
|
||||||
|
"""
|
||||||
|
A wrapper for dealing with a collection of DungeonCell folders.
|
||||||
|
Interfaces between Inquisitor and the ReadableItems produced by its
|
||||||
|
sources.
|
||||||
|
"""
|
||||||
def __init__(self, path):
|
def __init__(self, path):
|
||||||
"""
|
|
||||||
Serves as an interface between Inquisitor and a folder of
|
|
||||||
serialized readable items.
|
|
||||||
"""
|
|
||||||
self.path = path
|
self.path = path
|
||||||
self.log = logging.getLogger("inquisitor.dungeon")
|
# Initialize DungeonCells for each folder with a state file
|
||||||
|
self.cells = {}
|
||||||
def load_path(self, path):
|
for filename in os.listdir(self.path):
|
||||||
self.log.debug("Loading item from {}".format(path))
|
if not os.path.isdir(os.path.join(self.path, filename)):
|
||||||
with open(path, 'r', encoding='utf-8') as f:
|
|
||||||
item = ast.literal_eval(f.read())
|
|
||||||
return item
|
|
||||||
|
|
||||||
def load_item(self, source, itemid):
|
|
||||||
item_path = os.path.join(self.path, source, itemid + ".item")
|
|
||||||
return self.load_path(item_path)
|
|
||||||
|
|
||||||
def save_item(self, item):
|
|
||||||
path = os.path.join(self.path, item['source'], item['id'] + ".item")
|
|
||||||
self.log.debug("Saving item {} to {}".format(item['id'], path))
|
|
||||||
with open(path, 'w', encoding='utf-8') as f:
|
|
||||||
f.write(str(item))
|
|
||||||
|
|
||||||
def update(self, itemsource):
|
|
||||||
"""
|
|
||||||
Fetches items from the given source, saves new active items,
|
|
||||||
and clears out old inactive items.
|
|
||||||
"""
|
|
||||||
new_items = 0
|
|
||||||
self.log.info("Updating source {}".format(itemsource.SOURCE))
|
|
||||||
# Check if the source has a folder.
|
|
||||||
source_folder = os.path.join(self.path, itemsource.SOURCE)
|
|
||||||
source_state = os.path.join(source_folder, "state")
|
|
||||||
if not os.path.isdir(source_folder):
|
|
||||||
self.log.info("Creating folder {}".format(source_folder))
|
|
||||||
os.mkdir(source_folder)
|
|
||||||
# Initialize persistent state.
|
|
||||||
with open(source_state, 'w') as f:
|
|
||||||
f.write("{}")
|
|
||||||
# Load source persistent state.
|
|
||||||
state = self.load_path(source_state)
|
|
||||||
# Any inactive items that no longer show up as new should be
|
|
||||||
# removed. Track which items to check for inactivity.
|
|
||||||
extant_items_to_check = [
|
|
||||||
filename
|
|
||||||
for filename in os.listdir(source_folder)
|
|
||||||
if filename.endswith(".item")]
|
|
||||||
# Get the new items from the source.
|
|
||||||
source_items = itemsource.fetch_new(state)
|
|
||||||
with open(source_state, 'w', encoding='utf-8') as f:
|
|
||||||
f.write(str(state))
|
|
||||||
for source_item in source_items:
|
|
||||||
file_path = os.path.join(source_folder, source_item['id'] + ".item")
|
|
||||||
if os.path.isfile(file_path):
|
|
||||||
# Still-new items are exempt from activity checks.
|
|
||||||
extant_items_to_check.remove(source_item['id'] + ".item")
|
|
||||||
item = self.load_path(file_path)
|
|
||||||
if not item['active']:
|
|
||||||
# Don't reactivate inactive items.
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
new_items += 1
|
|
||||||
# Add new items and update active ones.
|
|
||||||
self.save_item(source_item)
|
|
||||||
# Check old items for inactivity.
|
|
||||||
for extant_item_filename in extant_items_to_check:
|
|
||||||
file_path = os.path.join(source_folder, extant_item_filename)
|
|
||||||
item = self.load_path(file_path)
|
|
||||||
if not item['active']:
|
|
||||||
# Remove old inactive items.
|
|
||||||
self.log.info("Deleting {}".format(file_path))
|
|
||||||
os.remove(file_path)
|
|
||||||
return new_items
|
|
||||||
|
|
||||||
def get_active_items(self):
|
|
||||||
source_folders = os.listdir(self.path)
|
|
||||||
items = []
|
|
||||||
for source_folder_name in source_folders:
|
|
||||||
items.extend(self.get_active_items_for_folder(source_folder_name))
|
|
||||||
return items
|
|
||||||
|
|
||||||
def get_active_items_for_folder(self, source):
|
|
||||||
source_folder = os.path.join(self.path, source)
|
|
||||||
item_filenames = os.listdir(source_folder)
|
|
||||||
items = []
|
|
||||||
for item_filename in item_filenames:
|
|
||||||
if not item_filename.endswith(".item"):
|
|
||||||
continue
|
continue
|
||||||
file_path = os.path.join(source_folder, item_filename)
|
if not os.path.isfile(os.path.join(self.path, filename, 'status')):
|
||||||
item = self.load_path(file_path)
|
continue
|
||||||
if item['active']:
|
self.cells[filename] = DungeonCell(self.path, filename)
|
||||||
items.append(item)
|
# Ensure Inquisitor's source is present
|
||||||
return items
|
if "inquisitor" not in self.cells:
|
||||||
|
self.cells["inquisitor"] = DungeonCell(self.path, 'inquisitor')
|
||||||
|
|
||||||
def deactivate_item(self, source, itemid):
|
def __getitem__(self, key):
|
||||||
item_path = os.path.join(self.path, source, itemid + ".item")
|
return self.cells[key]
|
||||||
if not os.path.isfile(item_path):
|
|
||||||
self.log.error("No item found: {}".format(item_path))
|
def __setitem__(self, key, value):
|
||||||
|
if type(value) is not DungeonCell:
|
||||||
|
raise TypeError("Can't store a '{}' as '{}': not a DungeonCell".format(type(value), key))
|
||||||
|
self.cells[key] = value
|
||||||
|
|
||||||
|
def __contains__(self, key):
|
||||||
|
return key in self.cells
|
||||||
|
|
||||||
|
def __iter__(self):
|
||||||
|
for name in self.cells:
|
||||||
|
yield name
|
||||||
|
|
||||||
|
def push_error_item(self, title, body=None):
|
||||||
|
logger.error(title)
|
||||||
|
item = create_item(
|
||||||
|
'inquisitor',
|
||||||
|
'{:x}'.format(random.getrandbits(16 * 4)),
|
||||||
|
title,
|
||||||
|
body="<pre>{}</pre>".format(body) if body else None)
|
||||||
|
self['inquisitor'][item['id']] = item
|
||||||
|
|
||||||
|
def try_load_source(self, sources_path, source_name):
|
||||||
|
"""
|
||||||
|
Tries to load the given source, creating a cell in the dungeon if
|
||||||
|
necessary. Returns the source and its DungeonCell if the source is
|
||||||
|
valid and None, None otherwise.
|
||||||
|
"""
|
||||||
|
# Check if the named source is present in the sources directory.
|
||||||
|
source_file_path = os.path.join(sources_path, source_name + ".py")
|
||||||
|
if not os.path.isfile(source_file_path):
|
||||||
|
msg = "Could not find source '{}'".format(source_name)
|
||||||
|
self.push_error_item(msg)
|
||||||
|
return None, None
|
||||||
|
# Try to import the source module.
|
||||||
|
try:
|
||||||
|
logger.debug("Loading module {}".format(source_file_path))
|
||||||
|
spec = importlib.util.spec_from_file_location("itemsource", source_file_path)
|
||||||
|
itemsource = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(itemsource)
|
||||||
|
if not hasattr(itemsource, 'fetch_new'):
|
||||||
|
raise ImportError("fetch_new missing")
|
||||||
|
except Exception:
|
||||||
|
msg = "Error importing source '{}'".format(source_name)
|
||||||
|
self.push_error_item(msg, traceback.format_exc())
|
||||||
|
return None, None
|
||||||
|
# Since the source is valid, get or create the source cell.
|
||||||
|
if source_name not in self:
|
||||||
|
self[source_name] = DungeonCell(self.path, source_name)
|
||||||
|
|
||||||
|
return itemsource, self[source_name]
|
||||||
|
|
||||||
|
def update(self, source_arg, args):
|
||||||
|
"""
|
||||||
|
Loads the given source and fetches new items. Clears old and inactive items.
|
||||||
|
"""
|
||||||
|
# Split off the source name from the fetch argument.
|
||||||
|
splits = source_arg.split(":", maxsplit=1)
|
||||||
|
source_name = splits[0]
|
||||||
|
source_args = splits[1] if len(splits) > 1 else None
|
||||||
|
|
||||||
|
# Load the source.
|
||||||
|
source, cell = self.try_load_source(args.srcdir, source_name)
|
||||||
|
if source is None or cell is None:
|
||||||
|
# try_load_source has already logged the error
|
||||||
|
return
|
||||||
|
|
||||||
|
# Update the cell from the source.
|
||||||
|
try:
|
||||||
|
new_count, deleted_count = cell.update_from_source(source, source_args)
|
||||||
|
logger.info("{} new item{}, {} deleted item{}".format(
|
||||||
|
new_count, "s" if new_count != 1 else "",
|
||||||
|
deleted_count, "s" if deleted_count != 1 else ""))
|
||||||
|
except:
|
||||||
|
msg = "Error fetching items from source '{}'".format(source_name)
|
||||||
|
self.push_error_item(msg, traceback.format_exc())
|
||||||
return
|
return
|
||||||
item = self.load_path(item_path)
|
|
||||||
item['active'] = False
|
|
||||||
self.save_item(item)
|
|
||||||
return item
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,28 @@
|
||||||
|
# Standard library imports
|
||||||
|
import importlib.util
|
||||||
|
import os
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
# Globals
|
||||||
|
logger = logging.getLogger("inquisitor.item")
|
||||||
|
|
||||||
|
|
||||||
|
def create_item(source, item_id, title, link=None, time=None, author=None, body=None):
|
||||||
|
item = {
|
||||||
|
'id': item_id,
|
||||||
|
'source': source,
|
||||||
|
'active': True,
|
||||||
|
'created': time.time(),
|
||||||
|
'title': title,
|
||||||
|
}
|
||||||
|
if link is not None:
|
||||||
|
item['link'] = link
|
||||||
|
if time is not None:
|
||||||
|
item['time'] = time
|
||||||
|
if author is not None:
|
||||||
|
item['author'] = author
|
||||||
|
if body is not None:
|
||||||
|
item['body'] = body
|
||||||
|
return item
|
||||||
Loading…
Reference in New Issue