Clean up source loading code

This commit is contained in:
Tim Van Baak 2020-08-06 14:43:46 -07:00
parent 6009a23283
commit d72c0326fb
1 changed files with 59 additions and 30 deletions

View File

@ -4,63 +4,90 @@ import importlib.util
import json import json
import sys import sys
from inquisitor import loader, timestamp, error from inquisitor import loader, timestamp, error
from inquisitor.configs import SOURCES_PATH, DUNGEON_PATH, logger from inquisitor.configs import SOURCES_PATH, DUNGEON_PATH, logger
def ensure_cell(name):
"""
Creates a cell in the dungeon. Idempotent.
"""
cell_path = os.path.join(DUNGEON_PATH, name)
if not os.path.isdir(cell_path):
logger.info(f'Creating cell for source "{name}"')
os.mkdir(cell_path)
state_path = os.path.join(cell_path, 'state')
if not os.path.isfile(state_path):
with open(state_path, 'w', encoding='utf8') as state:
json.dump({}, state)
def update_sources(*source_names): def update_sources(*source_names):
sys.path.append(SOURCES_PATH) """
Attempts to update each given source.
"""
for source_name in source_names: for source_name in source_names:
# Import the source
try: try:
source_module = load_source(source_name) source_module = load_source(source_name)
except Exception: except Exception:
error.as_item("Error importing source '{}'".format(source_name), traceback.format_exc()) error.as_item(
f'Error importing source "{source_name}"',
traceback.format_exc())
continue continue
cell_path = os.path.join(DUNGEON_PATH, source_name) # If it doesn't have a cell yet, create one
if not os.path.isdir(cell_path):
try: try:
logger.info("Creating cell for source '{}'".format(source_name)) ensure_cell(source_name)
os.mkdir(cell_path)
state_path = os.path.join(cell_path, "state")
with open(state_path, 'w', encoding='utf8') as f:
f.write(json.dumps({}))
except Exception: except Exception:
error.as_item("Error initializing source '{}'".format(source_name), traceback.format_exc()) error.as_item(
f'Error initializing source "{source_name}"',
traceback.format_exc())
continue continue
# Update the source
try: try:
logger.info("Updating source '{}'".format(source_name)) logger.info(f'Updating source "{source_name}"')
new_count, del_count = update_source(source_name, source_module.fetch_new) update_source(source_name, source_module.fetch_new)
logger.info("{} new item{}, {} deleted item{}".format(
new_count, "s" if new_count != 1 else "",
del_count, "s" if del_count != 1 else ""))
except Exception: except Exception:
error.as_item("Error updating source '{}'".format(source_name), traceback.format_exc()) error.as_item(
f'Error updating source "{source_name}"',
traceback.format_exc())
def load_source(source_name): def load_source(source_name):
""" """
Attempts to load the source module with the given name. Raises an exception on failure. Attempts to load the source module with the given name.
Raises an exception on failure.
""" """
# Push the sources directory # Push the sources directory.
cwd = os.getcwd() cwd = os.getcwd()
try: try:
os.chdir(SOURCES_PATH) os.chdir(SOURCES_PATH)
# Check if the named source is present. # Check if the named source is present.
source_file_name = source_name + ".py" source_file_name = source_name + '.py'
if not os.path.isfile(source_file_name): if not os.path.isfile(source_file_name):
raise FileNotFoundError("Missing '{}' in '{}'".format(source_name, SOURCES_PATH)) raise FileNotFoundError('Missing "{source_name}" in "{SOURCES_PATH}"')
# Try to import the source module.
logger.debug("Loading module {}".format(source_file_name)) # Import the source module by file path.
logger.debug('Loading module "{source_file_name}"')
spec = importlib.util.spec_from_file_location("itemsource", source_file_name) spec = importlib.util.spec_from_file_location("itemsource", source_file_name)
itemsource = importlib.util.module_from_spec(spec) itemsource = importlib.util.module_from_spec(spec)
spec.loader.exec_module(itemsource) spec.loader.exec_module(itemsource)
itemsource = importlib.import_module(source_name)
# Require fetch_new().
if not hasattr(itemsource, 'fetch_new'): if not hasattr(itemsource, 'fetch_new'):
raise ImportError("Missing fetch_new in '{}'".format(source_file_name)) raise ImportError('Missing fetch_new in "{source_file_name}"')
# Since the source is valid, get or create the source cell.
return itemsource return itemsource
finally: finally:
os.chdir(cwd) os.chdir(cwd)
def update_source(source_name, fetch_new): def update_source(source_name, fetch_new):
""" """
Attempts to update the given source. Raises an exception if the source does. Attempts to update the given source. Raises an exception if the source does.
@ -138,8 +165,10 @@ def update_source(source_name, fetch_new):
# Note update timestamp in state # Note update timestamp in state
state['last_updated'] = timestamp.now() state['last_updated'] = timestamp.now()
# Return counts # Log counts
return len(new_items), del_count logger.info("{} new item{}, {} deleted item{}".format(
len(new_items), "s" if len(new_items) != 1 else "",
del_count, "s" if del_count != 1 else ""))
def populate_new(source_name, item): def populate_new(source_name, item):
# id is required # id is required