Clean up source loading code
This commit is contained in:
parent
6009a23283
commit
d72c0326fb
|
@ -4,63 +4,90 @@ import importlib.util
|
||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
|
||||||
from inquisitor import loader, timestamp, error
|
from inquisitor import loader, timestamp, error
|
||||||
from inquisitor.configs import SOURCES_PATH, DUNGEON_PATH, logger
|
from inquisitor.configs import SOURCES_PATH, DUNGEON_PATH, logger
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_cell(name):
|
||||||
|
"""
|
||||||
|
Creates a cell in the dungeon. Idempotent.
|
||||||
|
"""
|
||||||
|
cell_path = os.path.join(DUNGEON_PATH, name)
|
||||||
|
if not os.path.isdir(cell_path):
|
||||||
|
logger.info(f'Creating cell for source "{name}"')
|
||||||
|
os.mkdir(cell_path)
|
||||||
|
state_path = os.path.join(cell_path, 'state')
|
||||||
|
if not os.path.isfile(state_path):
|
||||||
|
with open(state_path, 'w', encoding='utf8') as state:
|
||||||
|
json.dump({}, state)
|
||||||
|
|
||||||
|
|
||||||
def update_sources(*source_names):
|
def update_sources(*source_names):
|
||||||
sys.path.append(SOURCES_PATH)
|
"""
|
||||||
|
Attempts to update each given source.
|
||||||
|
"""
|
||||||
for source_name in source_names:
|
for source_name in source_names:
|
||||||
|
# Import the source
|
||||||
try:
|
try:
|
||||||
source_module = load_source(source_name)
|
source_module = load_source(source_name)
|
||||||
except Exception:
|
except Exception:
|
||||||
error.as_item("Error importing source '{}'".format(source_name), traceback.format_exc())
|
error.as_item(
|
||||||
|
f'Error importing source "{source_name}"',
|
||||||
|
traceback.format_exc())
|
||||||
continue
|
continue
|
||||||
|
|
||||||
cell_path = os.path.join(DUNGEON_PATH, source_name)
|
# If it doesn't have a cell yet, create one
|
||||||
if not os.path.isdir(cell_path):
|
|
||||||
try:
|
try:
|
||||||
logger.info("Creating cell for source '{}'".format(source_name))
|
ensure_cell(source_name)
|
||||||
os.mkdir(cell_path)
|
|
||||||
state_path = os.path.join(cell_path, "state")
|
|
||||||
with open(state_path, 'w', encoding='utf8') as f:
|
|
||||||
f.write(json.dumps({}))
|
|
||||||
except Exception:
|
except Exception:
|
||||||
error.as_item("Error initializing source '{}'".format(source_name), traceback.format_exc())
|
error.as_item(
|
||||||
|
f'Error initializing source "{source_name}"',
|
||||||
|
traceback.format_exc())
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# Update the source
|
||||||
try:
|
try:
|
||||||
logger.info("Updating source '{}'".format(source_name))
|
logger.info(f'Updating source "{source_name}"')
|
||||||
new_count, del_count = update_source(source_name, source_module.fetch_new)
|
update_source(source_name, source_module.fetch_new)
|
||||||
logger.info("{} new item{}, {} deleted item{}".format(
|
|
||||||
new_count, "s" if new_count != 1 else "",
|
|
||||||
del_count, "s" if del_count != 1 else ""))
|
|
||||||
except Exception:
|
except Exception:
|
||||||
error.as_item("Error updating source '{}'".format(source_name), traceback.format_exc())
|
error.as_item(
|
||||||
|
f'Error updating source "{source_name}"',
|
||||||
|
traceback.format_exc())
|
||||||
|
|
||||||
|
|
||||||
def load_source(source_name):
|
def load_source(source_name):
|
||||||
"""
|
"""
|
||||||
Attempts to load the source module with the given name. Raises an exception on failure.
|
Attempts to load the source module with the given name.
|
||||||
|
Raises an exception on failure.
|
||||||
"""
|
"""
|
||||||
# Push the sources directory
|
# Push the sources directory.
|
||||||
cwd = os.getcwd()
|
cwd = os.getcwd()
|
||||||
try:
|
try:
|
||||||
os.chdir(SOURCES_PATH)
|
os.chdir(SOURCES_PATH)
|
||||||
|
|
||||||
# Check if the named source is present.
|
# Check if the named source is present.
|
||||||
source_file_name = source_name + ".py"
|
source_file_name = source_name + '.py'
|
||||||
if not os.path.isfile(source_file_name):
|
if not os.path.isfile(source_file_name):
|
||||||
raise FileNotFoundError("Missing '{}' in '{}'".format(source_name, SOURCES_PATH))
|
raise FileNotFoundError('Missing "{source_name}" in "{SOURCES_PATH}"')
|
||||||
# Try to import the source module.
|
|
||||||
logger.debug("Loading module {}".format(source_file_name))
|
# Import the source module by file path.
|
||||||
|
logger.debug('Loading module "{source_file_name}"')
|
||||||
spec = importlib.util.spec_from_file_location("itemsource", source_file_name)
|
spec = importlib.util.spec_from_file_location("itemsource", source_file_name)
|
||||||
itemsource = importlib.util.module_from_spec(spec)
|
itemsource = importlib.util.module_from_spec(spec)
|
||||||
spec.loader.exec_module(itemsource)
|
spec.loader.exec_module(itemsource)
|
||||||
|
itemsource = importlib.import_module(source_name)
|
||||||
|
|
||||||
|
# Require fetch_new().
|
||||||
if not hasattr(itemsource, 'fetch_new'):
|
if not hasattr(itemsource, 'fetch_new'):
|
||||||
raise ImportError("Missing fetch_new in '{}'".format(source_file_name))
|
raise ImportError('Missing fetch_new in "{source_file_name}"')
|
||||||
# Since the source is valid, get or create the source cell.
|
|
||||||
return itemsource
|
return itemsource
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
os.chdir(cwd)
|
os.chdir(cwd)
|
||||||
|
|
||||||
|
|
||||||
def update_source(source_name, fetch_new):
|
def update_source(source_name, fetch_new):
|
||||||
"""
|
"""
|
||||||
Attempts to update the given source. Raises an exception if the source does.
|
Attempts to update the given source. Raises an exception if the source does.
|
||||||
|
@ -138,8 +165,10 @@ def update_source(source_name, fetch_new):
|
||||||
# Note update timestamp in state
|
# Note update timestamp in state
|
||||||
state['last_updated'] = timestamp.now()
|
state['last_updated'] = timestamp.now()
|
||||||
|
|
||||||
# Return counts
|
# Log counts
|
||||||
return len(new_items), del_count
|
logger.info("{} new item{}, {} deleted item{}".format(
|
||||||
|
len(new_items), "s" if len(new_items) != 1 else "",
|
||||||
|
del_count, "s" if del_count != 1 else ""))
|
||||||
|
|
||||||
def populate_new(source_name, item):
|
def populate_new(source_name, item):
|
||||||
# id is required
|
# id is required
|
||||||
|
|
Loading…
Reference in New Issue