{}".format("\n\n".join(errors)),
- 'created': None,
- }
- active_items.insert(0, read_ex)
+ logger.info("Returning {} of {} items".format(len(active_items), total))
+ if errors:
+ read_ex = {
+ "title": "Read errors",
+ "body": "{}".format("\n\n".join(errors)),
+ "created": None,
+ }
+ active_items.insert(0, read_ex)
- if total > 0:
- # Create the feed control item
- link_table = ["{}'.format(body)
- path = os.path.join(DUNGEON_PATH, 'inquisitor', iid + ".item")
- logger.error(json.dumps(item))
- with open(path, 'w') as f:
- f.write(json.dumps(item, indent=2))
+ iid = "{:x}".format(random.getrandbits(16 * 4))
+ item = {
+ "id": iid,
+ "source": "inquisitor",
+ "title": title,
+ "active": True,
+ "created": timestamp.now(),
+ "tags": ["inquisitor", "error"],
+ }
+ if body is not None:
+ item["body"] = "{}".format(body)
+ path = os.path.join(DUNGEON_PATH, "inquisitor", iid + ".item")
+ logger.error(json.dumps(item))
+ with open(path, "w") as f:
+ f.write(json.dumps(item, indent=2))
diff --git a/inquisitor/loader.py b/inquisitor/loader.py
index 414620c..0632c5d 100644
--- a/inquisitor/loader.py
+++ b/inquisitor/loader.py
@@ -7,180 +7,180 @@ from inquisitor import error
from inquisitor import timestamp
-class WritethroughDict():
- """A wrapper for a dictionary saved to the file system."""
+class WritethroughDict:
+ """A wrapper for a dictionary saved to the file system."""
- @staticmethod
- def create(path, item):
- """
- Creates a writethrough dictionary from a dictionary in memory and
- initializes a file to save it.
- """
- if os.path.isfile(path):
- raise FileExistsError(path)
- wd = WritethroughDict(path, item)
- wd.flush()
- return wd
+ @staticmethod
+ def create(path, item):
+ """
+ Creates a writethrough dictionary from a dictionary in memory and
+ initializes a file to save it.
+ """
+ if os.path.isfile(path):
+ raise FileExistsError(path)
+ wd = WritethroughDict(path, item)
+ wd.flush()
+ return wd
- @staticmethod
- def load(path):
- """
- Creates a writethrough dictionary from an existing file in the
- file system.
- """
- if not os.path.isfile(path):
- raise FileNotFoundError(path)
- with open(path) as f:
- item = json.load(f)
- return WritethroughDict(path, item)
+ @staticmethod
+ def load(path):
+ """
+ Creates a writethrough dictionary from an existing file in the
+ file system.
+ """
+ if not os.path.isfile(path):
+ raise FileNotFoundError(path)
+ with open(path) as f:
+ item = json.load(f)
+ return WritethroughDict(path, item)
- def __init__(self, path, item):
- self.path = path
- self.item = item
+ def __init__(self, path, item):
+ self.path = path
+ self.item = item
- def __getitem__(self, key):
- return self.item[key]
+ def __getitem__(self, key):
+ return self.item[key]
- def get(self, *args, **kwargs):
- return self.item.get(*args, **kwargs)
+ def get(self, *args, **kwargs):
+ return self.item.get(*args, **kwargs)
- def __setitem__(self, key, value):
- self.item[key] = value
- self.flush()
+ def __setitem__(self, key, value):
+ self.item[key] = value
+ self.flush()
- def __contains__(self, key):
- return key in self.item
+ def __contains__(self, key):
+ return key in self.item
- def __repr__(self):
- return repr(self.item)
+ def __repr__(self):
+ return repr(self.item)
- def __str__(self):
- return str(self.item)
+ def __str__(self):
+ return str(self.item)
- def flush(self):
- s = json.dumps(self.item, indent=2)
- with open(self.path, 'w', encoding="utf8") as f:
- f.write(s)
+ def flush(self):
+ s = json.dumps(self.item, indent=2)
+ with open(self.path, "w", encoding="utf8") as f:
+ f.write(s)
def load_state(source_name):
- """Loads the state dictionary for a source."""
- state_path = os.path.join(DUNGEON_PATH, source_name, "state")
- return WritethroughDict.load(state_path)
+ """Loads the state dictionary for a source."""
+ state_path = os.path.join(DUNGEON_PATH, source_name, "state")
+ return WritethroughDict.load(state_path)
def load_item(source_name, item_id):
- """Loads an item from a source."""
- item_path = os.path.join(DUNGEON_PATH, source_name, f'{item_id}.item')
- return WritethroughDict.load(item_path)
+ """Loads an item from a source."""
+ item_path = os.path.join(DUNGEON_PATH, source_name, f"{item_id}.item")
+ return WritethroughDict.load(item_path)
def item_exists(source_name, item_id):
- """
- Checks for the existence of an item.
- """
- item_path = os.path.join(DUNGEON_PATH, source_name, f'{item_id}.item')
- return os.path.isfile(item_path)
+ """
+ Checks for the existence of an item.
+ """
+ item_path = os.path.join(DUNGEON_PATH, source_name, f"{item_id}.item")
+ return os.path.isfile(item_path)
def get_item_ids(cell_name):
- """
- Returns a list of item ids in the given cell.
- """
- cell_path = os.path.join(DUNGEON_PATH, cell_name)
- return [
- filename[:-5]
- for filename in os.listdir(cell_path)
- if filename.endswith('.item')
- ]
+ """
+ Returns a list of item ids in the given cell.
+ """
+ cell_path = os.path.join(DUNGEON_PATH, cell_name)
+ return [
+ filename[:-5]
+ for filename in os.listdir(cell_path)
+ if filename.endswith(".item")
+ ]
def new_item(source_name, item):
- """
- Creates a new item with the fields in the provided dictionary.
- Initializes other fields to their default values.
- """
- # id is required
- if 'id' not in item:
- raise Exception(f'Cannot create item with no id. Value = {item}')
+ """
+ Creates a new item with the fields in the provided dictionary.
+ Initializes other fields to their default values.
+ """
+ # id is required
+ if "id" not in item:
+ raise Exception(f"Cannot create item with no id. Value = {item}")
- # source must be filled in, so if it is absent it is auto-populated with
- # source_name. Note: this allows sources to fill in a different source.
- if 'source' not in item:
- item['source'] = source_name
+ # source must be filled in, so if it is absent it is auto-populated with
+ # source_name. Note: this allows sources to fill in a different source.
+ if "source" not in item:
+ item["source"] = source_name
- # active is forced to True for new items
- item['active'] = True
+ # active is forced to True for new items
+ item["active"] = True
- # created is forced to the current timestamp
- item['created'] = timestamp.now()
+ # created is forced to the current timestamp
+ item["created"] = timestamp.now()
- # title is auto-populated with the id if missing
- if 'title' not in item:
- item['title'] = item['id']
+ # title is auto-populated with the id if missing
+ if "title" not in item:
+ item["title"] = item["id"]
- # tags is auto-populated if missing (not if empty!)
- if 'tags' not in item:
- item['tags'] = [source_name]
+ # tags is auto-populated if missing (not if empty!)
+ if "tags" not in item:
+ item["tags"] = [source_name]
- # All other fields are optional.
- item_path = os.path.join(DUNGEON_PATH, item['source'], f'{item["id"]}.item')
- return WritethroughDict.create(item_path, item)
+ # All other fields are optional.
+ item_path = os.path.join(DUNGEON_PATH, item["source"], f'{item["id"]}.item')
+ return WritethroughDict.create(item_path, item)
def delete_item(source_name, item_id):
- """
- Delete an item.
- """
- item_path = os.path.join(DUNGEON_PATH, source_name, f'{item_id}.item')
- os.remove(item_path)
+ """
+ Delete an item.
+ """
+ item_path = os.path.join(DUNGEON_PATH, source_name, f"{item_id}.item")
+ os.remove(item_path)
def load_items(source_name):
- """
- Returns a map of ids to items and a list of unreadable files.
- """
- cell_path = os.path.join(DUNGEON_PATH, source_name)
- items = {}
- errors = []
- for filename in os.listdir(cell_path):
- if filename.endswith('.item'):
- try:
- item = load_item(source_name, filename[:-5])
- items[item['id']] = item
- except Exception:
- errors.append(filename)
- return items, errors
+ """
+ Returns a map of ids to items and a list of unreadable files.
+ """
+ cell_path = os.path.join(DUNGEON_PATH, source_name)
+ items = {}
+ errors = []
+ for filename in os.listdir(cell_path):
+ if filename.endswith(".item"):
+ try:
+ item = load_item(source_name, filename[:-5])
+ items[item["id"]] = item
+ except Exception:
+ errors.append(filename)
+ return items, errors
def load_active_items(source_names):
- """
- Returns a list of active items and a list of unreadable items. If
- `source_names` is defined, load only from sources in that list.
- """
- items = []
- errors = []
- now = timestamp.now()
- check_list = source_names or os.listdir(DUNGEON_PATH)
- for source_name in check_list:
- source_path = os.path.join(DUNGEON_PATH, source_name)
- if not os.path.isdir(source_path):
- logger.warning(f'Skipping nonexistent source {source_name}')
- continue
- for filename in os.listdir(source_path):
- if not filename.endswith('.item'):
- continue
- try:
- item = load_item(source_name, filename[:-5])
- # The time-to-show field hides items until an expiry date.
- if 'tts' in item:
- tts_date = item['created'] + item['tts']
- if now < tts_date:
- continue
- # Don't show inactive items
- if not item['active']:
- continue
- items.append(item)
- except Exception:
- errors.append(filename)
- return items, errors
+ """
+ Returns a list of active items and a list of unreadable items. If
+ `source_names` is defined, load only from sources in that list.
+ """
+ items = []
+ errors = []
+ now = timestamp.now()
+ check_list = source_names or os.listdir(DUNGEON_PATH)
+ for source_name in check_list:
+ source_path = os.path.join(DUNGEON_PATH, source_name)
+ if not os.path.isdir(source_path):
+ logger.warning(f"Skipping nonexistent source {source_name}")
+ continue
+ for filename in os.listdir(source_path):
+ if not filename.endswith(".item"):
+ continue
+ try:
+ item = load_item(source_name, filename[:-5])
+ # The time-to-show field hides items until an expiry date.
+ if "tts" in item:
+ tts_date = item["created"] + item["tts"]
+ if now < tts_date:
+ continue
+ # Don't show inactive items
+ if not item["active"]:
+ continue
+ items.append(item)
+ except Exception:
+ errors.append(filename)
+ return items, errors
diff --git a/inquisitor/sources.py b/inquisitor/sources.py
index dd45f7a..0a25f1d 100644
--- a/inquisitor/sources.py
+++ b/inquisitor/sources.py
@@ -10,229 +10,234 @@ from inquisitor.configs import SOURCES_PATH, DUNGEON_PATH, logger
USE_NEWEST = (
- 'title',
- 'tags',
- 'link',
- 'time'
- 'author',
- 'body',
- 'ttl',
- 'ttd',
- 'tts',
+ "title",
+ "tags",
+ "link",
+ "time" "author",
+ "body",
+ "ttl",
+ "ttd",
+ "tts",
)
class InquisitorStubSource:
- """A dummy source-like object for clearing out ad-hoc inquisitor items"""
- def fetch_new(self, state):
- return []
+ """A dummy source-like object for clearing out ad-hoc inquisitor items"""
+
+ def fetch_new(self, state):
+ return []
def ensure_cell(name):
- """
- Creates a cell in the dungeon. Idempotent.
- """
- cell_path = os.path.join(DUNGEON_PATH, name)
- if not os.path.isdir(cell_path):
- logger.info(f'Creating cell for source "{name}"')
- os.mkdir(cell_path)
- state_path = os.path.join(cell_path, 'state')
- if not os.path.isfile(state_path):
- with open(state_path, 'w', encoding='utf8') as state:
- json.dump({}, state)
+ """
+ Creates a cell in the dungeon. Idempotent.
+ """
+ cell_path = os.path.join(DUNGEON_PATH, name)
+ if not os.path.isdir(cell_path):
+ logger.info(f'Creating cell for source "{name}"')
+ os.mkdir(cell_path)
+ state_path = os.path.join(cell_path, "state")
+ if not os.path.isfile(state_path):
+ with open(state_path, "w", encoding="utf8") as state:
+ json.dump({}, state)
def update_sources(*source_names):
- """
- Attempts to update each given source.
- """
- for source_name in source_names:
- # Import the source
- try:
- source_module = load_source(source_name)
- except Exception:
- error.as_item(
- f'Error importing source "{source_name}"',
- traceback.format_exc())
- continue
+ """
+ Attempts to update each given source.
+ """
+ for source_name in source_names:
+ # Import the source
+ try:
+ source_module = load_source(source_name)
+ except Exception:
+ error.as_item(
+ f'Error importing source "{source_name}"', traceback.format_exc()
+ )
+ continue
- # If it doesn't have a cell yet, create one
- try:
- ensure_cell(source_name)
- except Exception:
- error.as_item(
- f'Error initializing source "{source_name}"',
- traceback.format_exc())
- continue
+ # If it doesn't have a cell yet, create one
+ try:
+ ensure_cell(source_name)
+ except Exception:
+ error.as_item(
+ f'Error initializing source "{source_name}"', traceback.format_exc()
+ )
+ continue
- # Update the source
- try:
- logger.info(f'Updating source "{source_name}"')
- update_source(source_name, source_module)
- except Exception:
- error.as_item(
- f'Error updating source "{source_name}"',
- traceback.format_exc())
+ # Update the source
+ try:
+ logger.info(f'Updating source "{source_name}"')
+ update_source(source_name, source_module)
+ except Exception:
+ error.as_item(
+ f'Error updating source "{source_name}"', traceback.format_exc()
+ )
def load_source(source_name):
- """
- Attempts to load the source module with the given name.
- Raises an exception on failure.
- """
- if source_name == 'inquisitor':
- return InquisitorStubSource()
+ """
+ Attempts to load the source module with the given name.
+ Raises an exception on failure.
+ """
+ if source_name == "inquisitor":
+ return InquisitorStubSource()
- cwd = os.getcwd()
- try:
- # Push the sources directory.
- os.chdir(SOURCES_PATH)
- # Make the sources directory importable while working with sources.
- if SOURCES_PATH not in sys.path:
- sys.path.insert(0, SOURCES_PATH)
+ cwd = os.getcwd()
+ try:
+ # Push the sources directory.
+ os.chdir(SOURCES_PATH)
+ # Make the sources directory importable while working with sources.
+ if SOURCES_PATH not in sys.path:
+ sys.path.insert(0, SOURCES_PATH)
- # Check if the named source is present.
- source_file_name = source_name + '.py'
- if not os.path.isfile(source_file_name):
- raise FileNotFoundError(f'Missing "{source_name}" in "{SOURCES_PATH}"')
+ # Check if the named source is present.
+ source_file_name = source_name + ".py"
+ if not os.path.isfile(source_file_name):
+ raise FileNotFoundError(f'Missing "{source_name}" in "{SOURCES_PATH}"')
- # Import the source module by file path.
- logger.debug(f'Loading module "{source_file_name}"')
- spec = importlib.util.spec_from_file_location(source_name, source_file_name)
- itemsource = importlib.util.module_from_spec(spec)
- spec.loader.exec_module(itemsource)
- itemsource = importlib.import_module(source_name)
+ # Import the source module by file path.
+ logger.debug(f'Loading module "{source_file_name}"')
+ spec = importlib.util.spec_from_file_location(source_name, source_file_name)
+ itemsource = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(itemsource)
+ itemsource = importlib.import_module(source_name)
- # Require fetch_new().
- if not hasattr(itemsource, 'fetch_new'):
- raise ImportError(f'Missing fetch_new in "{source_file_name}"')
+ # Require fetch_new().
+ if not hasattr(itemsource, "fetch_new"):
+ raise ImportError(f'Missing fetch_new in "{source_file_name}"')
- return itemsource
+ return itemsource
- finally:
- os.chdir(cwd)
- if SOURCES_PATH in sys.path:
- sys.path.remove(SOURCES_PATH)
+ finally:
+ os.chdir(cwd)
+ if SOURCES_PATH in sys.path:
+ sys.path.remove(SOURCES_PATH)
def update_source(source_name, source):
- """
- Attempts to update the given source. Raises an exception if the source does.
- """
- # Get a list of item ids that already existed in this source's cell.
- prior_ids = loader.get_item_ids(source_name)
- logger.debug(f'Found {len(prior_ids)} prior items')
+ """
+ Attempts to update the given source. Raises an exception if the source does.
+ """
+ # Get a list of item ids that already existed in this source's cell.
+ prior_ids = loader.get_item_ids(source_name)
+ logger.debug(f"Found {len(prior_ids)} prior items")
- # Get the feed items from the source's fetch method.
- state = loader.load_state(source_name)
- fetched = source.fetch_new(state)
- state.flush()
- logger.debug(f'Fetched {len(fetched)} items')
- fetched_items = {item['id']: item for item in fetched}
+ # Get the feed items from the source's fetch method.
+ state = loader.load_state(source_name)
+ fetched = source.fetch_new(state)
+ state.flush()
+ logger.debug(f"Fetched {len(fetched)} items")
+ fetched_items = {item["id"]: item for item in fetched}
- # Determine which items are new and which are updates.
- # We query the file system here instead of checking against this source's
- # item ids from above because sources are allowed to generate in other
- # sources' cells.
- new_items = []
- updated_items = []
- for item in fetched:
- item_source = item.get('source', source_name)
- if loader.item_exists(item_source, item['id']):
- updated_items.append(item)
- else:
- new_items.append(item)
+ # Determine which items are new and which are updates.
+ # We query the file system here instead of checking against this source's
+ # item ids from above because sources are allowed to generate in other
+ # sources' cells.
+ new_items = []
+ updated_items = []
+ for item in fetched:
+ item_source = item.get("source", source_name)
+ if loader.item_exists(item_source, item["id"]):
+ updated_items.append(item)
+ else:
+ new_items.append(item)
- # Write all the new items to the source's cell.
- has_create_handler = hasattr(source, 'on_create')
- for item in new_items:
- item_source = item.get('source', source_name)
- created_item = loader.new_item(item_source, item)
- if has_create_handler:
- # Because some sources do not return items more than once,
- # exceptions in the on-create handler must be squashed.
- try:
- source.on_create(state, created_item)
- except:
- error.as_item(
- f'Exception in {source_name}.on_create',
- traceback.format_exc())
+ # Write all the new items to the source's cell.
+ has_create_handler = hasattr(source, "on_create")
+ for item in new_items:
+ item_source = item.get("source", source_name)
+ created_item = loader.new_item(item_source, item)
+ if has_create_handler:
+ # Because some sources do not return items more than once,
+ # exceptions in the on-create handler must be squashed.
+ try:
+ source.on_create(state, created_item)
+ except:
+ error.as_item(
+ f"Exception in {source_name}.on_create", traceback.format_exc()
+ )
- # Update the other items using the fetched items' values.
- for new_item in updated_items:
- old_item = loader.load_item(new_item['source'], new_item['id'])
- for field in USE_NEWEST:
- if field in new_item and old_item[field] != new_item[field]:
- old_item[field] = new_item[field]
- if 'callback' in new_item:
- old_callback = old_item.get('callback', {})
- # Because of the way this update happens, any fields that are set
- # in the callback when the item is new will keep their original
- # values, as those values reappear in new_item on subsequent
- # updates.
- old_item['callback'] = {**old_item['callback'], **new_item['callback']}
+ # Update the other items using the fetched items' values.
+ for new_item in updated_items:
+ old_item = loader.load_item(new_item["source"], new_item["id"])
+ for field in USE_NEWEST:
+ if field in new_item and old_item[field] != new_item[field]:
+ old_item[field] = new_item[field]
+ if "callback" in new_item:
+ old_callback = old_item.get("callback", {})
+ # Because of the way this update happens, any fields that are set
+ # in the callback when the item is new will keep their original
+ # values, as those values reappear in new_item on subsequent
+ # updates.
+ old_item["callback"] = {**old_item["callback"], **new_item["callback"]}
- # In general, items are removed when they are old (not found in the last
- # fetch) and inactive. Some item fields can change this basic behavior.
- del_count = 0
- now = timestamp.now()
- has_delete_handler = hasattr(source, 'on_delete')
- fetched_ids = [item['id'] for item in updated_items]
- old_item_ids = [
- item_id for item_id in prior_ids
- if item_id not in fetched_ids]
- for item_id in old_item_ids:
- item = loader.load_item(source_name, item_id)
- remove = not item['active']
- # The time-to-live field protects an item from removal until expiry.
- # This is mainly used to avoid old items resurfacing when their source
- # cannot guarantee monotonicity.
- if 'ttl' in item:
- ttl_date = item['created'] + item['ttl']
- if ttl_date > now:
- continue
- # The time-to-die field can force an active item to be removed.
- if 'ttd' in item:
- ttd_date = item['created'] + item['ttd']
- if ttd_date < now:
- remove = True
- # Items to be removed are deleted
- if remove:
- try:
- if has_delete_handler:
- # Run the delete handler so exceptions prevent deletions
- source.on_delete(state, item)
- loader.delete_item(source_name, item['id'])
- del_count += 1
- except:
- error.as_item(
- f'Failed to delete {source_name}/{item["id"]}',
- traceback.format_exc())
+ # In general, items are removed when they are old (not found in the last
+ # fetch) and inactive. Some item fields can change this basic behavior.
+ del_count = 0
+ now = timestamp.now()
+ has_delete_handler = hasattr(source, "on_delete")
+ fetched_ids = [item["id"] for item in updated_items]
+ old_item_ids = [item_id for item_id in prior_ids if item_id not in fetched_ids]
+ for item_id in old_item_ids:
+ item = loader.load_item(source_name, item_id)
+ remove = not item["active"]
+ # The time-to-live field protects an item from removal until expiry.
+ # This is mainly used to avoid old items resurfacing when their source
+ # cannot guarantee monotonicity.
+ if "ttl" in item:
+ ttl_date = item["created"] + item["ttl"]
+ if ttl_date > now:
+ continue
+ # The time-to-die field can force an active item to be removed.
+ if "ttd" in item:
+ ttd_date = item["created"] + item["ttd"]
+ if ttd_date < now:
+ remove = True
+ # Items to be removed are deleted
+ if remove:
+ try:
+ if has_delete_handler:
+ # Run the delete handler so exceptions prevent deletions
+ source.on_delete(state, item)
+ loader.delete_item(source_name, item["id"])
+ del_count += 1
+ except:
+ error.as_item(
+ f'Failed to delete {source_name}/{item["id"]}',
+ traceback.format_exc(),
+ )
- # Note update timestamp in state
- state['last_updated'] = timestamp.now()
+ # Note update timestamp in state
+ state["last_updated"] = timestamp.now()
- # Log counts
- logger.info("{} new item{}, {} deleted item{}".format(
- len(new_items), "s" if len(new_items) != 1 else "",
- del_count, "s" if del_count != 1 else ""))
+ # Log counts
+ logger.info(
+ "{} new item{}, {} deleted item{}".format(
+ len(new_items),
+ "s" if len(new_items) != 1 else "",
+ del_count,
+ "s" if del_count != 1 else "",
+ )
+ )
def item_callback(source_name, itemid):
- try:
- # Load the module with the callback function
- source_module = load_source(source_name)
- if not hasattr(source_module, 'callback'):
- raise ImportError(f"Missing callback in '{source_name}'")
- # Load the source state and the origin item
- state = loader.load_state(source_name)
- item = loader.load_item(source_name, itemid)
- # Execute callback
- source_module.callback(state, item)
- # Save any changes
- item.flush()
- state.flush()
- except Exception:
- error.as_item(
- f"Error executing callback for {source_name}/{itemid}",
- traceback.format_exc())
+ try:
+ # Load the module with the callback function
+ source_module = load_source(source_name)
+ if not hasattr(source_module, "callback"):
+ raise ImportError(f"Missing callback in '{source_name}'")
+ # Load the source state and the origin item
+ state = loader.load_state(source_name)
+ item = loader.load_item(source_name, itemid)
+ # Execute callback
+ source_module.callback(state, item)
+ # Save any changes
+ item.flush()
+ state.flush()
+ except Exception:
+ error.as_item(
+ f"Error executing callback for {source_name}/{itemid}",
+ traceback.format_exc(),
+ )
diff --git a/inquisitor/templates.py b/inquisitor/templates.py
index bd30102..a3ce408 100644
--- a/inquisitor/templates.py
+++ b/inquisitor/templates.py
@@ -17,212 +17,221 @@ import requests
# Module imports
from inquisitor import CACHE_PATH
-logger = logging.getLogger('inquisitor.templates')
+logger = logging.getLogger("inquisitor.templates")
def cache_image(source, url, filename):
- # Define some paths
- path = os.path.join(CACHE_PATH, source)
- file_path = os.path.join(path, filename)
- cached_url = f'/cache/{source}/{filename}'
- # Ensure cache folder
- if not os.path.isdir(path):
- os.mkdir(path)
- # Fetch url
- logger.info(f'Caching {url} to {file_path}')
- response = requests.get(url)
- # Write file to disk
- with open(file_path, 'wb') as f:
- f.write(response.content)
- # Return the inquisitor path to the file
- return cached_url
+ # Define some paths
+ path = os.path.join(CACHE_PATH, source)
+ file_path = os.path.join(path, filename)
+ cached_url = f"/cache/{source}/{filename}"
+ # Ensure cache folder
+ if not os.path.isdir(path):
+ os.mkdir(path)
+ # Fetch url
+ logger.info(f"Caching {url} to {file_path}")
+ response = requests.get(url)
+ # Write file to disk
+ with open(file_path, "wb") as f:
+ f.write(response.content)
+ # Return the inquisitor path to the file
+ return cached_url
class LinearCrawler:
- """
- An engine for generating items from web sources that link content
- together in a linear fashion, such as webcomics.
- """
- def fetch_new(self, state):
- items = []
- max_iter = self.max_iterations() - 1
- new = self.try_fetch(state)
- items.extend(new)
- for iter in range(max_iter):
- sleep(1)
- # If we've already gotten some items out of this fetch, we don't
- # want to lose them and have the state still be set to the next
- # page, so we wrap further calls in a try block and force return
- # if we hit an error.
- try:
- new = self.try_fetch(state)
- except:
- new = []
- items.extend(new)
- # Cut out early if there was nothing returned
- if not new:
- break
- return items
+ """
+ An engine for generating items from web sources that link content
+ together in a linear fashion, such as webcomics.
+ """
- def try_fetch(self, state):
- # Check for whether a new page should be crawled
- if 'current_page' not in state:
- next_page = self.get_start_url()
- else:
- current = state['current_page']
- response = requests.get(current)
- soup = BeautifulSoup(response.text, features='html.parser')
- next_page = self.get_next_page_url(current, soup)
- if not next_page:
- return [] # nothing new
+ def fetch_new(self, state):
+ items = []
+ max_iter = self.max_iterations() - 1
+ new = self.try_fetch(state)
+ items.extend(new)
+ for iter in range(max_iter):
+ sleep(1)
+ # If we've already gotten some items out of this fetch, we don't
+ # want to lose them and have the state still be set to the next
+ # page, so we wrap further calls in a try block and force return
+ # if we hit an error.
+ try:
+ new = self.try_fetch(state)
+ except:
+ new = []
+ items.extend(new)
+ # Cut out early if there was nothing returned
+ if not new:
+ break
+ return items
- # Download the new page
- logger.info('Fetching ' + next_page)
- response = requests.get(next_page)
- soup = BeautifulSoup(response.text, features="html.parser")
+ def try_fetch(self, state):
+ # Check for whether a new page should be crawled
+ if "current_page" not in state:
+ next_page = self.get_start_url()
+ else:
+ current = state["current_page"]
+ response = requests.get(current)
+ soup = BeautifulSoup(response.text, features="html.parser")
+ next_page = self.get_next_page_url(current, soup)
+ if not next_page:
+ return [] # nothing new
- # Create an item from the page
- item = self.make_item(next_page, soup)
+ # Download the new page
+ logger.info("Fetching " + next_page)
+ response = requests.get(next_page)
+ soup = BeautifulSoup(response.text, features="html.parser")
- # Update the state and return the item
- state['current_page'] = next_page
- return [item]
+ # Create an item from the page
+ item = self.make_item(next_page, soup)
- def max_iterations(self):
- return 3
+ # Update the state and return the item
+ state["current_page"] = next_page
+ return [item]
- def get_start_url(self):
- raise NotImplementedError('get_start_url is required')
+ def max_iterations(self):
+ return 3
- def get_next_page_url(self, url, soup):
- raise NotImplementedError('get_next_page_url is required')
+ def get_start_url(self):
+ raise NotImplementedError("get_start_url is required")
- def make_item(self, url, soup):
- raise NotImplementedError('make_item is required')
+ def get_next_page_url(self, url, soup):
+ raise NotImplementedError("get_next_page_url is required")
+
+ def make_item(self, url, soup):
+ raise NotImplementedError("make_item is required")
class RedditScraper:
- """
- An engine for generating items from subreddits.
- Requires defining source, subreddit_name
- fetch new with RedditScraper.fetch_new(state, __name__, reddit)
- """
- @staticmethod
- def fetch_new(state, name, reddit):
- items = []
- for name, obj in inspect.getmembers(sys.modules[name]):
- if (inspect.isclass(obj)
- and issubclass(obj, RedditScraper)
- and obj is not RedditScraper
- ):
- sub_items = obj(reddit).get_items()
- items.extend(sub_items)
- return items
+ """
+ An engine for generating items from subreddits.
+ Requires defining source, subreddit_name
+ fetch new with RedditScraper.fetch_new(state, __name__, reddit)
+ """
- def __init__(self, reddit):
- self.reddit = reddit
+ @staticmethod
+ def fetch_new(state, name, reddit):
+ items = []
+ for name, obj in inspect.getmembers(sys.modules[name]):
+ if (
+ inspect.isclass(obj)
+ and issubclass(obj, RedditScraper)
+ and obj is not RedditScraper
+ ):
+ sub_items = obj(reddit).get_items()
+ items.extend(sub_items)
+ return items
- def get_items(self):
- sub_name = self.subreddit_name
- logger.info(f'Fetching posts from r/{sub_name}')
- subreddit = self.reddit.subreddit(sub_name)
- posts = self.subreddit_page(subreddit)
- items = []
- for post in posts:
- if self.filter_post(post):
- items.append(self.item_from_post(post))
- return items
+ def __init__(self, reddit):
+ self.reddit = reddit
- def item_from_post(self, post):
- item = {
- 'source': self.source,
- 'id': post.id,
- 'title': self.get_title(post),
- 'link': self.get_link(post),
- 'time': post.created_utc,
- 'author': '/u/' + (post.author.name if post.author else "[deleted]"),
- 'body': self.get_body(post),
- 'tags': self.get_tags(post),
- 'ttl': self.get_ttl(post),
- }
- ttl = self.get_ttl(post)
- if ttl is not None: item['ttl'] = ttl
- ttd = self.get_ttd(post)
- if ttd is not None: item['ttd'] = ttd
- tts = self.get_tts(post)
- if tts is not None: item['tts'] = tts
- callback = self.get_callback(post)
- if callback is not None: item['callback'] = callback
- return item
+ def get_items(self):
+ sub_name = self.subreddit_name
+ logger.info(f"Fetching posts from r/{sub_name}")
+ subreddit = self.reddit.subreddit(sub_name)
+ posts = self.subreddit_page(subreddit)
+ items = []
+ for post in posts:
+ if self.filter_post(post):
+ items.append(self.item_from_post(post))
+ return items
- def subreddit_page(self, subreddit):
- return subreddit.hot(limit=25)
+ def item_from_post(self, post):
+ item = {
+ "source": self.source,
+ "id": post.id,
+ "title": self.get_title(post),
+ "link": self.get_link(post),
+ "time": post.created_utc,
+ "author": "/u/" + (post.author.name if post.author else "[deleted]"),
+ "body": self.get_body(post),
+ "tags": self.get_tags(post),
+ "ttl": self.get_ttl(post),
+ }
+ ttl = self.get_ttl(post)
+ if ttl is not None:
+ item["ttl"] = ttl
+ ttd = self.get_ttd(post)
+ if ttd is not None:
+ item["ttd"] = ttd
+ tts = self.get_tts(post)
+ if tts is not None:
+ item["tts"] = tts
+ callback = self.get_callback(post)
+ if callback is not None:
+ item["callback"] = callback
+ return item
- def filter_post(self, post):
- return True
+ def subreddit_page(self, subreddit):
+ return subreddit.hot(limit=25)
- def get_title(self, post):
- s = '[S] ' if post.spoiler else ''
- nsfw = '[NSFW] ' if post.over_18 else ''
- return f'{s}{nsfw}/{post.subreddit_name_prefixed}: {post.title}'
+ def filter_post(self, post):
+ return True
- def get_link(self, post):
- return f'https://reddit.com{post.permalink}'
+ def get_title(self, post):
+ s = "[S] " if post.spoiler else ""
+ nsfw = "[NSFW] " if post.over_18 else ""
+ return f"{s}{nsfw}/{post.subreddit_name_prefixed}: {post.title}"
- def get_body(self, post):
- parts = []
- if not post.is_self:
- parts.append(f'link: {post.url}')
- if hasattr(post, 'preview'):
- try:
- previews = post.preview['images'][0]['resolutions']
- small_previews = [p for p in previews if p['width'] < 800]
- preview = sorted(small_previews, key=lambda p:-p['width'])[0]
- parts.append(f'{preview_body}
') - return ''.join(parts) + def get_link(self, post): + return f"https://reddit.com{post.permalink}" - def get_tags(self, post): - tags = ['reddit', post.subreddit_name_prefixed[2:]] - if post.over_18: - tags.append('nsfw') - return tags + def get_body(self, post): + parts = [] + if not post.is_self: + parts.append(f'link: {post.url}') + if hasattr(post, "preview"): + try: + previews = post.preview["images"][0]["resolutions"] + small_previews = [p for p in previews if p["width"] < 800] + preview = sorted(small_previews, key=lambda p: -p["width"])[0] + parts.append(f'
{preview_body}
") + return "".join(parts) - def get_ttl(self, post): - return 60 * 60 * 24 * 7 # 1 week + def get_tags(self, post): + tags = ["reddit", post.subreddit_name_prefixed[2:]] + if post.over_18: + tags.append("nsfw") + return tags - def get_ttd(self, post): - return None + def get_ttl(self, post): + return 60 * 60 * 24 * 7 # 1 week - def get_tts(self, post): - return None + def get_ttd(self, post): + return None - def get_callback(self, post): - return None + def get_tts(self, post): + return None - def callback(self, state, item): - raise NotImplementedError('callback') + def get_callback(self, post): + return None - def on_create(self, state, item): - raise NotImplementedError('on_create') + def callback(self, state, item): + raise NotImplementedError("callback") - def on_delete(self, state, item): - raise NotImplementedError('on_delete') + def on_create(self, state, item): + raise NotImplementedError("on_create") + + def on_delete(self, state, item): + raise NotImplementedError("on_delete") diff --git a/inquisitor/timestamp.py b/inquisitor/timestamp.py index 111bc43..3128432 100644 --- a/inquisitor/timestamp.py +++ b/inquisitor/timestamp.py @@ -1,9 +1,11 @@ import time import datetime + def now(): - return int(time.time()) + return int(time.time()) + def stamp_to_readable(ts, formatstr="%Y-%m-%d %H:%M:%S"): - dt = datetime.datetime.fromtimestamp(ts) - return dt.strftime(formatstr) + dt = datetime.datetime.fromtimestamp(ts) + return dt.strftime(formatstr)