From fe0967d625246d019321bf79c87bf1b0c74359c8 Mon Sep 17 00:00:00 2001 From: Tim Van Baak Date: Wed, 8 Jan 2020 21:41:38 -0800 Subject: [PATCH] Missed adding config package to source control --- amanuensis/cli.py | 3 +- amanuensis/config/__init__.py | 62 ++++++++++++++++++++++++++++++++ amanuensis/config/init.py | 56 +++++++++++++++++++++++++++++ amanuensis/config/loader.py | 68 +++++++++++++++++++++++++++++++++++ 4 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 amanuensis/config/__init__.py create mode 100644 amanuensis/config/init.py create mode 100644 amanuensis/config/loader.py diff --git a/amanuensis/cli.py b/amanuensis/cli.py index 8855607..757673e 100644 --- a/amanuensis/cli.py +++ b/amanuensis/cli.py @@ -98,10 +98,9 @@ def command_generate_secret(args): import os import config - from config.loader import WritableConfig secret_key = os.urandom(32) - with WritableConfig(os.path.join(config.CONFIG_DIR, "config.json")) as cfg: + with config.json_rw("config.json") as cfg: cfg['secret_key'] = secret_key.hex() config.logger.info("Regenerated Flask secret key") diff --git a/amanuensis/config/__init__.py b/amanuensis/config/__init__.py new file mode 100644 index 0000000..0fd3abf --- /dev/null +++ b/amanuensis/config/__init__.py @@ -0,0 +1,62 @@ +# Standard library imports +import json +import logging +import os + +# Module imports +from errors import MissingConfigError, MalformedConfigError +import config.init +import config.loader + + +# Environment variable name constants +ENV_SECRET_KEY = "AMANUENSIS_SECRET_KEY" +ENV_CONFIG_DIR = "AMANUENSIS_CONFIG_DIR" +ENV_LOG_FILE = "AMANUENSIS_LOG_FILE" +ENV_LOG_FILE_SIZE = "AMANUENSIS_LOG_FILE_SIZE" +ENV_LOG_FILE_NUM = "AMANUENSIS_LOG_FILE_NUM" + +# +# The config directory can be set by cli input, so the config infrastructure +# needs to wait for initialization before it can load any configs. +# +CONFIG_DIR = None +GLOBAL_CONFIG = None +logger = None + +def init_config(args): + """ + Initializes the config infrastructure to read configs from the + directory given by args.config_dir. Initializes logging. + """ + global CONFIG_DIR, GLOBAL_CONFIG, logger + CONFIG_DIR = args.config_dir + config.init.verify_config_dir(CONFIG_DIR) + with config.loader.json_ro(os.path.join(CONFIG_DIR, "config.json")) as cfg: + GLOBAL_CONFIG = cfg + config.init.init_logging(args, GLOBAL_CONFIG['logging']) + logger = logging.getLogger("amanuensis") + +def get(key): + return GLOBAL_CONFIG[key] + +def prepend(path): + return os.path.join(CONFIG_DIR, path) + +def open_sh(path, mode): + return config.loader.open_sh(prepend(path), mode) + +def open_ex(path, mode): + return config.loader.open_ex(prepend(path), mode) + +def json_ro(path): + return config.loader.json_ro(prepend(path)) + +def json_rw(path): + return config.loader.json_rw(prepend(path)) + +def json(*args, mode='r'): + if not args[-1].endswith(".json"): + args[-1] = args[-1] + ".json" + path = os.path.join(CONFIG_DIR, *args) + diff --git a/amanuensis/config/init.py b/amanuensis/config/init.py new file mode 100644 index 0000000..cf7d903 --- /dev/null +++ b/amanuensis/config/init.py @@ -0,0 +1,56 @@ +# Standard library imports +import copy +import json +import logging.config +import os +import pkg_resources + +# Module imports +from errors import MissingConfigError, MalformedConfigError +from config.loader import json_ro + + +def verify_config_dir(config_dir): + """ + Verifies that the given directory has a valid global config in it and + returns the global config if so + """ + # Check that config dir exists + if not os.path.isdir(config_dir): + raise MissingConfigError("Config directory not found: {}".format(config_dir)) + # Check that global config file exists + global_config_path = os.path.join(config_dir, "config.json") + if not os.path.isfile(global_config_path): + raise MissingConfigError("Config directory missing global config file: {}".format(config_dir)) + # Check that global config file has all the default settings + def_cfg_s = pkg_resources.resource_stream("__main__", "resources/default_config.json") + def_cfg = json.load(def_cfg_s) + with json_ro(global_config_path) as global_config_file: + for key in def_cfg.keys(): + if key not in global_config_file.keys(): + raise MalformedConfigError("Missing '{}' in global config. If you updated Amanuensis, run init --update to pick up new config keys".format(key)) + # Configs verified + return True + +def init_logging(args, logging_config): + """ + Initializes logging by using the logging section of the global config + file. + """ + # Get the logging config section + cfg = copy.deepcopy(logging_config) + # Apply any commandline settings to what was defined in the config file + handlers = cfg['loggers']['amanuensis']['handlers'] + if args.verbose: + if 'cli-basic' in handlers: + handlers.remove('cli_basic') + handlers.append('cli_verbose') + if args.log_file: + cfg['handlers']['file']['filename'] = args.log_file + handlers.append("file") + # Load the config + try: + logging.config.dictConfig(cfg) + except: + raise MalformedConfigError("Failed to load logging config") + diff --git a/amanuensis/config/loader.py b/amanuensis/config/loader.py new file mode 100644 index 0000000..b1d9df4 --- /dev/null +++ b/amanuensis/config/loader.py @@ -0,0 +1,68 @@ +# Standard library imports +from collections import OrderedDict +import fcntl +import json +import os + +# Module imports +from errors import ReadOnlyError + +class ReadOnlyOrderedDict(OrderedDict): + """An ordered dictionary that cannot be modified""" + def __readonly__(self, *args, **kwargs): + raise ReadOnlyError("Cannot modify a ReadOnlyOrderedDict") + + def __init__(self, *args, **kwargs): + super(ReadOnlyOrderedDict, self).__init__(*args, **kwargs) + self.__setitem__ = self.__readonly__ + self.__delitem__ = self.__readonly__ + self.pop = self.__readonly__ + self.popitem = self.__readonly__ + self.clear = self.__readonly__ + self.update = self.__readonly__ + self.setdefault = self.__readonly__ + +class open_lock(): + def __init__(self, path, mode, lock_type): + self.fd = open(path, mode, encoding='utf8') + fcntl.lockf(self.fd, lock_type) + + def __enter__(self): + return self.fd + + def __exit__(self, exc_type, exc_value, traceback): + fcntl.lockf(self.fd, fcntl.LOCK_UN) + self.fd.close() + +class open_sh(open_lock): + def __init__(self, path, mode): + super().__init__(path, mode, fcntl.LOCK_SH) + +class open_ex(open_lock): + def __init__(self, path, mode): + super().__init__(path, mode, fcntl.LOCK_EX) + +class json_ro(open_sh): + def __init__(self, path): + super().__init__(path, 'r') + self.config = None + + def __enter__(self): + self.config = json.load(self.fd, object_pairs_hook=ReadOnlyOrderedDict) + return self.config + +class json_rw(open_ex): + def __init__(self, path): + super().__init__(path, 'r+') + self.config = None + + def __enter__(self): + self.config = json.load(self.fd, object_pairs_hook=OrderedDict) + return self.config + + def __exit__(self, exc_type, exc_value, traceback): + self.fd.seek(0) + json.dump(self.config, self.fd, allow_nan=False, indent='\t') + self.fd.truncate() + super().__exit__(exc_type, exc_value, traceback) +