Compare commits
8 Commits
Author | SHA1 | Date |
---|---|---|
Tim Van Baak | c1e27b73b0 | |
Tim Van Baak | d9a5f6b7c2 | |
Tim Van Baak | 4e574a5e8f | |
Tim Van Baak | b0096691d8 | |
Tim Van Baak | 675e019f72 | |
Tim Van Baak | 5ecb187c92 | |
Tim Van Baak | 7528c2465f | |
Tim Van Baak | 25958b123f |
16
flake.lock
16
flake.lock
|
@ -16,6 +16,21 @@
|
||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"my-flake": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1670879933,
|
||||||
|
"narHash": "sha256-V45PH0cnFLilx66x4td5qQnWNn/V/6/6b7FQDIHvdyI=",
|
||||||
|
"owner": "Jaculabilis",
|
||||||
|
"repo": "my-flake",
|
||||||
|
"rev": "2b2cd07a6d971b15fc5f65d6d963d0da551a5892",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "Jaculabilis",
|
||||||
|
"repo": "my-flake",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
"nixpkgs": {
|
"nixpkgs": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1669833724,
|
"lastModified": 1669833724,
|
||||||
|
@ -35,6 +50,7 @@
|
||||||
"root": {
|
"root": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"flake-compat": "flake-compat",
|
"flake-compat": "flake-compat",
|
||||||
|
"my-flake": "my-flake",
|
||||||
"nixpkgs": "nixpkgs"
|
"nixpkgs": "nixpkgs"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
76
flake.nix
76
flake.nix
|
@ -1,5 +1,6 @@
|
||||||
{
|
{
|
||||||
inputs = {
|
inputs = {
|
||||||
|
my-flake.url = "github:Jaculabilis/my-flake";
|
||||||
nixpkgs.url = "github:NixOS/nixpkgs?ref=refs/tags/22.11";
|
nixpkgs.url = "github:NixOS/nixpkgs?ref=refs/tags/22.11";
|
||||||
flake-compat = {
|
flake-compat = {
|
||||||
url = "github:edolstra/flake-compat";
|
url = "github:edolstra/flake-compat";
|
||||||
|
@ -7,21 +8,68 @@
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
outputs = { self, nixpkgs, flake-compat }:
|
outputs = { self, my-flake, nixpkgs, flake-compat }:
|
||||||
let
|
let
|
||||||
system = "x86_64-linux";
|
systems = [ "aarch64-linux" "x86_64-linux" ];
|
||||||
pkgs = nixpkgs.legacyPackages.${system};
|
each = system:
|
||||||
in
|
let
|
||||||
{
|
pkgs = (import nixpkgs {
|
||||||
packages.${system}.default =
|
inherit system;
|
||||||
(pkgs.poetry2nix.mkPoetryApplication {
|
overlays = [ self.overlays.default ];
|
||||||
projectDir = ./.;
|
});
|
||||||
}).dependencyEnv;
|
in {
|
||||||
|
packages.${system} = {
|
||||||
defaultPackage.${system} = self.packages.${system}.default;
|
default = self.packages.${system}.inquisitor;
|
||||||
|
inquisitor = pkgs.inquisitor;
|
||||||
devShell.${system} = pkgs.mkShell {
|
env = pkgs.inquisitor.dependencyEnv;
|
||||||
buildInputs = [ (pkgs.python3.withPackages (p: [p.poetry])) ];
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
devShells.${system} = {
|
||||||
|
default = self.devShells.${system}.inquisitor;
|
||||||
|
inquisitor = pkgs.mkShell {
|
||||||
|
buildInputs = [ (pkgs.python3.withPackages (p: [p.poetry])) ];
|
||||||
|
shellHook = ''
|
||||||
|
PS1="(inquisitor) $PS1"
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
sources = pkgs.mkShell {
|
||||||
|
buildInputs = [ self.packages.${system}.env ];
|
||||||
|
shellHook = ''
|
||||||
|
PS1="(sources) $PS1"
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
checks.${system}.test-module = let
|
||||||
|
test-lib = import "${nixpkgs}/nixos/lib/testing-python.nix" {
|
||||||
|
inherit system;
|
||||||
|
};
|
||||||
|
in
|
||||||
|
test-lib.makeTest {
|
||||||
|
name = "inquisitor-test-module";
|
||||||
|
nodes = {
|
||||||
|
host = { ... }: {
|
||||||
|
imports = [ self.nixosModules.default ];
|
||||||
|
services.inquisitor.enable = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
testScript = ''
|
||||||
|
start_all()
|
||||||
|
host.wait_for_unit("multi-user.target")
|
||||||
|
# Check that the setup script ran
|
||||||
|
host.succeed("[ -e /var/lib/inquisitor ]")
|
||||||
|
# Check that the service came up
|
||||||
|
host.succeed("systemctl is-enabled inquisitor")
|
||||||
|
host.succeed("systemctl is-active inquisitor")
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
in (my-flake.outputs-for each systems) // {
|
||||||
|
overlays.default = final: prev: {
|
||||||
|
inquisitor = final.poetry2nix.mkPoetryApplication {
|
||||||
|
projectDir = ./.;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
nixosModules.default = import ./module.nix self;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
@ -236,6 +236,11 @@ def command_help(args):
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def nocommand(args):
|
||||||
|
print("command required")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
"""CLI entry point"""
|
"""CLI entry point"""
|
||||||
# Enable piping
|
# Enable piping
|
||||||
|
@ -283,8 +288,11 @@ def main():
|
||||||
add_logging_handler(verbose=args.verbose, log_filename=None)
|
add_logging_handler(verbose=args.verbose, log_filename=None)
|
||||||
|
|
||||||
# Execute command
|
# Execute command
|
||||||
if args.command:
|
try:
|
||||||
sys.exit(commands[args.command](args.args))
|
command = commands.get(args.command, nocommand)
|
||||||
else:
|
sys.exit(command(args.args))
|
||||||
print("command required")
|
except BrokenPipeError:
|
||||||
sys.exit(0)
|
# See https://docs.python.org/3.10/library/signal.html#note-on-sigpipe
|
||||||
|
devnull = os.open(os.devnull, os.O_WRONLY)
|
||||||
|
os.dup2(devnull, sys.stdout.fileno())
|
||||||
|
sys.exit(1)
|
||||||
|
|
|
@ -42,7 +42,8 @@ def read_config_file(config_path):
|
||||||
"""
|
"""
|
||||||
# Parse the config file into key-value pairs
|
# Parse the config file into key-value pairs
|
||||||
if not os.path.isfile(config_path):
|
if not os.path.isfile(config_path):
|
||||||
raise FileNotFoundError(f'No config file found at {config_path}')
|
|
||||||
|
raise FileNotFoundError(f'No config file found at {config_path}, try setting {CONFIG_ENVVAR}')
|
||||||
accumulated_configs = {}
|
accumulated_configs = {}
|
||||||
current_key = None
|
current_key = None
|
||||||
with open(config_path, 'r', encoding='utf8') as cfg:
|
with open(config_path, 'r', encoding='utf8') as cfg:
|
||||||
|
|
|
@ -0,0 +1,154 @@
|
||||||
|
flake: { config, lib, pkgs, ... }:
|
||||||
|
|
||||||
|
let
|
||||||
|
inherit (lib) mkIf mkOption types;
|
||||||
|
|
||||||
|
cfg = config.services.inquisitor;
|
||||||
|
in
|
||||||
|
{
|
||||||
|
options = {
|
||||||
|
services.inquisitor = {
|
||||||
|
enable = mkOption {
|
||||||
|
type = types.bool;
|
||||||
|
default = true;
|
||||||
|
description = "Enable the Inquisitor aggregator.";
|
||||||
|
};
|
||||||
|
|
||||||
|
listen.addr = mkOption {
|
||||||
|
type = types.str;
|
||||||
|
default = "0.0.0.0";
|
||||||
|
description = "Listen address passed to nginx.";
|
||||||
|
};
|
||||||
|
|
||||||
|
listen.port = mkOption {
|
||||||
|
type = types.port;
|
||||||
|
default = 80;
|
||||||
|
description = "Listen port passed to nginx.";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
config =
|
||||||
|
let
|
||||||
|
# Get the inquisitor package from the flake.
|
||||||
|
inquisitor = flake.packages.${pkgs.stdenv.hostPlatform.system}.env;
|
||||||
|
|
||||||
|
# Define the inquisitor state directory.
|
||||||
|
stateDir = "/var/lib/inquisitor";
|
||||||
|
|
||||||
|
# Define an scp helper for item callbacks to use.
|
||||||
|
scp-helper = pkgs.writeShellScriptBin "scp-helper" ''
|
||||||
|
${pkgs.openssh}/bin/scp -i ${stateDir}/.ssh/inquisitor.key -oStrictHostKeyChecking=no "$@"
|
||||||
|
'';
|
||||||
|
|
||||||
|
# Define the inquisitor service user.
|
||||||
|
svcUser = {
|
||||||
|
name = "inquisitor";
|
||||||
|
group = "inquisitor";
|
||||||
|
description = "Inquisitor service user";
|
||||||
|
isSystemUser = true;
|
||||||
|
shell = pkgs.bashInteractive;
|
||||||
|
packages = [ inquisitor pkgs.cron ];
|
||||||
|
};
|
||||||
|
|
||||||
|
# Create a config file pointing to the state directory.
|
||||||
|
inqConfig = pkgs.writeTextFile {
|
||||||
|
name = "inquisitor.conf";
|
||||||
|
text = ''
|
||||||
|
DataPath = ${stateDir}/data/
|
||||||
|
SourcePath = ${stateDir}/sources/
|
||||||
|
CachePath = ${stateDir}/cache/
|
||||||
|
Verbose = false
|
||||||
|
LogFile = ${stateDir}/inquisitor.log
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
# Create a setup script to ensure the service directory state.
|
||||||
|
inqSetup = pkgs.writeShellScript "inquisitor-setup.sh" ''
|
||||||
|
# Ensure the required directories exist.
|
||||||
|
${pkgs.coreutils}/bin/mkdir -p ${stateDir}/data/inquisitor/
|
||||||
|
${pkgs.coreutils}/bin/mkdir -p ${stateDir}/sources/
|
||||||
|
${pkgs.coreutils}/bin/mkdir -p ${stateDir}/cache/
|
||||||
|
if [ ! -f ${stateDir}/data/inquisitor/state ]; then
|
||||||
|
${pkgs.coreutils}/bin/echo "{}" > ${stateDir}/data/inquisitor/state
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Ensure the service owns the folders.
|
||||||
|
${pkgs.coreutils}/bin/chown -R ${svcUser.name} ${stateDir}
|
||||||
|
|
||||||
|
# Ensure the scp helper is present
|
||||||
|
if [ -f ${stateDir}/scp-helper ]; then
|
||||||
|
${pkgs.coreutils}/bin/rm ${stateDir}/scp-helper
|
||||||
|
fi
|
||||||
|
ln -s -t ${stateDir} ${scp-helper}/bin/scp-helper
|
||||||
|
'';
|
||||||
|
|
||||||
|
# Create a run script for the service.
|
||||||
|
inqRun = pkgs.writeShellScript "inquisitor-run.sh" ''
|
||||||
|
cd ${stateDir}
|
||||||
|
${inquisitor}/bin/gunicorn \
|
||||||
|
--bind=localhost:24133 \
|
||||||
|
--workers=4 \
|
||||||
|
--timeout 120 \
|
||||||
|
--log-level debug \
|
||||||
|
"inquisitor.app:wsgi()"
|
||||||
|
'';
|
||||||
|
|
||||||
|
# Create a wrapper to execute the cli as the service user.
|
||||||
|
# (needed to avoid creating files in the state dir the service can't read)
|
||||||
|
inqWrapper = pkgs.writeShellScriptBin "inq" ''
|
||||||
|
sudo --user=${svcUser.name} ${inquisitor}/bin/inquisitor "$@"
|
||||||
|
'';
|
||||||
|
in mkIf cfg.enable
|
||||||
|
{
|
||||||
|
users.users.inquisitor = svcUser;
|
||||||
|
users.groups.inquisitor = {};
|
||||||
|
|
||||||
|
# Link the config in /etc to avoid envvar shenanigans
|
||||||
|
environment.etc."inquisitor.conf".source = inqConfig;
|
||||||
|
|
||||||
|
# Give all users the wrapper program.
|
||||||
|
environment.systemPackages = [ inqWrapper ];
|
||||||
|
# Allow the sudo in the cli wrapper without password.
|
||||||
|
security.sudo.extraRules = [{
|
||||||
|
commands = [{
|
||||||
|
command = "${inquisitor}/bin/inquisitor";
|
||||||
|
options = [ "NOPASSWD" ];
|
||||||
|
}];
|
||||||
|
runAs = svcUser.name;
|
||||||
|
groups = [ "users" ];
|
||||||
|
}];
|
||||||
|
|
||||||
|
# Run the setup script on activation.
|
||||||
|
system.activationScripts.inquisitorSetup = "${inqSetup}";
|
||||||
|
|
||||||
|
# Set up the inquisitor service.
|
||||||
|
systemd.services.inquisitor = {
|
||||||
|
description = "Inquisitor server";
|
||||||
|
script = "${inqRun}";
|
||||||
|
serviceConfig = {
|
||||||
|
User = svcUser.name;
|
||||||
|
Type = "simple";
|
||||||
|
};
|
||||||
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
after = [ "network.target" ];
|
||||||
|
enable = true;
|
||||||
|
};
|
||||||
|
|
||||||
|
# Set up the nginx reverse proxy to the server.
|
||||||
|
services.nginx.enable = true;
|
||||||
|
services.nginx.virtualHosts.inquisitorHost = {
|
||||||
|
listen = [ cfg.listen ];
|
||||||
|
locations."/".extraConfig = ''
|
||||||
|
access_log /var/log/nginx/access.inquisitor.log;
|
||||||
|
proxy_buffering off;
|
||||||
|
proxy_pass http://localhost:24133/;
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
networking.firewall.allowedTCPPorts = [ cfg.listen.port ];
|
||||||
|
|
||||||
|
# Enable cron so the service can use it to schedule fetches.
|
||||||
|
services.cron.enable = true;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
|
@ -172,14 +172,6 @@ dev = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.4.0,<0.5)", "betamax-seria
|
||||||
lint = ["black", "flake8", "flynt", "pre-commit", "pydocstyle"]
|
lint = ["black", "flake8", "flynt", "pre-commit", "pydocstyle"]
|
||||||
test = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.4.0,<0.5)", "betamax-serializers (>=0.2.0,<0.3)", "mock (>=0.8)", "pytest", "testfixtures (>4.13.2,<7)"]
|
test = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.4.0,<0.5)", "betamax-serializers (>=0.2.0,<0.3)", "mock (>=0.8)", "pytest", "testfixtures (>4.13.2,<7)"]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "protobuf"
|
|
||||||
version = "4.21.12"
|
|
||||||
description = ""
|
|
||||||
category = "main"
|
|
||||||
optional = false
|
|
||||||
python-versions = ">=3.7"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "requests"
|
name = "requests"
|
||||||
version = "2.28.1"
|
version = "2.28.1"
|
||||||
|
@ -289,7 +281,7 @@ dev = ["praw", "gunicorn", "feedparser"]
|
||||||
[metadata]
|
[metadata]
|
||||||
lock-version = "1.1"
|
lock-version = "1.1"
|
||||||
python-versions = "^3.10"
|
python-versions = "^3.10"
|
||||||
content-hash = "f7202e31b8297ce5c72738178aa28256ba6a10d03dc741804a10f1a8ba952d2b"
|
content-hash = "2d6e1c8843f9821ef246fdeeeef572bcdaa2db452bbaa74531e8af91f8d89bb5"
|
||||||
|
|
||||||
[metadata.files]
|
[metadata.files]
|
||||||
beautifulsoup4 = [
|
beautifulsoup4 = [
|
||||||
|
@ -386,22 +378,6 @@ prawcore = [
|
||||||
{file = "prawcore-2.3.0-py3-none-any.whl", hash = "sha256:48c17db447fa06a13ca3e722201f443031437708daa736c05a1df895fbcceff5"},
|
{file = "prawcore-2.3.0-py3-none-any.whl", hash = "sha256:48c17db447fa06a13ca3e722201f443031437708daa736c05a1df895fbcceff5"},
|
||||||
{file = "prawcore-2.3.0.tar.gz", hash = "sha256:daf1ccd4b7a80dc4e6567d48336d782e94a9a6dad83770fc2edf76dc9a84f56d"},
|
{file = "prawcore-2.3.0.tar.gz", hash = "sha256:daf1ccd4b7a80dc4e6567d48336d782e94a9a6dad83770fc2edf76dc9a84f56d"},
|
||||||
]
|
]
|
||||||
protobuf = [
|
|
||||||
{file = "protobuf-4.21.12-cp310-abi3-win32.whl", hash = "sha256:b135410244ebe777db80298297a97fbb4c862c881b4403b71bac9d4107d61fd1"},
|
|
||||||
{file = "protobuf-4.21.12-cp310-abi3-win_amd64.whl", hash = "sha256:89f9149e4a0169cddfc44c74f230d7743002e3aa0b9472d8c28f0388102fc4c2"},
|
|
||||||
{file = "protobuf-4.21.12-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:299ea899484ee6f44604deb71f424234f654606b983cb496ea2a53e3c63ab791"},
|
|
||||||
{file = "protobuf-4.21.12-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:d1736130bce8cf131ac7957fa26880ca19227d4ad68b4888b3be0dea1f95df97"},
|
|
||||||
{file = "protobuf-4.21.12-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:78a28c9fa223998472886c77042e9b9afb6fe4242bd2a2a5aced88e3f4422aa7"},
|
|
||||||
{file = "protobuf-4.21.12-cp37-cp37m-win32.whl", hash = "sha256:3d164928ff0727d97022957c2b849250ca0e64777ee31efd7d6de2e07c494717"},
|
|
||||||
{file = "protobuf-4.21.12-cp37-cp37m-win_amd64.whl", hash = "sha256:f45460f9ee70a0ec1b6694c6e4e348ad2019275680bd68a1d9314b8c7e01e574"},
|
|
||||||
{file = "protobuf-4.21.12-cp38-cp38-win32.whl", hash = "sha256:6ab80df09e3208f742c98443b6166bcb70d65f52cfeb67357d52032ea1ae9bec"},
|
|
||||||
{file = "protobuf-4.21.12-cp38-cp38-win_amd64.whl", hash = "sha256:1f22ac0ca65bb70a876060d96d914dae09ac98d114294f77584b0d2644fa9c30"},
|
|
||||||
{file = "protobuf-4.21.12-cp39-cp39-win32.whl", hash = "sha256:27f4d15021da6d2b706ddc3860fac0a5ddaba34ab679dc182b60a8bb4e1121cc"},
|
|
||||||
{file = "protobuf-4.21.12-cp39-cp39-win_amd64.whl", hash = "sha256:237216c3326d46808a9f7c26fd1bd4b20015fb6867dc5d263a493ef9a539293b"},
|
|
||||||
{file = "protobuf-4.21.12-py2.py3-none-any.whl", hash = "sha256:a53fd3f03e578553623272dc46ac2f189de23862e68565e83dde203d41b76fc5"},
|
|
||||||
{file = "protobuf-4.21.12-py3-none-any.whl", hash = "sha256:b98d0148f84e3a3c569e19f52103ca1feacdac0d2df8d6533cf983d1fda28462"},
|
|
||||||
{file = "protobuf-4.21.12.tar.gz", hash = "sha256:7cd532c4566d0e6feafecc1059d04c7915aec8e182d1cf7adee8b24ef1e2e6ab"},
|
|
||||||
]
|
|
||||||
requests = [
|
requests = [
|
||||||
{file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"},
|
{file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"},
|
||||||
{file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"},
|
{file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"},
|
||||||
|
|
|
@ -14,7 +14,6 @@ beautifulsoup4 = "^4.11.1"
|
||||||
praw = {version = "^7.6.1", optional = true}
|
praw = {version = "^7.6.1", optional = true}
|
||||||
gunicorn = {version = "^20.1.0", optional = true}
|
gunicorn = {version = "^20.1.0", optional = true}
|
||||||
feedparser = {version = "^6.0.10", optional = true}
|
feedparser = {version = "^6.0.10", optional = true}
|
||||||
protobuf = "^4.21.12"
|
|
||||||
|
|
||||||
[tool.poetry.extras]
|
[tool.poetry.extras]
|
||||||
dev = ["praw", "gunicorn", "feedparser"]
|
dev = ["praw", "gunicorn", "feedparser"]
|
||||||
|
|
Loading…
Reference in New Issue