From 312e231ea6b2557e0aeb6e84df56230d9c9fe58f Mon Sep 17 00:00:00 2001 From: Tim Van Baak Date: Wed, 21 Jun 2023 18:32:44 -0700 Subject: [PATCH] Add managed crontab support --- README.md | 11 ++-- demo/alice/currenttime/intake.json | 3 +- demo/default.nix | 5 +- intake/app.py | 11 +++- intake/cli.py | 20 +++++++- intake/crontab.py | 82 ++++++++++++++++++++++++++++++ module.nix | 5 +- 7 files changed, 126 insertions(+), 11 deletions(-) create mode 100644 intake/crontab.py diff --git a/README.md b/README.md index 68fda11..9955592 100644 --- a/README.md +++ b/README.md @@ -41,17 +41,20 @@ intake }, "env": { "...": "..." - } + }, + "cron": "* * * * *" } ``` -Each key under `action` defines an action that can be taken for the source. `action` must be present with a `fetch` action. `env` is optional. +Each key under `action` defines an action that can be taken for the source. A minimal `intake.json` must contain a `fetch` action with an `exe`. All other configs are optional. + +The `fetch` action is executed by `intake update`. All other actions are executable with `intake action`. `intake crontab` will create a crontab entry for each source with a `cron` config defined. The value of `cron` is the crontab spec according to which the source should be updated. ## Interface for source programs -Intake interacts with sources by executing the actions defined in the source's `intake.json`. The `fetch` action is required and used to check for new feed items. +Intake interacts with sources by executing the actions defined in the source's `intake.json`. The `fetch` action is required and used to check for new feed items when `intake update` is executed. -When any action is executed, intake executes the `exe` program for the action with the corresponding `args` as arguments. The process's working directory is set to the source's folder, i.e. the folder containing `intake.json`. The process's environment is as follows: +To execute an action, intake executes the `exe` program for the action with the corresponding `args` (if present) as arguments. The process's working directory is set to the source's folder, i.e. the folder containing `intake.json`. The process's environment is as follows: * intake's environment is inherited. * `STATE_PATH` is set to the absolute path of `state`. diff --git a/demo/alice/currenttime/intake.json b/demo/alice/currenttime/intake.json index 40b0a0c..ca72d1d 100644 --- a/demo/alice/currenttime/intake.json +++ b/demo/alice/currenttime/intake.json @@ -4,5 +4,6 @@ "exe": "currenttime.sh", "args": [] } - } + }, + "cron": "* * * * *" } diff --git a/demo/default.nix b/demo/default.nix index 1e964a4..c09dcd9 100644 --- a/demo/default.nix +++ b/demo/default.nix @@ -8,17 +8,16 @@ isNormalUser = true; password = "alpha"; uid = 1000; + packages = [ pkgs.intake ]; }; users.users.bob = { isNormalUser = true; password = "beta"; uid = 1001; + packages = [ pkgs.intake ]; }; - # Put intake on both users' PATH - environment.systemPackages = [ pkgs.intake ]; - # Set up intake for both users with an entry point at port 8080 services.intake = { listen.port = 8080; diff --git a/intake/app.py b/intake/app.py index e4698e0..2afd212 100644 --- a/intake/app.py +++ b/intake/app.py @@ -7,7 +7,16 @@ import json import os import time -from flask import Flask, render_template, request, jsonify, abort, redirect, url_for, current_app +from flask import ( + Flask, + render_template, + request, + jsonify, + abort, + redirect, + url_for, + current_app, +) from intake.core import intake_data_dir from intake.source import LocalSource, execute_action, Item diff --git a/intake/cli.py b/intake/cli.py index d23e323..5d6f227 100644 --- a/intake/cli.py +++ b/intake/cli.py @@ -11,11 +11,11 @@ import subprocess import sys from intake.core import intake_data_dir +from intake.crontab import update_crontab_entries from intake.source import fetch_items, LocalSource, update_items, execute_action from intake.types import InvalidConfigException, SourceUpdateException - def cmd_edit(cmd_args): """Open a source's config for editing.""" parser = argparse.ArgumentParser( @@ -296,6 +296,24 @@ def cmd_passwd(cmd_args): return 0 +def cmd_crontab(cmd_args): + """Update cron with configured source cron entries.""" + parser = argparse.ArgumentParser( + prog="intake crontab", + description=cmd_crontab.__doc__, + ) + parser.add_argument( + "--data", + "-d", + help="Path to the intake data directory containing source directories", + ) + args = parser.parse_args(cmd_args) + + data_path: Path = Path(args.data) if args.data else intake_data_dir() + update_crontab_entries(data_path) + return 0 + + def cmd_run(cmd_args): """Run the default Flask server.""" parser = argparse.ArgumentParser( diff --git a/intake/crontab.py b/intake/crontab.py new file mode 100644 index 0000000..e3d004f --- /dev/null +++ b/intake/crontab.py @@ -0,0 +1,82 @@ +from pathlib import Path +import os +import subprocess + +from intake.source import LocalSource + + +INTAKE_CRON_BEGIN = "### begin intake-managed crontab entries" +INTAKE_CRON_END = "### end intake-managed crontab entries" + + +def get_desired_crons(data_path: Path): + """ + Get a list of sources and crontab specs from the data directory. + """ + for child in data_path.iterdir(): + if not (child / "intake.json").exists(): + continue + source = LocalSource(data_path, child.name) + config = source.get_config() + if cron := config.get("cron"): + yield f"{cron} . /etc/profile; intake update -s {source.source_name}" + + +def update_crontab_entries(data_path: Path): + """ + Update the intake-managed section of the user's crontab. + """ + # If there is no crontab command available, quit early. + crontab_exists = subprocess.run(["command", "-v" "crontab"], shell=True) + if crontab_exists.returncode: + print("Could not update crontab") + return + + # Get the current crontab + get_crontab = subprocess.run( + ["crontab", "-e"], + env={**os.environ, "EDITOR": "cat"}, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + crontab_lines = get_crontab.stdout.decode("utf-8").splitlines() + + # Splice the intake crons into the crontab + new_crontab_lines = [] + section_found = False + in_section = False + for i in range(len(crontab_lines)): + + if not section_found and crontab_lines[i] == INTAKE_CRON_BEGIN: + section_found = True + in_section = True + # Open the section and add everything + new_crontab_lines.append(INTAKE_CRON_BEGIN) + new_crontab_lines.extend(get_desired_crons(data_path)) + + elif crontab_lines[i] == INTAKE_CRON_END: + new_crontab_lines.append(INTAKE_CRON_END) + in_section = False + + elif not in_section: + new_crontab_lines.append(crontab_lines[i]) + + # If the splice mark was never found, append the whole section to the end + if not section_found: + new_crontab_lines.append(INTAKE_CRON_BEGIN) + new_crontab_lines.extend(get_desired_crons(data_path)) + new_crontab_lines.append(INTAKE_CRON_END) + + # Save the updated crontab + new_crontab: bytes = "\n".join(new_crontab_lines).encode("utf8") + save_crontab = subprocess.Popen( + ["crontab", "-"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + (stdout, stderr) = save_crontab.communicate(new_crontab) + for line in stdout.decode("utf8").splitlines(): + print("[stdout]", line) + for line in stderr.decode("utf8").splitlines(): + print("[stderr]", line) diff --git a/module.nix b/module.nix index 4406c81..2bdb2df 100644 --- a/module.nix +++ b/module.nix @@ -74,7 +74,7 @@ in { intakeDir = "/etc/intake"; intakePwd = "${intakeDir}/htpasswd"; in { - # Apply the overlay so intake is included inpkgs. + # Apply the overlay so intake is included in pkgs. nixpkgs.overlays = [ flake.overlays.default ]; # Define a user group for access to the htpasswd file. nginx needs to be able to read it. @@ -105,6 +105,9 @@ in { }; in mkMerge (map addPackagesToUser enabledUserNames); + # Enable cron + services.cron.enable = true; + # Define a user service for each configured user systemd.services = let