feat(523): the backup job - an hourly tick that runs at the chosen hour, Back up now, and the last 30 runs

services/backup_job.py reads its config when it fires, so startup reads
nothing (#4181) and a changed hour needs no reschedule. A run is held to
a 60-minute deadline (a hung NFS mount would otherwise hold the lock and
skip every later backup), never runs twice at once, and keeps its outcome
- file, size, duration, or the reason in words - in the bounded
backup_runs setting, which no backup carries. summarise() gives the page
its verdict: overdue when on and no success for 26h.

Admin API: GET/PUT /api/admin/backups, POST /backups/check (tests unsaved
form values, 45s deadline), POST /backups/run (202, background).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/scribe/app.py

@@ -304,6 +304,11 @@ def create_app() -> Quart:
                 asyncio.get_running_loop(), await get_maintenance_hour()
             )
 
+            # Scheduled backups (milestone 523). An hourly tick that reads its
+            # config when it fires, so starting it reads nothing here (#4181).
+            from scribe.services.backup_job import start_backup_scheduler
+            start_backup_scheduler(asyncio.get_running_loop())
+
             # Diagnostic instrumentation — heartbeat, signal handlers, asyncio
             # exception hook. Cheap (~1 log line/min), high diagnostic value when
             # the app crashes mysteriously. See services/diagnostics.py.
@@ -330,6 +335,8 @@ def create_app() -> Quart:
             stop_db_maintenance_scheduler,
         )
         stop_db_maintenance_scheduler()
+        from scribe.services.backup_job import stop_backup_scheduler
+        stop_backup_scheduler()
         from scribe.services.diagnostics import stop_diagnostics
         stop_diagnostics()
 

src/scribe/routes/admin.py

@@ -315,6 +315,120 @@ async def run_db_maintenance_now():
     return jsonify(summary)
 
 
+# ── Scheduled backups (milestone 523) ────────────────────────────────────────
+# One dated, uncompressed tar per run to a folder or an S3 bucket — see
+# services/backup_file.py for why it is shaped that way, backup_destination.py
+# for where it goes, backup_job.py for when.
+
+# How long "Test" waits on the destination. A folder on a hung NFS mount has
+# no timeout of its own (rule 156), and the request must come back either way.
+_BACKUP_CHECK_TIMEOUT = 45.0
+
+
+async def _backups_payload() -> dict:
+    from datetime import datetime, timezone
+
+    from scribe.services import backup_destination as dest
+    from scribe.services import backup_job as job
+
+    config = await dest.load_config()
+    secret, stored = await dest.load_secret()
+    runs = await job.list_runs()
+    return {
+        "config": config.to_dict(),
+        # Never the secret itself: whether one is kept, and whether this
+        # instance can still open it (it can't after a restore onto another
+        # SECRET_KEY, and then it has to be entered again).
+        "secret": {"set": stored, "readable": secret is not None},
+        "problems": config.problems(has_secret=secret is not None),
+        "running": job.is_running(),
+        "runs": runs,
+        **job.summarise(runs, enabled=config.enabled, now=datetime.now(timezone.utc)),
+    }
+
+
+@admin_bp.route("/backups", methods=["GET"])
+@admin_required
+async def get_backups():
+    """The backup destination, its schedule, and the recent runs."""
+    return jsonify(await _backups_payload())
+
+
+@admin_bp.route("/backups", methods=["PUT"])
+@admin_required
+async def update_backups():
+    """Change the destination or schedule. `s3_secret` replaces the stored
+    secret when it is sent and isn't the mask; `clear_s3_secret` removes it."""
+    from scribe.services import backup_destination as dest
+
+    data = await request.get_json() or {}
+    try:
+        config = dest.apply_changes(await dest.load_config(), data)
+    except ValueError as exc:
+        return jsonify({"error": str(exc)}), 400
+    await dest.save_config(config)
+    secret = data.get("s3_secret")
+    if data.get("clear_s3_secret"):
+        await dest.save_secret("")
+    elif isinstance(secret, str) and secret and secret != SECRET_MASK:
+        await dest.save_secret(secret)
+    uid = get_current_user_id()
+    await log_audit(
+        "backup_config", user_id=uid, username=g.user.username,
+        ip_address=request.remote_addr,
+        details={**config.to_dict(), "secret_changed": bool(
+            data.get("clear_s3_secret") or (isinstance(secret, str) and secret and secret != SECRET_MASK)
+        )},
+    )
+    return jsonify(await _backups_payload())
+
+
+@admin_bp.route("/backups/check", methods=["POST"])
+@admin_required
+async def check_backups_destination():
+    """Ask the destination whether it would take a backup, without sending
+    one. The body may carry unsaved form values, so a destination can be
+    tested before it is saved; the stored secret is used unless one is sent."""
+    from scribe.services import backup_destination as dest
+
+    data = await request.get_json() or {}
+    try:
+        config = dest.apply_changes(await dest.load_config(), data)
+    except ValueError as exc:
+        return jsonify({"ok": False, "message": str(exc)})
+    secret = data.get("s3_secret")
+    if not (isinstance(secret, str) and secret and secret != SECRET_MASK):
+        secret, _ = await dest.load_secret()
+    try:
+        message = await asyncio.wait_for(
+            dest.check_destination(config, secret), _BACKUP_CHECK_TIMEOUT,
+        )
+    except dest.DestinationError as exc:
+        return jsonify({"ok": False, "message": str(exc)})
+    except (TimeoutError, asyncio.TimeoutError):
+        return jsonify({"ok": False, "message": (
+            f"The destination didn't answer within {int(_BACKUP_CHECK_TIMEOUT)} seconds."
+        )})
+    return jsonify({"ok": True, "message": message})
+
+
+@admin_bp.route("/backups/run", methods=["POST"])
+@admin_required
+async def run_backup_now():
+    """Start a backup now. It runs in the background; the page follows it
+    through GET /backups (`running`, then the newest run)."""
+    from scribe.services import backup_job as job
+
+    try:
+        job.start_backup_now()
+    except job.BackupBusy:
+        return jsonify({"error": "A backup is already running."}), 409
+    uid = get_current_user_id()
+    await log_audit("backup_run", user_id=uid, username=g.user.username,
+                    ip_address=request.remote_addr, details={})
+    return jsonify({"status": "started"}), 202
+
+
 @admin_bp.route("/invitations", methods=["POST"])
 @admin_required
 async def create_invite():

src/scribe/services/backup.py

@@ -776,7 +776,11 @@ def _rule_version_rows(rows) -> list[dict]:
 # under SECRET_KEY, but a backup is exactly the thing copied off-site — it must
 # not carry the key that writes to the place it is copied to. A restore leaves
 # the destination configured and asks for the secret again.
-_SETTINGS_NOT_EXPORTED = frozenset({"backup_s3_secret"})
+#
+# `backup_runs` is the scheduled backup's own run history: a record of what
+# THIS instance sent where. Restored onto another instance it would claim
+# backups that instance never made.
+_SETTINGS_NOT_EXPORTED = frozenset({"backup_s3_secret", "backup_runs"})
 
 
 def _setting_rows(rows) -> list[dict]:

src/scribe/services/backup_destination.py

@@ -108,6 +108,32 @@ class BackupConfig:
         return out
 
 
+def apply_changes(current: BackupConfig, data: dict) -> BackupConfig:
+    """`current` with the fields an admin sent changed, checked strictly.
+
+    `from_dict` forgives a stored value it can't read; a value a person just
+    typed is refused instead, with the reason, so a typo in the hour does not
+    silently become 02:00. Keys that aren't config fields (the secret, say)
+    are ignored here.
+    """
+    names = {f.name for f in fields(BackupConfig)}
+    changes = {k: v for k, v in (data or {}).items() if k in names}
+    if "hour" in changes:
+        try:
+            hour = int(changes["hour"])
+        except (TypeError, ValueError):
+            raise ValueError("The hour must be a whole number from 0 to 23.") from None
+        if not 0 <= hour <= 23:
+            raise ValueError("The hour must be from 0 to 23 (UTC).")
+        changes["hour"] = hour
+    if "destination" in changes and changes["destination"] not in DESTINATION_KINDS:
+        raise ValueError("The destination must be a folder or an S3 bucket.")
+    for name in ("enabled", "s3_verify_tls"):
+        if name in changes and not isinstance(changes[name], bool):
+            raise ValueError(f"{name} must be true or false.")
+    return BackupConfig.from_dict({**current.to_dict(), **changes})
+
+
 async def load_config() -> BackupConfig:
     raw = await get_admin_setting(CONFIG_SETTING, "")
     try:

src/scribe/services/backup_job.py

@@ -0,0 +1,188 @@
+"""The scheduled backup: when it runs, running it, and what each run did.
+
+ONE APScheduler job (services/scheduler.py) fires at the top of every hour and
+reads the config at fire time: when backups are on and the hour is the chosen
+one, it runs. Reading at fire time rather than scheduling the chosen hour
+keeps the app's startup free of a database read (#4181 — a lifespan hook that
+waits on the database is a worker that never serves), and a changed hour
+needs no reschedule.
+
+A run exports the full backup, writes it as the dated tar
+(services/backup_file.py) and delivers it (services/backup_destination.py).
+Its outcome is kept as one entry in a bounded list — the newest
+`KEEP_RUNS` — in the `backup_runs` admin setting, like DB maintenance keeps
+its last run. A failure is logged as an error AND kept with its reason in
+words, so the Settings page can say what went wrong without anyone reading
+logs.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import time
+from datetime import datetime, timedelta, timezone
+
+from apscheduler.triggers.cron import CronTrigger
+
+from scribe.services.backup_destination import (
+    DestinationError, deliver, load_config, load_secret,
+)
+from scribe.services.scheduler import ScheduledJob
+from scribe.services.settings import get_admin_setting, set_admin_setting
+
+logger = logging.getLogger(__name__)
+
+RUNS_SETTING = "backup_runs"
+KEEP_RUNS = 30
+# A daily backup that hasn't succeeded for this long is overdue: a day plus
+# room for one slow run, so a backup that lands at 02:40 instead of 02:05
+# does not flicker the page into a warning.
+STALE_AFTER = timedelta(hours=26)
+# The whole run's deadline (rule 156: a wait with no deadline is a bug). The
+# S3 upload has its own 15-minute timeout, but a folder on a hung NFS mount
+# has none, and a run that never returns holds the lock — every later backup
+# would be skipped as "already running", silently, forever.
+RUN_DEADLINE = timedelta(minutes=60)
+
+_LOCK = asyncio.Lock()
+_TASKS: set[asyncio.Task] = set()
+
+
+class BackupBusy(Exception):
+    """A backup is already running."""
+
+
+def is_running() -> bool:
+    return _LOCK.locked()
+
+
+async def list_runs() -> list[dict]:
+    """The kept runs, newest first."""
+    raw = await get_admin_setting(RUNS_SETTING, "")
+    try:
+        runs = json.loads(raw) if raw else []
+    except ValueError:
+        return []
+    return [r for r in runs if isinstance(r, dict)] if isinstance(runs, list) else []
+
+
+async def _record(run: dict) -> None:
+    runs = [run, *await list_runs()][:KEEP_RUNS]
+    await set_admin_setting(RUNS_SETTING, json.dumps(runs))
+
+
+def summarise(runs: list[dict], *, enabled: bool, now: datetime) -> dict:
+    """What the page leads with: when the last good backup was, and whether
+    backups are overdue. Pure, so the stale rule is tested on its own."""
+    last_ok = next((r for r in runs if r.get("status") == "ok"), None)
+    last_ok_at = _parse(last_ok.get("finished_at")) if last_ok else None
+    if not enabled:
+        overdue = False
+    elif last_ok_at is None:
+        # On, and never succeeded: overdue once the first scheduled chance
+        # has passed. Before any run at all, there is nothing to be late for.
+        overdue = bool(runs)
+    else:
+        overdue = now - last_ok_at > STALE_AFTER
+    return {
+        "last_run": runs[0] if runs else None,
+        "last_success": last_ok,
+        "overdue": overdue,
+    }
+
+
+def _parse(value) -> datetime | None:
+    try:
+        return datetime.fromisoformat(value)
+    except (TypeError, ValueError):
+        return None
+
+
+async def run_backup(trigger: str) -> dict:
+    """Run one backup now and keep its outcome. Raises BackupBusy when one is
+    already running; never raises for a failed backup — that is the outcome.
+    """
+    if _LOCK.locked():
+        raise BackupBusy()
+    async with _LOCK:
+        started = datetime.now(timezone.utc)
+        clock = time.monotonic()
+        run: dict = {"started_at": started.isoformat(), "trigger": trigger}
+        try:
+            config = await load_config()
+            secret, _ = await load_secret()
+            run["destination"] = config.destination
+            delivered = await asyncio.wait_for(
+                _export_and_deliver(config, secret, started), RUN_DEADLINE.total_seconds(),
+            )
+            run.update(status="ok", file=delivered.name,
+                       size_bytes=delivered.size_bytes, where=delivered.where)
+        except (TimeoutError, asyncio.TimeoutError):
+            run.update(status="failed", error=(
+                f"The backup didn't finish within {int(RUN_DEADLINE.total_seconds() // 60)} "
+                "minutes and was abandoned. The destination may be hanging."
+            ))
+        except DestinationError as exc:
+            run.update(status="failed", error=str(exc))
+        except Exception as exc:
+            logger.exception("backup run failed")
+            run.update(status="failed", error=f"The backup failed: {exc}")
+        run["finished_at"] = datetime.now(timezone.utc).isoformat()
+        run["duration_s"] = round(time.monotonic() - clock, 1)
+        if run["status"] == "ok":
+            logger.info("backup: wrote %s (%d bytes) to %s in %.1fs",
+                        run["file"], run["size_bytes"], run["where"], run["duration_s"])
+        else:
+            logger.error("backup: %s", run["error"])
+        try:
+            await _record(run)
+        except Exception:
+            # The backup itself is done (or failed) either way; losing its
+            # entry in the history must not turn into an unhandled error in a
+            # scheduler thread. The log line above still says what happened.
+            logger.exception("backup: could not keep this run in the history")
+        return run
+
+
+async def _export_and_deliver(config, secret, started):
+    from scribe.services.backup import export_full_backup
+    payload = await export_full_backup()
+    return await deliver(
+        payload, config, secret, started,
+        app_version=os.environ.get("APP_VERSION", "dev"),
+    )
+
+
+def start_backup_now() -> None:
+    """Start a manual backup in the background (the request returns at once;
+    the page reads progress from `is_running` and the run list). Raises
+    BackupBusy rather than queueing a second run behind the first."""
+    if _LOCK.locked():
+        raise BackupBusy()
+    task = asyncio.get_running_loop().create_task(run_backup("manual"))
+    _TASKS.add(task)
+    task.add_done_callback(_TASKS.discard)
+
+
+async def _hourly_check() -> None:
+    config = await load_config()
+    if not config.enabled or datetime.now(timezone.utc).hour != config.hour:
+        return
+    try:
+        await run_backup("scheduled")
+    except BackupBusy:
+        logger.info("backup: a run is already going; the scheduled one is skipped")
+
+
+_JOB = ScheduledJob("backup_hourly_check", _hourly_check, label="Backup")
+
+
+def start_backup_scheduler(loop: asyncio.AbstractEventLoop) -> None:
+    _JOB.start(loop, CronTrigger(minute=0, timezone="UTC"),
+               describe="hourly; runs at the configured UTC hour")
+
+
+def stop_backup_scheduler() -> None:
+    _JOB.stop()