feat(523): the backup job - an hourly tick that runs at the chosen hour, Back up now, and the last 30 runs

services/backup_job.py reads its config when it fires, so startup reads
nothing (#4181) and a changed hour needs no reschedule. A run is held to
a 60-minute deadline (a hung NFS mount would otherwise hold the lock and
skip every later backup), never runs twice at once, and keeps its outcome
- file, size, duration, or the reason in words - in the bounded
backup_runs setting, which no backup carries. summarise() gives the page
its verdict: overdue when on and no success for 26h.

Admin API: GET/PUT /api/admin/backups, POST /backups/check (tests unsaved
form values, 45s deadline), POST /backups/run (202, background).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/scribe/services/backup.py

@@ -776,7 +776,11 @@ def _rule_version_rows(rows) -> list[dict]:
 # under SECRET_KEY, but a backup is exactly the thing copied off-site — it must
 # not carry the key that writes to the place it is copied to. A restore leaves
 # the destination configured and asks for the secret again.
-_SETTINGS_NOT_EXPORTED = frozenset({"backup_s3_secret"})
+#
+# `backup_runs` is the scheduled backup's own run history: a record of what
+# THIS instance sent where. Restored onto another instance it would claim
+# backups that instance never made.
+_SETTINGS_NOT_EXPORTED = frozenset({"backup_s3_secret", "backup_runs"})
 
 
 def _setting_rows(rows) -> list[dict]:

src/scribe/services/backup_destination.py

@@ -108,6 +108,32 @@ class BackupConfig:
         return out
 
 
+def apply_changes(current: BackupConfig, data: dict) -> BackupConfig:
+    """`current` with the fields an admin sent changed, checked strictly.
+
+    `from_dict` forgives a stored value it can't read; a value a person just
+    typed is refused instead, with the reason, so a typo in the hour does not
+    silently become 02:00. Keys that aren't config fields (the secret, say)
+    are ignored here.
+    """
+    names = {f.name for f in fields(BackupConfig)}
+    changes = {k: v for k, v in (data or {}).items() if k in names}
+    if "hour" in changes:
+        try:
+            hour = int(changes["hour"])
+        except (TypeError, ValueError):
+            raise ValueError("The hour must be a whole number from 0 to 23.") from None
+        if not 0 <= hour <= 23:
+            raise ValueError("The hour must be from 0 to 23 (UTC).")
+        changes["hour"] = hour
+    if "destination" in changes and changes["destination"] not in DESTINATION_KINDS:
+        raise ValueError("The destination must be a folder or an S3 bucket.")
+    for name in ("enabled", "s3_verify_tls"):
+        if name in changes and not isinstance(changes[name], bool):
+            raise ValueError(f"{name} must be true or false.")
+    return BackupConfig.from_dict({**current.to_dict(), **changes})
+
+
 async def load_config() -> BackupConfig:
     raw = await get_admin_setting(CONFIG_SETTING, "")
     try:

src/scribe/services/backup_job.py

@@ -0,0 +1,188 @@
+"""The scheduled backup: when it runs, running it, and what each run did.
+
+ONE APScheduler job (services/scheduler.py) fires at the top of every hour and
+reads the config at fire time: when backups are on and the hour is the chosen
+one, it runs. Reading at fire time rather than scheduling the chosen hour
+keeps the app's startup free of a database read (#4181 — a lifespan hook that
+waits on the database is a worker that never serves), and a changed hour
+needs no reschedule.
+
+A run exports the full backup, writes it as the dated tar
+(services/backup_file.py) and delivers it (services/backup_destination.py).
+Its outcome is kept as one entry in a bounded list — the newest
+`KEEP_RUNS` — in the `backup_runs` admin setting, like DB maintenance keeps
+its last run. A failure is logged as an error AND kept with its reason in
+words, so the Settings page can say what went wrong without anyone reading
+logs.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import time
+from datetime import datetime, timedelta, timezone
+
+from apscheduler.triggers.cron import CronTrigger
+
+from scribe.services.backup_destination import (
+    DestinationError, deliver, load_config, load_secret,
+)
+from scribe.services.scheduler import ScheduledJob
+from scribe.services.settings import get_admin_setting, set_admin_setting
+
+logger = logging.getLogger(__name__)
+
+RUNS_SETTING = "backup_runs"
+KEEP_RUNS = 30
+# A daily backup that hasn't succeeded for this long is overdue: a day plus
+# room for one slow run, so a backup that lands at 02:40 instead of 02:05
+# does not flicker the page into a warning.
+STALE_AFTER = timedelta(hours=26)
+# The whole run's deadline (rule 156: a wait with no deadline is a bug). The
+# S3 upload has its own 15-minute timeout, but a folder on a hung NFS mount
+# has none, and a run that never returns holds the lock — every later backup
+# would be skipped as "already running", silently, forever.
+RUN_DEADLINE = timedelta(minutes=60)
+
+_LOCK = asyncio.Lock()
+_TASKS: set[asyncio.Task] = set()
+
+
+class BackupBusy(Exception):
+    """A backup is already running."""
+
+
+def is_running() -> bool:
+    return _LOCK.locked()
+
+
+async def list_runs() -> list[dict]:
+    """The kept runs, newest first."""
+    raw = await get_admin_setting(RUNS_SETTING, "")
+    try:
+        runs = json.loads(raw) if raw else []
+    except ValueError:
+        return []
+    return [r for r in runs if isinstance(r, dict)] if isinstance(runs, list) else []
+
+
+async def _record(run: dict) -> None:
+    runs = [run, *await list_runs()][:KEEP_RUNS]
+    await set_admin_setting(RUNS_SETTING, json.dumps(runs))
+
+
+def summarise(runs: list[dict], *, enabled: bool, now: datetime) -> dict:
+    """What the page leads with: when the last good backup was, and whether
+    backups are overdue. Pure, so the stale rule is tested on its own."""
+    last_ok = next((r for r in runs if r.get("status") == "ok"), None)
+    last_ok_at = _parse(last_ok.get("finished_at")) if last_ok else None
+    if not enabled:
+        overdue = False
+    elif last_ok_at is None:
+        # On, and never succeeded: overdue once the first scheduled chance
+        # has passed. Before any run at all, there is nothing to be late for.
+        overdue = bool(runs)
+    else:
+        overdue = now - last_ok_at > STALE_AFTER
+    return {
+        "last_run": runs[0] if runs else None,
+        "last_success": last_ok,
+        "overdue": overdue,
+    }
+
+
+def _parse(value) -> datetime | None:
+    try:
+        return datetime.fromisoformat(value)
+    except (TypeError, ValueError):
+        return None
+
+
+async def run_backup(trigger: str) -> dict:
+    """Run one backup now and keep its outcome. Raises BackupBusy when one is
+    already running; never raises for a failed backup — that is the outcome.
+    """
+    if _LOCK.locked():
+        raise BackupBusy()
+    async with _LOCK:
+        started = datetime.now(timezone.utc)
+        clock = time.monotonic()
+        run: dict = {"started_at": started.isoformat(), "trigger": trigger}
+        try:
+            config = await load_config()
+            secret, _ = await load_secret()
+            run["destination"] = config.destination
+            delivered = await asyncio.wait_for(
+                _export_and_deliver(config, secret, started), RUN_DEADLINE.total_seconds(),
+            )
+            run.update(status="ok", file=delivered.name,
+                       size_bytes=delivered.size_bytes, where=delivered.where)
+        except (TimeoutError, asyncio.TimeoutError):
+            run.update(status="failed", error=(
+                f"The backup didn't finish within {int(RUN_DEADLINE.total_seconds() // 60)} "
+                "minutes and was abandoned. The destination may be hanging."
+            ))
+        except DestinationError as exc:
+            run.update(status="failed", error=str(exc))
+        except Exception as exc:
+            logger.exception("backup run failed")
+            run.update(status="failed", error=f"The backup failed: {exc}")
+        run["finished_at"] = datetime.now(timezone.utc).isoformat()
+        run["duration_s"] = round(time.monotonic() - clock, 1)
+        if run["status"] == "ok":
+            logger.info("backup: wrote %s (%d bytes) to %s in %.1fs",
+                        run["file"], run["size_bytes"], run["where"], run["duration_s"])
+        else:
+            logger.error("backup: %s", run["error"])
+        try:
+            await _record(run)
+        except Exception:
+            # The backup itself is done (or failed) either way; losing its
+            # entry in the history must not turn into an unhandled error in a
+            # scheduler thread. The log line above still says what happened.
+            logger.exception("backup: could not keep this run in the history")
+        return run
+
+
+async def _export_and_deliver(config, secret, started):
+    from scribe.services.backup import export_full_backup
+    payload = await export_full_backup()
+    return await deliver(
+        payload, config, secret, started,
+        app_version=os.environ.get("APP_VERSION", "dev"),
+    )
+
+
+def start_backup_now() -> None:
+    """Start a manual backup in the background (the request returns at once;
+    the page reads progress from `is_running` and the run list). Raises
+    BackupBusy rather than queueing a second run behind the first."""
+    if _LOCK.locked():
+        raise BackupBusy()
+    task = asyncio.get_running_loop().create_task(run_backup("manual"))
+    _TASKS.add(task)
+    task.add_done_callback(_TASKS.discard)
+
+
+async def _hourly_check() -> None:
+    config = await load_config()
+    if not config.enabled or datetime.now(timezone.utc).hour != config.hour:
+        return
+    try:
+        await run_backup("scheduled")
+    except BackupBusy:
+        logger.info("backup: a run is already going; the scheduled one is skipped")
+
+
+_JOB = ScheduledJob("backup_hourly_check", _hourly_check, label="Backup")
+
+
+def start_backup_scheduler(loop: asyncio.AbstractEventLoop) -> None:
+    _JOB.start(loop, CronTrigger(minute=0, timezone="UTC"),
+               describe="hourly; runs at the configured UTC hour")
+
+
+def stop_backup_scheduler() -> None:
+    _JOB.stop()