feat(backup): daily backups to an S3 bucket, Settings → Backups, and the guide (#5673, #5675)

#5673 "S3 client and the backup job: settings, schedule, history, metrics":
- steward/backup/s3.py: SigV4 over httpx, path-style; put and list only
  (an append-only key is enough; retention is the bucket's). Signing is held
  to AWS's published examples.
- steward/backup/job.py: a 15-minute tick. The daily backup goes at the hour
  chosen on the reader's clock; a missed day is made up, and a failure is
  retried an hour later. History of the last 30 runs, audit
  (backup.made / backup.failed), and backup_age_secs / backup_bytes under
  "Steward itself", resource backup, for alert rules.
- Settings → Backups: verdict first, then the bucket (secret sealed; a
  blank box keeps it), the schedule, Test the bucket, Back up now, and the
  recent runs.

#5675 "Docs: backing up to Archives, and restoring": docs/core/backups.md.

Also: the audit page now words the Prometheus target actions from #5668,
which showed as codes, and a guard fails on any logged action without
wording.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/backup/job.py

@@ -1,10 +1,40 @@
-"""Steward's own backups as a scheduled job: make the file, send it, record
-how it went (milestone 507)."""
+"""Steward's own backups as a scheduled job: make the file, send it to the
+bucket, record how it went (#5673).
+
+Once a day at the hour chosen in Settings → Backups, in the zone of whoever
+saved it. A day missed while Steward was down is made up on the next tick;
+a failed run is tried again an hour later, never sooner, so a target that is
+down isn't pressed. Each run is kept in `backup.history` (newest first, the
+last HISTORY_KEEP), audited, and leaves two metrics under source "steward",
+resource "backup": backup_age_secs (since the last success, every tick while
+backups are on, so an alert rule can watch it) and backup_bytes.
+"""
 from __future__ import annotations
 
+import asyncio
+import logging
+import tempfile
+import time
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
 from sqlalchemy import text
 from sqlalchemy.ext.asyncio import create_async_engine
 
+from .dump import BackupError, make_backup
+from .s3 import S3Error, S3Target, list_objects, put_file
+
+logger = logging.getLogger(__name__)
+
+TICK_SECONDS = 15 * 60
+RETRY_AFTER = timedelta(hours=1)
+HISTORY_KEEP = 30
+PREFIX = "steward-"
+
+# One backup at a time, whether the schedule or a person started it.
+_running = asyncio.Lock()
+_background: set[asyncio.Task] = set()
+
 
 async def migration_heads(database_url: str) -> list[str]:
     """The migration revisions the database is at, for the manifest."""
@@ -14,3 +44,208 @@ async def migration_heads(database_url: str) -> list[str]:
             return list((await conn.execute(text("SELECT version_num FROM alembic_version"))).scalars())
     finally:
         await engine.dispose()
+
+
+def target_from(settings: dict) -> S3Target | None:
+    """The bucket the settings name, or None while they're incomplete."""
+    endpoint = (settings.get("backup.s3_endpoint") or "").strip()
+    bucket = (settings.get("backup.s3_bucket") or "").strip()
+    access = (settings.get("backup.s3_access_key") or "").strip()
+    secret = settings.get("backup.s3_secret_key") or ""
+    if not (endpoint and bucket and access and secret):
+        return None
+    return S3Target(endpoint, bucket, access, secret,
+                    region=(settings.get("backup.s3_region") or "us-east-1").strip(),
+                    verify_tls=bool(settings.get("backup.s3_verify_tls", True)))
+
+
+def _moment(raw) -> datetime | None:
+    try:
+        return datetime.fromisoformat(raw) if raw else None
+    except (TypeError, ValueError):
+        return None
+
+
+def last_slot(now: datetime, hour: int, tz) -> datetime:
+    """The latest scheduled moment at or before `now`: today's hour in `tz`,
+    or yesterday's when today's hasn't come yet."""
+    local = now.astimezone(tz)
+    slot = local.replace(hour=hour, minute=0, second=0, microsecond=0)
+    if slot > local:
+        slot -= timedelta(days=1)
+    return slot.astimezone(timezone.utc)
+
+
+def is_due(now: datetime, hour: int, tz, last_success: datetime | None,
+           last_attempt: datetime | None) -> bool:
+    """Due when no backup has succeeded since the latest slot, and the last
+    try (if it failed) was at least RETRY_AFTER ago."""
+    slot = last_slot(now, hour, tz)
+    if last_success is not None and last_success >= slot:
+        return False
+    return last_attempt is None or now - last_attempt >= RETRY_AFTER
+
+
+def history_with(history: list, entry: dict) -> list:
+    return ([entry] + list(history or []))[:HISTORY_KEEP]
+
+
+async def _record(app, entry: dict, user_id: str | None, username: str) -> None:
+    from steward.core.alerts import MetricBatch
+    from steward.core.audit import log_audit
+    from steward.core.settings import get_setting, set_setting
+
+    async with app.db_sessionmaker() as db:
+        async with db.begin():
+            history = await get_setting(db, "backup.history") or []
+            await set_setting(db, "backup.history", history_with(history, entry))
+            await set_setting(db, "backup.last_attempt_at", entry["at"])
+            if entry["ok"]:
+                await set_setting(db, "backup.last_success_at", entry["at"])
+                batch = MetricBatch()
+                batch.add(source_module="steward", resource_name="backup",
+                          metric_name="backup_bytes", value=float(entry["size"]))
+                batch.add(source_module="steward", resource_name="backup",
+                          metric_name="backup_age_secs", value=0.0)
+                await batch.record(db)
+    await log_audit(app, user_id, username, "backup.made" if entry["ok"] else "backup.failed",
+                    entity_type="backup", entity_id=entry.get("name") or None,
+                    detail={k: entry[k] for k in ("name", "size", "secs", "error", "by") if entry.get(k)})
+
+
+async def run_backup(app, by: str = "schedule", user_id: str | None = None) -> dict:
+    """Make one backup and send it. Returns the history entry it recorded."""
+    from steward.core.settings import get_all_settings
+
+    async with _running:
+        started = time.monotonic()
+        at = datetime.now(timezone.utc)
+        entry = {"at": at.isoformat(), "ok": False, "by": by, "name": "", "size": 0,
+                 "secs": 0.0, "error": ""}
+        async with app.db_sessionmaker() as db:
+            settings = await get_all_settings(db)
+        target = target_from(settings)
+        try:
+            if target is None:
+                raise BackupError("The bucket isn't set up: fill in the endpoint, bucket and keys.")
+            db_url = app.config["DATABASE_URL"]
+            heads = await migration_heads(db_url)
+            with tempfile.TemporaryDirectory(prefix="steward-backup-") as tmp:
+                made = await make_backup(db_url, app.config["SECRET_KEY"], heads, Path(tmp), now=at)
+                await put_file(target, made.name, made.path)
+            entry.update(ok=True, name=made.name, size=made.size)
+        except (BackupError, S3Error) as exc:
+            entry["error"] = str(exc)
+        except Exception as exc:     # recorded and logged: a run never takes the scheduler down
+            logger.exception("Backup failed")
+            entry["error"] = f"Unexpected error: {exc}"
+        entry["secs"] = round(time.monotonic() - started, 1)
+        if entry["ok"]:
+            logger.info("Backup %s sent (%d bytes, %.1fs)", entry["name"], entry["size"], entry["secs"])
+        else:
+            logger.warning("Backup failed: %s", entry["error"])
+        await _record(app, entry, user_id, by if by != "schedule" else "")
+        return entry
+
+
+def start_backup(app, by: str, user_id: str | None) -> bool:
+    """Start a backup in the background for a person; False if one is running."""
+    if _running.locked():
+        return False
+    task = asyncio.get_running_loop().create_task(run_backup(app, by=by, user_id=user_id))
+    _background.add(task)
+    task.add_done_callback(_background.discard)
+    return True
+
+
+def is_running() -> bool:
+    return _running.locked()
+
+
+async def tick(app, now: datetime | None = None) -> None:
+    """The scheduled task: record how old the newest backup is, and make one
+    when it is due."""
+    from steward.core.alerts import MetricBatch
+    from steward.core.formatting import zone
+    from steward.core.settings import get_all_settings
+
+    now = now or datetime.now(timezone.utc)
+    async with app.db_sessionmaker() as db:
+        settings = await get_all_settings(db)
+    if not settings.get("backup.enabled"):
+        return
+    last_success = _moment(settings.get("backup.last_success_at"))
+    if last_success is not None:
+        async with app.db_sessionmaker() as db:
+            async with db.begin():
+                batch = MetricBatch()
+                batch.add(source_module="steward", resource_name="backup",
+                          metric_name="backup_age_secs", value=(now - last_success).total_seconds())
+                await batch.record(db)
+    if _running.locked():
+        return
+    if is_due(now, int(settings.get("backup.schedule_hour", 3)), zone(settings.get("backup.schedule_tz")),
+              last_success, _moment(settings.get("backup.last_attempt_at"))):
+        await run_backup(app)
+
+
+async def test_target(target: S3Target) -> tuple[bool, str]:
+    """Settings → Backups' Test button: can Steward reach the bucket with
+    these keys? A listing refused as AccessDenied still proves the keys:
+    an append-only key may not list, and uploads anyway."""
+    try:
+        found = await list_objects(target, PREFIX, limit=1000)
+    except S3Error as exc:
+        said = str(exc)
+        if "AccessDenied" in said:
+            return True, ("Reached the bucket and the keys were accepted. They may not list "
+                          "what is there, which an append-only key can't; uploads still work.")
+        return False, said
+    if not found:
+        return True, f"Reached {target.bucket}; no Steward backups there yet."
+    newest = max(found, key=lambda o: o.key)
+    return True, f"Reached {target.bucket}: {len(found)} Steward backups, the newest {newest.key}."
+
+
+# ── What Settings → Backups shows ────────────────────────────────────────────
+
+def shown_hour(hour: int, saved_tz, view_tz, now: datetime | None = None) -> int:
+    """The schedule's hour as a reader in `view_tz` sees it today."""
+    local = (now or datetime.now(timezone.utc)).astimezone(saved_tz)
+    return local.replace(hour=hour, minute=0, second=0, microsecond=0).astimezone(view_tz).hour
+
+
+def history_rows(history: list) -> list[dict]:
+    """The stored history with its times read back as datetimes."""
+    return [entry | {"at": _moment(entry.get("at"))} for entry in history or []]
+
+
+def verdict(settings: dict, running: bool) -> dict:
+    """One line for the top of the page: {"tone": ok|warn|bad|off, "text", "moment"}.
+    `moment` is the time the line is about, shown as "ago" by the template,
+    which ends the sentence; a failure also carries `failed`, its `error` and
+    `good` (the newest success, or None)."""
+    rows = history_rows(settings.get("backup.history") or [])
+    last_ok = _moment(settings.get("backup.last_success_at"))
+    if running:
+        return {"tone": "ok", "text": "A backup is being made now", "moment": None}
+    if not settings.get("backup.enabled"):
+        if target_from(settings) is None:
+            return {"tone": "off", "text": "Backups are off, and no bucket is set up", "moment": None}
+        return {"tone": "off", "text": "Scheduled backups are off; Back up now still works", "moment": None}
+    if target_from(settings) is None:
+        return {"tone": "bad", "text": "Backups are on, but the bucket isn't set up", "moment": None}
+    if rows and not rows[0]["ok"]:
+        # `good`: when the newest backup that did go was sent (None: never).
+        return {"tone": "bad", "text": "The last backup failed", "moment": rows[0]["at"],
+                "error": rows[0]["error"], "good": last_ok, "failed": True}
+    if last_ok is None:
+        return {"tone": "warn", "text": "No backup has been sent yet; the first goes within 15 minutes",
+                "moment": None}
+    return {"tone": "ok", "text": f"The newest backup is {rows[0]['name']} ({_size(rows[0]['size'])}), sent",
+            "moment": last_ok}
+
+
+def _size(n) -> str:
+    from steward.core.formatting import human_bytes
+    return human_bytes(n)

steward/backup/s3.py

@@ -0,0 +1,191 @@
+"""A small S3 client for sending backups: AWS Signature Version 4 over httpx,
+path-style addressing (endpoint/bucket/key), the three calls a backup needs.
+
+Fabled Archives speaks S3 with path-style buckets and append-only keys by
+default (System 180 ruling: backups go there with an append-only key), so
+Steward only ever puts and lists; it never deletes. Retention is the
+target's business.
+
+Written by hand rather than with boto3: three calls don't justify a large
+dependency, and signing is a pure function tests can hold to AWS's own
+published examples.
+"""
+from __future__ import annotations
+
+import hashlib
+import hmac
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import quote
+
+import httpx
+
+EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
+TIMEOUT_SECONDS = 30.0
+# A backup is megabytes; a slow link still gets this long to send one.
+UPLOAD_TIMEOUT_SECONDS = 15 * 60
+CHUNK = 1 << 20
+
+
+class S3Error(Exception):
+    """Why a call to the bucket failed, said to the person."""
+
+
+@dataclass(frozen=True)
+class S3Target:
+    endpoint: str          # http(s)://host[:port], no bucket
+    bucket: str
+    access_key: str
+    secret_key: str
+    region: str = "us-east-1"
+    verify_tls: bool = True
+
+    def url(self, key: str = "") -> str:
+        base = f"{self.endpoint.rstrip('/')}/{self.bucket}"
+        return f"{base}/{key}" if key else f"{base}/"
+
+
+@dataclass(frozen=True)
+class StoredObject:
+    key: str
+    size: int
+    last_modified: str
+
+
+# ── Signing ──────────────────────────────────────────────────────────────────
+
+def _encode(text: str, keep_slash: bool = False) -> str:
+    return quote(text, safe="-_.~" + ("/" if keep_slash else ""))
+
+
+def _hmac(key: bytes, msg: str) -> bytes:
+    return hmac.new(key, msg.encode(), hashlib.sha256).digest()
+
+
+def sign(method: str, url: str, headers: dict[str, str], payload_sha256: str,
+         access_key: str, secret_key: str, region: str, at: datetime,
+         service: str = "s3") -> dict[str, str]:
+    """The request's headers with x-amz-date, x-amz-content-sha256 and
+    Authorization added. Every header given is signed, Host always."""
+    u = httpx.URL(url)
+    stamp = at.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    day = stamp[:8]
+    out = dict(headers)
+    out["x-amz-date"] = stamp
+    out["x-amz-content-sha256"] = payload_sha256
+    signed = {k.lower(): " ".join(str(v).split()) for k, v in out.items()}
+    signed["host"] = u.netloc.decode()
+    names = sorted(signed)
+
+    query = sorted((_encode(k), _encode(v)) for k, v in u.params.multi_items())
+    canonical = "\n".join([
+        method.upper(),
+        _encode(u.path or "/", keep_slash=True),
+        "&".join(f"{k}={v}" for k, v in query),
+        "".join(f"{n}:{signed[n]}\n" for n in names),
+        ";".join(names),
+        payload_sha256,
+    ])
+    scope = f"{day}/{region}/{service}/aws4_request"
+    to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope,
+                         hashlib.sha256(canonical.encode()).hexdigest()])
+    key = _hmac(_hmac(_hmac(_hmac(f"AWS4{secret_key}".encode(), day), region), service), "aws4_request")
+    signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
+    out["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, "
+                            f"SignedHeaders={';'.join(names)}, Signature={signature}")
+    return out
+
+
+# ── Calls ────────────────────────────────────────────────────────────────────
+
+def _says(response: httpx.Response) -> str:
+    """An S3 error response in words: its Code and Message when it has them."""
+    try:
+        root = ET.fromstring(response.content)
+        code = root.findtext("Code") or ""
+        message = root.findtext("Message") or ""
+    except ET.ParseError:
+        code, message = "", ""
+    hints = {
+        "SignatureDoesNotMatch": "the secret key is wrong",
+        "InvalidAccessKeyId": "the access key isn't known there",
+        "NoSuchBucket": "there is no such bucket",
+        "AccessDenied": "the key isn't allowed to do that",
+    }
+    said = hints.get(code) or message or response.reason_phrase
+    return f"HTTP {response.status_code}{f' {code}' if code else ''}: {said}."
+
+
+def _reach_error(target: S3Target, exc: httpx.HTTPError) -> S3Error:
+    if isinstance(exc, httpx.TimeoutException):
+        return S3Error(f"{target.endpoint} didn't answer in time.")
+    return S3Error(f"Couldn't reach {target.endpoint}: {exc}.")
+
+
+def file_sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        while chunk := f.read(CHUNK):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+async def put_file(target: S3Target, key: str, path: Path, *,
+                   transport: httpx.AsyncBaseTransport | None = None,
+                   now: datetime | None = None) -> None:
+    """Upload one file as one object."""
+    import asyncio
+    digest = await asyncio.to_thread(file_sha256, path)
+    size = path.stat().st_size
+    url = target.url(key)
+    headers = sign("PUT", url, {"Content-Length": str(size), "Content-Type": "application/x-tar"},
+                   digest, target.access_key, target.secret_key, target.region,
+                   now or datetime.now(timezone.utc))
+
+    async def body():
+        with open(path, "rb") as f:
+            while chunk := f.read(CHUNK):
+                yield chunk
+
+    try:
+        async with httpx.AsyncClient(verify=target.verify_tls, timeout=UPLOAD_TIMEOUT_SECONDS,
+                                     transport=transport) as client:
+            response = await client.put(url, headers=headers, content=body())
+    except httpx.HTTPError as exc:
+        raise _reach_error(target, exc) from None
+    if response.status_code // 100 != 2:
+        raise S3Error(f"Upload of {key} refused: {_says(response)}")
+
+
+async def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *,
+                       transport: httpx.AsyncBaseTransport | None = None,
+                       now: datetime | None = None) -> list[StoredObject]:
+    """Up to `limit` objects under `prefix` (ListObjectsV2, one page)."""
+    params = {"list-type": "2", "max-keys": str(limit)}
+    if prefix:
+        params["prefix"] = prefix
+    url = str(httpx.URL(target.url(), params=params))
+    headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
+                   target.region, now or datetime.now(timezone.utc))
+    try:
+        async with httpx.AsyncClient(verify=target.verify_tls, timeout=TIMEOUT_SECONDS,
+                                     transport=transport) as client:
+            response = await client.get(url, headers=headers)
+    except httpx.HTTPError as exc:
+        raise _reach_error(target, exc) from None
+    if response.status_code != 200:
+        raise S3Error(f"Listing {target.bucket} refused: {_says(response)}")
+    try:
+        root = ET.fromstring(response.content)
+    except ET.ParseError:
+        raise S3Error(f"{target.endpoint} answered a listing with something that isn't S3's XML.") from None
+    out = []
+    for item in root.iter():
+        if item.tag.rsplit("}", 1)[-1] != "Contents":
+            continue
+        fields = {child.tag.rsplit("}", 1)[-1]: (child.text or "") for child in item}
+        out.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0),
+                                fields.get("LastModified", "")))
+    return out