feat(backup): daily backups to an S3 bucket, Settings → Backups, and the guide (#5673, #5675)

#5673 "S3 client and the backup job: settings, schedule, history, metrics":
- steward/backup/s3.py: SigV4 over httpx, path-style; put and list only
  (an append-only key is enough; retention is the bucket's). Signing is held
  to AWS's published examples.
- steward/backup/job.py: a 15-minute tick. The daily backup goes at the hour
  chosen on the reader's clock; a missed day is made up, and a failure is
  retried an hour later. History of the last 30 runs, audit
  (backup.made / backup.failed), and backup_age_secs / backup_bytes under
  "Steward itself", resource backup, for alert rules.
- Settings → Backups: verdict first, then the bucket (secret sealed; a
  blank box keeps it), the schedule, Test the bucket, Back up now, and the
  recent runs.

#5675 "Docs: backing up to Archives, and restoring": docs/core/backups.md.

Also: the audit page now words the Prometheus target actions from #5668,
which showed as codes, and a guard fails on any logged action without
wording.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/service.py

@@ -28,7 +28,8 @@ CORE_METRICS: dict[str, list[str]] = {
     "dns":     ["is_up", "response_ms"],
     "http":    ["is_up", "response_ms"],
     "steward": ["ingest_posts", "ingest_failed", "ingest_slowest_ms", "db_roundtrip_ms",
-                "unwatched_secs", "cleanup_secs", "open_fds", "open_fds_pct"],
+                "unwatched_secs", "cleanup_secs", "open_fds", "open_fds_pct",
+                "backup_age_secs", "backup_bytes"],
 }
 
 

steward/app.py

@@ -441,12 +441,25 @@ def _register_core_tasks(app: Quart) -> None:
         from .core.self_monitor import record_self_metrics
         await record_self_metrics(app)
 
+    async def run_backup_tick():
+        from .backup.job import tick
+        await tick(app)
+
     async def run_email_outbox():
         from .core.outbox import send_due
         await send_due(app)
 
+    from .backup.job import TICK_SECONDS as BACKUP_TICK_S
     from .core.outbox import SENDER_TICK_S
     app._task_registry.extend([
+        # Steward's own daily backup to its bucket (#5673): a tick that
+        # records the newest backup's age and makes one when it is due.
+        ScheduledTask(
+            name="backup",
+            coro_factory=run_backup_tick,
+            interval_seconds=BACKUP_TICK_S,
+            run_on_startup=False,
+        ),
         # Every email is a row until the mail server accepts it (#5506); this
         # sends what is due and retries what failed. On startup too, so what a
         # restart left unsent goes out at once.

steward/audit/service.py

@@ -187,6 +187,11 @@ ACTIONS = {
     "snmp.device_created": "added SNMP device",
     "snmp.device_updated": "changed SNMP device",
     "snmp.device_deleted": "deleted SNMP device",
+    "prometheus.target_created": "added Prometheus target",
+    "prometheus.target_updated": "changed Prometheus target",
+    "prometheus.target_deleted": "deleted Prometheus target",
+    "backup.made": "sent a backup",
+    "backup.failed": "tried to send a backup, which failed:",
 }
 
 # The filter's choices: an area, matched as a prefix of the action code.
@@ -194,7 +199,8 @@ AREAS = (("auth.", "Sign-ins"), ("account.", "Accounts"), ("user.", "Users"), ("
          ("alert_rule.", "Alert rules"), ("maintenance_window.", "Maintenance"),
          ("ansible.run", "Playbook runs"), ("ansible.", "Ansible (all)"),
          ("settings.", "Settings"), ("plugin.", "Plugins"), ("api_token.", "AI access"),
-         ("docker.", "Docker"), ("snmp.", "SNMP"))
+         ("docker.", "Docker"), ("snmp.", "SNMP"), ("prometheus.", "Prometheus"),
+         ("backup.", "Backups"))
 
 
 def action_text(action: str) -> str:
@@ -211,7 +217,8 @@ _LABELS = {"check": "dry run", "interval_seconds": "every", "interval_s": "every
 _METHODS = {"local": "password", "ldap": "LDAP", "oidc": "single sign-on"}
 _SECTIONS = {"general": "General", "thresholds": "Thresholds", "notifications": "Notifications",
              "ansible.managed_key": "Ansible: managed SSH key",
-             "ansible.credentials": "Ansible: credentials", "auth": "Authentication"}
+             "ansible.credentials": "Ansible: credentials", "auth": "Authentication",
+             "backups": "Backups"}
 
 
 def _plain(value) -> str:

steward/backup/job.py

@@ -1,10 +1,40 @@
-"""Steward's own backups as a scheduled job: make the file, send it, record
-how it went (milestone 507)."""
+"""Steward's own backups as a scheduled job: make the file, send it to the
+bucket, record how it went (#5673).
+
+Once a day at the hour chosen in Settings → Backups, in the zone of whoever
+saved it. A day missed while Steward was down is made up on the next tick;
+a failed run is tried again an hour later, never sooner, so a target that is
+down isn't pressed. Each run is kept in `backup.history` (newest first, the
+last HISTORY_KEEP), audited, and leaves two metrics under source "steward",
+resource "backup": backup_age_secs (since the last success, every tick while
+backups are on, so an alert rule can watch it) and backup_bytes.
+"""
 from __future__ import annotations
 
+import asyncio
+import logging
+import tempfile
+import time
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
 from sqlalchemy import text
 from sqlalchemy.ext.asyncio import create_async_engine
 
+from .dump import BackupError, make_backup
+from .s3 import S3Error, S3Target, list_objects, put_file
+
+logger = logging.getLogger(__name__)
+
+TICK_SECONDS = 15 * 60
+RETRY_AFTER = timedelta(hours=1)
+HISTORY_KEEP = 30
+PREFIX = "steward-"
+
+# One backup at a time, whether the schedule or a person started it.
+_running = asyncio.Lock()
+_background: set[asyncio.Task] = set()
+
 
 async def migration_heads(database_url: str) -> list[str]:
     """The migration revisions the database is at, for the manifest."""
@@ -14,3 +44,208 @@ async def migration_heads(database_url: str) -> list[str]:
             return list((await conn.execute(text("SELECT version_num FROM alembic_version"))).scalars())
     finally:
         await engine.dispose()
+
+
+def target_from(settings: dict) -> S3Target | None:
+    """The bucket the settings name, or None while they're incomplete."""
+    endpoint = (settings.get("backup.s3_endpoint") or "").strip()
+    bucket = (settings.get("backup.s3_bucket") or "").strip()
+    access = (settings.get("backup.s3_access_key") or "").strip()
+    secret = settings.get("backup.s3_secret_key") or ""
+    if not (endpoint and bucket and access and secret):
+        return None
+    return S3Target(endpoint, bucket, access, secret,
+                    region=(settings.get("backup.s3_region") or "us-east-1").strip(),
+                    verify_tls=bool(settings.get("backup.s3_verify_tls", True)))
+
+
+def _moment(raw) -> datetime | None:
+    try:
+        return datetime.fromisoformat(raw) if raw else None
+    except (TypeError, ValueError):
+        return None
+
+
+def last_slot(now: datetime, hour: int, tz) -> datetime:
+    """The latest scheduled moment at or before `now`: today's hour in `tz`,
+    or yesterday's when today's hasn't come yet."""
+    local = now.astimezone(tz)
+    slot = local.replace(hour=hour, minute=0, second=0, microsecond=0)
+    if slot > local:
+        slot -= timedelta(days=1)
+    return slot.astimezone(timezone.utc)
+
+
+def is_due(now: datetime, hour: int, tz, last_success: datetime | None,
+           last_attempt: datetime | None) -> bool:
+    """Due when no backup has succeeded since the latest slot, and the last
+    try (if it failed) was at least RETRY_AFTER ago."""
+    slot = last_slot(now, hour, tz)
+    if last_success is not None and last_success >= slot:
+        return False
+    return last_attempt is None or now - last_attempt >= RETRY_AFTER
+
+
+def history_with(history: list, entry: dict) -> list:
+    return ([entry] + list(history or []))[:HISTORY_KEEP]
+
+
+async def _record(app, entry: dict, user_id: str | None, username: str) -> None:
+    from steward.core.alerts import MetricBatch
+    from steward.core.audit import log_audit
+    from steward.core.settings import get_setting, set_setting
+
+    async with app.db_sessionmaker() as db:
+        async with db.begin():
+            history = await get_setting(db, "backup.history") or []
+            await set_setting(db, "backup.history", history_with(history, entry))
+            await set_setting(db, "backup.last_attempt_at", entry["at"])
+            if entry["ok"]:
+                await set_setting(db, "backup.last_success_at", entry["at"])
+                batch = MetricBatch()
+                batch.add(source_module="steward", resource_name="backup",
+                          metric_name="backup_bytes", value=float(entry["size"]))
+                batch.add(source_module="steward", resource_name="backup",
+                          metric_name="backup_age_secs", value=0.0)
+                await batch.record(db)
+    await log_audit(app, user_id, username, "backup.made" if entry["ok"] else "backup.failed",
+                    entity_type="backup", entity_id=entry.get("name") or None,
+                    detail={k: entry[k] for k in ("name", "size", "secs", "error", "by") if entry.get(k)})
+
+
+async def run_backup(app, by: str = "schedule", user_id: str | None = None) -> dict:
+    """Make one backup and send it. Returns the history entry it recorded."""
+    from steward.core.settings import get_all_settings
+
+    async with _running:
+        started = time.monotonic()
+        at = datetime.now(timezone.utc)
+        entry = {"at": at.isoformat(), "ok": False, "by": by, "name": "", "size": 0,
+                 "secs": 0.0, "error": ""}
+        async with app.db_sessionmaker() as db:
+            settings = await get_all_settings(db)
+        target = target_from(settings)
+        try:
+            if target is None:
+                raise BackupError("The bucket isn't set up: fill in the endpoint, bucket and keys.")
+            db_url = app.config["DATABASE_URL"]
+            heads = await migration_heads(db_url)
+            with tempfile.TemporaryDirectory(prefix="steward-backup-") as tmp:
+                made = await make_backup(db_url, app.config["SECRET_KEY"], heads, Path(tmp), now=at)
+                await put_file(target, made.name, made.path)
+            entry.update(ok=True, name=made.name, size=made.size)
+        except (BackupError, S3Error) as exc:
+            entry["error"] = str(exc)
+        except Exception as exc:     # recorded and logged: a run never takes the scheduler down
+            logger.exception("Backup failed")
+            entry["error"] = f"Unexpected error: {exc}"
+        entry["secs"] = round(time.monotonic() - started, 1)
+        if entry["ok"]:
+            logger.info("Backup %s sent (%d bytes, %.1fs)", entry["name"], entry["size"], entry["secs"])
+        else:
+            logger.warning("Backup failed: %s", entry["error"])
+        await _record(app, entry, user_id, by if by != "schedule" else "")
+        return entry
+
+
+def start_backup(app, by: str, user_id: str | None) -> bool:
+    """Start a backup in the background for a person; False if one is running."""
+    if _running.locked():
+        return False
+    task = asyncio.get_running_loop().create_task(run_backup(app, by=by, user_id=user_id))
+    _background.add(task)
+    task.add_done_callback(_background.discard)
+    return True
+
+
+def is_running() -> bool:
+    return _running.locked()
+
+
+async def tick(app, now: datetime | None = None) -> None:
+    """The scheduled task: record how old the newest backup is, and make one
+    when it is due."""
+    from steward.core.alerts import MetricBatch
+    from steward.core.formatting import zone
+    from steward.core.settings import get_all_settings
+
+    now = now or datetime.now(timezone.utc)
+    async with app.db_sessionmaker() as db:
+        settings = await get_all_settings(db)
+    if not settings.get("backup.enabled"):
+        return
+    last_success = _moment(settings.get("backup.last_success_at"))
+    if last_success is not None:
+        async with app.db_sessionmaker() as db:
+            async with db.begin():
+                batch = MetricBatch()
+                batch.add(source_module="steward", resource_name="backup",
+                          metric_name="backup_age_secs", value=(now - last_success).total_seconds())
+                await batch.record(db)
+    if _running.locked():
+        return
+    if is_due(now, int(settings.get("backup.schedule_hour", 3)), zone(settings.get("backup.schedule_tz")),
+              last_success, _moment(settings.get("backup.last_attempt_at"))):
+        await run_backup(app)
+
+
+async def test_target(target: S3Target) -> tuple[bool, str]:
+    """Settings → Backups' Test button: can Steward reach the bucket with
+    these keys? A listing refused as AccessDenied still proves the keys:
+    an append-only key may not list, and uploads anyway."""
+    try:
+        found = await list_objects(target, PREFIX, limit=1000)
+    except S3Error as exc:
+        said = str(exc)
+        if "AccessDenied" in said:
+            return True, ("Reached the bucket and the keys were accepted. They may not list "
+                          "what is there, which an append-only key can't; uploads still work.")
+        return False, said
+    if not found:
+        return True, f"Reached {target.bucket}; no Steward backups there yet."
+    newest = max(found, key=lambda o: o.key)
+    return True, f"Reached {target.bucket}: {len(found)} Steward backups, the newest {newest.key}."
+
+
+# ── What Settings → Backups shows ────────────────────────────────────────────
+
+def shown_hour(hour: int, saved_tz, view_tz, now: datetime | None = None) -> int:
+    """The schedule's hour as a reader in `view_tz` sees it today."""
+    local = (now or datetime.now(timezone.utc)).astimezone(saved_tz)
+    return local.replace(hour=hour, minute=0, second=0, microsecond=0).astimezone(view_tz).hour
+
+
+def history_rows(history: list) -> list[dict]:
+    """The stored history with its times read back as datetimes."""
+    return [entry | {"at": _moment(entry.get("at"))} for entry in history or []]
+
+
+def verdict(settings: dict, running: bool) -> dict:
+    """One line for the top of the page: {"tone": ok|warn|bad|off, "text", "moment"}.
+    `moment` is the time the line is about, shown as "ago" by the template,
+    which ends the sentence; a failure also carries `failed`, its `error` and
+    `good` (the newest success, or None)."""
+    rows = history_rows(settings.get("backup.history") or [])
+    last_ok = _moment(settings.get("backup.last_success_at"))
+    if running:
+        return {"tone": "ok", "text": "A backup is being made now", "moment": None}
+    if not settings.get("backup.enabled"):
+        if target_from(settings) is None:
+            return {"tone": "off", "text": "Backups are off, and no bucket is set up", "moment": None}
+        return {"tone": "off", "text": "Scheduled backups are off; Back up now still works", "moment": None}
+    if target_from(settings) is None:
+        return {"tone": "bad", "text": "Backups are on, but the bucket isn't set up", "moment": None}
+    if rows and not rows[0]["ok"]:
+        # `good`: when the newest backup that did go was sent (None: never).
+        return {"tone": "bad", "text": "The last backup failed", "moment": rows[0]["at"],
+                "error": rows[0]["error"], "good": last_ok, "failed": True}
+    if last_ok is None:
+        return {"tone": "warn", "text": "No backup has been sent yet; the first goes within 15 minutes",
+                "moment": None}
+    return {"tone": "ok", "text": f"The newest backup is {rows[0]['name']} ({_size(rows[0]['size'])}), sent",
+            "moment": last_ok}
+
+
+def _size(n) -> str:
+    from steward.core.formatting import human_bytes
+    return human_bytes(n)

steward/backup/s3.py

@@ -0,0 +1,191 @@
+"""A small S3 client for sending backups: AWS Signature Version 4 over httpx,
+path-style addressing (endpoint/bucket/key), the three calls a backup needs.
+
+Fabled Archives speaks S3 with path-style buckets and append-only keys by
+default (System 180 ruling: backups go there with an append-only key), so
+Steward only ever puts and lists; it never deletes. Retention is the
+target's business.
+
+Written by hand rather than with boto3: three calls don't justify a large
+dependency, and signing is a pure function tests can hold to AWS's own
+published examples.
+"""
+from __future__ import annotations
+
+import hashlib
+import hmac
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import quote
+
+import httpx
+
+EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
+TIMEOUT_SECONDS = 30.0
+# A backup is megabytes; a slow link still gets this long to send one.
+UPLOAD_TIMEOUT_SECONDS = 15 * 60
+CHUNK = 1 << 20
+
+
+class S3Error(Exception):
+    """Why a call to the bucket failed, said to the person."""
+
+
+@dataclass(frozen=True)
+class S3Target:
+    endpoint: str          # http(s)://host[:port], no bucket
+    bucket: str
+    access_key: str
+    secret_key: str
+    region: str = "us-east-1"
+    verify_tls: bool = True
+
+    def url(self, key: str = "") -> str:
+        base = f"{self.endpoint.rstrip('/')}/{self.bucket}"
+        return f"{base}/{key}" if key else f"{base}/"
+
+
+@dataclass(frozen=True)
+class StoredObject:
+    key: str
+    size: int
+    last_modified: str
+
+
+# ── Signing ──────────────────────────────────────────────────────────────────
+
+def _encode(text: str, keep_slash: bool = False) -> str:
+    return quote(text, safe="-_.~" + ("/" if keep_slash else ""))
+
+
+def _hmac(key: bytes, msg: str) -> bytes:
+    return hmac.new(key, msg.encode(), hashlib.sha256).digest()
+
+
+def sign(method: str, url: str, headers: dict[str, str], payload_sha256: str,
+         access_key: str, secret_key: str, region: str, at: datetime,
+         service: str = "s3") -> dict[str, str]:
+    """The request's headers with x-amz-date, x-amz-content-sha256 and
+    Authorization added. Every header given is signed, Host always."""
+    u = httpx.URL(url)
+    stamp = at.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    day = stamp[:8]
+    out = dict(headers)
+    out["x-amz-date"] = stamp
+    out["x-amz-content-sha256"] = payload_sha256
+    signed = {k.lower(): " ".join(str(v).split()) for k, v in out.items()}
+    signed["host"] = u.netloc.decode()
+    names = sorted(signed)
+
+    query = sorted((_encode(k), _encode(v)) for k, v in u.params.multi_items())
+    canonical = "\n".join([
+        method.upper(),
+        _encode(u.path or "/", keep_slash=True),
+        "&".join(f"{k}={v}" for k, v in query),
+        "".join(f"{n}:{signed[n]}\n" for n in names),
+        ";".join(names),
+        payload_sha256,
+    ])
+    scope = f"{day}/{region}/{service}/aws4_request"
+    to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope,
+                         hashlib.sha256(canonical.encode()).hexdigest()])
+    key = _hmac(_hmac(_hmac(_hmac(f"AWS4{secret_key}".encode(), day), region), service), "aws4_request")
+    signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
+    out["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, "
+                            f"SignedHeaders={';'.join(names)}, Signature={signature}")
+    return out
+
+
+# ── Calls ────────────────────────────────────────────────────────────────────
+
+def _says(response: httpx.Response) -> str:
+    """An S3 error response in words: its Code and Message when it has them."""
+    try:
+        root = ET.fromstring(response.content)
+        code = root.findtext("Code") or ""
+        message = root.findtext("Message") or ""
+    except ET.ParseError:
+        code, message = "", ""
+    hints = {
+        "SignatureDoesNotMatch": "the secret key is wrong",
+        "InvalidAccessKeyId": "the access key isn't known there",
+        "NoSuchBucket": "there is no such bucket",
+        "AccessDenied": "the key isn't allowed to do that",
+    }
+    said = hints.get(code) or message or response.reason_phrase
+    return f"HTTP {response.status_code}{f' {code}' if code else ''}: {said}."
+
+
+def _reach_error(target: S3Target, exc: httpx.HTTPError) -> S3Error:
+    if isinstance(exc, httpx.TimeoutException):
+        return S3Error(f"{target.endpoint} didn't answer in time.")
+    return S3Error(f"Couldn't reach {target.endpoint}: {exc}.")
+
+
+def file_sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        while chunk := f.read(CHUNK):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+async def put_file(target: S3Target, key: str, path: Path, *,
+                   transport: httpx.AsyncBaseTransport | None = None,
+                   now: datetime | None = None) -> None:
+    """Upload one file as one object."""
+    import asyncio
+    digest = await asyncio.to_thread(file_sha256, path)
+    size = path.stat().st_size
+    url = target.url(key)
+    headers = sign("PUT", url, {"Content-Length": str(size), "Content-Type": "application/x-tar"},
+                   digest, target.access_key, target.secret_key, target.region,
+                   now or datetime.now(timezone.utc))
+
+    async def body():
+        with open(path, "rb") as f:
+            while chunk := f.read(CHUNK):
+                yield chunk
+
+    try:
+        async with httpx.AsyncClient(verify=target.verify_tls, timeout=UPLOAD_TIMEOUT_SECONDS,
+                                     transport=transport) as client:
+            response = await client.put(url, headers=headers, content=body())
+    except httpx.HTTPError as exc:
+        raise _reach_error(target, exc) from None
+    if response.status_code // 100 != 2:
+        raise S3Error(f"Upload of {key} refused: {_says(response)}")
+
+
+async def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *,
+                       transport: httpx.AsyncBaseTransport | None = None,
+                       now: datetime | None = None) -> list[StoredObject]:
+    """Up to `limit` objects under `prefix` (ListObjectsV2, one page)."""
+    params = {"list-type": "2", "max-keys": str(limit)}
+    if prefix:
+        params["prefix"] = prefix
+    url = str(httpx.URL(target.url(), params=params))
+    headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
+                   target.region, now or datetime.now(timezone.utc))
+    try:
+        async with httpx.AsyncClient(verify=target.verify_tls, timeout=TIMEOUT_SECONDS,
+                                     transport=transport) as client:
+            response = await client.get(url, headers=headers)
+    except httpx.HTTPError as exc:
+        raise _reach_error(target, exc) from None
+    if response.status_code != 200:
+        raise S3Error(f"Listing {target.bucket} refused: {_says(response)}")
+    try:
+        root = ET.fromstring(response.content)
+    except ET.ParseError:
+        raise S3Error(f"{target.endpoint} answered a listing with something that isn't S3's XML.") from None
+    out = []
+    for item in root.iter():
+        if item.tag.rsplit("}", 1)[-1] != "Contents":
+            continue
+        fields = {child.tag.rsplit("}", 1)[-1]: (child.text or "") for child in item}
+        out.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0),
+                                fields.get("LastModified", "")))
+    return out

steward/core/formatting.py

@@ -286,6 +286,8 @@ METRICS: dict[str, tuple[str, str]] = {
     "db_roundtrip_ms": ("Database round trip", "a trivial query to Steward's database, connection included"),
     "unwatched_secs": ("Time not watching", "how long Steward wasn't recording before this minute; 0 when unbroken"),
     "cleanup_secs": ("Hourly cleanup took", "how long the last retention and rollup run took"),
+    "backup_age_secs": ("Newest backup's age", "time since Steward last sent a backup to its bucket"),
+    "backup_bytes": ("Backup size", "the size of the last backup sent"),
     "open_fds": ("Open files", "files Steward itself has open"),
     "open_fds_pct": ("Open files (% of limit)", "files Steward has open, against its limit"),
     "scrape_ms": ("Scrape time", "how long the target took to hand over its metrics"),

steward/core/settings.py

@@ -176,6 +176,20 @@ DEFAULTS: dict[str, Any] = {
     # so "Sunday 08:00" means 08:00 where they are (#5378).
     "reports.schedule_tz": "",
     "reports.last_sent_at": "",
+    # Steward's own backups (#5673), sent to an S3 bucket (Fabled Archives).
+    "backup.enabled": False,
+    "backup.s3_endpoint": "",
+    "backup.s3_bucket": "",
+    "backup.s3_region": "us-east-1",
+    "backup.s3_access_key": "",
+    "backup.s3_secret_key": "",
+    "backup.s3_verify_tls": True,
+    "backup.schedule_hour": 3,   # an hour in backup.schedule_tz
+    "backup.schedule_tz": "",
+    "backup.last_success_at": "",
+    "backup.last_attempt_at": "",
+    # Newest first: {at, ok, by, name, size, secs, error}.
+    "backup.history": [],
     # What the last hourly cleanup did (#5469), written by core/cleanup.py for
     # Settings → System: {"at": ISO time, "secs": float, "removed": [[what, n]]}.
     "cleanup.last_run": {},
@@ -194,6 +208,7 @@ SECRET_KEYS: set[str] = {
     # A webhook URL carries its own credential (Discord, Slack and ntfy put the
     # token in the path).
     "webhook.url",
+    "backup.s3_secret_key",
 }
 
 

steward/settings/routes.py

@@ -687,6 +687,92 @@ async def send_report_now():
     return await _reports_page(settings, {"ok": ok, "message": message})
 
 
+# ── Backups ───────────────────────────────────────────────────────────────────
+
+async def _backups_page(flash=None):
+    """Settings → Backups (#5673): how the newest backup went, the bucket and
+    the daily hour (in the reader's zone), and the runs before."""
+    from steward.backup import job
+    from steward.core.formatting import zone
+    async with current_app.db_sessionmaker() as db:
+        settings = await get_all_settings(db)
+    hour = job.shown_hour(int(settings.get("backup.schedule_hour", 3)),
+                          zone(settings.get("backup.schedule_tz")), zone(session.get("tz")))
+    return await render_template(
+        "settings/backups.html", settings=settings, flash=flash, shown_hour=hour,
+        verdict=job.verdict(settings, job.is_running()),
+        history=job.history_rows(settings.get("backup.history")),
+        has_secret=bool(settings.get("backup.s3_secret_key")))
+
+
+@settings_bp.get("/backups/")
+@require_role(UserRole.admin)
+async def backups():
+    return await _backups_page()
+
+
+@settings_bp.post("/backups/")
+@require_role(UserRole.admin)
+async def save_backups():
+    """The hour is what the reader picked on their own clock, so it is kept in
+    their zone. A blank secret key keeps the stored one."""
+    form = await request.form
+    endpoint = form.get("backup.s3_endpoint", "").strip()
+    if endpoint and not endpoint.startswith(("http://", "https://")):
+        return await _backups_page({"ok": False, "message":
+                                    "The endpoint starts with http:// or https://, e.g. http://archives:8081."})
+    try:
+        hour = int(form.get("backup.schedule_hour", 3))
+    except ValueError:
+        hour = 3
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            await set_setting(db, "backup.enabled", "backup.enabled" in form)
+            await set_setting(db, "backup.s3_endpoint", endpoint)
+            await set_setting(db, "backup.s3_bucket", form.get("backup.s3_bucket", "").strip())
+            await set_setting(db, "backup.s3_region", form.get("backup.s3_region", "").strip() or "us-east-1")
+            await set_setting(db, "backup.s3_access_key", form.get("backup.s3_access_key", "").strip())
+            secret = form.get("backup.s3_secret_key", "")
+            if secret.strip():
+                await set_setting(db, "backup.s3_secret_key", secret.strip())
+            elif "backup.clear_secret" in form:
+                await set_setting(db, "backup.s3_secret_key", "")
+            await set_setting(db, "backup.s3_verify_tls", "backup.s3_verify_tls" in form)
+            await set_setting(db, "backup.schedule_hour", min(max(hour, 0), 23))
+            await set_setting(db, "backup.schedule_tz", session.get("tz") or "")
+    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
+                    "settings.saved", detail={"section": "backups"})
+    return redirect(url_for("settings.backups"))
+
+
+@settings_bp.post("/backups/test/")
+@require_role(UserRole.admin)
+async def test_backups():
+    """Reach the saved bucket with the saved keys (the operator asked, so the
+    target is contacted)."""
+    from steward.backup import job
+    async with current_app.db_sessionmaker() as db:
+        settings = await get_all_settings(db)
+    target = job.target_from(settings)
+    if target is None:
+        return await _backups_page({"ok": False, "message":
+                                    "Save the endpoint, bucket and both keys first."})
+    ok, message = await job.test_target(target)
+    return await _backups_page({"ok": ok, "message": message})
+
+
+@settings_bp.post("/backups/now/")
+@require_role(UserRole.admin)
+async def backup_now():
+    from steward.backup import job
+    started = job.start_backup(current_app._get_current_object(),
+                               by=session.get("username", "") or "an admin",
+                               user_id=session.get("user_id"))
+    message = ("Making a backup now; it shows under Recent backups when it is sent."
+               if started else "A backup is already being made.")
+    return await _backups_page({"ok": started, "message": message})
+
+
 def _build_plugin_cfg_from_form(plugin: dict, form) -> dict:
     """Extract config values for one plugin from a form submission."""
     name = plugin["_dir"]

steward/templates/settings/_tabs.html

@@ -7,6 +7,7 @@
                  ("thresholds", "Thresholds & Retention", "/settings/thresholds/"),
                  ("notifications", "Notifications", "/settings/notifications/"),
                  ("reports", "Reports", "/settings/reports/"),
+                 ("backups", "Backups", "/settings/backups/"),
                  ("auth", "Authentication", "/settings/auth/"),
                  ("users", "Users", "/settings/users/"),
                  ("ansible", "Ansible", "/settings/ansible/"),

steward/templates/settings/backups.html

@@ -0,0 +1,138 @@
+{# settings/backups.html — Steward's own backups (#5673): a verdict, the bucket
+   they go to (Fabled Archives over S3) and the daily hour, then the recent
+   runs. Restoring is `steward restore`; see docs/core/backups.md. #}
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs %}
+{% from "settings/_tabs.html" import settings_tabs %}
+{% block title %}Settings — Backups — Steward{% endblock %}
+{% block section_tabs %}{{ settings_tabs("backups") }}{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("Backups", "")]) }}{% endblock %}
+{% block content %}
+{% set tone = {"ok": "tone-ok", "warn": "tone-warn", "bad": "tone-crit", "off": "tone-dim"}[verdict.tone] %}
+<p class="page-verdict"><span class="{{ tone }}">{{ verdict.text }}
+  {%- if verdict.moment %} {{ verdict.moment|ago }}{% endif %}.
+  {%- if verdict.failed %} {{ verdict.error }}
+    {% if verdict.good %}The newest good backup was sent {{ verdict.good|ago }}.{% else %}No backup has been sent yet.{% endif %}
+  {%- endif %}</span></p>
+
+{% if flash %}
+<div style="padding:0.75rem 1rem;border-radius:6px;margin-bottom:1.25rem;
+            background:{% if flash.ok %}var(--green-dim){% else %}var(--red-dim){% endif %};
+            border:1px solid {% if flash.ok %}var(--green){% else %}var(--red){% endif %};
+            color:{% if flash.ok %}var(--green){% else %}var(--red){% endif %};
+            font-size:0.88rem;">
+  {{ flash.message }}
+</div>
+{% endif %}
+
+<div style="max-width:640px;">
+  <form method="post" action="/settings/backups/">{{ csrf_field() }}
+  <div class="card" style="margin-bottom:1.25rem;">
+    <h2 class="section-title" style="margin-bottom:0.5rem;">Bucket</h2>
+    <p class="field-hint" style="margin-bottom:1rem;">
+      Each backup is one file, <code>steward-&lt;time&gt;.tar</code>: Steward's configuration, its audit log
+      and alert history, and its secret key. Metrics and check results are left out. Anyone holding a
+      copy can read every password and key Steward keeps, so keep the bucket private.
+    </p>
+
+    <div class="form-group">
+      <label for="b-endpoint">Endpoint</label>
+      <input type="url" id="b-endpoint" name="backup.s3_endpoint" value="{{ settings['backup.s3_endpoint'] }}"
+             placeholder="http://archives:8081">
+      <p class="field-hint">The S3 address, without the bucket. Requests are path-style: endpoint/bucket/file.</p>
+    </div>
+    <div style="display:grid;grid-template-columns:2fr 1fr;gap:1rem;">
+      <div class="form-group">
+        <label for="b-bucket">Bucket</label>
+        <input type="text" id="b-bucket" name="backup.s3_bucket" value="{{ settings['backup.s3_bucket'] }}"
+               placeholder="steward">
+      </div>
+      <div class="form-group">
+        <label for="b-region">Region</label>
+        <input type="text" id="b-region" name="backup.s3_region" value="{{ settings['backup.s3_region'] }}">
+      </div>
+    </div>
+    <div style="display:grid;grid-template-columns:1fr 1fr;gap:1rem;">
+      <div class="form-group">
+        <label for="b-access">Access key</label>
+        <input type="text" id="b-access" name="backup.s3_access_key" autocomplete="off"
+               value="{{ settings['backup.s3_access_key'] }}">
+      </div>
+      <div class="form-group">
+        <label for="b-secret">Secret key</label>
+        <input type="password" id="b-secret" name="backup.s3_secret_key" autocomplete="new-password"
+               placeholder="{% if has_secret %}kept; type to change{% endif %}">
+        {% if has_secret %}
+        <label style="display:flex;align-items:center;gap:0.4rem;font-weight:normal;margin-top:0.35rem;">
+          <input type="checkbox" name="backup.clear_secret"> Remove the stored secret key
+        </label>
+        {% endif %}
+      </div>
+    </div>
+    <p class="field-hint">An append-only key is enough: Steward only uploads and never deletes. How long
+      backups are kept is the bucket's own setting.</p>
+    <div class="form-group" style="display:flex;align-items:center;gap:0.75rem;">
+      <input type="checkbox" name="backup.s3_verify_tls" id="b-tls"
+             {% if settings['backup.s3_verify_tls'] %}checked{% endif %}>
+      <label for="b-tls" style="margin:0;">Check the endpoint's TLS certificate</label>
+    </div>
+
+    <h2 class="section-title" style="margin:1.5rem 0 0.75rem;">Schedule</h2>
+    <div class="form-group" style="display:flex;align-items:center;gap:0.75rem;">
+      <input type="checkbox" name="backup.enabled" id="b-enabled"
+             {% if settings['backup.enabled'] %}checked{% endif %}>
+      <label for="b-enabled" style="margin:0;">Back up every day</label>
+    </div>
+    <div class="form-group" style="max-width:220px;">
+      <label for="b-hour">At hour ({{ tz_label() }})</label>
+      <select id="b-hour" name="backup.schedule_hour">
+        {% for h in range(24) %}
+        <option value="{{ h }}" {% if shown_hour == h %}selected{% endif %}>{{ "%02d:00"|format(h) }}</option>
+        {% endfor %}
+      </select>
+    </div>
+    <p class="field-hint">A day missed while Steward was down is made up when it is back; a failed backup is
+      tried again an hour later. Watch it with an <a href="/alerts/rules/new">alert rule</a> on
+      "Steward itself", resource <code>backup</code>, metric "Newest backup's age".</p>
+
+    <div style="margin-top:1rem;"><button type="submit" class="btn">Save</button></div>
+  </div>
+  </form>
+
+  <div class="card" style="margin-bottom:1.25rem;display:flex;gap:0.75rem;flex-wrap:wrap;align-items:center;">
+    <form method="post" action="/settings/backups/test/">{{ csrf_field() }}
+      <button type="submit" class="btn btn-ghost">Test the bucket</button>
+    </form>
+    <form method="post" action="/settings/backups/now/">{{ csrf_field() }}
+      <button type="submit" class="btn btn-ghost">Back up now</button>
+    </form>
+    <p class="field-hint" style="margin:0;">Both use the saved settings.</p>
+  </div>
+</div>
+
+<h2 class="section-title" style="margin:1.5rem 0 1rem;">Recent backups</h2>
+{% if history %}
+<div class="card-flush">
+  <table class="table">
+    <thead><tr><th>When</th><th>Outcome</th><th>File</th><th>Size</th><th>Took</th><th>Started by</th></tr></thead>
+    <tbody>
+      {% for r in history %}
+      <tr>
+        <td class="nowrap">{{ r.at|when if r.at else "—" }}</td>
+        <td>{% if r.ok %}<span class="tone-ok">Sent</span>{% else %}<span class="tone-crit">Failed</span>
+          <div class="cell-sub">{{ r.error }}</div>{% endif %}</td>
+        <td class="cell-name"><span title="{{ r.name }}">{{ r.name or "—" }}</span></td>
+        <td class="nowrap">{{ r.size|bytes if r.ok else "—" }}</td>
+        <td class="nowrap">{{ r.secs|duration }}</td>
+        <td>{{ "the schedule" if r.by == "schedule" else r.by }}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+</div>
+{% else %}
+<p class="tone-dim">None yet.</p>
+{% endif %}
+<p class="field-hint" style="margin-top:0.75rem;">To bring Steward back from one, see
+  <code>steward restore</code> in the backups guide (docs/core/backups.md).</p>
+{% endblock %}