feat(mcp): Steward's own backups: the runs, the next one, checking and making one (#5836)

list_backups, check_backup and make_backup are Settings → Backups for an
agent, admin only like the page. make_backup runs the page's job in the
background and is audited as "<user> via MCP". Restoring stays in the
browser. get_overview names a scheduled backup that failed, is two hours
past its slot, or has no bucket.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/mcp/tools/__init__.py

@@ -12,9 +12,10 @@ each argument's empty value means, and what comes back.
 """
 from __future__ import annotations
 
-from . import alerts, anomalies, audit, automation, hosts, metrics, monitors, overview, session
+from . import alerts, anomalies, audit, automation, backups, hosts, metrics, monitors, overview, session
 
-_MODULES = (session, overview, hosts, monitors, alerts, anomalies, metrics, audit, automation)
+_MODULES = (session, overview, hosts, monitors, alerts, anomalies, metrics, audit, automation,
+            backups)
 
 
 def register_all(register_tool) -> None:

steward/mcp/tools/backups.py

@@ -0,0 +1,208 @@
+"""Steward's own backups (#5836): what was made, where it went, how its check
+came out, and making or checking one, as Settings → Backups does.
+
+Admin only, like the page: the bucket and its backups hold every secret
+Steward keeps. Restoring stays in the browser, behind its typed confirmation.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+from steward.backup import job
+from steward.backup.dump import BackupError
+from steward.backup.s3 import S3Error, list_objects
+from steward.core.settings import get_all_settings
+from steward.mcp._context import current_caller, mcp_app
+from steward.models.users import UserRole
+
+from ._audit import audit_mcp, mcp_username
+from ._shape import age_s, bounded, clamp_limit, db_session, iso, now_utc, redact_url
+
+ROWS_SHOWN = 20  # rows named per table in a check; the rest are counted
+
+
+def next_backup_at(settings: dict, now: datetime) -> datetime | None:
+    """When the schedule makes the next one: the next day's hour once today's
+    is done, an hour after a failed try, else the next 15-minute tick. None
+    while scheduled backups are off."""
+    from steward.core.formatting import zone
+    if not settings.get("backup.enabled"):
+        return None
+    hour, tz = int(settings.get("backup.schedule_hour", 3)), zone(settings.get("backup.schedule_tz"))
+    slot = job.last_slot(now, hour, tz)
+    last_success = job._moment(settings.get("backup.last_success_at"))
+    if last_success is not None and last_success >= slot:
+        return job.last_slot(slot + timedelta(hours=25), hour, tz)     # tomorrow's, across a DST change
+    last_attempt = job._moment(settings.get("backup.last_attempt_at"))
+    if last_attempt is not None and now - last_attempt < job.RETRY_AFTER:
+        return last_attempt + job.RETRY_AFTER
+    return now + timedelta(seconds=job.TICK_SECONDS)
+
+
+def _history_row(entry: dict, now: datetime) -> dict:
+    at = job._moment(entry.get("at"))
+    return {
+        "name": entry.get("name") or None, "ok": bool(entry.get("ok")),
+        "at": iso(at), "age_s": age_s(at, now), "by": entry.get("by") or "schedule",
+        "size_bytes": entry.get("size") or 0, "secs": entry.get("secs"),
+        "error": entry.get("error") or None, "check": entry.get("check"),
+    }
+
+
+async def list_backups(limit: int = 10, bucket: bool = False) -> dict:
+    """Steward's own backups: whether they're on, where they go, when the next
+    one is due, and the runs so far (newest first), each with the check made
+    on it just after it was sent.
+
+    `verdict` is the line Settings → Backups opens with. `running` is true
+    while one is being made. A run row: `name` (the file in the bucket),
+    `ok`, `at`, `by` (a person, or "schedule"), `size_bytes`, `secs`,
+    `error` and `check` (`ok` false means the backup's key didn't open all
+    its secrets; `changes` counts differences from what was in place, which
+    is not a fault).
+
+    A backup holds settings and keys, users and tokens, hosts, monitors,
+    alert rules, inventory and dashboards; no metrics, logs or history.
+    Steward never deletes backups (its key is append-only); how long they
+    are kept is set on the bucket's side.
+
+    bucket: true also lists the Steward backups in the bucket. An
+    append-only key may not list; then `stored_error` says so.
+    limit: runs to return, default 10, max 30 (all that are kept).
+    """
+    now = now_utc()
+    async with db_session() as db:
+        settings = await get_all_settings(db)
+    running = job.is_running()
+    target = job.target_from(settings)
+    verdict = job.verdict(settings, running)
+    history = [_history_row(e, now) for e in settings.get("backup.history") or []]
+    nxt = next_backup_at(settings, now)
+    out = {
+        "enabled": bool(settings.get("backup.enabled")),
+        "running": running,
+        "verdict": {"tone": verdict["tone"], "text": verdict["text"],
+                    "at": iso(verdict.get("moment")), "error": verdict.get("error")},
+        "bucket": ({"endpoint": redact_url(target.endpoint), "bucket": target.bucket,
+                    "region": target.region} if target else None),
+        "schedule": {"hour": int(settings.get("backup.schedule_hour", 3)),
+                     "time_zone": settings.get("backup.schedule_tz") or "UTC",
+                     "next_at": iso(nxt)},
+        "last_success_at": iso(job._moment(settings.get("backup.last_success_at"))),
+        "last_success_age_s": age_s(job._moment(settings.get("backup.last_success_at")), now),
+        **bounded(history, clamp_limit(limit, 10, job.HISTORY_KEEP), key="runs"),
+    }
+    if bucket:
+        if target is None:
+            out["stored_error"] = "The bucket isn't set up: fill in the endpoint, bucket and keys."
+        else:
+            try:
+                found = await list_objects(target, job.PREFIX, limit=1000)
+            except S3Error as exc:
+                out["stored_error"] = str(exc) + (
+                    " An append-only key can't list; the runs above are what was sent."
+                    if "AccessDenied" in str(exc) else "")
+            else:
+                stored = sorted((o for o in found if job.is_backup_name(o.key)),
+                                key=lambda o: o.key, reverse=True)
+                out["stored"] = [{"name": o.key, "size_bytes": o.size, "modified_at": o.last_modified}
+                                 for o in stored[:50]]
+                out["stored_total"] = len(stored)
+    return out
+
+
+def _table(t) -> dict:
+    row = {"table": t.table, "same": t.same, "changes": t.changes}
+    if t.note:
+        row["note"] = t.note
+    if t.differ:
+        row["differ"] = [
+            {"label": d.label, "columns": d.columns, "secret": d.secret,
+             **({"backup_value": d.backup_value, "here_value": d.here_value}
+                if not d.secret and (d.backup_value is not None or d.here_value is not None) else {})}
+            for d in t.differ[:ROWS_SHOWN]]
+    for key, rows in (("only_in_backup", t.only_backup), ("only_here", t.only_here)):
+        if rows:
+            row[key] = rows[:ROWS_SHOWN]
+    cut = sum(max(0, len(x) - ROWS_SHOWN) for x in (t.differ, t.only_backup, t.only_here))
+    if cut:
+        row["not_named"] = cut
+    return row
+
+
+def report_dict(report) -> dict:
+    """A CheckReport as the agent reads it: the verdict first, then only the
+    tables that differ."""
+    return {
+        "name": report.name,
+        "made_at": iso(report.made_at),
+        "ok": report.ok,
+        "summary": report.summary(),
+        "schema": report.schema,
+        "backup_revisions": report.backup_heads,
+        "here_revisions": report.here_heads,
+        "key_same": report.key_same,
+        "sealed": report.sealed,
+        "unopened": report.unopened,
+        "rows": report.rows,
+        "changes": report.changes,
+        "areas": [
+            {"area": a.name, "same": a.same, "changes": a.changes,
+             "tables": [_table(t) for t in a.tables if t.changes or t.note]}
+            for a in report.areas],
+    }
+
+
+async def check_backup(name: str) -> dict:
+    """Fetch one backup from the bucket and compare it with what is in place,
+    as Settings → Backups → Check does. Nothing changes here.
+
+    name: the file, e.g. "steward-20261015T030000Z.tar" (list_backups).
+
+    `ok` is whether the backup's own key opens every secret in it, so a
+    restore would bring them back. `summary` says it in a line. `schema` is
+    "same", "older" (made by an earlier Steward; a restore upgrades it) or
+    "newer" (made by a later Steward; this one can't restore it). `key_same`
+    is whether the backup carries this Steward's secret key. `changes`
+    counts rows that differ, exist only in the backup, or only here; `areas`
+    names them, table by table, with a setting's two values when it isn't
+    secret. A difference is what a restore would put back, not a fault.
+
+    An append-only key may not read files back; then this fails, and the
+    check can be made on the page with a downloaded copy instead.
+    """
+    try:
+        report = await job.check_stored(mcp_app(), (name or "").strip())
+    except (BackupError, S3Error) as exc:
+        said = str(exc)
+        if "AccessDenied" in said:
+            said += (" The bucket's key may not read backups back; download the file and "
+                     "check it on Settings → Backups instead.")
+        raise ValueError(said) from None
+    return report_dict(report)
+
+
+async def make_backup() -> dict:
+    """Make a backup now and send it to the bucket, as Settings → Backups →
+    Back up now does. It runs in the background: list_backups shows
+    `running`, then the new run at the top with its check. Audited.
+
+    Refused while one is already being made, or when no bucket is set up.
+    """
+    async with db_session() as db:
+        settings = await get_all_settings(db)
+    if job.target_from(settings) is None:
+        raise ValueError("The bucket isn't set up: fill in the endpoint, bucket and keys on "
+                         "Settings → Backups. Nothing was started.")
+    caller = current_caller()
+    if not job.start_backup(mcp_app(), by=mcp_username(caller.username), user_id=caller.user_id):
+        raise ValueError("A backup is already being made; list_backups shows it. Nothing was started.")
+    await audit_mcp("backup.started", "backup")
+    return {"started": True,
+            "summary": "Making a backup now; list_backups shows it once it is sent and checked."}
+
+
+def register(register_tool) -> None:
+    for fn in (list_backups, check_backup):
+        register_tool(fn, role=UserRole.admin, read_only=True)
+    register_tool(make_backup, role=UserRole.admin, read_only=False)

steward/mcp/tools/overview.py

@@ -7,8 +7,10 @@ from datetime import timedelta
 from sqlalchemy import select
 
 from steward.alerts import service as alert_svc
+from steward.backup import job as backup_job
 from steward.core import alerts
 from steward.core import anomalies as anom
+from steward.core.settings import get_all_settings
 from steward.hosts import service as host_svc
 from steward.mcp._context import mcp_app
 from steward.models.ansible import AnsibleRun, AnsibleRunStatus
@@ -22,6 +24,9 @@ from ._shape import age_s, db_session, iso, now_utc, rnd
 logger = logging.getLogger(__name__)
 
 FINDINGS_CAP = 50
+# A scheduled backup this long past its slot with none made is overdue: one
+# 15-minute tick and the hour a failed try waits, with room to spare.
+BACKUP_OVERDUE_AFTER = timedelta(hours=2)
 _SEVERITY = {"critical": 0, "warning": 1}
 
 
@@ -202,6 +207,41 @@ def anomaly_findings(open_now) -> list[dict]:
     return out
 
 
+def backup_findings(settings: dict, running: bool, now) -> list[dict]:
+    """Steward's own backups (#5836), while scheduled ones are on: the bucket
+    isn't set up, the last one failed (since the first of the failures in a
+    row), or none has been made since the day's slot."""
+    from steward.core.formatting import zone
+    if running or not settings.get("backup.enabled"):
+        return []
+    subject, nxt = "Steward backups", "list_backups()"
+    if backup_job.target_from(settings) is None:
+        return [finding("warning", "backup_not_set_up", subject,
+                        "Scheduled backups are on, but the bucket isn't set up, so none are made",
+                        next=nxt)]
+    history = settings.get("backup.history") or []
+    last_ok = backup_job._moment(settings.get("backup.last_success_at"))
+    good = (f"the newest good one was sent {iso(last_ok)}" if last_ok
+            else "none has been sent yet")
+    failing = []
+    for entry in history:
+        if entry.get("ok"):
+            break
+        failing.append(entry)
+    if failing:
+        tries = f" ({len(failing)} tries in a row)" if len(failing) > 1 else ""
+        return [finding("warning", "backup_failed", subject,
+                        f"The last backup failed{tries}: {failing[0].get('error') or 'no reason recorded'}; {good}",
+                        since=backup_job._moment(failing[-1].get("at")), next=nxt)]
+    slot = backup_job.last_slot(now, int(settings.get("backup.schedule_hour", 3)),
+                                zone(settings.get("backup.schedule_tz")))
+    if history and (last_ok is None or last_ok < slot) and now - slot >= BACKUP_OVERDUE_AFTER:
+        return [finding("warning", "backup_overdue", subject,
+                        f"No backup has been made since the scheduled one at {iso(slot)}; {good}",
+                        since=slot, next=nxt)]
+    return []
+
+
 async def _plugin_findings(server) -> tuple[list[dict], list[str]]:
     found, unavailable = [], []
     for plugin, source in sorted((getattr(server, "overview_sources", None) or {}).items()):
@@ -225,7 +265,8 @@ async def get_overview() -> dict:
     agents that stopped reporting, hosts over their CPU, memory or disk
     limits (all cutoffs are set under Settings → Thresholds), Ansible runs that failed
     in the last 24h, metrics going on now well above what is normal for them
-    at this hour (metric anomalies), and anything enabled plugins add
+    at this hour (metric anomalies), Steward's own scheduled backup failing or
+    overdue, and anything enabled plugins add
     (unhealthy containers, for one). Each finding carries the ids to follow
     and `next`, the call that shows its evidence.
 
@@ -254,6 +295,7 @@ async def get_overview() -> dict:
         windows = await alert_svc.list_maintenance_windows(db)
         open_anomalies = await anom.open_anomalies(db)
         to_review = await anom.to_review_count(db)
+        settings = await get_all_settings(db)
 
     thresholds = app.config.get("THRESHOLDS") or {}
     listed, silent, speaking = fold_agent_silence(active, agent)
@@ -261,7 +303,8 @@ async def get_overview() -> dict:
                 + alert_findings(listed, silent)
                 + agent_findings(hosts, agent, thresholds, silent, speaking)
                 + run_findings(runs)
-                + anomaly_findings(open_anomalies))
+                + anomaly_findings(open_anomalies)
+                + backup_findings(settings, backup_job.is_running(), now))
     plugin_found, unavailable = await _plugin_findings(getattr(app, "mcp_server", None))
     findings = rank(findings + plugin_found)