feat(mcp): Steward's own backups: the runs, the next one, checking and making one (#5836)
list_backups, check_backup and make_backup are Settings → Backups for an agent, admin only like the page. make_backup runs the page's job in the background and is audited as "<user> via MCP". Restoring stays in the browser. get_overview names a scheduled backup that failed, is two hours past its slot, or has no bucket. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -80,6 +80,11 @@ While backups are on, Steward records two metrics under the source
|
||||
A rule such as `backup_age_secs > 93600` (26 hours) fires when a daily backup
|
||||
has been missed.
|
||||
|
||||
An agent on the MCP sees the same without a rule. `get_overview` names a
|
||||
scheduled backup that failed, is two hours past its slot with none made, or has
|
||||
no bucket set up. `list_backups`, `check_backup` and `make_backup` are this
|
||||
page's admin tools (see [MCP](mcp.md)); restoring isn't among them.
|
||||
|
||||
## Checking a backup
|
||||
|
||||
**Settings → Backups → Check** compares a backup with what is in place, without
|
||||
|
||||
@@ -107,6 +107,16 @@ The loop for building one is save → `run_playbook` in check mode on one target
|
||||
→ `get_ansible_run` → fix and save again → apply with `check=false`. The audit
|
||||
row records the size, never the content.
|
||||
|
||||
`list_backups`, `check_backup` and `make_backup` are Settings → Backups for an
|
||||
agent (#5836), admin only like the page, since a backup holds every secret
|
||||
Steward keeps. `list_backups` gives the schedule, the next run and the last 30
|
||||
runs with the check each one got after it was sent (`bucket=true` also lists the
|
||||
bucket, which an append-only key may refuse). `check_backup` fetches one and
|
||||
compares it with what is in place, changing nothing. `make_backup` starts one in
|
||||
the background and is audited. Restoring stays in the browser, behind its typed
|
||||
confirmation. `get_overview` reports, for every role, a scheduled backup that
|
||||
failed, is overdue, or has no bucket set up.
|
||||
|
||||
The client sees each tool's full contract: arguments, what an empty value
|
||||
means, and what comes back. That contract is the tool's docstring, so read the
|
||||
docstrings in `steward/mcp/tools/` for the detail.
|
||||
|
||||
@@ -12,9 +12,10 @@ each argument's empty value means, and what comes back.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from . import alerts, anomalies, audit, automation, hosts, metrics, monitors, overview, session
|
||||
from . import alerts, anomalies, audit, automation, backups, hosts, metrics, monitors, overview, session
|
||||
|
||||
_MODULES = (session, overview, hosts, monitors, alerts, anomalies, metrics, audit, automation)
|
||||
_MODULES = (session, overview, hosts, monitors, alerts, anomalies, metrics, audit, automation,
|
||||
backups)
|
||||
|
||||
|
||||
def register_all(register_tool) -> None:
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Steward's own backups (#5836): what was made, where it went, how its check
|
||||
came out, and making or checking one, as Settings → Backups does.
|
||||
|
||||
Admin only, like the page: the bucket and its backups hold every secret
|
||||
Steward keeps. Restoring stays in the browser, behind its typed confirmation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from steward.backup import job
|
||||
from steward.backup.dump import BackupError
|
||||
from steward.backup.s3 import S3Error, list_objects
|
||||
from steward.core.settings import get_all_settings
|
||||
from steward.mcp._context import current_caller, mcp_app
|
||||
from steward.models.users import UserRole
|
||||
|
||||
from ._audit import audit_mcp, mcp_username
|
||||
from ._shape import age_s, bounded, clamp_limit, db_session, iso, now_utc, redact_url
|
||||
|
||||
ROWS_SHOWN = 20 # rows named per table in a check; the rest are counted
|
||||
|
||||
|
||||
def next_backup_at(settings: dict, now: datetime) -> datetime | None:
|
||||
"""When the schedule makes the next one: the next day's hour once today's
|
||||
is done, an hour after a failed try, else the next 15-minute tick. None
|
||||
while scheduled backups are off."""
|
||||
from steward.core.formatting import zone
|
||||
if not settings.get("backup.enabled"):
|
||||
return None
|
||||
hour, tz = int(settings.get("backup.schedule_hour", 3)), zone(settings.get("backup.schedule_tz"))
|
||||
slot = job.last_slot(now, hour, tz)
|
||||
last_success = job._moment(settings.get("backup.last_success_at"))
|
||||
if last_success is not None and last_success >= slot:
|
||||
return job.last_slot(slot + timedelta(hours=25), hour, tz) # tomorrow's, across a DST change
|
||||
last_attempt = job._moment(settings.get("backup.last_attempt_at"))
|
||||
if last_attempt is not None and now - last_attempt < job.RETRY_AFTER:
|
||||
return last_attempt + job.RETRY_AFTER
|
||||
return now + timedelta(seconds=job.TICK_SECONDS)
|
||||
|
||||
|
||||
def _history_row(entry: dict, now: datetime) -> dict:
|
||||
at = job._moment(entry.get("at"))
|
||||
return {
|
||||
"name": entry.get("name") or None, "ok": bool(entry.get("ok")),
|
||||
"at": iso(at), "age_s": age_s(at, now), "by": entry.get("by") or "schedule",
|
||||
"size_bytes": entry.get("size") or 0, "secs": entry.get("secs"),
|
||||
"error": entry.get("error") or None, "check": entry.get("check"),
|
||||
}
|
||||
|
||||
|
||||
async def list_backups(limit: int = 10, bucket: bool = False) -> dict:
|
||||
"""Steward's own backups: whether they're on, where they go, when the next
|
||||
one is due, and the runs so far (newest first), each with the check made
|
||||
on it just after it was sent.
|
||||
|
||||
`verdict` is the line Settings → Backups opens with. `running` is true
|
||||
while one is being made. A run row: `name` (the file in the bucket),
|
||||
`ok`, `at`, `by` (a person, or "schedule"), `size_bytes`, `secs`,
|
||||
`error` and `check` (`ok` false means the backup's key didn't open all
|
||||
its secrets; `changes` counts differences from what was in place, which
|
||||
is not a fault).
|
||||
|
||||
A backup holds settings and keys, users and tokens, hosts, monitors,
|
||||
alert rules, inventory and dashboards; no metrics, logs or history.
|
||||
Steward never deletes backups (its key is append-only); how long they
|
||||
are kept is set on the bucket's side.
|
||||
|
||||
bucket: true also lists the Steward backups in the bucket. An
|
||||
append-only key may not list; then `stored_error` says so.
|
||||
limit: runs to return, default 10, max 30 (all that are kept).
|
||||
"""
|
||||
now = now_utc()
|
||||
async with db_session() as db:
|
||||
settings = await get_all_settings(db)
|
||||
running = job.is_running()
|
||||
target = job.target_from(settings)
|
||||
verdict = job.verdict(settings, running)
|
||||
history = [_history_row(e, now) for e in settings.get("backup.history") or []]
|
||||
nxt = next_backup_at(settings, now)
|
||||
out = {
|
||||
"enabled": bool(settings.get("backup.enabled")),
|
||||
"running": running,
|
||||
"verdict": {"tone": verdict["tone"], "text": verdict["text"],
|
||||
"at": iso(verdict.get("moment")), "error": verdict.get("error")},
|
||||
"bucket": ({"endpoint": redact_url(target.endpoint), "bucket": target.bucket,
|
||||
"region": target.region} if target else None),
|
||||
"schedule": {"hour": int(settings.get("backup.schedule_hour", 3)),
|
||||
"time_zone": settings.get("backup.schedule_tz") or "UTC",
|
||||
"next_at": iso(nxt)},
|
||||
"last_success_at": iso(job._moment(settings.get("backup.last_success_at"))),
|
||||
"last_success_age_s": age_s(job._moment(settings.get("backup.last_success_at")), now),
|
||||
**bounded(history, clamp_limit(limit, 10, job.HISTORY_KEEP), key="runs"),
|
||||
}
|
||||
if bucket:
|
||||
if target is None:
|
||||
out["stored_error"] = "The bucket isn't set up: fill in the endpoint, bucket and keys."
|
||||
else:
|
||||
try:
|
||||
found = await list_objects(target, job.PREFIX, limit=1000)
|
||||
except S3Error as exc:
|
||||
out["stored_error"] = str(exc) + (
|
||||
" An append-only key can't list; the runs above are what was sent."
|
||||
if "AccessDenied" in str(exc) else "")
|
||||
else:
|
||||
stored = sorted((o for o in found if job.is_backup_name(o.key)),
|
||||
key=lambda o: o.key, reverse=True)
|
||||
out["stored"] = [{"name": o.key, "size_bytes": o.size, "modified_at": o.last_modified}
|
||||
for o in stored[:50]]
|
||||
out["stored_total"] = len(stored)
|
||||
return out
|
||||
|
||||
|
||||
def _table(t) -> dict:
|
||||
row = {"table": t.table, "same": t.same, "changes": t.changes}
|
||||
if t.note:
|
||||
row["note"] = t.note
|
||||
if t.differ:
|
||||
row["differ"] = [
|
||||
{"label": d.label, "columns": d.columns, "secret": d.secret,
|
||||
**({"backup_value": d.backup_value, "here_value": d.here_value}
|
||||
if not d.secret and (d.backup_value is not None or d.here_value is not None) else {})}
|
||||
for d in t.differ[:ROWS_SHOWN]]
|
||||
for key, rows in (("only_in_backup", t.only_backup), ("only_here", t.only_here)):
|
||||
if rows:
|
||||
row[key] = rows[:ROWS_SHOWN]
|
||||
cut = sum(max(0, len(x) - ROWS_SHOWN) for x in (t.differ, t.only_backup, t.only_here))
|
||||
if cut:
|
||||
row["not_named"] = cut
|
||||
return row
|
||||
|
||||
|
||||
def report_dict(report) -> dict:
|
||||
"""A CheckReport as the agent reads it: the verdict first, then only the
|
||||
tables that differ."""
|
||||
return {
|
||||
"name": report.name,
|
||||
"made_at": iso(report.made_at),
|
||||
"ok": report.ok,
|
||||
"summary": report.summary(),
|
||||
"schema": report.schema,
|
||||
"backup_revisions": report.backup_heads,
|
||||
"here_revisions": report.here_heads,
|
||||
"key_same": report.key_same,
|
||||
"sealed": report.sealed,
|
||||
"unopened": report.unopened,
|
||||
"rows": report.rows,
|
||||
"changes": report.changes,
|
||||
"areas": [
|
||||
{"area": a.name, "same": a.same, "changes": a.changes,
|
||||
"tables": [_table(t) for t in a.tables if t.changes or t.note]}
|
||||
for a in report.areas],
|
||||
}
|
||||
|
||||
|
||||
async def check_backup(name: str) -> dict:
|
||||
"""Fetch one backup from the bucket and compare it with what is in place,
|
||||
as Settings → Backups → Check does. Nothing changes here.
|
||||
|
||||
name: the file, e.g. "steward-20261015T030000Z.tar" (list_backups).
|
||||
|
||||
`ok` is whether the backup's own key opens every secret in it, so a
|
||||
restore would bring them back. `summary` says it in a line. `schema` is
|
||||
"same", "older" (made by an earlier Steward; a restore upgrades it) or
|
||||
"newer" (made by a later Steward; this one can't restore it). `key_same`
|
||||
is whether the backup carries this Steward's secret key. `changes`
|
||||
counts rows that differ, exist only in the backup, or only here; `areas`
|
||||
names them, table by table, with a setting's two values when it isn't
|
||||
secret. A difference is what a restore would put back, not a fault.
|
||||
|
||||
An append-only key may not read files back; then this fails, and the
|
||||
check can be made on the page with a downloaded copy instead.
|
||||
"""
|
||||
try:
|
||||
report = await job.check_stored(mcp_app(), (name or "").strip())
|
||||
except (BackupError, S3Error) as exc:
|
||||
said = str(exc)
|
||||
if "AccessDenied" in said:
|
||||
said += (" The bucket's key may not read backups back; download the file and "
|
||||
"check it on Settings → Backups instead.")
|
||||
raise ValueError(said) from None
|
||||
return report_dict(report)
|
||||
|
||||
|
||||
async def make_backup() -> dict:
|
||||
"""Make a backup now and send it to the bucket, as Settings → Backups →
|
||||
Back up now does. It runs in the background: list_backups shows
|
||||
`running`, then the new run at the top with its check. Audited.
|
||||
|
||||
Refused while one is already being made, or when no bucket is set up.
|
||||
"""
|
||||
async with db_session() as db:
|
||||
settings = await get_all_settings(db)
|
||||
if job.target_from(settings) is None:
|
||||
raise ValueError("The bucket isn't set up: fill in the endpoint, bucket and keys on "
|
||||
"Settings → Backups. Nothing was started.")
|
||||
caller = current_caller()
|
||||
if not job.start_backup(mcp_app(), by=mcp_username(caller.username), user_id=caller.user_id):
|
||||
raise ValueError("A backup is already being made; list_backups shows it. Nothing was started.")
|
||||
await audit_mcp("backup.started", "backup")
|
||||
return {"started": True,
|
||||
"summary": "Making a backup now; list_backups shows it once it is sent and checked."}
|
||||
|
||||
|
||||
def register(register_tool) -> None:
|
||||
for fn in (list_backups, check_backup):
|
||||
register_tool(fn, role=UserRole.admin, read_only=True)
|
||||
register_tool(make_backup, role=UserRole.admin, read_only=False)
|
||||
@@ -7,8 +7,10 @@ from datetime import timedelta
|
||||
from sqlalchemy import select
|
||||
|
||||
from steward.alerts import service as alert_svc
|
||||
from steward.backup import job as backup_job
|
||||
from steward.core import alerts
|
||||
from steward.core import anomalies as anom
|
||||
from steward.core.settings import get_all_settings
|
||||
from steward.hosts import service as host_svc
|
||||
from steward.mcp._context import mcp_app
|
||||
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
||||
@@ -22,6 +24,9 @@ from ._shape import age_s, db_session, iso, now_utc, rnd
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FINDINGS_CAP = 50
|
||||
# A scheduled backup this long past its slot with none made is overdue: one
|
||||
# 15-minute tick and the hour a failed try waits, with room to spare.
|
||||
BACKUP_OVERDUE_AFTER = timedelta(hours=2)
|
||||
_SEVERITY = {"critical": 0, "warning": 1}
|
||||
|
||||
|
||||
@@ -202,6 +207,41 @@ def anomaly_findings(open_now) -> list[dict]:
|
||||
return out
|
||||
|
||||
|
||||
def backup_findings(settings: dict, running: bool, now) -> list[dict]:
|
||||
"""Steward's own backups (#5836), while scheduled ones are on: the bucket
|
||||
isn't set up, the last one failed (since the first of the failures in a
|
||||
row), or none has been made since the day's slot."""
|
||||
from steward.core.formatting import zone
|
||||
if running or not settings.get("backup.enabled"):
|
||||
return []
|
||||
subject, nxt = "Steward backups", "list_backups()"
|
||||
if backup_job.target_from(settings) is None:
|
||||
return [finding("warning", "backup_not_set_up", subject,
|
||||
"Scheduled backups are on, but the bucket isn't set up, so none are made",
|
||||
next=nxt)]
|
||||
history = settings.get("backup.history") or []
|
||||
last_ok = backup_job._moment(settings.get("backup.last_success_at"))
|
||||
good = (f"the newest good one was sent {iso(last_ok)}" if last_ok
|
||||
else "none has been sent yet")
|
||||
failing = []
|
||||
for entry in history:
|
||||
if entry.get("ok"):
|
||||
break
|
||||
failing.append(entry)
|
||||
if failing:
|
||||
tries = f" ({len(failing)} tries in a row)" if len(failing) > 1 else ""
|
||||
return [finding("warning", "backup_failed", subject,
|
||||
f"The last backup failed{tries}: {failing[0].get('error') or 'no reason recorded'}; {good}",
|
||||
since=backup_job._moment(failing[-1].get("at")), next=nxt)]
|
||||
slot = backup_job.last_slot(now, int(settings.get("backup.schedule_hour", 3)),
|
||||
zone(settings.get("backup.schedule_tz")))
|
||||
if history and (last_ok is None or last_ok < slot) and now - slot >= BACKUP_OVERDUE_AFTER:
|
||||
return [finding("warning", "backup_overdue", subject,
|
||||
f"No backup has been made since the scheduled one at {iso(slot)}; {good}",
|
||||
since=slot, next=nxt)]
|
||||
return []
|
||||
|
||||
|
||||
async def _plugin_findings(server) -> tuple[list[dict], list[str]]:
|
||||
found, unavailable = [], []
|
||||
for plugin, source in sorted((getattr(server, "overview_sources", None) or {}).items()):
|
||||
@@ -225,7 +265,8 @@ async def get_overview() -> dict:
|
||||
agents that stopped reporting, hosts over their CPU, memory or disk
|
||||
limits (all cutoffs are set under Settings → Thresholds), Ansible runs that failed
|
||||
in the last 24h, metrics going on now well above what is normal for them
|
||||
at this hour (metric anomalies), and anything enabled plugins add
|
||||
at this hour (metric anomalies), Steward's own scheduled backup failing or
|
||||
overdue, and anything enabled plugins add
|
||||
(unhealthy containers, for one). Each finding carries the ids to follow
|
||||
and `next`, the call that shows its evidence.
|
||||
|
||||
@@ -254,6 +295,7 @@ async def get_overview() -> dict:
|
||||
windows = await alert_svc.list_maintenance_windows(db)
|
||||
open_anomalies = await anom.open_anomalies(db)
|
||||
to_review = await anom.to_review_count(db)
|
||||
settings = await get_all_settings(db)
|
||||
|
||||
thresholds = app.config.get("THRESHOLDS") or {}
|
||||
listed, silent, speaking = fold_agent_silence(active, agent)
|
||||
@@ -261,7 +303,8 @@ async def get_overview() -> dict:
|
||||
+ alert_findings(listed, silent)
|
||||
+ agent_findings(hosts, agent, thresholds, silent, speaking)
|
||||
+ run_findings(runs)
|
||||
+ anomaly_findings(open_anomalies))
|
||||
+ anomaly_findings(open_anomalies)
|
||||
+ backup_findings(settings, backup_job.is_running(), now))
|
||||
plugin_found, unavailable = await _plugin_findings(getattr(app, "mcp_server", None))
|
||||
findings = rank(findings + plugin_found)
|
||||
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Integration (#5836): the MCP's backup tools against real Postgres.
|
||||
|
||||
The history Steward keeps in its settings, read back through list_backups;
|
||||
make_backup refused with no bucket, then started (the run itself stubbed, as
|
||||
in test_backup_job.py) and audited; the overview names a failing backup.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
if not os.environ.get("STEWARD_DATABASE_URL"):
|
||||
pytest.skip("needs Postgres")
|
||||
from steward.app import create_app
|
||||
return create_app(testing=False)
|
||||
|
||||
|
||||
async def _configure(app, **over):
|
||||
from steward.core.settings import set_setting
|
||||
values = {"backup.enabled": True, "backup.s3_endpoint": "http://user:pw@archives:8081",
|
||||
"backup.s3_bucket": "steward", "backup.s3_access_key": "AK",
|
||||
"backup.s3_secret_key": "SK", "backup.schedule_hour": 3, "backup.schedule_tz": "UTC",
|
||||
"backup.history": [], "backup.last_success_at": "", "backup.last_attempt_at": "", **over}
|
||||
async with app.db_sessionmaker() as db:
|
||||
async with db.begin():
|
||||
for k, v in values.items():
|
||||
await set_setting(db, k, v)
|
||||
|
||||
|
||||
def test_backups_read_made_and_reported_through_mcp(app, monkeypatch):
|
||||
from sqlalchemy import select
|
||||
from steward.backup import job
|
||||
from steward.core.api_tokens import Caller
|
||||
from steward.mcp import _context
|
||||
from steward.mcp.tools import backups, overview
|
||||
from steward.models.audit import AuditEvent
|
||||
from steward.models.users import User, UserRole
|
||||
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
now = datetime.now(timezone.utc)
|
||||
good_at, failed_at = now - timedelta(days=1), now - timedelta(minutes=10)
|
||||
history = [
|
||||
{"at": failed_at.isoformat(), "ok": False, "by": "schedule", "name": "", "size": 0,
|
||||
"secs": 1.0, "error": "Upload of steward-x.tar refused: 503"},
|
||||
{"at": good_at.isoformat(), "ok": True, "by": f"bk-{tag}", "name": "steward-20261014T030000Z.tar",
|
||||
"size": 52000, "secs": 4.2, "error": "",
|
||||
"check": {"ok": True, "changes": 0, "rows": 310, "summary": "Matches what is in place."}},
|
||||
]
|
||||
made = []
|
||||
|
||||
async def _fake_run(app_, by="schedule", user_id=None):
|
||||
made.append((by, user_id))
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(job, "run_backup", _fake_run)
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
user = User(username=f"bk-{tag}", email=f"bk-{tag}@test", password_hash="x",
|
||||
role=UserRole.admin)
|
||||
s.add(user)
|
||||
await s.flush()
|
||||
user_id = user.id
|
||||
_context._app_ctx.set(app)
|
||||
_context._caller_ctx.set(Caller(user_id=user_id, username=f"bk-{tag}",
|
||||
role=UserRole.admin, token_id=9))
|
||||
|
||||
await _configure(app, **{"backup.s3_bucket": ""})
|
||||
with pytest.raises(ValueError, match="bucket isn't set up.*Nothing was started"):
|
||||
await backups.make_backup()
|
||||
|
||||
await _configure(app, **{"backup.history": history,
|
||||
"backup.last_success_at": good_at.isoformat(),
|
||||
"backup.last_attempt_at": failed_at.isoformat()})
|
||||
listed = await backups.list_backups()
|
||||
assert listed["enabled"] is True and listed["running"] is False
|
||||
assert listed["bucket"] == {"endpoint": "http://user:***@archives:8081", "bucket": "steward",
|
||||
"region": "us-east-1"}
|
||||
assert listed["verdict"]["tone"] == "bad" and "503" in listed["verdict"]["error"]
|
||||
assert listed["schedule"]["next_at"] is not None
|
||||
failed, good = listed["runs"]
|
||||
assert failed["ok"] is False and failed["by"] == "schedule" and failed["name"] is None
|
||||
assert good["name"] == "steward-20261014T030000Z.tar" and good["size_bytes"] == 52000
|
||||
assert good["check"]["rows"] == 310 and listed["last_success_age_s"] >= 86000
|
||||
assert (await backups.list_backups(limit=1))["truncated"] is True
|
||||
|
||||
started = await backups.make_backup()
|
||||
assert started["started"] is True
|
||||
await asyncio.sleep(0) # the background task runs the stub
|
||||
assert made == [(f"bk-{tag} via MCP", user_id)]
|
||||
|
||||
async with app.db_sessionmaker() as s:
|
||||
rows = list((await s.execute(select(AuditEvent).where(
|
||||
AuditEvent.user_id == user_id, AuditEvent.action == "backup.started"))).scalars())
|
||||
assert len(rows) == 1 and rows[0].username == f"bk-{tag} via MCP"
|
||||
|
||||
ov = await overview.get_overview()
|
||||
mine = [f for f in ov["findings"] if f["kind"] == "backup_failed"]
|
||||
assert len(mine) == 1 and "refused: 503" in mine[0]["summary"]
|
||||
|
||||
await _configure(app, **{"backup.enabled": False})
|
||||
|
||||
asyncio.run(_go())
|
||||
@@ -0,0 +1,109 @@
|
||||
"""The backup tools' pure parts (#5836): when the next backup is due, what the
|
||||
overview says about backups, how a check reads to an agent, and their roles.
|
||||
|
||||
Reading the history, starting one and the audit row run against Postgres in
|
||||
tests/integration/test_mcp_backups.py.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from steward.backup.check import AreaCheck, CheckReport, RowDiff, TableCheck
|
||||
from steward.mcp.tools import backups, overview
|
||||
from steward.models.users import UserRole
|
||||
|
||||
NOW = datetime(2026, 10, 15, 6, 0, tzinfo=timezone.utc) # three hours after the 03:00 slot
|
||||
SLOT = datetime(2026, 10, 15, 3, 0, tzinfo=timezone.utc)
|
||||
|
||||
BUCKET = {"backup.s3_endpoint": "http://archives:8081", "backup.s3_bucket": "steward",
|
||||
"backup.s3_access_key": "AK", "backup.s3_secret_key": "SK"}
|
||||
|
||||
|
||||
def _settings(**over) -> dict:
|
||||
return {"backup.enabled": True, "backup.schedule_hour": 3, "backup.schedule_tz": "UTC",
|
||||
**BUCKET, **over}
|
||||
|
||||
|
||||
def _entry(at: datetime, ok: bool, error: str = "") -> dict:
|
||||
return {"at": at.isoformat(), "ok": ok, "by": "schedule", "name": "steward-x.tar" if ok else "",
|
||||
"size": 1000 if ok else 0, "secs": 2.0, "error": error}
|
||||
|
||||
|
||||
def test_the_next_backup_is_tomorrows_slot_an_hour_after_a_failure_or_the_next_tick():
|
||||
assert backups.next_backup_at(_settings(**{"backup.enabled": False}), NOW) is None
|
||||
done = _settings(**{"backup.last_success_at": (SLOT + timedelta(minutes=5)).isoformat()})
|
||||
assert backups.next_backup_at(done, NOW) == SLOT + timedelta(days=1)
|
||||
tried = _settings(**{"backup.last_success_at": (SLOT - timedelta(days=1)).isoformat(),
|
||||
"backup.last_attempt_at": (NOW - timedelta(minutes=20)).isoformat()})
|
||||
assert backups.next_backup_at(tried, NOW) == NOW + timedelta(minutes=40)
|
||||
due = _settings(**{"backup.last_success_at": (SLOT - timedelta(days=1)).isoformat()})
|
||||
assert backups.next_backup_at(due, NOW) == NOW + timedelta(minutes=15)
|
||||
|
||||
|
||||
def test_the_overview_says_nothing_about_backups_that_are_off_running_or_fine():
|
||||
assert overview.backup_findings({"backup.enabled": False}, False, NOW) == []
|
||||
failed = _settings(**{"backup.history": [_entry(NOW, False, "refused")]})
|
||||
assert overview.backup_findings(failed, True, NOW) == []
|
||||
fine = _settings(**{"backup.history": [_entry(SLOT, True)],
|
||||
"backup.last_success_at": SLOT.isoformat()})
|
||||
assert overview.backup_findings(fine, False, NOW) == []
|
||||
assert overview.backup_findings(_settings(), False, NOW) == [] # never run yet: not overdue
|
||||
|
||||
|
||||
def test_backups_on_with_no_bucket_is_a_finding():
|
||||
settings = {"backup.enabled": True, "backup.schedule_hour": 3}
|
||||
[f] = overview.backup_findings(settings, False, NOW)
|
||||
assert f["kind"] == "backup_not_set_up" and f["next"] == "list_backups()"
|
||||
|
||||
|
||||
def test_failed_backups_are_counted_since_the_first_in_a_row():
|
||||
good = SLOT - timedelta(days=1)
|
||||
history = [_entry(NOW - timedelta(hours=1), False, "Upload of steward-x.tar refused: 500"),
|
||||
_entry(SLOT, False, "unreachable"), _entry(good, True)]
|
||||
[f] = overview.backup_findings(_settings(**{"backup.history": history,
|
||||
"backup.last_success_at": good.isoformat()}),
|
||||
False, NOW)
|
||||
assert f["kind"] == "backup_failed" and f["severity"] == "warning"
|
||||
assert "(2 tries in a row): Upload of steward-x.tar refused: 500" in f["summary"]
|
||||
assert f"the newest good one was sent {overview.iso(good)}" in f["summary"]
|
||||
assert f["since"] == "2026-10-15T03:00:00Z"
|
||||
|
||||
|
||||
def test_no_backup_two_hours_past_its_slot_is_overdue():
|
||||
yesterday = SLOT - timedelta(days=1)
|
||||
settings = _settings(**{"backup.history": [_entry(yesterday, True)],
|
||||
"backup.last_success_at": yesterday.isoformat()})
|
||||
[f] = overview.backup_findings(settings, False, NOW)
|
||||
assert f["kind"] == "backup_overdue" and f["since"] == "2026-10-15T03:00:00Z"
|
||||
assert overview.backup_findings(settings, False, SLOT + timedelta(minutes=90)) == []
|
||||
|
||||
|
||||
def test_a_check_names_only_the_tables_that_differ_and_keeps_secret_values_out():
|
||||
many = [f"host-{i}" for i in range(backups.ROWS_SHOWN + 5)]
|
||||
report = CheckReport(
|
||||
name="steward-20261015T030000Z.tar", manifest={"created_at": SLOT.isoformat()},
|
||||
schema="same", backup_heads=["a1"], here_heads=["a1"], key_same=True, sealed=4, unopened=0,
|
||||
areas=[AreaCheck("Settings", [
|
||||
TableCheck("app_settings", same=10, differ=[
|
||||
RowDiff("backup.schedule_hour", ["value_json"], False, "3", "4"),
|
||||
RowDiff("smtp.password", ["value_json"], True)]),
|
||||
TableCheck("users", same=2)]),
|
||||
AreaCheck("Hosts", [TableCheck("hosts", same=1, only_backup=many)])])
|
||||
out = backups.report_dict(report)
|
||||
assert out["ok"] is True and out["made_at"] == "2026-10-15T03:00:00Z"
|
||||
assert out["changes"] == 2 + len(many) and out["rows"] == 13 + out["changes"]
|
||||
settings_area, hosts_area = out["areas"]
|
||||
assert [t["table"] for t in settings_area["tables"]] == ["app_settings"]
|
||||
plain, secret = settings_area["tables"][0]["differ"]
|
||||
assert (plain["backup_value"], plain["here_value"]) == ("3", "4")
|
||||
assert secret == {"label": "smtp.password", "columns": ["value_json"], "secret": True}
|
||||
hosts = hosts_area["tables"][0]
|
||||
assert len(hosts["only_in_backup"]) == backups.ROWS_SHOWN and hosts["not_named"] == 5
|
||||
|
||||
|
||||
def test_backup_tools_are_admin_like_the_page(app):
|
||||
mcp = app.mcp_server
|
||||
for name, read_only in (("list_backups", True), ("check_backup", True), ("make_backup", False)):
|
||||
assert mcp.tool_roles[name] is UserRole.admin, name
|
||||
assert mcp._tool_manager.get_tool(name).annotations.read_only_hint is read_only, name
|
||||
assert (mcp._tool_manager.get_tool(name).description or "").strip(), name
|
||||
Reference in New Issue
Block a user