feat(mcp): Steward's own backups: the runs, the next one, checking and making one (#5836)
list_backups, check_backup and make_backup are Settings → Backups for an agent, admin only like the page. make_backup runs the page's job in the background and is audited as "<user> via MCP". Restoring stays in the browser. get_overview names a scheduled backup that failed, is two hours past its slot, or has no bucket. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -80,6 +80,11 @@ While backups are on, Steward records two metrics under the source
|
|||||||
A rule such as `backup_age_secs > 93600` (26 hours) fires when a daily backup
|
A rule such as `backup_age_secs > 93600` (26 hours) fires when a daily backup
|
||||||
has been missed.
|
has been missed.
|
||||||
|
|
||||||
|
An agent on the MCP sees the same without a rule. `get_overview` names a
|
||||||
|
scheduled backup that failed, is two hours past its slot with none made, or has
|
||||||
|
no bucket set up. `list_backups`, `check_backup` and `make_backup` are this
|
||||||
|
page's admin tools (see [MCP](mcp.md)); restoring isn't among them.
|
||||||
|
|
||||||
## Checking a backup
|
## Checking a backup
|
||||||
|
|
||||||
**Settings → Backups → Check** compares a backup with what is in place, without
|
**Settings → Backups → Check** compares a backup with what is in place, without
|
||||||
|
|||||||
@@ -107,6 +107,16 @@ The loop for building one is save → `run_playbook` in check mode on one target
|
|||||||
→ `get_ansible_run` → fix and save again → apply with `check=false`. The audit
|
→ `get_ansible_run` → fix and save again → apply with `check=false`. The audit
|
||||||
row records the size, never the content.
|
row records the size, never the content.
|
||||||
|
|
||||||
|
`list_backups`, `check_backup` and `make_backup` are Settings → Backups for an
|
||||||
|
agent (#5836), admin only like the page, since a backup holds every secret
|
||||||
|
Steward keeps. `list_backups` gives the schedule, the next run and the last 30
|
||||||
|
runs with the check each one got after it was sent (`bucket=true` also lists the
|
||||||
|
bucket, which an append-only key may refuse). `check_backup` fetches one and
|
||||||
|
compares it with what is in place, changing nothing. `make_backup` starts one in
|
||||||
|
the background and is audited. Restoring stays in the browser, behind its typed
|
||||||
|
confirmation. `get_overview` reports, for every role, a scheduled backup that
|
||||||
|
failed, is overdue, or has no bucket set up.
|
||||||
|
|
||||||
The client sees each tool's full contract: arguments, what an empty value
|
The client sees each tool's full contract: arguments, what an empty value
|
||||||
means, and what comes back. That contract is the tool's docstring, so read the
|
means, and what comes back. That contract is the tool's docstring, so read the
|
||||||
docstrings in `steward/mcp/tools/` for the detail.
|
docstrings in `steward/mcp/tools/` for the detail.
|
||||||
|
|||||||
@@ -12,9 +12,10 @@ each argument's empty value means, and what comes back.
|
|||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from . import alerts, anomalies, audit, automation, hosts, metrics, monitors, overview, session
|
from . import alerts, anomalies, audit, automation, backups, hosts, metrics, monitors, overview, session
|
||||||
|
|
||||||
_MODULES = (session, overview, hosts, monitors, alerts, anomalies, metrics, audit, automation)
|
_MODULES = (session, overview, hosts, monitors, alerts, anomalies, metrics, audit, automation,
|
||||||
|
backups)
|
||||||
|
|
||||||
|
|
||||||
def register_all(register_tool) -> None:
|
def register_all(register_tool) -> None:
|
||||||
|
|||||||
@@ -0,0 +1,208 @@
|
|||||||
|
"""Steward's own backups (#5836): what was made, where it went, how its check
|
||||||
|
came out, and making or checking one, as Settings → Backups does.
|
||||||
|
|
||||||
|
Admin only, like the page: the bucket and its backups hold every secret
|
||||||
|
Steward keeps. Restoring stays in the browser, behind its typed confirmation.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
|
from steward.backup import job
|
||||||
|
from steward.backup.dump import BackupError
|
||||||
|
from steward.backup.s3 import S3Error, list_objects
|
||||||
|
from steward.core.settings import get_all_settings
|
||||||
|
from steward.mcp._context import current_caller, mcp_app
|
||||||
|
from steward.models.users import UserRole
|
||||||
|
|
||||||
|
from ._audit import audit_mcp, mcp_username
|
||||||
|
from ._shape import age_s, bounded, clamp_limit, db_session, iso, now_utc, redact_url
|
||||||
|
|
||||||
|
ROWS_SHOWN = 20 # rows named per table in a check; the rest are counted
|
||||||
|
|
||||||
|
|
||||||
|
def next_backup_at(settings: dict, now: datetime) -> datetime | None:
|
||||||
|
"""When the schedule makes the next one: the next day's hour once today's
|
||||||
|
is done, an hour after a failed try, else the next 15-minute tick. None
|
||||||
|
while scheduled backups are off."""
|
||||||
|
from steward.core.formatting import zone
|
||||||
|
if not settings.get("backup.enabled"):
|
||||||
|
return None
|
||||||
|
hour, tz = int(settings.get("backup.schedule_hour", 3)), zone(settings.get("backup.schedule_tz"))
|
||||||
|
slot = job.last_slot(now, hour, tz)
|
||||||
|
last_success = job._moment(settings.get("backup.last_success_at"))
|
||||||
|
if last_success is not None and last_success >= slot:
|
||||||
|
return job.last_slot(slot + timedelta(hours=25), hour, tz) # tomorrow's, across a DST change
|
||||||
|
last_attempt = job._moment(settings.get("backup.last_attempt_at"))
|
||||||
|
if last_attempt is not None and now - last_attempt < job.RETRY_AFTER:
|
||||||
|
return last_attempt + job.RETRY_AFTER
|
||||||
|
return now + timedelta(seconds=job.TICK_SECONDS)
|
||||||
|
|
||||||
|
|
||||||
|
def _history_row(entry: dict, now: datetime) -> dict:
|
||||||
|
at = job._moment(entry.get("at"))
|
||||||
|
return {
|
||||||
|
"name": entry.get("name") or None, "ok": bool(entry.get("ok")),
|
||||||
|
"at": iso(at), "age_s": age_s(at, now), "by": entry.get("by") or "schedule",
|
||||||
|
"size_bytes": entry.get("size") or 0, "secs": entry.get("secs"),
|
||||||
|
"error": entry.get("error") or None, "check": entry.get("check"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def list_backups(limit: int = 10, bucket: bool = False) -> dict:
|
||||||
|
"""Steward's own backups: whether they're on, where they go, when the next
|
||||||
|
one is due, and the runs so far (newest first), each with the check made
|
||||||
|
on it just after it was sent.
|
||||||
|
|
||||||
|
`verdict` is the line Settings → Backups opens with. `running` is true
|
||||||
|
while one is being made. A run row: `name` (the file in the bucket),
|
||||||
|
`ok`, `at`, `by` (a person, or "schedule"), `size_bytes`, `secs`,
|
||||||
|
`error` and `check` (`ok` false means the backup's key didn't open all
|
||||||
|
its secrets; `changes` counts differences from what was in place, which
|
||||||
|
is not a fault).
|
||||||
|
|
||||||
|
A backup holds settings and keys, users and tokens, hosts, monitors,
|
||||||
|
alert rules, inventory and dashboards; no metrics, logs or history.
|
||||||
|
Steward never deletes backups (its key is append-only); how long they
|
||||||
|
are kept is set on the bucket's side.
|
||||||
|
|
||||||
|
bucket: true also lists the Steward backups in the bucket. An
|
||||||
|
append-only key may not list; then `stored_error` says so.
|
||||||
|
limit: runs to return, default 10, max 30 (all that are kept).
|
||||||
|
"""
|
||||||
|
now = now_utc()
|
||||||
|
async with db_session() as db:
|
||||||
|
settings = await get_all_settings(db)
|
||||||
|
running = job.is_running()
|
||||||
|
target = job.target_from(settings)
|
||||||
|
verdict = job.verdict(settings, running)
|
||||||
|
history = [_history_row(e, now) for e in settings.get("backup.history") or []]
|
||||||
|
nxt = next_backup_at(settings, now)
|
||||||
|
out = {
|
||||||
|
"enabled": bool(settings.get("backup.enabled")),
|
||||||
|
"running": running,
|
||||||
|
"verdict": {"tone": verdict["tone"], "text": verdict["text"],
|
||||||
|
"at": iso(verdict.get("moment")), "error": verdict.get("error")},
|
||||||
|
"bucket": ({"endpoint": redact_url(target.endpoint), "bucket": target.bucket,
|
||||||
|
"region": target.region} if target else None),
|
||||||
|
"schedule": {"hour": int(settings.get("backup.schedule_hour", 3)),
|
||||||
|
"time_zone": settings.get("backup.schedule_tz") or "UTC",
|
||||||
|
"next_at": iso(nxt)},
|
||||||
|
"last_success_at": iso(job._moment(settings.get("backup.last_success_at"))),
|
||||||
|
"last_success_age_s": age_s(job._moment(settings.get("backup.last_success_at")), now),
|
||||||
|
**bounded(history, clamp_limit(limit, 10, job.HISTORY_KEEP), key="runs"),
|
||||||
|
}
|
||||||
|
if bucket:
|
||||||
|
if target is None:
|
||||||
|
out["stored_error"] = "The bucket isn't set up: fill in the endpoint, bucket and keys."
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
found = await list_objects(target, job.PREFIX, limit=1000)
|
||||||
|
except S3Error as exc:
|
||||||
|
out["stored_error"] = str(exc) + (
|
||||||
|
" An append-only key can't list; the runs above are what was sent."
|
||||||
|
if "AccessDenied" in str(exc) else "")
|
||||||
|
else:
|
||||||
|
stored = sorted((o for o in found if job.is_backup_name(o.key)),
|
||||||
|
key=lambda o: o.key, reverse=True)
|
||||||
|
out["stored"] = [{"name": o.key, "size_bytes": o.size, "modified_at": o.last_modified}
|
||||||
|
for o in stored[:50]]
|
||||||
|
out["stored_total"] = len(stored)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _table(t) -> dict:
|
||||||
|
row = {"table": t.table, "same": t.same, "changes": t.changes}
|
||||||
|
if t.note:
|
||||||
|
row["note"] = t.note
|
||||||
|
if t.differ:
|
||||||
|
row["differ"] = [
|
||||||
|
{"label": d.label, "columns": d.columns, "secret": d.secret,
|
||||||
|
**({"backup_value": d.backup_value, "here_value": d.here_value}
|
||||||
|
if not d.secret and (d.backup_value is not None or d.here_value is not None) else {})}
|
||||||
|
for d in t.differ[:ROWS_SHOWN]]
|
||||||
|
for key, rows in (("only_in_backup", t.only_backup), ("only_here", t.only_here)):
|
||||||
|
if rows:
|
||||||
|
row[key] = rows[:ROWS_SHOWN]
|
||||||
|
cut = sum(max(0, len(x) - ROWS_SHOWN) for x in (t.differ, t.only_backup, t.only_here))
|
||||||
|
if cut:
|
||||||
|
row["not_named"] = cut
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def report_dict(report) -> dict:
|
||||||
|
"""A CheckReport as the agent reads it: the verdict first, then only the
|
||||||
|
tables that differ."""
|
||||||
|
return {
|
||||||
|
"name": report.name,
|
||||||
|
"made_at": iso(report.made_at),
|
||||||
|
"ok": report.ok,
|
||||||
|
"summary": report.summary(),
|
||||||
|
"schema": report.schema,
|
||||||
|
"backup_revisions": report.backup_heads,
|
||||||
|
"here_revisions": report.here_heads,
|
||||||
|
"key_same": report.key_same,
|
||||||
|
"sealed": report.sealed,
|
||||||
|
"unopened": report.unopened,
|
||||||
|
"rows": report.rows,
|
||||||
|
"changes": report.changes,
|
||||||
|
"areas": [
|
||||||
|
{"area": a.name, "same": a.same, "changes": a.changes,
|
||||||
|
"tables": [_table(t) for t in a.tables if t.changes or t.note]}
|
||||||
|
for a in report.areas],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def check_backup(name: str) -> dict:
|
||||||
|
"""Fetch one backup from the bucket and compare it with what is in place,
|
||||||
|
as Settings → Backups → Check does. Nothing changes here.
|
||||||
|
|
||||||
|
name: the file, e.g. "steward-20261015T030000Z.tar" (list_backups).
|
||||||
|
|
||||||
|
`ok` is whether the backup's own key opens every secret in it, so a
|
||||||
|
restore would bring them back. `summary` says it in a line. `schema` is
|
||||||
|
"same", "older" (made by an earlier Steward; a restore upgrades it) or
|
||||||
|
"newer" (made by a later Steward; this one can't restore it). `key_same`
|
||||||
|
is whether the backup carries this Steward's secret key. `changes`
|
||||||
|
counts rows that differ, exist only in the backup, or only here; `areas`
|
||||||
|
names them, table by table, with a setting's two values when it isn't
|
||||||
|
secret. A difference is what a restore would put back, not a fault.
|
||||||
|
|
||||||
|
An append-only key may not read files back; then this fails, and the
|
||||||
|
check can be made on the page with a downloaded copy instead.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
report = await job.check_stored(mcp_app(), (name or "").strip())
|
||||||
|
except (BackupError, S3Error) as exc:
|
||||||
|
said = str(exc)
|
||||||
|
if "AccessDenied" in said:
|
||||||
|
said += (" The bucket's key may not read backups back; download the file and "
|
||||||
|
"check it on Settings → Backups instead.")
|
||||||
|
raise ValueError(said) from None
|
||||||
|
return report_dict(report)
|
||||||
|
|
||||||
|
|
||||||
|
async def make_backup() -> dict:
|
||||||
|
"""Make a backup now and send it to the bucket, as Settings → Backups →
|
||||||
|
Back up now does. It runs in the background: list_backups shows
|
||||||
|
`running`, then the new run at the top with its check. Audited.
|
||||||
|
|
||||||
|
Refused while one is already being made, or when no bucket is set up.
|
||||||
|
"""
|
||||||
|
async with db_session() as db:
|
||||||
|
settings = await get_all_settings(db)
|
||||||
|
if job.target_from(settings) is None:
|
||||||
|
raise ValueError("The bucket isn't set up: fill in the endpoint, bucket and keys on "
|
||||||
|
"Settings → Backups. Nothing was started.")
|
||||||
|
caller = current_caller()
|
||||||
|
if not job.start_backup(mcp_app(), by=mcp_username(caller.username), user_id=caller.user_id):
|
||||||
|
raise ValueError("A backup is already being made; list_backups shows it. Nothing was started.")
|
||||||
|
await audit_mcp("backup.started", "backup")
|
||||||
|
return {"started": True,
|
||||||
|
"summary": "Making a backup now; list_backups shows it once it is sent and checked."}
|
||||||
|
|
||||||
|
|
||||||
|
def register(register_tool) -> None:
|
||||||
|
for fn in (list_backups, check_backup):
|
||||||
|
register_tool(fn, role=UserRole.admin, read_only=True)
|
||||||
|
register_tool(make_backup, role=UserRole.admin, read_only=False)
|
||||||
@@ -7,8 +7,10 @@ from datetime import timedelta
|
|||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|
||||||
from steward.alerts import service as alert_svc
|
from steward.alerts import service as alert_svc
|
||||||
|
from steward.backup import job as backup_job
|
||||||
from steward.core import alerts
|
from steward.core import alerts
|
||||||
from steward.core import anomalies as anom
|
from steward.core import anomalies as anom
|
||||||
|
from steward.core.settings import get_all_settings
|
||||||
from steward.hosts import service as host_svc
|
from steward.hosts import service as host_svc
|
||||||
from steward.mcp._context import mcp_app
|
from steward.mcp._context import mcp_app
|
||||||
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
||||||
@@ -22,6 +24,9 @@ from ._shape import age_s, db_session, iso, now_utc, rnd
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
FINDINGS_CAP = 50
|
FINDINGS_CAP = 50
|
||||||
|
# A scheduled backup this long past its slot with none made is overdue: one
|
||||||
|
# 15-minute tick and the hour a failed try waits, with room to spare.
|
||||||
|
BACKUP_OVERDUE_AFTER = timedelta(hours=2)
|
||||||
_SEVERITY = {"critical": 0, "warning": 1}
|
_SEVERITY = {"critical": 0, "warning": 1}
|
||||||
|
|
||||||
|
|
||||||
@@ -202,6 +207,41 @@ def anomaly_findings(open_now) -> list[dict]:
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def backup_findings(settings: dict, running: bool, now) -> list[dict]:
|
||||||
|
"""Steward's own backups (#5836), while scheduled ones are on: the bucket
|
||||||
|
isn't set up, the last one failed (since the first of the failures in a
|
||||||
|
row), or none has been made since the day's slot."""
|
||||||
|
from steward.core.formatting import zone
|
||||||
|
if running or not settings.get("backup.enabled"):
|
||||||
|
return []
|
||||||
|
subject, nxt = "Steward backups", "list_backups()"
|
||||||
|
if backup_job.target_from(settings) is None:
|
||||||
|
return [finding("warning", "backup_not_set_up", subject,
|
||||||
|
"Scheduled backups are on, but the bucket isn't set up, so none are made",
|
||||||
|
next=nxt)]
|
||||||
|
history = settings.get("backup.history") or []
|
||||||
|
last_ok = backup_job._moment(settings.get("backup.last_success_at"))
|
||||||
|
good = (f"the newest good one was sent {iso(last_ok)}" if last_ok
|
||||||
|
else "none has been sent yet")
|
||||||
|
failing = []
|
||||||
|
for entry in history:
|
||||||
|
if entry.get("ok"):
|
||||||
|
break
|
||||||
|
failing.append(entry)
|
||||||
|
if failing:
|
||||||
|
tries = f" ({len(failing)} tries in a row)" if len(failing) > 1 else ""
|
||||||
|
return [finding("warning", "backup_failed", subject,
|
||||||
|
f"The last backup failed{tries}: {failing[0].get('error') or 'no reason recorded'}; {good}",
|
||||||
|
since=backup_job._moment(failing[-1].get("at")), next=nxt)]
|
||||||
|
slot = backup_job.last_slot(now, int(settings.get("backup.schedule_hour", 3)),
|
||||||
|
zone(settings.get("backup.schedule_tz")))
|
||||||
|
if history and (last_ok is None or last_ok < slot) and now - slot >= BACKUP_OVERDUE_AFTER:
|
||||||
|
return [finding("warning", "backup_overdue", subject,
|
||||||
|
f"No backup has been made since the scheduled one at {iso(slot)}; {good}",
|
||||||
|
since=slot, next=nxt)]
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
async def _plugin_findings(server) -> tuple[list[dict], list[str]]:
|
async def _plugin_findings(server) -> tuple[list[dict], list[str]]:
|
||||||
found, unavailable = [], []
|
found, unavailable = [], []
|
||||||
for plugin, source in sorted((getattr(server, "overview_sources", None) or {}).items()):
|
for plugin, source in sorted((getattr(server, "overview_sources", None) or {}).items()):
|
||||||
@@ -225,7 +265,8 @@ async def get_overview() -> dict:
|
|||||||
agents that stopped reporting, hosts over their CPU, memory or disk
|
agents that stopped reporting, hosts over their CPU, memory or disk
|
||||||
limits (all cutoffs are set under Settings → Thresholds), Ansible runs that failed
|
limits (all cutoffs are set under Settings → Thresholds), Ansible runs that failed
|
||||||
in the last 24h, metrics going on now well above what is normal for them
|
in the last 24h, metrics going on now well above what is normal for them
|
||||||
at this hour (metric anomalies), and anything enabled plugins add
|
at this hour (metric anomalies), Steward's own scheduled backup failing or
|
||||||
|
overdue, and anything enabled plugins add
|
||||||
(unhealthy containers, for one). Each finding carries the ids to follow
|
(unhealthy containers, for one). Each finding carries the ids to follow
|
||||||
and `next`, the call that shows its evidence.
|
and `next`, the call that shows its evidence.
|
||||||
|
|
||||||
@@ -254,6 +295,7 @@ async def get_overview() -> dict:
|
|||||||
windows = await alert_svc.list_maintenance_windows(db)
|
windows = await alert_svc.list_maintenance_windows(db)
|
||||||
open_anomalies = await anom.open_anomalies(db)
|
open_anomalies = await anom.open_anomalies(db)
|
||||||
to_review = await anom.to_review_count(db)
|
to_review = await anom.to_review_count(db)
|
||||||
|
settings = await get_all_settings(db)
|
||||||
|
|
||||||
thresholds = app.config.get("THRESHOLDS") or {}
|
thresholds = app.config.get("THRESHOLDS") or {}
|
||||||
listed, silent, speaking = fold_agent_silence(active, agent)
|
listed, silent, speaking = fold_agent_silence(active, agent)
|
||||||
@@ -261,7 +303,8 @@ async def get_overview() -> dict:
|
|||||||
+ alert_findings(listed, silent)
|
+ alert_findings(listed, silent)
|
||||||
+ agent_findings(hosts, agent, thresholds, silent, speaking)
|
+ agent_findings(hosts, agent, thresholds, silent, speaking)
|
||||||
+ run_findings(runs)
|
+ run_findings(runs)
|
||||||
+ anomaly_findings(open_anomalies))
|
+ anomaly_findings(open_anomalies)
|
||||||
|
+ backup_findings(settings, backup_job.is_running(), now))
|
||||||
plugin_found, unavailable = await _plugin_findings(getattr(app, "mcp_server", None))
|
plugin_found, unavailable = await _plugin_findings(getattr(app, "mcp_server", None))
|
||||||
findings = rank(findings + plugin_found)
|
findings = rank(findings + plugin_found)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,113 @@
|
|||||||
|
"""Integration (#5836): the MCP's backup tools against real Postgres.
|
||||||
|
|
||||||
|
The history Steward keeps in its settings, read back through list_backups;
|
||||||
|
make_backup refused with no bucket, then started (the run itself stubbed, as
|
||||||
|
in test_backup_job.py) and audited; the overview names a failing backup.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def app():
|
||||||
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
||||||
|
pytest.skip("needs Postgres")
|
||||||
|
from steward.app import create_app
|
||||||
|
return create_app(testing=False)
|
||||||
|
|
||||||
|
|
||||||
|
async def _configure(app, **over):
|
||||||
|
from steward.core.settings import set_setting
|
||||||
|
values = {"backup.enabled": True, "backup.s3_endpoint": "http://user:pw@archives:8081",
|
||||||
|
"backup.s3_bucket": "steward", "backup.s3_access_key": "AK",
|
||||||
|
"backup.s3_secret_key": "SK", "backup.schedule_hour": 3, "backup.schedule_tz": "UTC",
|
||||||
|
"backup.history": [], "backup.last_success_at": "", "backup.last_attempt_at": "", **over}
|
||||||
|
async with app.db_sessionmaker() as db:
|
||||||
|
async with db.begin():
|
||||||
|
for k, v in values.items():
|
||||||
|
await set_setting(db, k, v)
|
||||||
|
|
||||||
|
|
||||||
|
def test_backups_read_made_and_reported_through_mcp(app, monkeypatch):
|
||||||
|
from sqlalchemy import select
|
||||||
|
from steward.backup import job
|
||||||
|
from steward.core.api_tokens import Caller
|
||||||
|
from steward.mcp import _context
|
||||||
|
from steward.mcp.tools import backups, overview
|
||||||
|
from steward.models.audit import AuditEvent
|
||||||
|
from steward.models.users import User, UserRole
|
||||||
|
|
||||||
|
tag = uuid.uuid4().hex[:8]
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
good_at, failed_at = now - timedelta(days=1), now - timedelta(minutes=10)
|
||||||
|
history = [
|
||||||
|
{"at": failed_at.isoformat(), "ok": False, "by": "schedule", "name": "", "size": 0,
|
||||||
|
"secs": 1.0, "error": "Upload of steward-x.tar refused: 503"},
|
||||||
|
{"at": good_at.isoformat(), "ok": True, "by": f"bk-{tag}", "name": "steward-20261014T030000Z.tar",
|
||||||
|
"size": 52000, "secs": 4.2, "error": "",
|
||||||
|
"check": {"ok": True, "changes": 0, "rows": 310, "summary": "Matches what is in place."}},
|
||||||
|
]
|
||||||
|
made = []
|
||||||
|
|
||||||
|
async def _fake_run(app_, by="schedule", user_id=None):
|
||||||
|
made.append((by, user_id))
|
||||||
|
return {}
|
||||||
|
|
||||||
|
monkeypatch.setattr(job, "run_backup", _fake_run)
|
||||||
|
|
||||||
|
async def _go():
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
async with s.begin():
|
||||||
|
user = User(username=f"bk-{tag}", email=f"bk-{tag}@test", password_hash="x",
|
||||||
|
role=UserRole.admin)
|
||||||
|
s.add(user)
|
||||||
|
await s.flush()
|
||||||
|
user_id = user.id
|
||||||
|
_context._app_ctx.set(app)
|
||||||
|
_context._caller_ctx.set(Caller(user_id=user_id, username=f"bk-{tag}",
|
||||||
|
role=UserRole.admin, token_id=9))
|
||||||
|
|
||||||
|
await _configure(app, **{"backup.s3_bucket": ""})
|
||||||
|
with pytest.raises(ValueError, match="bucket isn't set up.*Nothing was started"):
|
||||||
|
await backups.make_backup()
|
||||||
|
|
||||||
|
await _configure(app, **{"backup.history": history,
|
||||||
|
"backup.last_success_at": good_at.isoformat(),
|
||||||
|
"backup.last_attempt_at": failed_at.isoformat()})
|
||||||
|
listed = await backups.list_backups()
|
||||||
|
assert listed["enabled"] is True and listed["running"] is False
|
||||||
|
assert listed["bucket"] == {"endpoint": "http://user:***@archives:8081", "bucket": "steward",
|
||||||
|
"region": "us-east-1"}
|
||||||
|
assert listed["verdict"]["tone"] == "bad" and "503" in listed["verdict"]["error"]
|
||||||
|
assert listed["schedule"]["next_at"] is not None
|
||||||
|
failed, good = listed["runs"]
|
||||||
|
assert failed["ok"] is False and failed["by"] == "schedule" and failed["name"] is None
|
||||||
|
assert good["name"] == "steward-20261014T030000Z.tar" and good["size_bytes"] == 52000
|
||||||
|
assert good["check"]["rows"] == 310 and listed["last_success_age_s"] >= 86000
|
||||||
|
assert (await backups.list_backups(limit=1))["truncated"] is True
|
||||||
|
|
||||||
|
started = await backups.make_backup()
|
||||||
|
assert started["started"] is True
|
||||||
|
await asyncio.sleep(0) # the background task runs the stub
|
||||||
|
assert made == [(f"bk-{tag} via MCP", user_id)]
|
||||||
|
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
rows = list((await s.execute(select(AuditEvent).where(
|
||||||
|
AuditEvent.user_id == user_id, AuditEvent.action == "backup.started"))).scalars())
|
||||||
|
assert len(rows) == 1 and rows[0].username == f"bk-{tag} via MCP"
|
||||||
|
|
||||||
|
ov = await overview.get_overview()
|
||||||
|
mine = [f for f in ov["findings"] if f["kind"] == "backup_failed"]
|
||||||
|
assert len(mine) == 1 and "refused: 503" in mine[0]["summary"]
|
||||||
|
|
||||||
|
await _configure(app, **{"backup.enabled": False})
|
||||||
|
|
||||||
|
asyncio.run(_go())
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
"""The backup tools' pure parts (#5836): when the next backup is due, what the
|
||||||
|
overview says about backups, how a check reads to an agent, and their roles.
|
||||||
|
|
||||||
|
Reading the history, starting one and the audit row run against Postgres in
|
||||||
|
tests/integration/test_mcp_backups.py.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from steward.backup.check import AreaCheck, CheckReport, RowDiff, TableCheck
|
||||||
|
from steward.mcp.tools import backups, overview
|
||||||
|
from steward.models.users import UserRole
|
||||||
|
|
||||||
|
NOW = datetime(2026, 10, 15, 6, 0, tzinfo=timezone.utc) # three hours after the 03:00 slot
|
||||||
|
SLOT = datetime(2026, 10, 15, 3, 0, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
BUCKET = {"backup.s3_endpoint": "http://archives:8081", "backup.s3_bucket": "steward",
|
||||||
|
"backup.s3_access_key": "AK", "backup.s3_secret_key": "SK"}
|
||||||
|
|
||||||
|
|
||||||
|
def _settings(**over) -> dict:
|
||||||
|
return {"backup.enabled": True, "backup.schedule_hour": 3, "backup.schedule_tz": "UTC",
|
||||||
|
**BUCKET, **over}
|
||||||
|
|
||||||
|
|
||||||
|
def _entry(at: datetime, ok: bool, error: str = "") -> dict:
|
||||||
|
return {"at": at.isoformat(), "ok": ok, "by": "schedule", "name": "steward-x.tar" if ok else "",
|
||||||
|
"size": 1000 if ok else 0, "secs": 2.0, "error": error}
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_next_backup_is_tomorrows_slot_an_hour_after_a_failure_or_the_next_tick():
|
||||||
|
assert backups.next_backup_at(_settings(**{"backup.enabled": False}), NOW) is None
|
||||||
|
done = _settings(**{"backup.last_success_at": (SLOT + timedelta(minutes=5)).isoformat()})
|
||||||
|
assert backups.next_backup_at(done, NOW) == SLOT + timedelta(days=1)
|
||||||
|
tried = _settings(**{"backup.last_success_at": (SLOT - timedelta(days=1)).isoformat(),
|
||||||
|
"backup.last_attempt_at": (NOW - timedelta(minutes=20)).isoformat()})
|
||||||
|
assert backups.next_backup_at(tried, NOW) == NOW + timedelta(minutes=40)
|
||||||
|
due = _settings(**{"backup.last_success_at": (SLOT - timedelta(days=1)).isoformat()})
|
||||||
|
assert backups.next_backup_at(due, NOW) == NOW + timedelta(minutes=15)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_overview_says_nothing_about_backups_that_are_off_running_or_fine():
|
||||||
|
assert overview.backup_findings({"backup.enabled": False}, False, NOW) == []
|
||||||
|
failed = _settings(**{"backup.history": [_entry(NOW, False, "refused")]})
|
||||||
|
assert overview.backup_findings(failed, True, NOW) == []
|
||||||
|
fine = _settings(**{"backup.history": [_entry(SLOT, True)],
|
||||||
|
"backup.last_success_at": SLOT.isoformat()})
|
||||||
|
assert overview.backup_findings(fine, False, NOW) == []
|
||||||
|
assert overview.backup_findings(_settings(), False, NOW) == [] # never run yet: not overdue
|
||||||
|
|
||||||
|
|
||||||
|
def test_backups_on_with_no_bucket_is_a_finding():
|
||||||
|
settings = {"backup.enabled": True, "backup.schedule_hour": 3}
|
||||||
|
[f] = overview.backup_findings(settings, False, NOW)
|
||||||
|
assert f["kind"] == "backup_not_set_up" and f["next"] == "list_backups()"
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_backups_are_counted_since_the_first_in_a_row():
|
||||||
|
good = SLOT - timedelta(days=1)
|
||||||
|
history = [_entry(NOW - timedelta(hours=1), False, "Upload of steward-x.tar refused: 500"),
|
||||||
|
_entry(SLOT, False, "unreachable"), _entry(good, True)]
|
||||||
|
[f] = overview.backup_findings(_settings(**{"backup.history": history,
|
||||||
|
"backup.last_success_at": good.isoformat()}),
|
||||||
|
False, NOW)
|
||||||
|
assert f["kind"] == "backup_failed" and f["severity"] == "warning"
|
||||||
|
assert "(2 tries in a row): Upload of steward-x.tar refused: 500" in f["summary"]
|
||||||
|
assert f"the newest good one was sent {overview.iso(good)}" in f["summary"]
|
||||||
|
assert f["since"] == "2026-10-15T03:00:00Z"
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_backup_two_hours_past_its_slot_is_overdue():
|
||||||
|
yesterday = SLOT - timedelta(days=1)
|
||||||
|
settings = _settings(**{"backup.history": [_entry(yesterday, True)],
|
||||||
|
"backup.last_success_at": yesterday.isoformat()})
|
||||||
|
[f] = overview.backup_findings(settings, False, NOW)
|
||||||
|
assert f["kind"] == "backup_overdue" and f["since"] == "2026-10-15T03:00:00Z"
|
||||||
|
assert overview.backup_findings(settings, False, SLOT + timedelta(minutes=90)) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_check_names_only_the_tables_that_differ_and_keeps_secret_values_out():
|
||||||
|
many = [f"host-{i}" for i in range(backups.ROWS_SHOWN + 5)]
|
||||||
|
report = CheckReport(
|
||||||
|
name="steward-20261015T030000Z.tar", manifest={"created_at": SLOT.isoformat()},
|
||||||
|
schema="same", backup_heads=["a1"], here_heads=["a1"], key_same=True, sealed=4, unopened=0,
|
||||||
|
areas=[AreaCheck("Settings", [
|
||||||
|
TableCheck("app_settings", same=10, differ=[
|
||||||
|
RowDiff("backup.schedule_hour", ["value_json"], False, "3", "4"),
|
||||||
|
RowDiff("smtp.password", ["value_json"], True)]),
|
||||||
|
TableCheck("users", same=2)]),
|
||||||
|
AreaCheck("Hosts", [TableCheck("hosts", same=1, only_backup=many)])])
|
||||||
|
out = backups.report_dict(report)
|
||||||
|
assert out["ok"] is True and out["made_at"] == "2026-10-15T03:00:00Z"
|
||||||
|
assert out["changes"] == 2 + len(many) and out["rows"] == 13 + out["changes"]
|
||||||
|
settings_area, hosts_area = out["areas"]
|
||||||
|
assert [t["table"] for t in settings_area["tables"]] == ["app_settings"]
|
||||||
|
plain, secret = settings_area["tables"][0]["differ"]
|
||||||
|
assert (plain["backup_value"], plain["here_value"]) == ("3", "4")
|
||||||
|
assert secret == {"label": "smtp.password", "columns": ["value_json"], "secret": True}
|
||||||
|
hosts = hosts_area["tables"][0]
|
||||||
|
assert len(hosts["only_in_backup"]) == backups.ROWS_SHOWN and hosts["not_named"] == 5
|
||||||
|
|
||||||
|
|
||||||
|
def test_backup_tools_are_admin_like_the_page(app):
|
||||||
|
mcp = app.mcp_server
|
||||||
|
for name, read_only in (("list_backups", True), ("check_backup", True), ("make_backup", False)):
|
||||||
|
assert mcp.tool_roles[name] is UserRole.admin, name
|
||||||
|
assert mcp._tool_manager.get_tool(name).annotations.read_only_hint is read_only, name
|
||||||
|
assert (mcp._tool_manager.get_tool(name).description or "").strip(), name
|
||||||
Reference in New Issue
Block a user