feat(mcp): Steward's own backups: the runs, the next one, checking and making one (#5836)
list_backups, check_backup and make_backup are Settings → Backups for an agent, admin only like the page. make_backup runs the page's job in the background and is audited as "<user> via MCP". Restoring stays in the browser. get_overview names a scheduled backup that failed, is two hours past its slot, or has no bucket. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -7,8 +7,10 @@ from datetime import timedelta
|
||||
from sqlalchemy import select
|
||||
|
||||
from steward.alerts import service as alert_svc
|
||||
from steward.backup import job as backup_job
|
||||
from steward.core import alerts
|
||||
from steward.core import anomalies as anom
|
||||
from steward.core.settings import get_all_settings
|
||||
from steward.hosts import service as host_svc
|
||||
from steward.mcp._context import mcp_app
|
||||
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
||||
@@ -22,6 +24,9 @@ from ._shape import age_s, db_session, iso, now_utc, rnd
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FINDINGS_CAP = 50
|
||||
# A scheduled backup this long past its slot with none made is overdue: one
|
||||
# 15-minute tick and the hour a failed try waits, with room to spare.
|
||||
BACKUP_OVERDUE_AFTER = timedelta(hours=2)
|
||||
_SEVERITY = {"critical": 0, "warning": 1}
|
||||
|
||||
|
||||
@@ -202,6 +207,41 @@ def anomaly_findings(open_now) -> list[dict]:
|
||||
return out
|
||||
|
||||
|
||||
def backup_findings(settings: dict, running: bool, now) -> list[dict]:
|
||||
"""Steward's own backups (#5836), while scheduled ones are on: the bucket
|
||||
isn't set up, the last one failed (since the first of the failures in a
|
||||
row), or none has been made since the day's slot."""
|
||||
from steward.core.formatting import zone
|
||||
if running or not settings.get("backup.enabled"):
|
||||
return []
|
||||
subject, nxt = "Steward backups", "list_backups()"
|
||||
if backup_job.target_from(settings) is None:
|
||||
return [finding("warning", "backup_not_set_up", subject,
|
||||
"Scheduled backups are on, but the bucket isn't set up, so none are made",
|
||||
next=nxt)]
|
||||
history = settings.get("backup.history") or []
|
||||
last_ok = backup_job._moment(settings.get("backup.last_success_at"))
|
||||
good = (f"the newest good one was sent {iso(last_ok)}" if last_ok
|
||||
else "none has been sent yet")
|
||||
failing = []
|
||||
for entry in history:
|
||||
if entry.get("ok"):
|
||||
break
|
||||
failing.append(entry)
|
||||
if failing:
|
||||
tries = f" ({len(failing)} tries in a row)" if len(failing) > 1 else ""
|
||||
return [finding("warning", "backup_failed", subject,
|
||||
f"The last backup failed{tries}: {failing[0].get('error') or 'no reason recorded'}; {good}",
|
||||
since=backup_job._moment(failing[-1].get("at")), next=nxt)]
|
||||
slot = backup_job.last_slot(now, int(settings.get("backup.schedule_hour", 3)),
|
||||
zone(settings.get("backup.schedule_tz")))
|
||||
if history and (last_ok is None or last_ok < slot) and now - slot >= BACKUP_OVERDUE_AFTER:
|
||||
return [finding("warning", "backup_overdue", subject,
|
||||
f"No backup has been made since the scheduled one at {iso(slot)}; {good}",
|
||||
since=slot, next=nxt)]
|
||||
return []
|
||||
|
||||
|
||||
async def _plugin_findings(server) -> tuple[list[dict], list[str]]:
|
||||
found, unavailable = [], []
|
||||
for plugin, source in sorted((getattr(server, "overview_sources", None) or {}).items()):
|
||||
@@ -225,7 +265,8 @@ async def get_overview() -> dict:
|
||||
agents that stopped reporting, hosts over their CPU, memory or disk
|
||||
limits (all cutoffs are set under Settings → Thresholds), Ansible runs that failed
|
||||
in the last 24h, metrics going on now well above what is normal for them
|
||||
at this hour (metric anomalies), and anything enabled plugins add
|
||||
at this hour (metric anomalies), Steward's own scheduled backup failing or
|
||||
overdue, and anything enabled plugins add
|
||||
(unhealthy containers, for one). Each finding carries the ids to follow
|
||||
and `next`, the call that shows its evidence.
|
||||
|
||||
@@ -254,6 +295,7 @@ async def get_overview() -> dict:
|
||||
windows = await alert_svc.list_maintenance_windows(db)
|
||||
open_anomalies = await anom.open_anomalies(db)
|
||||
to_review = await anom.to_review_count(db)
|
||||
settings = await get_all_settings(db)
|
||||
|
||||
thresholds = app.config.get("THRESHOLDS") or {}
|
||||
listed, silent, speaking = fold_agent_silence(active, agent)
|
||||
@@ -261,7 +303,8 @@ async def get_overview() -> dict:
|
||||
+ alert_findings(listed, silent)
|
||||
+ agent_findings(hosts, agent, thresholds, silent, speaking)
|
||||
+ run_findings(runs)
|
||||
+ anomaly_findings(open_anomalies))
|
||||
+ anomaly_findings(open_anomalies)
|
||||
+ backup_findings(settings, backup_job.is_running(), now))
|
||||
plugin_found, unavailable = await _plugin_findings(getattr(app, "mcp_server", None))
|
||||
findings = rank(findings + plugin_found)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user