feat: the Backups screen says whether backups are keeping up, and the System view warns when they aren't
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 18s
CI and images / frontend-build (push) Successful in 25s
CI and images / backend-lint-and-test (push) Successful in 45s
CI and images / integration (push) Successful in 2m31s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m54s
CI and images / smoke-web (push) Successful in 48s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped

- A Backups row on Settings → System and the nav dot turn amber when the
  newest database dump is more than a day old or failed, or the weekly
  bucket check found files missing that no media run has sent since.
  No row when backups aren't set up.
- The weekly bucket check is kept as a run (kind "reconcile") with its
  counts, trimmed to the last 12, and listed on the Backups card.
- The Backups card leads with the same verdict and the newest dump's size,
  shows bytes beside the running media job's file count, and lets each run
  open its list of failed files with why.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-07 17:51:39 -05:00
co-authored by Claude Opus 5.5
parent 9715143492
commit 66045794f0
11 changed files with 469 additions and 14 deletions
+120
View File
@@ -0,0 +1,120 @@
"""Are the backups keeping up? One row on the System view, and the nav dot.
The backups run on their own at night, so the only way to notice one has
stopped is to be told. The System view's health endpoint asks `backup_part`,
which answers with a part like any other: `ok`, or `stale` with the sentence
that says what is wrong. It warns when
- the newest database backup is more than a day old, or there is none;
- the newest database backup failed;
- the weekly check of the off-site bucket found files missing from it, and no
media run has finished since to send them again.
No part at all when backups aren't set up (no off-site destination and the
nightly backup off): someone who chose not to back up is not told so every
fifteen seconds.
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from sqlalchemy import select
from ..models import BackupRun, ImportSettings
from . import backup_destination
# A nightly backup is due every 24 hours. The hour of slack covers the dump
# itself and a beat tick that lands a little late.
DUMP_DUE_SECONDS = 25 * 3600
@dataclass(frozen=True)
class BackupFacts:
expected: bool
offsite: bool
last_dump_at: datetime | None = None
last_dump_error: str | None = None
missing: int = 0
def _age(seconds: float) -> str:
hours = int(seconds // 3600)
if hours >= 48:
return f"{hours // 24} days"
if hours >= 1:
return f"{hours} h"
return f"{max(1, int(seconds // 60))} min"
def verdict(facts: BackupFacts, now: datetime) -> dict | None:
"""The part for these facts, or None when backups aren't set up."""
if not facts.expected:
return None
where = "to the off-site bucket" if facts.offsite else "on this server"
problems = []
if facts.last_dump_error:
problems.append(f"The last database backup failed: {facts.last_dump_error}")
if facts.last_dump_at is None:
if not facts.last_dump_error:
problems.append(f"No database backup has been taken {where} yet")
else:
age = (now - facts.last_dump_at).total_seconds()
if age >= DUMP_DUE_SECONDS:
problems.append(f"The newest database backup {where} is {_age(age)} old")
if facts.missing:
problems.append(
f"The bucket check found {facts.missing:,} file(s) missing from the bucket; "
"the next media run sends them again"
)
part = {"key": "backup", "kind": "backup", "name": "Backups"}
if problems:
return {**part, "state": "stale", "detail": ". ".join(problems)}
age = (now - facts.last_dump_at).total_seconds()
return {**part, "state": "ok",
"detail": f"The newest database backup {where} is {_age(age)} old"}
def gather(session) -> BackupFacts:
"""Read the facts `verdict` judges, from a sync session."""
settings = ImportSettings.load_sync(session)
offsite = backup_destination.describe(settings)["configured"]
expected = offsite or bool(settings.backup_db_nightly_enabled)
if not expected:
return BackupFacts(expected=False, offsite=False)
def newest(kind: str, statuses: tuple[str, ...]):
return session.execute(
select(BackupRun).where(BackupRun.kind == kind)
.where(BackupRun.status.in_(statuses))
.order_by(BackupRun.started_at.desc()).limit(1)
).scalar_one_or_none()
last_ok = newest("db", ("ok",))
last_done = newest("db", ("ok", "error"))
failed = None
if last_done is not None and last_done.status == "error":
failed = last_done.error or "no reason recorded"
missing = 0
if offsite:
check = newest("reconcile", ("ok",))
dropped = int((check.manifest or {}).get("dropped") or 0) if check else 0
if dropped:
sent_since = session.execute(
select(BackupRun.id).where(BackupRun.kind == "media")
.where(BackupRun.status == "ok")
.where(BackupRun.finished_at > check.finished_at).limit(1)
).scalar_one_or_none()
missing = 0 if sent_since is not None else dropped
return BackupFacts(
expected=True, offsite=offsite,
last_dump_at=(last_ok.finished_at or last_ok.started_at) if last_ok else None,
last_dump_error=failed, missing=missing,
)
def backup_part(session, now: datetime) -> dict | None:
return verdict(gather(session), now)