feat: the Backups screen says whether backups are keeping up, and the System view warns when they aren't
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 18s
CI and images / frontend-build (push) Successful in 25s
CI and images / backend-lint-and-test (push) Successful in 45s
CI and images / integration (push) Successful in 2m31s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m54s
CI and images / smoke-web (push) Successful in 48s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped

- A Backups row on Settings → System and the nav dot turn amber when the
  newest database dump is more than a day old or failed, or the weekly
  bucket check found files missing that no media run has sent since.
  No row when backups aren't set up.
- The weekly bucket check is kept as a run (kind "reconcile") with its
  counts, trimmed to the last 12, and listed on the Backups card.
- The Backups card leads with the same verdict and the newest dump's size,
  shows bytes beside the running media job's file count, and lets each run
  open its list of failed files with why.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-07 17:51:39 -05:00
co-authored by Claude Opus 5.5
parent 9715143492
commit 66045794f0
11 changed files with 469 additions and 14 deletions
+120
View File
@@ -0,0 +1,120 @@
"""The Backups row on the System view (services/backup_health.py): quiet while
the backups keep up, a warning that says what is wrong when the newest dump is
more than a day old, the last one failed, or the bucket check found files
missing that no run has sent again since."""
from datetime import UTC, datetime, timedelta
import pytest
from backend.app.models import BackupRun
from backend.app.services import backup_health
from backend.app.services.backup_health import BackupFacts, verdict
NOW = datetime(2026, 3, 4, 12, 0, tzinfo=UTC)
def test_no_row_when_backups_are_not_set_up():
assert verdict(BackupFacts(expected=False, offsite=False), NOW) is None
def test_a_dump_from_last_night_is_ok_and_says_its_age():
part = verdict(BackupFacts(expected=True, offsite=True,
last_dump_at=NOW - timedelta(hours=9)), NOW)
assert part["state"] == "ok"
assert part["detail"] == "The newest database backup to the off-site bucket is 9 h old"
def test_a_dump_older_than_a_day_warns():
part = verdict(BackupFacts(expected=True, offsite=False,
last_dump_at=NOW - timedelta(days=3)), NOW)
assert part["state"] == "stale"
assert part["detail"] == "The newest database backup on this server is 3 days old"
def test_no_dump_yet_warns():
part = verdict(BackupFacts(expected=True, offsite=True), NOW)
assert part["state"] == "stale"
assert "No database backup has been taken to the off-site bucket yet" in part["detail"]
def test_a_failed_dump_warns_with_its_reason_beside_the_age():
part = verdict(BackupFacts(expected=True, offsite=True,
last_dump_at=NOW - timedelta(hours=30),
last_dump_error="S3Error: HTTP 403"), NOW)
assert part["state"] == "stale"
assert "The last database backup failed: S3Error: HTTP 403" in part["detail"]
assert "is 30 h old" in part["detail"]
def test_files_missing_from_the_bucket_warn():
part = verdict(BackupFacts(expected=True, offsite=True,
last_dump_at=NOW - timedelta(hours=2), missing=1234), NOW)
assert part["state"] == "stale"
assert "found 1,234 file(s) missing from the bucket" in part["detail"]
# ── reading the facts ───────────────────────────────────────────────────────
def _run(db_sync, kind, status, at, **kw):
row = BackupRun(kind=kind, status=status, triggered_by="nightly", started_at=at,
finished_at=at + timedelta(minutes=5), **kw)
db_sync.add(row)
db_sync.commit()
return row
@pytest.fixture
def offsite(monkeypatch):
monkeypatch.setattr("backend.app.services.backup_destination.describe",
lambda settings: {"configured": True})
@pytest.mark.integration
def test_the_facts_come_from_the_newest_runs(db_sync, offsite):
now = datetime.now(UTC)
_run(db_sync, "db", "ok", now - timedelta(days=2))
newest = _run(db_sync, "db", "ok", now - timedelta(hours=3))
_run(db_sync, "db", "running", now - timedelta(minutes=1))
facts = backup_health.gather(db_sync)
assert facts.expected and facts.offsite
assert facts.last_dump_at == newest.finished_at
assert facts.last_dump_error is None
assert facts.missing == 0
@pytest.mark.integration
def test_the_newest_finished_dump_failing_is_reported(db_sync, offsite):
now = datetime.now(UTC)
_run(db_sync, "db", "ok", now - timedelta(days=2))
_run(db_sync, "db", "error", now - timedelta(hours=3), error="DumpFailed: disk full")
facts = backup_health.gather(db_sync)
assert facts.last_dump_error == "DumpFailed: disk full"
@pytest.mark.integration
def test_missing_files_count_until_a_media_run_finishes_after_the_check(db_sync, offsite):
now = datetime.now(UTC)
check = _run(db_sync, "reconcile", "ok", now - timedelta(hours=5),
manifest={"listed": 10, "dropped": 3, "adopted": 0})
_run(db_sync, "media", "ok", now - timedelta(days=1))
assert backup_health.gather(db_sync).missing == 3
_run(db_sync, "media", "ok", check.finished_at + timedelta(minutes=1))
assert backup_health.gather(db_sync).missing == 0
@pytest.mark.integration
def test_nothing_is_expected_without_a_destination_or_the_nightly(db_sync, monkeypatch):
monkeypatch.setattr("backend.app.services.backup_destination.describe",
lambda settings: {"configured": False})
assert backup_health.gather(db_sync).expected is False
@pytest.mark.integration
async def test_the_system_view_lists_the_backups_row(client, offsite):
body = await (await client.get("/api/system/health")).get_json()
part = next(p for p in body["parts"] if p["key"] == "backup")
assert part["state"] == "stale"
assert part["detail"].startswith("No database backup has been taken")
+30
View File
@@ -324,3 +324,33 @@ def test_reconcile_resends_what_the_bucket_lost_and_adopts_what_it_has(
"listed": 2, "dropped": 1, "adopted": 1,
}
assert set(db_sync.execute(select(BackupBlob.sha256)).scalars()) == {_sha("b")}
def test_the_weekly_check_is_kept_as_a_run_with_its_counts(db_sync, run_env, monkeypatch):
from backend.app.tasks import backup as tasks
monkeypatch.setattr(backup_media, "reconcile",
lambda session, target: {"listed": 9, "dropped": 2, "adopted": 1})
out = tasks.reconcile_media_backup()
row = db_sync.get(BackupRun, out["backup_run_id"])
db_sync.refresh(row)
assert (row.kind, row.status, row.triggered_by) == ("reconcile", "ok", "weekly")
assert row.finished_at is not None
assert {k: row.manifest[k] for k in ("listed", "dropped", "adopted")} == {
"listed": 9, "dropped": 2, "adopted": 1,
}
assert row.manifest["offsite"]["bucket"] == "curator"
def test_a_weekly_check_that_fails_is_kept_with_why(db_sync, run_env, monkeypatch):
from backend.app.tasks import backup as tasks
def unreachable(session, target):
raise S3Error("HTTP 503: busy.")
monkeypatch.setattr(backup_media, "reconcile", unreachable)
with pytest.raises(S3Error):
tasks.reconcile_media_backup()
row = db_sync.execute(select(BackupRun).where(BackupRun.kind == "reconcile")).scalar_one()
assert row.status == "error"
assert row.error == "S3Error: HTTP 503: busy."