feat: the Backups screen says whether backups are keeping up, and the System view warns when they aren't
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 18s
CI and images / frontend-build (push) Successful in 25s
CI and images / backend-lint-and-test (push) Successful in 45s
CI and images / integration (push) Successful in 2m31s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m54s
CI and images / smoke-web (push) Successful in 48s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 18s
CI and images / frontend-build (push) Successful in 25s
CI and images / backend-lint-and-test (push) Successful in 45s
CI and images / integration (push) Successful in 2m31s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m54s
CI and images / smoke-web (push) Successful in 48s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
- A Backups row on Settings → System and the nav dot turn amber when the newest database dump is more than a day old or failed, or the weekly bucket check found files missing that no media run has sent since. No row when backups aren't set up. - The weekly bucket check is kept as a run (kind "reconcile") with its counts, trimmed to the last 12, and listed on the Backups card. - The Backups card leads with the same verdict and the newest dump's size, shows bytes beside the running media job's file count, and lets each run open its list of failed files with why. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,120 @@
|
||||
"""The Backups row on the System view (services/backup_health.py): quiet while
|
||||
the backups keep up, a warning that says what is wrong when the newest dump is
|
||||
more than a day old, the last one failed, or the bucket check found files
|
||||
missing that no run has sent again since."""
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.app.models import BackupRun
|
||||
from backend.app.services import backup_health
|
||||
from backend.app.services.backup_health import BackupFacts, verdict
|
||||
|
||||
NOW = datetime(2026, 3, 4, 12, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
def test_no_row_when_backups_are_not_set_up():
|
||||
assert verdict(BackupFacts(expected=False, offsite=False), NOW) is None
|
||||
|
||||
|
||||
def test_a_dump_from_last_night_is_ok_and_says_its_age():
|
||||
part = verdict(BackupFacts(expected=True, offsite=True,
|
||||
last_dump_at=NOW - timedelta(hours=9)), NOW)
|
||||
assert part["state"] == "ok"
|
||||
assert part["detail"] == "The newest database backup to the off-site bucket is 9 h old"
|
||||
|
||||
|
||||
def test_a_dump_older_than_a_day_warns():
|
||||
part = verdict(BackupFacts(expected=True, offsite=False,
|
||||
last_dump_at=NOW - timedelta(days=3)), NOW)
|
||||
assert part["state"] == "stale"
|
||||
assert part["detail"] == "The newest database backup on this server is 3 days old"
|
||||
|
||||
|
||||
def test_no_dump_yet_warns():
|
||||
part = verdict(BackupFacts(expected=True, offsite=True), NOW)
|
||||
assert part["state"] == "stale"
|
||||
assert "No database backup has been taken to the off-site bucket yet" in part["detail"]
|
||||
|
||||
|
||||
def test_a_failed_dump_warns_with_its_reason_beside_the_age():
|
||||
part = verdict(BackupFacts(expected=True, offsite=True,
|
||||
last_dump_at=NOW - timedelta(hours=30),
|
||||
last_dump_error="S3Error: HTTP 403"), NOW)
|
||||
assert part["state"] == "stale"
|
||||
assert "The last database backup failed: S3Error: HTTP 403" in part["detail"]
|
||||
assert "is 30 h old" in part["detail"]
|
||||
|
||||
|
||||
def test_files_missing_from_the_bucket_warn():
|
||||
part = verdict(BackupFacts(expected=True, offsite=True,
|
||||
last_dump_at=NOW - timedelta(hours=2), missing=1234), NOW)
|
||||
assert part["state"] == "stale"
|
||||
assert "found 1,234 file(s) missing from the bucket" in part["detail"]
|
||||
|
||||
|
||||
# ── reading the facts ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _run(db_sync, kind, status, at, **kw):
|
||||
row = BackupRun(kind=kind, status=status, triggered_by="nightly", started_at=at,
|
||||
finished_at=at + timedelta(minutes=5), **kw)
|
||||
db_sync.add(row)
|
||||
db_sync.commit()
|
||||
return row
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def offsite(monkeypatch):
|
||||
monkeypatch.setattr("backend.app.services.backup_destination.describe",
|
||||
lambda settings: {"configured": True})
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_the_facts_come_from_the_newest_runs(db_sync, offsite):
|
||||
now = datetime.now(UTC)
|
||||
_run(db_sync, "db", "ok", now - timedelta(days=2))
|
||||
newest = _run(db_sync, "db", "ok", now - timedelta(hours=3))
|
||||
_run(db_sync, "db", "running", now - timedelta(minutes=1))
|
||||
|
||||
facts = backup_health.gather(db_sync)
|
||||
assert facts.expected and facts.offsite
|
||||
assert facts.last_dump_at == newest.finished_at
|
||||
assert facts.last_dump_error is None
|
||||
assert facts.missing == 0
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_the_newest_finished_dump_failing_is_reported(db_sync, offsite):
|
||||
now = datetime.now(UTC)
|
||||
_run(db_sync, "db", "ok", now - timedelta(days=2))
|
||||
_run(db_sync, "db", "error", now - timedelta(hours=3), error="DumpFailed: disk full")
|
||||
facts = backup_health.gather(db_sync)
|
||||
assert facts.last_dump_error == "DumpFailed: disk full"
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_missing_files_count_until_a_media_run_finishes_after_the_check(db_sync, offsite):
|
||||
now = datetime.now(UTC)
|
||||
check = _run(db_sync, "reconcile", "ok", now - timedelta(hours=5),
|
||||
manifest={"listed": 10, "dropped": 3, "adopted": 0})
|
||||
_run(db_sync, "media", "ok", now - timedelta(days=1))
|
||||
assert backup_health.gather(db_sync).missing == 3
|
||||
|
||||
_run(db_sync, "media", "ok", check.finished_at + timedelta(minutes=1))
|
||||
assert backup_health.gather(db_sync).missing == 0
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_nothing_is_expected_without_a_destination_or_the_nightly(db_sync, monkeypatch):
|
||||
monkeypatch.setattr("backend.app.services.backup_destination.describe",
|
||||
lambda settings: {"configured": False})
|
||||
assert backup_health.gather(db_sync).expected is False
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_the_system_view_lists_the_backups_row(client, offsite):
|
||||
body = await (await client.get("/api/system/health")).get_json()
|
||||
part = next(p for p in body["parts"] if p["key"] == "backup")
|
||||
assert part["state"] == "stale"
|
||||
assert part["detail"].startswith("No database backup has been taken")
|
||||
Reference in New Issue
Block a user