feat: the Backups screen says whether backups are keeping up, and the System view warns when they aren't
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 18s
CI and images / frontend-build (push) Successful in 25s
CI and images / backend-lint-and-test (push) Successful in 45s
CI and images / integration (push) Successful in 2m31s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m54s
CI and images / smoke-web (push) Successful in 48s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped

- A Backups row on Settings → System and the nav dot turn amber when the
  newest database dump is more than a day old or failed, or the weekly
  bucket check found files missing that no media run has sent since.
  No row when backups aren't set up.
- The weekly bucket check is kept as a run (kind "reconcile") with its
  counts, trimmed to the last 12, and listed on the Backups card.
- The Backups card leads with the same verdict and the newest dump's size,
  shows bytes beside the running media job's file count, and lets each run
  open its list of failed files with why.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-07 17:51:39 -05:00
co-authored by Claude Opus 5.5
parent 9715143492
commit 66045794f0
11 changed files with 469 additions and 14 deletions
+34 -6
View File
@@ -280,9 +280,10 @@ def prune_backups() -> dict:
"""Daily Beat. Per-kind retention from ImportSettings.
Returns {"db_deleted": N, "images_deleted": M, "media_deleted": L,
"restore_deleted": R, "files_unlinked": K}. Media and restore runs leave
no files here; only their history is trimmed, to the latest
MEDIA_RUNS_KEPT and RESTORE_RUNS_KEPT.
"restore_deleted": R, "reconcile_deleted": C, "files_unlinked": K}. Media,
restore and reconcile runs leave no files here; only their history is
trimmed, to the latest MEDIA_RUNS_KEPT, RESTORE_RUNS_KEPT and
RECONCILE_RUNS_KEPT.
Tagged rows (tag IS NOT NULL) are never pruned.
Status='running' / 'restoring' rows are never pruned — the
recover_stalled_backup_runs sweep flips truly-stuck ones to
@@ -292,7 +293,7 @@ def prune_backups() -> dict:
"""
SessionLocal = _sync_session_factory()
counts = {"db_deleted": 0, "images_deleted": 0, "media_deleted": 0,
"restore_deleted": 0, "files_unlinked": 0}
"restore_deleted": 0, "reconcile_deleted": 0, "files_unlinked": 0}
with SessionLocal() as session:
s = ImportSettings.load_sync(session)
for kind, keep in (
@@ -300,6 +301,7 @@ def prune_backups() -> dict:
("images", s.backup_images_keep_last_n),
("media", MEDIA_RUNS_KEPT),
("restore", RESTORE_RUNS_KEPT),
("reconcile", RECONCILE_RUNS_KEPT),
):
candidates = session.execute(
select(BackupRun)
@@ -365,6 +367,8 @@ MEDIA_CHUNK_SECONDS = 45 * 60
MEDIA_RUNS_KEPT = 60
# A failed file is listed on the run, up to this many; the count covers all.
MEDIA_FAILURES_LISTED = 20
# The weekly bucket check keeps about a quarter's history.
RECONCILE_RUNS_KEPT = 12
def _offsite_manifest(target) -> dict:
@@ -585,7 +589,11 @@ def reconcile_media_backup() -> dict:
"""Weekly Beat. Compare what the off-site bucket holds under `media/` with
what FabledCurator believes it sent: a missing object is sent again by
the next run, and an object already there is not sent twice. Skipped
while a media run is going, and when no destination is set up."""
while a media run is going, and when no destination is set up.
Each check is kept as a run of kind "reconcile" with its counts
({listed, dropped, adopted}), so the Backups card lists it and the System
view can warn when files were missing from the bucket."""
SessionLocal = _sync_session_factory()
with SessionLocal() as session:
target = backup_destination.saved_target(ImportSettings.load_sync(session))
@@ -597,7 +605,27 @@ def reconcile_media_backup() -> dict:
).scalar_one_or_none()
if going is not None:
return {"skipped": "a media backup is running"}
return backup_media.reconcile(session, target)
now = datetime.now(UTC)
row = BackupRun(kind="reconcile", status="running", triggered_by="weekly",
started_at=now, last_progress_at=now,
manifest={"offsite": _offsite_manifest(target)})
session.add(row)
session.commit()
run_id = row.id
try:
counts = backup_media.reconcile(session, target)
except (SoftTimeLimitExceeded, Exception) as exc:
session.rollback()
row = session.get(BackupRun, run_id)
if row is not None:
_mark_failed(session, row, exc)
raise
row = session.get(BackupRun, run_id)
row.status = "ok"
row.finished_at = datetime.now(UTC)
row.manifest = {**(row.manifest or {}), **counts}
session.commit()
return {"backup_run_id": run_id, **counts}
# ── Restoring from the off-site bucket ───────────────────────────────────────