feat: test a database backup without touching the install

"Test a backup" on the Backups card loads a dump into a separate, temporary
database on the same Postgres server, the same way a restore does, checks every
table against the row counts recorded with the dump, brings the copy up to this
version, and drops it. The live database is only read.

Every dump now records its row counts, read in the snapshot pg_dump dumps from
(pg_export_snapshot + --snapshot), so the comparison is exact. Off-site the
counts sit beside the dump as <name>.counts.json; locally they go in the
manifest file. A dump taken before this is compared with the live database
instead.

A test cut off by a restart is ended, and its database dropped, when the long
maintenance lane starts. A test doesn't block backups; a restore and a second
test wait for it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/tasks/backup.py

@@ -26,6 +26,7 @@ from ..services import (
     backup_restore,
     backup_s3,
     backup_service,
+    backup_verify,
 )
 from ._sync_engine import get_sync_engine
 from ._sync_engine import sync_session_factory as _sync_session_factory
@@ -86,9 +87,7 @@ def backup_db_task(self, *, tag: str | None = None,
             target = backup_destination.saved_target(settings)
             limit_mbps = settings.backup_upload_limit_mbps
         if target is not None:
-            result = backup_service.backup_db_offsite(
-                db_url=cfg.database_url_sync, target=target, limit_mbps=limit_mbps,
-            )
+            result = _dump_offsite(cfg, target, limit_mbps)
         else:
             result = backup_service.backup_db(
                 db_url=cfg.database_url_sync, images_root=IMAGES_ROOT,
@@ -280,10 +279,10 @@ def prune_backups() -> dict:
     """Daily Beat. Per-kind retention from ImportSettings.
 
     Returns {"db_deleted": N, "images_deleted": M, "media_deleted": L,
-    "restore_deleted": R, "reconcile_deleted": C, "files_unlinked": K}. Media,
-    restore and reconcile runs leave no files here; only their history is
-    trimmed, to the latest MEDIA_RUNS_KEPT, RESTORE_RUNS_KEPT and
-    RECONCILE_RUNS_KEPT.
+    "restore_deleted": R, "reconcile_deleted": C, "test_deleted": T,
+    "files_unlinked": K}. Media, restore, reconcile and test runs leave no
+    files here; only their history is trimmed, to the latest MEDIA_RUNS_KEPT,
+    RESTORE_RUNS_KEPT, RECONCILE_RUNS_KEPT and TEST_RUNS_KEPT.
     Tagged rows (tag IS NOT NULL) are never pruned.
     Status='running' / 'restoring' rows are never pruned — the
     recover_stalled_backup_runs sweep flips truly-stuck ones to
@@ -293,7 +292,8 @@ def prune_backups() -> dict:
     """
     SessionLocal = _sync_session_factory()
     counts = {"db_deleted": 0, "images_deleted": 0, "media_deleted": 0,
-              "restore_deleted": 0, "reconcile_deleted": 0, "files_unlinked": 0}
+              "restore_deleted": 0, "reconcile_deleted": 0, "test_deleted": 0,
+              "files_unlinked": 0}
     with SessionLocal() as session:
         s = ImportSettings.load_sync(session)
         for kind, keep in (
@@ -302,6 +302,7 @@ def prune_backups() -> dict:
             ("media", MEDIA_RUNS_KEPT),
             ("restore", RESTORE_RUNS_KEPT),
             ("reconcile", RECONCILE_RUNS_KEPT),
+            ("test", TEST_RUNS_KEPT),
         ):
             candidates = session.execute(
                 select(BackupRun)
@@ -371,6 +372,21 @@ MEDIA_FAILURES_LISTED = 20
 RECONCILE_RUNS_KEPT = 12
 
 
+def _dump_offsite(cfg, target, limit_mbps) -> dict:
+    """Dump the database to the bucket, with its row counts beside it for a
+    test restore to check against. The dump stands without its counts, so a
+    failure to store them is logged, not raised."""
+    result = backup_service.backup_db_offsite(
+        db_url=cfg.database_url_sync, target=target, limit_mbps=limit_mbps)
+    if result.get("counts"):
+        try:
+            backup_verify.store_counts(target, result["key"], result["counts"])
+        except (backup_s3.S3Error, OSError):
+            log.warning("backup: the row counts for %s weren't stored", result["key"],
+                        exc_info=True)
+    return result
+
+
 def _offsite_manifest(target) -> dict:
     return {"endpoint": target.endpoint, "bucket": target.bucket,
             "key": backup_media.PREFIX}
@@ -439,6 +455,11 @@ def resume_offsite_runs() -> dict:
             ).scalars()
             if (r.manifest or {}).get("phase") == "files"
         ]
+    try:
+        tests = end_cut_off_tests()
+    except Exception:  # noqa: BLE001 — resuming the runs below matters more
+        log.exception("couldn't end a test restore cut off by the restart")
+        tests = 0
     for run_id in media:
         backup_media_task.delay(run_id=run_id, resume=True)
     for run_id in restores:
@@ -446,7 +467,7 @@ def resume_offsite_runs() -> dict:
     if media or restores:
         log.info("resumed off-site runs after a restart: media %s, restore %s",
                  media, restores)
-    return {"media": media, "restore": restores}
+    return {"media": media, "restore": restores, "tests_ended": tests}
 
 
 @celery.task(
@@ -647,7 +668,11 @@ def backup_busy(session) -> str | None:
     ).first()
     if row is None:
         return None
-    return "a restore is going" if row.kind == "restore" else f"a {row.kind} backup is running"
+    if row.kind == "restore":
+        return "a restore is going"
+    if row.kind == "test":
+        return "a test restore is running"
+    return f"a {row.kind} backup is running"
 
 
 def restore_going(session) -> bool:
@@ -657,6 +682,134 @@ def restore_going(session) -> bool:
     ).first() is not None
 
 
+# ── Test restore ─────────────────────────────────────────────────────────────
+
+TEST_RUNS_KEPT = 20
+
+
+@celery.task(
+    name="backend.app.tasks.backup.restore_test_task",
+    bind=True,
+    max_retries=0,
+    # Downloading a dump, loading it and rebuilding its indexes; a restore of
+    # the database alone, with room to spare.
+    soft_time_limit=4 * 3600, time_limit=4 * 3600 + 900,
+)
+def restore_test_task(self, *, dump_key: str | None = None,
+                      source_backup_run_id: int | None = None) -> dict:
+    """Test a database dump without touching this install: load it into a
+    temporary database, check every table against the counts recorded with
+    the dump, bring it up to this version, and drop it again
+    (services/backup_verify.py). Tests the bucket's dump at `dump_key` (the
+    newest when None), or, given `source_backup_run_id`, a dump on this
+    server. The run ends "ok" when every table matched, "error" with what
+    didn't otherwise. Returns {'backup_run_id': N, ...}."""
+    SessionLocal = _sync_session_factory()
+    cfg = get_config()
+    now = datetime.now(UTC)
+    with SessionLocal() as session:
+        if why := restore_test_blocked(session):
+            return {"skipped": why}
+        row = BackupRun(kind="test", status="running", triggered_by="manual",
+                        started_at=now, last_progress_at=now, manifest={"phase": "fetching"})
+        session.add(row)
+        session.commit()
+        run_id = row.id
+
+    def note(phase: str, **more) -> None:
+        with SessionLocal() as session:
+            r = session.get(BackupRun, run_id)
+            _note(session, r, {**r.manifest, "phase": phase, **more})
+
+    staged = offsite = None
+    try:
+        if source_backup_run_id is not None:
+            dump_path, recorded, source = _local_dump(SessionLocal, source_backup_run_id)
+        else:
+            with SessionLocal() as session:
+                settings = ImportSettings.load_sync(session)
+                target = backup_destination.saved_target(settings)
+                limit_mbps = settings.backup_upload_limit_mbps
+            if target is None:
+                raise backup_s3.S3Error("No off-site destination is set up.")
+            dump = backup_restore.find_dump(target, dump_key)
+            source = {"key": dump.key, "ts": dump.ts, "size_bytes": dump.size_bytes}
+            offsite = {**_offsite_manifest(target), "key": dump.key}
+            note("fetching", dump=source, offsite=offsite)
+            recorded = backup_verify.fetch_counts(target, dump.key)
+            staged = backup_restore.fetch_dump(
+                target, dump, IMAGES_ROOT / "_backups", backup_s3.Pacer(limit_mbps))
+            dump_path = staged
+        note("loading", dump=source)
+        result = backup_verify.trial_restore(
+            db_url=cfg.database_url_sync, dump_path=dump_path, recorded=recorded,
+            note=note)
+    except (SoftTimeLimitExceeded, Exception) as exc:
+        with SessionLocal() as session:
+            r = session.get(BackupRun, run_id)
+            if r is not None:
+                _mark_failed(session, r, exc)
+        raise
+    finally:
+        if staged is not None:
+            staged.unlink(missing_ok=True)
+
+    with SessionLocal() as session:
+        r = session.get(BackupRun, run_id)
+        r.status = "error" if result["mismatched"] else "ok"
+        r.error = backup_verify.summary(result) if result["mismatched"] else None
+        r.finished_at = datetime.now(UTC)
+        r.size_bytes = source.get("size_bytes")
+        r.manifest = {"phase": "done", "dump": source, "offsite": offsite,
+                      "summary": backup_verify.summary(result), **result}
+        session.commit()
+    return {"backup_run_id": run_id, "mismatched": result["mismatched"]}
+
+
+def _local_dump(SessionLocal, run_id: int) -> tuple[Path, dict | None, dict]:
+    """A dump on this server: its path, its recorded counts, and what the
+    test run says it tested."""
+    with SessionLocal() as session:
+        src = session.get(BackupRun, run_id)
+        if src is None or src.kind != "db" or src.status != "ok" or not src.sql_path:
+            raise backup_s3.S3Error(f"Backup {run_id} isn't a database dump on this server.")
+        path = Path(src.sql_path)
+        if not path.is_file():
+            raise backup_s3.S3Error(f"{path} is no longer on this server.")
+        recorded = backup_verify.local_counts((src.manifest or {}).get("manifest_path"))
+        ts = (src.manifest or {}).get("ts")
+        return path, recorded, {"run_id": run_id, "ts": ts, "size_bytes": src.size_bytes}
+
+
+def restore_test_blocked(session) -> str | None:
+    """Why a test restore can't start right now, or None. A backup going
+    doesn't stop one: the test only reads this install's database."""
+    if restore_going(session):
+        return "a restore is going"
+    going = session.execute(
+        select(BackupRun.id).where(BackupRun.kind == "test")
+        .where(BackupRun.status == "running").limit(1)
+    ).first()
+    return "a test restore is already running" if going else None
+
+
+def end_cut_off_tests() -> int:
+    """End every test restore a restart cut short, and drop the temporary
+    database it left. Called when the long-maintenance lane starts, where the
+    tests run, so none is alive then."""
+    with _sync_session_factory()() as session:
+        ended = session.execute(
+            update(BackupRun).where(BackupRun.kind == "test")
+            .where(BackupRun.status == "running")
+            .values(status="error", finished_at=datetime.now(UTC),
+                    error="Interrupted: the server restarted during the test. Test again.")
+        ).rowcount or 0
+        session.commit()
+    if ended:
+        backup_verify.drop_scratch(get_config().database_url_sync)
+    return ended
+
+
 def _library_is_empty(session) -> bool:
     return session.execute(text(
         "SELECT NOT EXISTS (SELECT 1 FROM image_record) "
@@ -749,8 +902,7 @@ def _restore_database(SessionLocal, run_id: int, target, limit_mbps, dump_key) -
             # dump, and would sit in the list as the newest.
             if not _library_is_empty(session):
                 session.commit()
-                saved = backup_service.backup_db_offsite(
-                    db_url=cfg.database_url_sync, target=target, limit_mbps=limit_mbps)
+                saved = _dump_offsite(cfg, target, limit_mbps)
                 m["saved_before"] = saved["key"]
             _note(session, row, m)
             started_at, manifest = row.started_at, dict(m)