feat: test a database backup without touching the install

"Test a backup" on the Backups card loads a dump into a separate, temporary
database on the same Postgres server, the same way a restore does, checks every
table against the row counts recorded with the dump, brings the copy up to this
version, and drops it. The live database is only read.

Every dump now records its row counts, read in the snapshot pg_dump dumps from
(pg_export_snapshot + --snapshot), so the comparison is exact. Off-site the
counts sit beside the dump as <name>.counts.json; locally they go in the
manifest file. A dump taken before this is compared with the live database
instead.

A test cut off by a restart is ended, and its database dropped, when the long
maintenance lane starts. A test doesn't block backups; a restore and a second
test wait for it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -197,6 +197,14 @@ goes again in the next run. When the newest database backup is more than a day
 old or failed, or the check found files missing, the **Backups** row on
 **Settings → System** and the dot beside the brand turn amber and say why.
 
+**Test a backup** proves a dump restores without touching your install: the dump
+is loaded into a separate, temporary database on the same Postgres server,
+every table is checked against the row counts recorded as the dump was taken,
+the copy is brought up to this version, and then it is deleted. It needs about
+as much free space on the database's disk as the database itself, while it
+runs. A dump taken before counts were recorded is shown beside the live
+database instead.
+
 To rebuild after losing the server, start a fresh install (first boot as
 above), enter the same destination, and under **Restore from the bucket** choose
 a dump. The database is replaced, then every file is fetched back to its place

backend/app/api/system_backup.py

@@ -20,7 +20,7 @@ system_backup_bp = Blueprint(
     "system_backup", __name__, url_prefix="/api/system/backup",
 )
 
-_KINDS = frozenset({"db", "images", "media", "restore", "reconcile"})
+_KINDS = frozenset({"db", "images", "media", "restore", "reconcile", "test"})
 _TAG_MAX_LEN = 64
 _BACKUP_SETTINGS_FIELDS = (
     "backup_db_nightly_enabled",
@@ -191,10 +191,52 @@ async def list_offsite_dumps():
         dumps = await asyncio.to_thread(backup_restore.list_dumps, target)
     except S3Error as exc:
         return _bad("destination_unreachable", detail=str(exc), status=502)
-    return jsonify({"dumps": [
-        {"key": d.key, "ts": d.ts, "size_bytes": d.size_bytes, "confirm": restore_token(d.ts)}
-        for d in dumps
-    ]})
+    return jsonify({
+        "dumps": [
+            {"key": d.key, "ts": d.ts, "size_bytes": d.size_bytes, "counted": d.counted,
+             "confirm": restore_token(d.ts)}
+            for d in dumps
+        ],
+        # A test restore needs about this much free space on the database's
+        # disk while it runs.
+        "database_bytes": await _database_bytes(),
+    })
+
+
+async def _database_bytes() -> int:
+    from sqlalchemy import text
+
+    async with get_session() as session:
+        return (await session.execute(
+            text("SELECT pg_database_size(current_database())"))).scalar_one()
+
+
+async def _restore_test_blocked() -> str | None:
+    from ..tasks.backup import restore_test_blocked
+
+    async with get_session() as session:
+        return await session.run_sync(restore_test_blocked)
+
+
+@system_backup_bp.route("/offsite/test", methods=["POST"])
+async def restore_test_offsite():
+    """Test a dump from the bucket without touching this install: it is
+    loaded into a temporary database, checked table by table, and dropped.
+    Body: {key}, a dump's key from /offsite/dumps, or {} for the newest.
+    Needs no confirmation; refused while a restore or another test is going."""
+    body = await request.get_json(silent=True) or {}
+    key = body.get("key")
+    if key is not None and (not isinstance(key, str) or not key.startswith("database/")):
+        return _bad("invalid_key", detail="key must be a dump's key from /offsite/dumps")
+    _, err = await _saved_target()
+    if err:
+        return err
+    busy = await _restore_test_blocked()
+    if busy:
+        return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
+    from ..tasks.backup import restore_test_task
+    restore_test_task.delay(dump_key=key)
+    return jsonify({"status": "dispatched"}), 202
 
 
 @system_backup_bp.route("/offsite/restore", methods=["POST"])
@@ -335,6 +377,25 @@ async def trigger_restore(run_id: int):
     return jsonify({"status": "dispatched", "kind": kind}), 202
 
 
+@system_backup_bp.route("/runs/<int:run_id>/test", methods=["POST"])
+async def restore_test_local(run_id: int):
+    """Test a database dump kept on this server, the same way as one in the
+    bucket. Needs no confirmation."""
+    async with get_session() as session:
+        row = await session.get(BackupRun, run_id)
+        if row is None:
+            return _bad("not_found", status=404)
+        if row.kind != "db" or row.status != "ok" or not row.sql_path:
+            return _bad("not_testable",
+                        detail="only a finished database dump on this server can be tested here")
+    busy = await _restore_test_blocked()
+    if busy:
+        return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
+    from ..tasks.backup import restore_test_task
+    restore_test_task.delay(source_backup_run_id=run_id)
+    return jsonify({"status": "dispatched"}), 202
+
+
 @system_backup_bp.route("/runs/<int:run_id>", methods=["DELETE"])
 async def delete_run(run_id: int):
     body = await request.get_json(silent=True) or {}

backend/app/services/backup_restore.py

@@ -12,6 +12,7 @@ only what a damaged library is missing.
 """
 from __future__ import annotations
 
+import os
 import re
 import shutil
 import subprocess
@@ -38,19 +39,23 @@ class Dump:
     key: str
     ts: str           # 20261107T134258Z, UTC
     size_bytes: int
+    counted: bool = False   # its row counts sit beside it, for a test restore
 
 
 def list_dumps(target: S3Target) -> list[Dump]:
     """Every database dump in the bucket, newest first."""
-    dumps, token = [], None
+    found, keys, token = [], set(), None
     while True:
         page = list_objects(target, DUMP_PREFIX, 1000, continuation=token)
         for o in page.objects:
+            keys.add(o.key)
             if m := _DUMP_NAME.match(o.key):
-                dumps.append(Dump(o.key, m.group(1), o.size))
+                found.append((o.key, m.group(1), o.size))
         token = page.next_token
         if not token:
             break
+    dumps = [Dump(k, ts, size, counted=k.removesuffix(".dump") + ".counts.json" in keys)
+             for k, ts, size in found]
     return sorted(dumps, key=lambda d: d.ts, reverse=True)
 
 
@@ -76,16 +81,18 @@ def fetch_dump(target: S3Target, dump: Dump, into: Path, pacer: Pacer | None) ->
     return path
 
 
-def migrate() -> None:
+def migrate(db_name: str | None = None) -> None:
     """Bring a restored database up to this install's schema. A dump taken by
     an older FabledCurator gains the migrations it predates; one from a newer
     FabledCurator fails here, with alembic's reason, because this install
-    doesn't know its schema."""
+    doesn't know its schema. `db_name` names another database on the same
+    server to bring up instead of this install's own."""
+    env = {**os.environ, "DB_NAME": db_name} if db_name else None
     with tempfile.TemporaryFile() as err:
         proc = subprocess.run(
             [sys.executable, "-m", "alembic", "-c", str(_ALEMBIC_INI), "upgrade", "head"],
             cwd=_ALEMBIC_INI.parent, stdout=subprocess.DEVNULL, stderr=err,
-            timeout=_MIGRATE_TIMEOUT_S,
+            timeout=_MIGRATE_TIMEOUT_S, env=env,
         )
         if proc.returncode != 0:
             err.seek(0)

backend/app/services/backup_service.py

@@ -17,14 +17,18 @@ lifecycle + soft/hard time limits + retention bookkeeping.
 from __future__ import annotations
 
 import json
+import logging
 import os
 import shutil
 import subprocess
 import tempfile
 import time
+from contextlib import contextmanager
 from datetime import UTC, datetime
 from pathlib import Path
 
+log = logging.getLogger(__name__)
+
 _BACKUPS_DIRNAME = "_backups"
 
 # Excluded from the images tarball, and each for its own reason (#4233, #4234):
@@ -65,12 +69,83 @@ _IMAGES_SUBPROCESS_TIMEOUT_S = 21000       # ~5.8 hr, under the 6-hr images soft
 _KILL_REAP_GRACE_S = 10
 
 
+# Tables whose rows stay out of every dump. `credential` holds account logins,
+# sealed with the key in /images/secrets: a restored install keeps the table,
+# empty, and you sign in to each account again.
+DATA_LEFT_OUT = ("credential",)
+
 # Every database dump, local or off-site: custom format (-Fc, compressed and
 # restored with pg_restore), without ownership or grants so it loads into any
-# install. The `credential` table's rows (account logins, sealed with the key
-# in /images/secrets) stay out: a restored install keeps the table, empty, and
-# you sign in to each account again.
-_PG_DUMP = ["pg_dump", "--no-owner", "--no-acl", "-Fc", "--exclude-table-data=credential"]
+# install.
+_PG_DUMP = ["pg_dump", "--no-owner", "--no-acl", "-Fc",
+            *(f"--exclude-table-data={t}" for t in DATA_LEFT_OUT)]
+
+
+def table_counts(conn) -> dict[str, int]:
+    """How many rows each table in the public schema holds, read on a psycopg
+    connection, in the order of the tables' names."""
+    from psycopg import sql
+
+    names = [r[0] for r in conn.execute(
+        "SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename")]
+    return {
+        n: conn.execute(sql.SQL("SELECT count(*) FROM {}").format(sql.Identifier(n))).fetchone()[0]
+        for n in names
+    }
+
+
+def database_counts(conn) -> dict:
+    """{"alembic": the schema's revision, "tables": {name: rows}}."""
+    has_version = conn.execute("SELECT to_regclass('public.alembic_version')").fetchone()[0]
+    alembic = (conn.execute("SELECT version_num FROM alembic_version").fetchone()
+               if has_version else None)
+    return {"alembic": alembic[0] if alembic else None, "tables": table_counts(conn)}
+
+
+def read_counts(db_url: str) -> dict:
+    """database_counts for the database at `db_url`, as it is now."""
+    import psycopg
+
+    with psycopg.connect(_libpq_url(db_url)) as conn:
+        return database_counts(conn)
+
+
+class DumpSnapshot:
+    """A read-only transaction whose snapshot pg_dump dumps from (it is handed
+    over with --snapshot), so rows counted here are exactly the rows the dump
+    holds. A test restore checks every table against these counts."""
+
+    def __init__(self, conn):
+        self.conn = conn
+        self.id = conn.execute("SELECT pg_export_snapshot()").fetchone()[0]
+
+    def counts(self) -> dict | None:
+        """The counts, or None when they can't be read: the dump is good
+        without them, and a test restore then says none were recorded."""
+        try:
+            return self._counts()
+        except Exception:  # noqa: BLE001 — see above
+            log.exception("backup: the dump's row counts couldn't be read")
+            return None
+
+    def _counts(self) -> dict:
+        counts = database_counts(self.conn)
+        for t in DATA_LEFT_OUT:
+            if t in counts["tables"]:
+                counts["tables"][t] = 0      # the dump holds the table, not its rows
+        return counts
+
+
+@contextmanager
+def dump_snapshot(db_url: str):
+    """Open the snapshot a dump is taken from. It stays open until the dump
+    is finished: pg_dump can only take it up while it is."""
+    import psycopg
+
+    with psycopg.connect(_libpq_url(db_url)) as conn:
+        conn.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ
+        conn.read_only = True
+        yield DumpSnapshot(conn)
 
 
 def _run_bounded(cmd: list[str], timeout: int) -> None:
@@ -128,7 +203,7 @@ def _file_size_or_none(path: Path) -> int | None:
 def _write_manifest(
     out_dir: Path, *, kind: str, ts: str,
     tag: str | None, triggered_by: str,
-    artifact_path: Path,
+    artifact_path: Path, counts: dict | None = None,
 ) -> Path:
     manifest = {
         "kind": kind,
@@ -138,6 +213,8 @@ def _write_manifest(
         "created_at": datetime.now(UTC).isoformat(),
         "artifact_path": str(artifact_path),
     }
+    if counts is not None:
+        manifest["counts"] = counts
     mf = out_dir / f"fc_{kind}_{ts}.json"
     mf.write_text(json.dumps(manifest, indent=2))
     return mf
@@ -163,17 +240,20 @@ def backup_db(
     os.close(fd)
     tmp_path = Path(tmp_name)
     try:
-        _run_bounded(
-            _gently([*_PG_DUMP, "-f", str(tmp_path), _libpq_url(db_url)]),
-            _DB_SUBPROCESS_TIMEOUT_S,
-        )
+        with dump_snapshot(db_url) as snap:
+            _run_bounded(
+                _gently([*_PG_DUMP, f"--snapshot={snap.id}", "-f", str(tmp_path),
+                         _libpq_url(db_url)]),
+                _DB_SUBPROCESS_TIMEOUT_S,
+            )
+            counts = snap.counts()
         shutil.move(str(tmp_path), str(sql_path))
     finally:
         if tmp_path.exists():
             tmp_path.unlink(missing_ok=True)
     manifest_path = _write_manifest(
         out_dir, kind="db", ts=ts, tag=tag, triggered_by=triggered_by,
-        artifact_path=sql_path,
+        artifact_path=sql_path, counts=counts,
     )
     return {
         "kind": "db",
@@ -182,6 +262,7 @@ def backup_db(
         "tar_path": None,
         "manifest_path": str(manifest_path),
         "size_bytes": _file_size_or_none(sql_path),
+        "counts": counts,
     }
 
 
@@ -214,15 +295,24 @@ def backup_db_offsite(*, db_url: str, target, limit_mbps: int | None = None) ->
     copy staged on this server. The object appears only once pg_dump has
     exited cleanly and every part has arrived; a dump that fails partway
     leaves nothing in the bucket, and raises DumpFailed with pg_dump's own
-    last line. Returns a dict for the caller to record in BackupRun."""
+    last line. Returns a dict for the caller to record in BackupRun, with
+    `counts`: each table's rows in the dump, for the caller to store beside
+    it."""
+    ts = _now_ts()
+    with dump_snapshot(db_url) as snap:
+        result = _stream_dump(db_url, target, ts, limit_mbps, snap.id)
+        result["counts"] = snap.counts()
+    return result
+
+
+def _stream_dump(db_url: str, target, ts: str, limit_mbps: int | None, snapshot: str) -> dict:
     from .backup_s3 import PART_SIZE, Pacer, Upload
 
-    ts = _now_ts()
     key = offsite_dump_key(ts)
     deadline = time.monotonic() + _DB_SUBPROCESS_TIMEOUT_S
     with tempfile.TemporaryFile() as err:
         proc = subprocess.Popen(
-            _gently([*_PG_DUMP, _libpq_url(db_url)]),
+            _gently([*_PG_DUMP, f"--snapshot={snapshot}", _libpq_url(db_url)]),
             stdout=subprocess.PIPE, stderr=err,
         )
         upload = None

backend/app/services/backup_verify.py

@@ -0,0 +1,162 @@
+"""Testing a database dump without touching this install.
+
+The dump is loaded into a separate, temporary database on the same Postgres
+server, using the same steps a real restore takes. Every table is counted
+there and compared, the copy is brought up to this version, and the copy is
+then dropped. The live database is only read, to show its counts alongside.
+
+Every dump records how many rows each table held, counted in the same
+snapshot pg_dump dumped from, so the comparison is exact. For a dump in the
+bucket the counts sit beside it as `<dump name>.counts.json`; for a dump on
+this server they are in its manifest file. A dump taken before counts were
+recorded can still be tested: it is then compared with the live database,
+where a difference is what changed after the dump was taken, not a fault.
+
+The temporary database needs about as much free space on the database's disk
+as the live one takes, and only for the length of the test.
+"""
+from __future__ import annotations
+
+import json
+import tempfile
+from collections.abc import Callable
+from pathlib import Path
+
+from sqlalchemy.engine import make_url
+
+from . import backup_restore, backup_service
+from .backup_s3 import ObjectMissing, S3Target, get_file, put_file
+
+SCRATCH_SUFFIX = "_restore_test"
+
+
+def counts_key(dump_key: str) -> str:
+    """Where a dump's row counts sit in the bucket, beside the dump."""
+    return dump_key.removesuffix(".dump") + ".counts.json"
+
+
+def store_counts(target: S3Target, dump_key: str, counts: dict) -> None:
+    with tempfile.TemporaryDirectory() as d:
+        path = Path(d) / "counts.json"
+        path.write_text(json.dumps(counts))
+        put_file(target, counts_key(dump_key), path, content_type="application/json")
+
+
+def fetch_counts(target: S3Target, dump_key: str) -> dict | None:
+    """The counts recorded with a dump in the bucket, or None for a dump that
+    has none."""
+    with tempfile.TemporaryDirectory() as d:
+        path = Path(d) / "counts.json"
+        try:
+            get_file(target, counts_key(dump_key), path)
+        except ObjectMissing:
+            return None
+        return json.loads(path.read_text())
+
+
+def local_counts(manifest_path: str | None) -> dict | None:
+    """The counts in a dump's manifest file on this server, or None."""
+    try:
+        return json.loads(Path(manifest_path).read_text()).get("counts")
+    except (TypeError, OSError, ValueError):
+        return None
+
+
+def scratch_name(db_url: str) -> str:
+    """The temporary database's name: the live one's, plus SCRATCH_SUFFIX."""
+    return f"{make_url(db_url).database}{SCRATCH_SUFFIX}"
+
+
+def scratch_url(db_url: str) -> str:
+    return make_url(db_url).set(database=scratch_name(db_url)).render_as_string(
+        hide_password=False)
+
+
+def _on_server(db_url: str, statement) -> None:
+    """Run a CREATE or DROP DATABASE, which can't run inside a transaction."""
+    import psycopg
+
+    with psycopg.connect(backup_service._libpq_url(db_url), autocommit=True) as conn:
+        conn.execute(statement)
+
+
+def drop_scratch(db_url: str) -> None:
+    """Drop the temporary database if it is there, ending any connection to it."""
+    from psycopg import sql
+
+    _on_server(db_url, sql.SQL("DROP DATABASE IF EXISTS {} WITH (FORCE)").format(
+        sql.Identifier(scratch_name(db_url))))
+
+
+def _create_scratch(db_url: str) -> None:
+    from psycopg import sql
+
+    _on_server(db_url, sql.SQL("CREATE DATABASE {} TEMPLATE template0").format(
+        sql.Identifier(scratch_name(db_url))))
+
+
+def trial_restore(*, db_url: str, dump_path: Path, recorded: dict | None,
+              note: Callable[[str], None] = lambda phase: None) -> dict:
+    """Load the dump at `dump_path` into the temporary database, compare it,
+    bring it up to this version, and drop it. `note` is told each phase as it
+    starts: "loading", "comparing", "upgrading". Raises RestoreFailed (or
+    S3Error from the upgrade) when the dump doesn't load or can't be brought
+    up; the temporary database is dropped either way."""
+    scratch = scratch_url(db_url)
+    if make_url(scratch).database == make_url(db_url).database:
+        raise ValueError("The test database would be the live one.")
+    drop_scratch(db_url)          # one left by a test cut off by a restart
+    _create_scratch(db_url)
+    try:
+        note("loading")
+        backup_service.restore_db(db_url=scratch, sql_path=dump_path)
+        note("comparing")
+        restored = backup_service.read_counts(scratch)
+        live = backup_service.read_counts(db_url)
+        note("upgrading")
+        backup_restore.migrate(db_name=scratch_name(db_url))
+        upgraded = backup_service.read_counts(scratch)["alembic"]
+    finally:
+        drop_scratch(db_url)
+    return {**compare(recorded, restored, live),
+            "alembic_dump": restored["alembic"], "alembic_now": upgraded}
+
+
+def compare(recorded: dict | None, restored: dict, live: dict) -> dict:
+    """Each table's rows in the dump (as recorded), in the restored copy, and
+    in the live database. With counts recorded, every table must come back
+    with exactly its count; `mismatched` names those that didn't."""
+    rec = (recorded or {}).get("tables")
+    names = sorted(set(restored["tables"]) | set(rec or {}) | set(live["tables"]))
+    tables = [
+        {"table": n, "dump": rec.get(n) if rec is not None else None,
+         "restored": restored["tables"].get(n), "live": live["tables"].get(n)}
+        for n in names
+    ]
+    mismatched = [t["table"] for t in tables
+                  if rec is not None and t["table"] in rec and t["restored"] != t["dump"]]
+    return {
+        "counted": rec is not None,
+        "tables": tables,
+        "mismatched": mismatched,
+        "rows_restored": sum(v for v in restored["tables"].values()),
+    }
+
+
+def summary(result: dict) -> str:
+    """One sentence on how a test went."""
+    n = len([t for t in result["tables"] if t["restored"] is not None])
+    rows = f"{result['rows_restored']:,}"
+    if result["mismatched"]:
+        shown = ", ".join(
+            f"{t['table']} ({t['dump']:,} in the dump, "
+            + ("table missing)" if t["restored"] is None else f"{t['restored']:,} restored)")
+            for t in result["tables"] if t["table"] in result["mismatched"][:5]
+        )
+        more = len(result["mismatched"]) - 5
+        return (f"{len(result['mismatched'])} table(s) came back different: {shown}"
+                + (f", and {more} more" if more > 0 else "") + ".")
+    if result["counted"]:
+        return f"All {n} tables came back with the {rows} rows the dump recorded."
+    return (f"The dump loaded cleanly: {n} tables, {rows} rows. It was taken before "
+            "dumps recorded their counts, so it is shown beside the live database instead.")

backend/app/tasks/backup.py

@@ -26,6 +26,7 @@ from ..services import (
     backup_restore,
     backup_s3,
     backup_service,
+    backup_verify,
 )
 from ._sync_engine import get_sync_engine
 from ._sync_engine import sync_session_factory as _sync_session_factory
@@ -86,9 +87,7 @@ def backup_db_task(self, *, tag: str | None = None,
             target = backup_destination.saved_target(settings)
             limit_mbps = settings.backup_upload_limit_mbps
         if target is not None:
-            result = backup_service.backup_db_offsite(
-                db_url=cfg.database_url_sync, target=target, limit_mbps=limit_mbps,
-            )
+            result = _dump_offsite(cfg, target, limit_mbps)
         else:
             result = backup_service.backup_db(
                 db_url=cfg.database_url_sync, images_root=IMAGES_ROOT,
@@ -280,10 +279,10 @@ def prune_backups() -> dict:
     """Daily Beat. Per-kind retention from ImportSettings.
 
     Returns {"db_deleted": N, "images_deleted": M, "media_deleted": L,
-    "restore_deleted": R, "reconcile_deleted": C, "files_unlinked": K}. Media,
-    restore and reconcile runs leave no files here; only their history is
-    trimmed, to the latest MEDIA_RUNS_KEPT, RESTORE_RUNS_KEPT and
-    RECONCILE_RUNS_KEPT.
+    "restore_deleted": R, "reconcile_deleted": C, "test_deleted": T,
+    "files_unlinked": K}. Media, restore, reconcile and test runs leave no
+    files here; only their history is trimmed, to the latest MEDIA_RUNS_KEPT,
+    RESTORE_RUNS_KEPT, RECONCILE_RUNS_KEPT and TEST_RUNS_KEPT.
     Tagged rows (tag IS NOT NULL) are never pruned.
     Status='running' / 'restoring' rows are never pruned — the
     recover_stalled_backup_runs sweep flips truly-stuck ones to
@@ -293,7 +292,8 @@ def prune_backups() -> dict:
     """
     SessionLocal = _sync_session_factory()
     counts = {"db_deleted": 0, "images_deleted": 0, "media_deleted": 0,
-              "restore_deleted": 0, "reconcile_deleted": 0, "files_unlinked": 0}
+              "restore_deleted": 0, "reconcile_deleted": 0, "test_deleted": 0,
+              "files_unlinked": 0}
     with SessionLocal() as session:
         s = ImportSettings.load_sync(session)
         for kind, keep in (
@@ -302,6 +302,7 @@ def prune_backups() -> dict:
             ("media", MEDIA_RUNS_KEPT),
             ("restore", RESTORE_RUNS_KEPT),
             ("reconcile", RECONCILE_RUNS_KEPT),
+            ("test", TEST_RUNS_KEPT),
         ):
             candidates = session.execute(
                 select(BackupRun)
@@ -371,6 +372,21 @@ MEDIA_FAILURES_LISTED = 20
 RECONCILE_RUNS_KEPT = 12
 
 
+def _dump_offsite(cfg, target, limit_mbps) -> dict:
+    """Dump the database to the bucket, with its row counts beside it for a
+    test restore to check against. The dump stands without its counts, so a
+    failure to store them is logged, not raised."""
+    result = backup_service.backup_db_offsite(
+        db_url=cfg.database_url_sync, target=target, limit_mbps=limit_mbps)
+    if result.get("counts"):
+        try:
+            backup_verify.store_counts(target, result["key"], result["counts"])
+        except (backup_s3.S3Error, OSError):
+            log.warning("backup: the row counts for %s weren't stored", result["key"],
+                        exc_info=True)
+    return result
+
+
 def _offsite_manifest(target) -> dict:
     return {"endpoint": target.endpoint, "bucket": target.bucket,
             "key": backup_media.PREFIX}
@@ -439,6 +455,11 @@ def resume_offsite_runs() -> dict:
             ).scalars()
             if (r.manifest or {}).get("phase") == "files"
         ]
+    try:
+        tests = end_cut_off_tests()
+    except Exception:  # noqa: BLE001 — resuming the runs below matters more
+        log.exception("couldn't end a test restore cut off by the restart")
+        tests = 0
     for run_id in media:
         backup_media_task.delay(run_id=run_id, resume=True)
     for run_id in restores:
@@ -446,7 +467,7 @@ def resume_offsite_runs() -> dict:
     if media or restores:
         log.info("resumed off-site runs after a restart: media %s, restore %s",
                  media, restores)
-    return {"media": media, "restore": restores}
+    return {"media": media, "restore": restores, "tests_ended": tests}
 
 
 @celery.task(
@@ -647,7 +668,11 @@ def backup_busy(session) -> str | None:
     ).first()
     if row is None:
         return None
-    return "a restore is going" if row.kind == "restore" else f"a {row.kind} backup is running"
+    if row.kind == "restore":
+        return "a restore is going"
+    if row.kind == "test":
+        return "a test restore is running"
+    return f"a {row.kind} backup is running"
 
 
 def restore_going(session) -> bool:
@@ -657,6 +682,134 @@ def restore_going(session) -> bool:
     ).first() is not None
 
 
+# ── Test restore ─────────────────────────────────────────────────────────────
+
+TEST_RUNS_KEPT = 20
+
+
+@celery.task(
+    name="backend.app.tasks.backup.restore_test_task",
+    bind=True,
+    max_retries=0,
+    # Downloading a dump, loading it and rebuilding its indexes; a restore of
+    # the database alone, with room to spare.
+    soft_time_limit=4 * 3600, time_limit=4 * 3600 + 900,
+)
+def restore_test_task(self, *, dump_key: str | None = None,
+                      source_backup_run_id: int | None = None) -> dict:
+    """Test a database dump without touching this install: load it into a
+    temporary database, check every table against the counts recorded with
+    the dump, bring it up to this version, and drop it again
+    (services/backup_verify.py). Tests the bucket's dump at `dump_key` (the
+    newest when None), or, given `source_backup_run_id`, a dump on this
+    server. The run ends "ok" when every table matched, "error" with what
+    didn't otherwise. Returns {'backup_run_id': N, ...}."""
+    SessionLocal = _sync_session_factory()
+    cfg = get_config()
+    now = datetime.now(UTC)
+    with SessionLocal() as session:
+        if why := restore_test_blocked(session):
+            return {"skipped": why}
+        row = BackupRun(kind="test", status="running", triggered_by="manual",
+                        started_at=now, last_progress_at=now, manifest={"phase": "fetching"})
+        session.add(row)
+        session.commit()
+        run_id = row.id
+
+    def note(phase: str, **more) -> None:
+        with SessionLocal() as session:
+            r = session.get(BackupRun, run_id)
+            _note(session, r, {**r.manifest, "phase": phase, **more})
+
+    staged = offsite = None
+    try:
+        if source_backup_run_id is not None:
+            dump_path, recorded, source = _local_dump(SessionLocal, source_backup_run_id)
+        else:
+            with SessionLocal() as session:
+                settings = ImportSettings.load_sync(session)
+                target = backup_destination.saved_target(settings)
+                limit_mbps = settings.backup_upload_limit_mbps
+            if target is None:
+                raise backup_s3.S3Error("No off-site destination is set up.")
+            dump = backup_restore.find_dump(target, dump_key)
+            source = {"key": dump.key, "ts": dump.ts, "size_bytes": dump.size_bytes}
+            offsite = {**_offsite_manifest(target), "key": dump.key}
+            note("fetching", dump=source, offsite=offsite)
+            recorded = backup_verify.fetch_counts(target, dump.key)
+            staged = backup_restore.fetch_dump(
+                target, dump, IMAGES_ROOT / "_backups", backup_s3.Pacer(limit_mbps))
+            dump_path = staged
+        note("loading", dump=source)
+        result = backup_verify.trial_restore(
+            db_url=cfg.database_url_sync, dump_path=dump_path, recorded=recorded,
+            note=note)
+    except (SoftTimeLimitExceeded, Exception) as exc:
+        with SessionLocal() as session:
+            r = session.get(BackupRun, run_id)
+            if r is not None:
+                _mark_failed(session, r, exc)
+        raise
+    finally:
+        if staged is not None:
+            staged.unlink(missing_ok=True)
+
+    with SessionLocal() as session:
+        r = session.get(BackupRun, run_id)
+        r.status = "error" if result["mismatched"] else "ok"
+        r.error = backup_verify.summary(result) if result["mismatched"] else None
+        r.finished_at = datetime.now(UTC)
+        r.size_bytes = source.get("size_bytes")
+        r.manifest = {"phase": "done", "dump": source, "offsite": offsite,
+                      "summary": backup_verify.summary(result), **result}
+        session.commit()
+    return {"backup_run_id": run_id, "mismatched": result["mismatched"]}
+
+
+def _local_dump(SessionLocal, run_id: int) -> tuple[Path, dict | None, dict]:
+    """A dump on this server: its path, its recorded counts, and what the
+    test run says it tested."""
+    with SessionLocal() as session:
+        src = session.get(BackupRun, run_id)
+        if src is None or src.kind != "db" or src.status != "ok" or not src.sql_path:
+            raise backup_s3.S3Error(f"Backup {run_id} isn't a database dump on this server.")
+        path = Path(src.sql_path)
+        if not path.is_file():
+            raise backup_s3.S3Error(f"{path} is no longer on this server.")
+        recorded = backup_verify.local_counts((src.manifest or {}).get("manifest_path"))
+        ts = (src.manifest or {}).get("ts")
+        return path, recorded, {"run_id": run_id, "ts": ts, "size_bytes": src.size_bytes}
+
+
+def restore_test_blocked(session) -> str | None:
+    """Why a test restore can't start right now, or None. A backup going
+    doesn't stop one: the test only reads this install's database."""
+    if restore_going(session):
+        return "a restore is going"
+    going = session.execute(
+        select(BackupRun.id).where(BackupRun.kind == "test")
+        .where(BackupRun.status == "running").limit(1)
+    ).first()
+    return "a test restore is already running" if going else None
+
+
+def end_cut_off_tests() -> int:
+    """End every test restore a restart cut short, and drop the temporary
+    database it left. Called when the long-maintenance lane starts, where the
+    tests run, so none is alive then."""
+    with _sync_session_factory()() as session:
+        ended = session.execute(
+            update(BackupRun).where(BackupRun.kind == "test")
+            .where(BackupRun.status == "running")
+            .values(status="error", finished_at=datetime.now(UTC),
+                    error="Interrupted: the server restarted during the test. Test again.")
+        ).rowcount or 0
+        session.commit()
+    if ended:
+        backup_verify.drop_scratch(get_config().database_url_sync)
+    return ended
+
+
 def _library_is_empty(session) -> bool:
     return session.execute(text(
         "SELECT NOT EXISTS (SELECT 1 FROM image_record) "
@@ -749,8 +902,7 @@ def _restore_database(SessionLocal, run_id: int, target, limit_mbps, dump_key) -
             # dump, and would sit in the list as the newest.
             if not _library_is_empty(session):
                 session.commit()
-                saved = backup_service.backup_db_offsite(
-                    db_url=cfg.database_url_sync, target=target, limit_mbps=limit_mbps)
+                saved = _dump_offsite(cfg, target, limit_mbps)
                 m["saved_before"] = saved["key"]
             _note(session, row, m)
             started_at, manifest = row.started_at, dict(m)

frontend/src/components/settings/BackupCard.vue

@@ -78,10 +78,15 @@
         @restore="onRestore"
         @delete="onDelete"
         @tag="onTag"
+        @test="(r) => testSection?.start({ runId: r.id })"
       />
 
       <v-divider class="my-4" />
 
+      <BackupTestSection ref="testSection" :bucket="offsite?.bucket || ''" @started="pollOnce" />
+
+      <v-divider class="my-4" />
+
       <!-- Off-site: images and attachments as one file per fingerprint -->
       <template v-if="offsite">
         <h3 class="fc-section-title">Images and attachments</h3>
@@ -159,43 +164,11 @@
         </v-alert>
         <BackupRunsTable :runs="store.restoreRuns" empty-text="No restores yet." />
 
-        <v-dialog v-model="dumpsOpen" max-width="560">
-          <v-card>
-            <v-card-title>Dumps in {{ offsite.bucket }}</v-card-title>
-            <v-card-text>
-              <v-alert
-                v-if="dumpsError" type="warning" variant="tonal" density="compact" class="mb-3"
-              >{{ dumpsError }}</v-alert>
-              <v-table v-else density="compact">
-                <thead>
-                  <tr><th>Taken</th><th class="text-right">Size</th><th /></tr>
-                </thead>
-                <tbody>
-                  <tr v-for="d in dumps" :key="d.key">
-                    <td :title="d.key">{{ dumpWhen(d.ts) }}</td>
-                    <td class="text-right" style="font-variant-numeric: tabular-nums;">
-                      {{ humanBytes(d.size_bytes) }}
-                    </td>
-                    <td class="text-right">
-                      <v-btn size="small" variant="text" color="warning" @click="onPickDump(d)">
-                        Restore…
-                      </v-btn>
-                    </td>
-                  </tr>
-                  <tr v-if="!dumps.length">
-                    <td colspan="3" class="text-center fc-muted py-4">
-                      The bucket holds no database dumps yet.
-                    </td>
-                  </tr>
-                </tbody>
-              </v-table>
-            </v-card-text>
-            <v-card-actions>
-              <v-spacer />
-              <v-btn variant="text" @click="dumpsOpen = false">Close</v-btn>
-            </v-card-actions>
-          </v-card>
-        </v-dialog>
+        <DumpPickerDialog
+          v-model="dumpsOpen" action="restore"
+          :bucket="offsite.bucket" :dumps="dumps" :error="dumpsError"
+          @pick="onPickDump"
+        />
       </template>
 
       <template v-else>
@@ -246,8 +219,10 @@ import { useSystemHealthStore } from '../../stores/systemHealth.js'
 import BackupConfirmModal from '../modal/DestructiveConfirmModal.vue'
 import MaintenanceTile from '../common/MaintenanceTile.vue'
 import { humanBytes } from '../../utils/bytes.js'
-import { formatDateTime, formatRelative } from '../../utils/date.js'
+import { formatDumpTime, formatRelative } from '../../utils/date.js'
 import BackupRunsTable from './BackupRunsTable.vue'
+import BackupTestSection from './BackupTestSection.vue'
+import DumpPickerDialog from './DumpPickerDialog.vue'
 
 const store = useBackupStore()
 const healthStore = useSystemHealthStore()
@@ -313,6 +288,7 @@ const dumpsError = ref('')
 const dumps = ref([])
 const pickedDump = ref(null)
 const fetchTriggering = ref(false)
+const testSection = ref(null)
 // The notice that a restore was asked for stays up until its run is over.
 watch(runningRestore, (now, before) => {
   if (before && !now) restoreDispatched.value = false
@@ -324,14 +300,6 @@ function restoreLine(m) {
     + `${n('present')} already there, ${n('missing')} not in the bucket, ${n('failed')} failed`
 }
 
-// 20261107T134258Z → the reader's local date and time.
-function dumpWhen(ts) {
-  const iso = ts.replace(
-    /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/, '$1-$2-$3T$4:$5:$6Z',
-  )
-  return formatDateTime(iso)
-}
-
 async function openDumps() {
   dumpsLoading.value = true
   dumpsError.value = ''
@@ -354,7 +322,7 @@ function onPickDump(d) {
   confirmRunId.value = ''
   confirmToken.value = d.confirm
   confirmDescription.value = (
-    `Replaces this install's database with the dump taken ${dumpWhen(d.ts)}, `
+    `Replaces this install's database with the dump taken ${formatDumpTime(d.ts)}, `
     + 'then fetches the library\'s files from the bucket.'
   )
   confirmOpen.value = true
@@ -380,6 +348,7 @@ const COVERAGE_EVERY = 6
 function pollOnce() {
   if (document.hidden) return
   store.loadRuns('db')
+  store.loadRuns('test')
   if (offsite.value) {
     store.loadRuns('media')
     store.loadRuns('restore')

frontend/src/components/settings/BackupRunsTable.vue

@@ -47,7 +47,7 @@
           <span v-else class="fc-muted">—</span>
         </td>
         <td class="text-right">
-          <span v-if="r.offsite" class="fc-muted">—</span>
+          <span v-if="r.offsite || !actions" class="fc-muted">—</span>
           <KebabMenu v-else :label="`Actions for backup ${r.id}`">
             <v-list-item
               :title="r.tag ? 'Untag' : 'Tag…'"
@@ -57,6 +57,11 @@
               title="Restore…" :disabled="r.status !== 'ok'"
               @click="$emit('restore', r)"
             />
+            <v-list-item
+              v-if="r.kind === 'db'"
+              title="Test restore" :disabled="r.status !== 'ok'"
+              @click="$emit('test', r)"
+            />
             <v-list-item
               title="Delete…"
               @click="$emit('delete', r)"
@@ -96,8 +101,11 @@ import KebabMenu from '../common/KebabMenu.vue'
 defineProps({
   runs: { type: Array, default: () => [] },
   emptyText: { type: String, default: 'No backups yet.' },
+  // False for runs that are records, not backups (tests): no Tag, Restore
+  // or Delete menu.
+  actions: { type: Boolean, default: true },
 })
-defineEmits(['restore', 'delete', 'tag'])
+defineEmits(['restore', 'delete', 'tag', 'test'])
 
 // Which runs have their failed-file list open.
 const open = reactive({})

frontend/src/components/settings/BackupTestSection.vue

@@ -0,0 +1,197 @@
+<template>
+  <div>
+    <h3 class="fc-section-title">Test a backup</h3>
+    <p class="fc-muted text-body-2 mb-2">
+      Load a database dump into a separate, temporary database, check that every
+      table came back with the rows it held when the dump was taken, and bring it
+      up to this version. The temporary database is then deleted. This install's
+      database and library aren't touched.
+      <template v-if="store.databaseBytes">
+        It needs about {{ humanBytes(store.databaseBytes) }} free on the
+        database's disk while it runs.
+      </template>
+    </p>
+    <div class="fc-settings-row mb-3">
+      <v-btn
+        color="accent" variant="tonal" rounded="pill"
+        prepend-icon="mdi-database-check-outline"
+        :disabled="blocked || (!bucket && !newestLocal)" :loading="starting"
+        @click="startNewest"
+      >Test the newest dump</v-btn>
+      <v-btn
+        v-if="bucket"
+        variant="text" rounded="pill"
+        :disabled="blocked" :loading="dumpsLoading"
+        @click="openDumps"
+      >Choose a dump…</v-btn>
+    </div>
+
+    <v-alert
+      v-if="running || dispatched"
+      type="info" variant="tonal" density="compact" class="mb-3"
+    >
+      Testing the dump<template v-if="running?.manifest?.dump?.ts">
+        taken {{ formatDumpTime(running.manifest.dump.ts) }}</template>:
+      {{ phaseText }}.
+    </v-alert>
+
+    <!-- The newest finished test: its verdict, and every table on request. -->
+    <div v-if="last" class="mb-3">
+      <div class="d-flex align-start text-body-2">
+        <v-icon
+          size="small" class="mr-2 mt-1"
+          :color="last.status === 'ok' ? 'success' : 'error'"
+        >{{ last.status === 'ok' ? 'mdi-check-circle' : 'mdi-close-circle' }}</v-icon>
+        <div>
+          <strong>Last test</strong>
+          ({{ formatRelative(last.finished_at) }}<template v-if="last.manifest?.dump?.ts">,
+            dump taken {{ formatDumpTime(last.manifest.dump.ts) }}</template>):
+          {{ last.manifest?.summary || last.error }}
+          <div v-if="upgradeLine" class="text-caption fc-muted">{{ upgradeLine }}</div>
+        </div>
+      </div>
+      <v-btn
+        v-if="tables.length"
+        size="x-small" variant="text" class="mt-1"
+        :append-icon="showTables ? 'mdi-chevron-up' : 'mdi-chevron-down'"
+        @click="showTables = !showTables"
+      >{{ showTables ? 'Hide' : 'Show' }} each table</v-btn>
+      <v-table v-if="showTables" density="compact" class="fc-test-tables mt-1">
+        <thead>
+          <tr>
+            <th>Table</th>
+            <th class="text-right">In the dump</th>
+            <th class="text-right">Restored</th>
+            <th class="text-right">Live now</th>
+          </tr>
+        </thead>
+        <tbody>
+          <tr
+            v-for="t in tables" :key="t.table"
+            :class="{ 'text-error': mismatched.has(t.table) }"
+          >
+            <td><code>{{ t.table }}</code></td>
+            <td class="text-right">{{ num(t.dump) }}</td>
+            <td class="text-right">{{ num(t.restored) }}</td>
+            <td class="text-right fc-muted">{{ num(t.live) }}</td>
+          </tr>
+        </tbody>
+      </v-table>
+    </div>
+
+    <BackupRunsTable :runs="store.testRuns" :actions="false" empty-text="No tests yet." />
+
+    <DumpPickerDialog
+      v-if="bucket"
+      v-model="dumpsOpen" action="test"
+      :bucket="bucket" :dumps="dumps" :error="dumpsError"
+      @pick="onPick"
+    />
+  </div>
+</template>
+
+<script setup>
+import { computed, ref, watch } from 'vue'
+
+import { useBackupStore } from '../../stores/backup.js'
+import { humanBytes } from '../../utils/bytes.js'
+import { formatDumpTime, formatRelative } from '../../utils/date.js'
+import BackupRunsTable from './BackupRunsTable.vue'
+import DumpPickerDialog from './DumpPickerDialog.vue'
+
+const props = defineProps({
+  // The off-site bucket's name when one is set up; otherwise the dumps on
+  // this server are tested.
+  bucket: { type: String, default: '' },
+})
+const emit = defineEmits(['started'])
+
+const store = useBackupStore()
+
+const running = computed(() => store.testRuns.find((r) => r.status === 'running'))
+const last = computed(() => store.testRuns.find((r) => r.status !== 'running'))
+const restoring = computed(() => store.restoreRuns.some((r) => r.status === 'restoring'))
+const blocked = computed(() => Boolean(running.value || restoring.value))
+const newestLocal = computed(() => store.dbRuns.find((r) => r.status === 'ok' && !r.offsite))
+
+const PHASES = {
+  fetching: 'downloading it from the bucket',
+  loading: 'loading it into a temporary database',
+  comparing: 'counting every table',
+  upgrading: 'bringing it up to this version',
+}
+const phaseText = computed(() => PHASES[running.value?.manifest?.phase] || 'starting')
+
+const tables = computed(() => last.value?.manifest?.tables || [])
+const mismatched = computed(() => new Set(last.value?.manifest?.mismatched || []))
+const showTables = ref(false)
+const upgradeLine = computed(() => {
+  const m = last.value?.manifest
+  if (!m?.alembic_now) return ''
+  return m.alembic_dump === m.alembic_now
+    ? 'Its schema is already this version\'s.'
+    : `Its schema was brought up from ${m.alembic_dump || 'none'} to ${m.alembic_now}.`
+})
+
+function num(n) {
+  return n == null ? '—' : n.toLocaleString()
+}
+
+// "Testing…" shows from the click until the run appears.
+const dispatched = ref(false)
+watch(running, (now) => { if (now) dispatched.value = false })
+
+const starting = ref(false)
+async function start(what) {
+  starting.value = true
+  try {
+    await store.testDump(what)
+    dispatched.value = true
+    emit('started')
+  } catch {
+    // store.lastError is shown on the card.
+  } finally {
+    starting.value = false
+  }
+}
+function startNewest() {
+  start(props.bucket ? {} : { runId: newestLocal.value.id })
+}
+
+const dumpsOpen = ref(false)
+const dumpsLoading = ref(false)
+const dumpsError = ref('')
+const dumps = ref([])
+async function openDumps() {
+  dumpsLoading.value = true
+  dumpsError.value = ''
+  try {
+    dumps.value = await store.listOffsiteDumps()
+  } catch (e) {
+    dumps.value = []
+    dumpsError.value = e.message
+  } finally {
+    dumpsLoading.value = false
+    dumpsOpen.value = true
+  }
+}
+function onPick(d) {
+  dumpsOpen.value = false
+  start({ key: d.key })
+}
+
+defineExpose({ start })
+</script>
+
+<style scoped>
+.fc-section-title {
+  font-family: 'Fraunces', Georgia, serif;
+  font-size: 16px; font-weight: 500;
+  color: rgb(var(--v-theme-on-surface));
+  margin-bottom: 8px;
+}
+.fc-settings-row {
+  display: flex; gap: 12px; align-items: center; flex-wrap: wrap;
+}
+.fc-test-tables { background: transparent; font-variant-numeric: tabular-nums; }
+</style>

frontend/src/components/settings/DumpPickerDialog.vue

@@ -0,0 +1,72 @@
+<template>
+  <!-- The database dumps in the off-site bucket, newest first, each with one
+       action: restore it, or test it. -->
+  <v-dialog :model-value="modelValue" max-width="560" @update:model-value="$emit('update:modelValue', $event)">
+    <v-card>
+      <v-card-title>Dumps in {{ bucket }}</v-card-title>
+      <v-card-text>
+        <v-alert
+          v-if="error" type="warning" variant="tonal" density="compact" class="mb-3"
+        >{{ error }}</v-alert>
+        <template v-else>
+          <v-table density="compact">
+            <thead>
+              <tr><th>Taken</th><th class="text-right">Size</th><th /></tr>
+            </thead>
+            <tbody>
+              <tr v-for="d in dumps" :key="d.key">
+                <td :title="d.key">
+                  {{ formatDumpTime(d.ts) }}
+                  <v-icon
+                    v-if="d.counted" size="x-small" class="ml-1" color="success"
+                    title="Its row counts were recorded, so a test checks every table exactly"
+                  >mdi-check-decagram-outline</v-icon>
+                </td>
+                <td class="text-right fc-tabular">{{ humanBytes(d.size_bytes) }}</td>
+                <td class="text-right">
+                  <v-btn
+                    size="small" variant="text" :color="action === 'restore' ? 'warning' : 'accent'"
+                    @click="$emit('pick', d)"
+                  >{{ action === 'restore' ? 'Restore…' : 'Test' }}</v-btn>
+                </td>
+              </tr>
+              <tr v-if="!dumps.length">
+                <td colspan="3" class="text-center fc-muted py-4">
+                  The bucket holds no database dumps yet.
+                </td>
+              </tr>
+            </tbody>
+          </v-table>
+          <p v-if="action === 'test' && dumps.some((d) => d.counted)" class="fc-muted text-caption mt-2">
+            <v-icon size="x-small" color="success">mdi-check-decagram-outline</v-icon>
+            marks a dump whose row counts were recorded as it was taken. A dump
+            without them is compared with the live database instead.
+          </p>
+        </template>
+      </v-card-text>
+      <v-card-actions>
+        <v-spacer />
+        <v-btn variant="text" @click="$emit('update:modelValue', false)">Close</v-btn>
+      </v-card-actions>
+    </v-card>
+  </v-dialog>
+</template>
+
+<script setup>
+import { humanBytes } from '../../utils/bytes.js'
+import { formatDumpTime } from '../../utils/date.js'
+
+defineProps({
+  modelValue: { type: Boolean, default: false },
+  bucket: { type: String, default: '' },
+  dumps: { type: Array, default: () => [] },
+  error: { type: String, default: '' },
+  // 'restore' or 'test': what choosing a dump does.
+  action: { type: String, default: 'restore' },
+})
+defineEmits(['update:modelValue', 'pick'])
+</script>
+
+<style scoped>
+.fc-tabular { font-variant-numeric: tabular-nums; }
+</style>

frontend/src/stores/backup.js

@@ -12,15 +12,17 @@ export const useBackupStore = defineStore('backup', () => {
   const restoreRuns = ref([])
   // The weekly check of the bucket against what was sent.
   const reconcileRuns = ref([])
+  // Test restores: a dump loaded into a temporary database and checked.
+  const testRuns = ref([])
   const settings = ref(null)
   const loading = ref({
     dbRuns: false, imagesRuns: false, mediaRuns: false, restoreRuns: false,
-    reconcileRuns: false, settings: false,
+    reconcileRuns: false, testRuns: false, settings: false,
   })
   const lastError = ref(null)
   const RUNS = {
     db: dbRuns, images: imagesRuns, media: mediaRuns, restore: restoreRuns,
-    reconcile: reconcileRuns,
+    reconcile: reconcileRuns, test: testRuns,
   }
 
   async function loadRuns(kind) {
@@ -149,12 +151,34 @@ export const useBackupStore = defineStore('backup', () => {
   }
 
   // Restoring from the bucket. Dumps arrive newest first, each with the
-  // `confirm` text a restore of it asks for.
+  // `confirm` text a restore of it asks for, and `counted` when its row
+  // counts were recorded beside it.
+  // How big the live database is: a test restore needs about that much free
+  // space on the database's disk while it runs.
+  const databaseBytes = ref(null)
+
   async function listOffsiteDumps() {
     const body = await api.get('/api/system/backup/offsite/dumps')
+    databaseBytes.value = body.database_bytes ?? null
     return body.dumps || []
   }
 
+  // Test a dump without touching this install: from the bucket by key (the
+  // newest when null), or one kept on this server by its run id.
+  async function testDump({ key = null, runId = null } = {}) {
+    lastError.value = null
+    try {
+      if (runId != null) {
+        await api.post(`/api/system/backup/runs/${runId}/test`, { body: {} })
+      } else {
+        await api.post('/api/system/backup/offsite/test', { body: key ? { key } : {} })
+      }
+    } catch (e) {
+      lastError.value = e.message
+      throw e
+    }
+  }
+
   async function restoreFromOffsite(key, confirm) {
     lastError.value = null
     try {
@@ -176,8 +200,9 @@ export const useBackupStore = defineStore('backup', () => {
   }
 
   return {
-    dbRuns, imagesRuns, mediaRuns, restoreRuns, reconcileRuns, mediaCoverage, loadMediaCoverage,
-    listOffsiteDumps, restoreFromOffsite, fetchMissingFiles,
+    dbRuns, imagesRuns, mediaRuns, restoreRuns, reconcileRuns, testRuns,
+    mediaCoverage, loadMediaCoverage,
+    listOffsiteDumps, databaseBytes, restoreFromOffsite, fetchMissingFiles, testDump,
     settings, loading, lastError,
     loadRuns, triggerBackup, restore, deleteRun, setTag,
     loadSettings, patchSettings,

frontend/src/utils/date.js

@@ -61,3 +61,13 @@ export function formatLocalDate (iso) {
     year: 'numeric', month: 'short', day: 'numeric',
   })
 }
+
+// A dump's compact UTC stamp (20261107T134258Z, as in its file name) as the
+// reader's local date and time.
+export function formatDumpTime (ts) {
+  if (typeof ts !== 'string') return ''
+  const iso = ts.replace(
+    /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/, '$1-$2-$3T$4:$5:$6Z',
+  )
+  return formatDateTime(iso)
+}

frontend/test/components/backupTestSection.spec.js

@@ -0,0 +1,59 @@
+// @vitest-environment happy-dom
+import { describe, it, expect } from 'vitest'
+
+import BackupTestSection from '../../src/components/settings/BackupTestSection.vue'
+import { useBackupStore } from '../../src/stores/backup.js'
+import { mountWithStore } from '../support/mountComponent.js'
+
+const FINISHED = {
+  id: 9, kind: 'test', status: 'error', started_at: new Date().toISOString(),
+  finished_at: new Date().toISOString(), duration_seconds: 300, size_bytes: 2048,
+  error: '1 table(s) came back different: post (7 in the dump, 6 restored).',
+  offsite: null,
+  manifest: {
+    phase: 'done',
+    dump: { ts: '20261107T030000Z' },
+    summary: '1 table(s) came back different: post (7 in the dump, 6 restored).',
+    counted: true,
+    mismatched: ['post'],
+    alembic_dump: '0120', alembic_now: '0121',
+    tables: [
+      { table: 'artist', dump: 10, restored: 10, live: 12 },
+      { table: 'post', dump: 7, restored: 6, live: 7 },
+    ],
+  },
+}
+
+function mount(runs) {
+  return mountWithStore(BackupTestSection, () => {
+    useBackupStore().testRuns = runs
+  }, { props: { bucket: 'curator' } })
+}
+
+describe('BackupTestSection', () => {
+  it('leads with the last test\'s verdict and how its schema was brought up', () => {
+    const t = mount([FINISHED]).text()
+    expect(t).toContain('Last test')
+    expect(t).toContain('post (7 in the dump, 6 restored)')
+    expect(t).toContain('brought up from 0120 to 0121')
+    expect(t).not.toContain('artist')
+  })
+
+  it('lists every table on request, with the dump, restored and live counts', async () => {
+    const w = mount([FINISHED])
+    const btn = w.findAll('v-btn').find((b) => b.text().includes('each table'))
+    await btn.trigger('click')
+    const rows = w.findAll('tr').filter((r) => r.text().includes('post'))
+    expect(rows.some((r) => r.classes().includes('text-error'))).toBe(true)
+    expect(w.text()).toContain('artist')
+  })
+
+  it('says what a running test is doing', () => {
+    const running = {
+      ...FINISHED, id: 10, status: 'running', finished_at: null, error: null,
+      manifest: { phase: 'loading', dump: { ts: '20261107T030000Z' } },
+    }
+    const t = mount([running, FINISHED]).text()
+    expect(t).toContain('loading it into a temporary database')
+  })
+})

tests/test_backup_restore.py

@@ -12,7 +12,7 @@ import pytest
 from sqlalchemy import func, select
 
 from backend.app.models import Artist, BackupRun, ImageRecord, Post, PostAttachment
-from backend.app.services import backup_restore
+from backend.app.services import backup_restore, backup_verify
 from backend.app.services.backup_s3 import ListPage, ObjectMissing, S3Error, S3Target, StoredObject
 from tests.fake_s3 import FakeS3
 
@@ -44,11 +44,13 @@ def test_dumps_are_listed_newest_first_and_other_keys_ignored(monkeypatch):
     page = ListPage([
         StoredObject("database/fabledcurator-20261101T034000Z.dump", 10, ""),
         StoredObject("database/fabledcurator-20261107T134258Z.dump", 30, ""),
+        StoredObject("database/fabledcurator-20261107T134258Z.counts.json", 2, ""),
         StoredObject("database/notes.txt", 1, ""),
     ], None)
     monkeypatch.setattr(backup_restore, "list_objects", lambda *a, **kw: page)
     dumps = backup_restore.list_dumps(TARGET)
     assert [d.ts for d in dumps] == ["20261107T134258Z", "20261101T034000Z"]
+    assert [d.counted for d in dumps] == [True, False]
     assert backup_restore.find_dump(TARGET, "latest").size_bytes == 30
     assert backup_restore.find_dump(TARGET, dumps[1].key).ts == "20261101T034000Z"
     with pytest.raises(S3Error, match="not among"):
@@ -291,7 +293,9 @@ def test_a_library_comes_back_whole_from_the_bucket(db_sync, tmp_path, monkeypat
         monkeypatch.setattr(backup_destination, "saved_target", lambda settings: s3.target())
         dumped = tasks.backup_db_task(triggered_by="manual")
         tasks.backup_media_task(triggered_by="manual")
-        dump_key = [k for k in s3.objects if k.startswith("database/")][0]
+        dump_key = [k for k in s3.objects if k.endswith(".dump")][0]
+        # The dump's row counts sit beside it, for a test restore.
+        assert backup_verify.counts_key(dump_key) in s3.objects
         assert len([k for k in s3.objects if k.startswith("media/")]) == 3
 
         # Lose the files, and change the database after the dump.
@@ -306,7 +310,7 @@ def test_a_library_comes_back_whole_from_the_bucket(db_sync, tmp_path, monkeypat
         assert queued == [{"run_id": first["backup_run_id"]}]
         done = tasks.restore_offsite_task(run_id=first["backup_run_id"])
         assert done["continues"] is False
-        dumps_now = [k for k in s3.objects if k.startswith("database/")]
+        dumps_now = [k for k in s3.objects if k.endswith(".dump")]
 
     # The database is the dump's: a fresh session, since the restore ended
     # every connection that was open.

tests/test_backup_service.py

@@ -58,9 +58,36 @@ def fake_subprocess(monkeypatch):
     return calls
 
 
+class _Snapshot:
+    """Stands in for the snapshot a dump is taken from."""
+    id = "00000003-0000001B-1"
+    COUNTS = {"alembic": "0121", "tables": {"credential": 0, "image_record": 2}}
+
+    def counts(self):
+        return self.COUNTS
+
+
+@pytest.fixture(autouse=True)
+def fake_snapshot(monkeypatch):
+    from contextlib import contextmanager
+
+    @contextmanager
+    def _snapshot(db_url):
+        yield _Snapshot()
+    monkeypatch.setattr(backup_service, "dump_snapshot", _snapshot)
+
+
 # --- backup_db -------------------------------------------------------
 
 
+def test_a_dump_is_taken_from_the_snapshot_its_row_counts_come_from(tmp_path, fake_subprocess):
+    result = backup_service.backup_db(db_url="postgresql://u@h/d", images_root=tmp_path)
+    assert f"--snapshot={_Snapshot.id}" in fake_subprocess[0]
+    assert result["counts"] == _Snapshot.COUNTS
+    manifest = json.loads(Path(result["manifest_path"]).read_text())
+    assert manifest["counts"] == _Snapshot.COUNTS
+
+
 def test_backup_db_writes_sql_and_manifest(tmp_path, fake_subprocess):
     result = backup_service.backup_db(
         db_url="postgresql://test@x/db",
@@ -448,6 +475,8 @@ def test_a_dump_streams_to_the_bucket_in_parts_with_nothing_kept_here(offsite, m
     assert dump.cmd[0] == "nice" and "pg_dump" in dump.cmd and "-Fc" in dump.cmd
     assert "-f" not in dump.cmd                  # stdout, not a file
     assert dump.cmd[-1] == "postgresql://u:p@h/d"
+    assert f"--snapshot={_Snapshot.id}" in dump.cmd
+    assert result["counts"] == _Snapshot.COUNTS
 
 
 def test_a_failed_dump_leaves_nothing_in_the_bucket_and_says_why(offsite, monkeypatch):

tests/test_backup_verify.py

@@ -0,0 +1,268 @@
+"""Testing a dump without touching this install (services/backup_verify.py and
+restore_test_task): the dump is loaded into a temporary database, every table
+is checked against the counts recorded with the dump, and the temporary
+database is dropped. The round trips at the bottom dump the test database to
+an in-process bucket with the real pg_dump and test it with the real
+pg_restore."""
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+
+import pytest
+from sqlalchemy import func, select, text
+
+from backend.app.models import Artist, BackupRun
+from backend.app.services import backup_service, backup_verify
+from backend.app.services.backup_verify import compare, summary
+from tests.fake_s3 import FakeS3
+
+LIVE = {"alembic": "0121", "tables": {"artist": 12, "image_record": 40, "post": 7}}
+
+
+def _counts(**tables):
+    return {"alembic": "0121", "tables": tables}
+
+
+def test_a_dump_that_comes_back_whole_matches_every_table():
+    recorded = _counts(artist=10, image_record=40, post=7)
+    result = compare(recorded, _counts(artist=10, image_record=40, post=7), LIVE)
+    assert result["counted"] and result["mismatched"] == []
+    assert result["rows_restored"] == 57
+    artist = next(t for t in result["tables"] if t["table"] == "artist")
+    assert artist == {"table": "artist", "dump": 10, "restored": 10, "live": 12}
+    assert summary(result) == "All 3 tables came back with the 57 rows the dump recorded."
+
+
+def test_a_table_with_rows_missing_or_gone_is_named():
+    recorded = _counts(artist=10, image_record=40, post=7)
+    result = compare(recorded, _counts(artist=10, image_record=39), LIVE)
+    assert result["mismatched"] == ["image_record", "post"]
+    assert summary(result) == (
+        "2 table(s) came back different: image_record (40 in the dump, 39 restored), "
+        "post (7 in the dump, table missing).")
+
+
+def test_a_dump_without_counts_is_shown_beside_the_live_database():
+    result = compare(None, _counts(artist=10, image_record=40), LIVE)
+    assert not result["counted"] and result["mismatched"] == []
+    assert next(t for t in result["tables"] if t["table"] == "post")["restored"] is None
+    assert summary(result).startswith("The dump loaded cleanly: 2 tables, 50 rows.")
+
+
+def test_the_counts_sit_beside_the_dump_and_the_test_database_beside_the_live_one():
+    key = "database/fabledcurator-20261107T134258Z.dump"
+    assert backup_verify.counts_key(key) == "database/fabledcurator-20261107T134258Z.counts.json"
+    url = "postgresql+psycopg://fc:pw@postgres:5432/fabledcurator"
+    assert backup_verify.scratch_name(url) == "fabledcurator_restore_test"
+    assert backup_verify.scratch_url(url) == (
+        "postgresql+psycopg://fc:pw@postgres:5432/fabledcurator_restore_test")
+
+
+# ── the task ────────────────────────────────────────────────────────────────
+
+
+def _local_dump(db_sync, tmp_path, counts):
+    dump = tmp_path / "fc_db_20261107T030000Z.dump"
+    dump.write_bytes(b"a dump")
+    manifest = tmp_path / "fc_db_20261107T030000Z.json"
+    manifest.write_text(json.dumps({"kind": "db", "counts": counts}))
+    row = BackupRun(kind="db", status="ok", triggered_by="manual", sql_path=str(dump),
+                    started_at=datetime.now(UTC),
+                    size_bytes=6, manifest={"ts": "20261107T030000Z",
+                                            "manifest_path": str(manifest)})
+    db_sync.add(row)
+    db_sync.commit()
+    return row
+
+
+@pytest.mark.integration
+def test_a_dump_on_this_server_is_tested_against_its_manifest(db_sync, tmp_path, monkeypatch):
+    from backend.app.tasks import backup as tasks
+    recorded = _counts(artist=3)
+    src = _local_dump(db_sync, tmp_path, recorded)
+    seen = {}
+
+    def _trial(*, db_url, dump_path, recorded, note):
+        seen.update(dump_path=dump_path, recorded=recorded)
+        note("comparing")
+        return {**compare(recorded, _counts(artist=3), _counts(artist=5)),
+                "alembic_dump": "0121", "alembic_now": "0121"}
+    monkeypatch.setattr(backup_verify, "trial_restore", _trial)
+
+    out = tasks.restore_test_task(source_backup_run_id=src.id)
+    db_sync.expire_all()
+    row = db_sync.get(BackupRun, out["backup_run_id"])
+    assert seen == {"dump_path": Path(src.sql_path), "recorded": recorded}
+    assert (row.kind, row.status, row.error) == ("test", "ok", None)
+    assert row.manifest["summary"] == "All 1 tables came back with the 3 rows the dump recorded."
+    assert row.manifest["dump"]["run_id"] == src.id
+    assert Path(src.sql_path).exists()          # the dump itself stays
+
+
+@pytest.mark.integration
+def test_a_test_that_finds_a_difference_ends_in_error_saying_which(db_sync, tmp_path, monkeypatch):
+    from backend.app.tasks import backup as tasks
+    src = _local_dump(db_sync, tmp_path, _counts(artist=3))
+    monkeypatch.setattr(backup_verify, "trial_restore", lambda *, recorded, **kw: {
+        **compare(recorded, _counts(artist=2), _counts(artist=2)),
+        "alembic_dump": "0121", "alembic_now": "0121"})
+    out = tasks.restore_test_task(source_backup_run_id=src.id)
+    db_sync.expire_all()
+    row = db_sync.get(BackupRun, out["backup_run_id"])
+    assert row.status == "error"
+    assert row.error == "1 table(s) came back different: artist (3 in the dump, 2 restored)."
+
+
+@pytest.mark.integration
+def test_a_dump_that_wont_load_is_recorded_with_why(db_sync, tmp_path, monkeypatch):
+    from backend.app.tasks import backup as tasks
+    src = _local_dump(db_sync, tmp_path, None)
+
+    def _fails(**kw):
+        raise backup_service.RestoreFailed("pg_restore: error: could not read input file")
+    monkeypatch.setattr(backup_verify, "trial_restore", _fails)
+    with pytest.raises(backup_service.RestoreFailed):
+        tasks.restore_test_task(source_backup_run_id=src.id)
+    row = db_sync.execute(select(BackupRun).where(BackupRun.kind == "test")).scalar_one()
+    assert row.status == "error"
+    assert row.error == "RestoreFailed: pg_restore: error: could not read input file"
+
+
+@pytest.mark.integration
+def test_no_test_starts_while_a_restore_or_another_test_is_going(db_sync, tmp_path):
+    from backend.app.tasks import backup as tasks
+    src = _local_dump(db_sync, tmp_path, None)
+    db_sync.add(BackupRun(kind="test", status="running", triggered_by="manual",
+                    started_at=datetime.now(UTC)))
+    db_sync.commit()
+    assert tasks.restore_test_task(source_backup_run_id=src.id) == {
+        "skipped": "a test restore is already running"}
+    assert tasks.backup_busy(db_sync) == "a test restore is running"
+
+
+@pytest.mark.integration
+def test_a_test_cut_off_by_a_restart_is_ended_and_its_database_dropped(db_sync, monkeypatch):
+    from backend.app.tasks import backup as tasks
+    dropped = []
+    monkeypatch.setattr(backup_verify, "drop_scratch", lambda url: dropped.append(url))
+    row = BackupRun(kind="test", status="running", triggered_by="manual",
+                    started_at=datetime.now(UTC))
+    db_sync.add(row)
+    db_sync.commit()
+    assert tasks.end_cut_off_tests() == 1
+    db_sync.expire_all()
+    assert db_sync.get(BackupRun, row.id).status == "error"
+    assert len(dropped) == 1
+
+
+# ── the screen's endpoints ──────────────────────────────────────────────────
+
+
+@pytest.mark.integration
+async def test_a_dump_on_this_server_can_be_tested_from_the_screen(client, db_sync, tmp_path,
+                                                                   monkeypatch):
+    from backend.app.tasks import backup as tasks
+    queued = []
+    monkeypatch.setattr(tasks.restore_test_task, "delay", lambda **kw: queued.append(kw))
+    src = _local_dump(db_sync, tmp_path, None)
+    media = BackupRun(kind="media", status="ok", triggered_by="manual",
+                      started_at=datetime.now(UTC))
+    db_sync.add(media)
+    db_sync.commit()
+
+    r = await client.post(f"/api/system/backup/runs/{media.id}/test", json={})
+    assert r.status_code == 400
+    r = await client.post(f"/api/system/backup/runs/{src.id}/test", json={})
+    assert r.status_code == 202
+    assert queued == [{"source_backup_run_id": src.id}]
+    r = await client.post("/api/system/backup/offsite/test", json={"key": "../etc"})
+    assert r.status_code == 400
+
+
+# ── round trips with the real pg_dump and pg_restore ────────────────────────
+
+
+def _scratch_exists(db_sync) -> bool:
+    from backend.app.config import get_config
+    name = backup_verify.scratch_name(get_config().database_url_sync)
+    return db_sync.execute(text("SELECT 1 FROM pg_database WHERE datname = :n"),
+                           {"n": name}).first() is not None
+
+
+@pytest.mark.integration
+def test_the_counts_are_read_in_the_snapshot_the_dump_is_taken_from(db_sync):
+    from backend.app.config import get_config
+    db_sync.add(Artist(name="Bea", slug="bea"))
+    db_sync.commit()
+    with backup_service.dump_snapshot(get_config().database_url_sync) as snap:
+        before = snap.counts()
+        db_sync.add(Artist(name="Later", slug="later"))
+        db_sync.commit()
+        assert snap.counts() == before
+    assert before["tables"]["artist"] == 1
+    assert before["tables"]["credential"] == 0
+    assert before["alembic"]
+
+
+@pytest.mark.integration
+def test_a_dump_tested_from_the_bucket_comes_back_whole_and_leaves_no_trace(
+    db_sync, tmp_path, monkeypatch,
+):
+    from backend.app.services import backup_destination
+    from backend.app.tasks import backup as tasks
+    db_sync.add(Artist(name="Bea", slug="bea"))
+    db_sync.commit()
+    monkeypatch.setattr(tasks, "IMAGES_ROOT", tmp_path)
+
+    with FakeS3() as s3:
+        monkeypatch.setattr(backup_destination, "saved_target", lambda settings: s3.target())
+        tasks.backup_db_task(triggered_by="manual")
+        dump_key = next(k for k in s3.objects if k.endswith(".dump"))
+        # The database moves on; the test compares with the dump's own counts.
+        db_sync.add(Artist(name="Later", slug="later"))
+        db_sync.commit()
+        out = tasks.restore_test_task(dump_key=dump_key)
+
+    with tasks._sync_session_factory()() as s:
+        row = s.get(BackupRun, out["backup_run_id"])
+        assert row.status == "ok", row.error
+        m = row.manifest
+        assert m["counted"] and m["mismatched"] == []
+        artist = next(t for t in m["tables"] if t["table"] == "artist")
+        assert (artist["dump"], artist["restored"], artist["live"]) == (1, 1, 2)
+        assert m["alembic_dump"] == m["alembic_now"]
+        assert m["offsite"]["key"] == dump_key
+        # This install is as it was, and the test left nothing behind.
+        assert s.execute(select(func.count()).select_from(Artist)).scalar() == 2
+        assert not _scratch_exists(s)
+    assert not list((tmp_path / "_backups").glob("*"))
+
+
+@pytest.mark.integration
+def test_a_dump_whose_counts_dont_match_fails_and_one_without_is_compared_with_live(
+    db_sync, tmp_path, monkeypatch,
+):
+    from backend.app.services import backup_destination
+    from backend.app.tasks import backup as tasks
+    db_sync.add(Artist(name="Bea", slug="bea"))
+    db_sync.commit()
+    monkeypatch.setattr(tasks, "IMAGES_ROOT", tmp_path)
+
+    with FakeS3() as s3:
+        monkeypatch.setattr(backup_destination, "saved_target", lambda settings: s3.target())
+        tasks.backup_db_task(triggered_by="manual")
+        dump_key = next(k for k in s3.objects if k.endswith(".dump"))
+        counts = json.loads(s3.objects[backup_verify.counts_key(dump_key)])
+        counts["tables"]["artist"] = 5
+        s3.objects[backup_verify.counts_key(dump_key)] = json.dumps(counts).encode()
+        wrong = tasks.restore_test_task(dump_key=dump_key)
+        del s3.objects[backup_verify.counts_key(dump_key)]
+        uncounted = tasks.restore_test_task(dump_key=dump_key)
+
+    with tasks._sync_session_factory()() as s:
+        row = s.get(BackupRun, wrong["backup_run_id"])
+        assert row.status == "error"
+        assert row.error == "1 table(s) came back different: artist (5 in the dump, 1 restored)."
+        row = s.get(BackupRun, uncounted["backup_run_id"])
+        assert row.status == "ok" and row.manifest["counted"] is False
+        assert not _scratch_exists(s)