feat: test a database backup without touching the install

"Test a backup" on the Backups card loads a dump into a separate, temporary
database on the same Postgres server, the same way a restore does, checks every
table against the row counts recorded with the dump, brings the copy up to this
version, and drops it. The live database is only read.

Every dump now records its row counts, read in the snapshot pg_dump dumps from
(pg_export_snapshot + --snapshot), so the comparison is exact. Off-site the
counts sit beside the dump as <name>.counts.json; locally they go in the
manifest file. A dump taken before this is compared with the live database
instead.

A test cut off by a restart is ended, and its database dropped, when the long
maintenance lane starts. A test doesn't block backups; a restore and a second
test wait for it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/api/system_backup.py

@@ -20,7 +20,7 @@ system_backup_bp = Blueprint(
     "system_backup", __name__, url_prefix="/api/system/backup",
 )
 
-_KINDS = frozenset({"db", "images", "media", "restore", "reconcile"})
+_KINDS = frozenset({"db", "images", "media", "restore", "reconcile", "test"})
 _TAG_MAX_LEN = 64
 _BACKUP_SETTINGS_FIELDS = (
     "backup_db_nightly_enabled",
@@ -191,10 +191,52 @@ async def list_offsite_dumps():
         dumps = await asyncio.to_thread(backup_restore.list_dumps, target)
     except S3Error as exc:
         return _bad("destination_unreachable", detail=str(exc), status=502)
-    return jsonify({"dumps": [
-        {"key": d.key, "ts": d.ts, "size_bytes": d.size_bytes, "confirm": restore_token(d.ts)}
-        for d in dumps
-    ]})
+    return jsonify({
+        "dumps": [
+            {"key": d.key, "ts": d.ts, "size_bytes": d.size_bytes, "counted": d.counted,
+             "confirm": restore_token(d.ts)}
+            for d in dumps
+        ],
+        # A test restore needs about this much free space on the database's
+        # disk while it runs.
+        "database_bytes": await _database_bytes(),
+    })
+
+
+async def _database_bytes() -> int:
+    from sqlalchemy import text
+
+    async with get_session() as session:
+        return (await session.execute(
+            text("SELECT pg_database_size(current_database())"))).scalar_one()
+
+
+async def _restore_test_blocked() -> str | None:
+    from ..tasks.backup import restore_test_blocked
+
+    async with get_session() as session:
+        return await session.run_sync(restore_test_blocked)
+
+
+@system_backup_bp.route("/offsite/test", methods=["POST"])
+async def restore_test_offsite():
+    """Test a dump from the bucket without touching this install: it is
+    loaded into a temporary database, checked table by table, and dropped.
+    Body: {key}, a dump's key from /offsite/dumps, or {} for the newest.
+    Needs no confirmation; refused while a restore or another test is going."""
+    body = await request.get_json(silent=True) or {}
+    key = body.get("key")
+    if key is not None and (not isinstance(key, str) or not key.startswith("database/")):
+        return _bad("invalid_key", detail="key must be a dump's key from /offsite/dumps")
+    _, err = await _saved_target()
+    if err:
+        return err
+    busy = await _restore_test_blocked()
+    if busy:
+        return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
+    from ..tasks.backup import restore_test_task
+    restore_test_task.delay(dump_key=key)
+    return jsonify({"status": "dispatched"}), 202
 
 
 @system_backup_bp.route("/offsite/restore", methods=["POST"])
@@ -335,6 +377,25 @@ async def trigger_restore(run_id: int):
     return jsonify({"status": "dispatched", "kind": kind}), 202
 
 
+@system_backup_bp.route("/runs/<int:run_id>/test", methods=["POST"])
+async def restore_test_local(run_id: int):
+    """Test a database dump kept on this server, the same way as one in the
+    bucket. Needs no confirmation."""
+    async with get_session() as session:
+        row = await session.get(BackupRun, run_id)
+        if row is None:
+            return _bad("not_found", status=404)
+        if row.kind != "db" or row.status != "ok" or not row.sql_path:
+            return _bad("not_testable",
+                        detail="only a finished database dump on this server can be tested here")
+    busy = await _restore_test_blocked()
+    if busy:
+        return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
+    from ..tasks.backup import restore_test_task
+    restore_test_task.delay(source_backup_run_id=run_id)
+    return jsonify({"status": "dispatched"}), 202
+
+
 @system_backup_bp.route("/runs/<int:run_id>", methods=["DELETE"])
 async def delete_run(run_id: int):
     body = await request.get_json(silent=True) or {}