feat: restore a new install from the off-site bucket, database first, then every file

Settings → Maintenance → Backups gains "Restore from the bucket": pick one of
the bucket's dated dumps, type its confirmation, and FabledCurator saves the
current database to the bucket, replaces it with the dump, migrates it to this
version, then fetches every image and attachment back to the path its row
records. Each file is checked against its sha256 on arrival and moved into
place only then; a file already on disk is left alone, so "Fetch missing
files" repairs a library that lost files without touching the database.

- services/backup_restore.py: list/find/fetch dumps, migrate, keep this
  install's destination (the dump's sealed secret can't be opened here),
  library_files work list, restore_file (one fetch per sha, copies for
  duplicate paths, refuses paths outside /images).
- restore_offsite_task: the database phase, then 45-minute file chunks with a
  heartbeat; a failed load leaves an empty migrated database and says why.
  The dump's own run row is marked ok, other runs then going "Interrupted".
- backup_s3.get_file: sha/size check on arrival, ObjectMissing for a 404,
  download pacing.
- backup_service: every dump leaves the credential table's rows out;
  restore_db reports pg_restore's last line, stops at the first error, ends
  other connections and waits at most 60 s for locks.
- Backups and orphan repair wait while a restore runs; the stall sweep judges
  restores by heartbeat; restore runs are pruned to 20.
- CLI: python -m backend.app.scripts.restore list|database|files.
- README: backup and restore section; logins are in no backup.
- Tests: a round trip through an in-process S3 bucket (tests/fake_s3.py) with
  real pg_dump/pg_restore, plus file, run, API and get_file cases.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/api/system_backup.py

@@ -20,7 +20,7 @@ system_backup_bp = Blueprint(
     "system_backup", __name__, url_prefix="/api/system/backup",
 )
 
-_KINDS = frozenset({"db", "images", "media"})
+_KINDS = frozenset({"db", "images", "media", "restore"})
 _TAG_MAX_LEN = 64
 _BACKUP_SETTINGS_FIELDS = (
     "backup_db_nightly_enabled",
@@ -142,6 +142,102 @@ async def media_coverage():
     return jsonify({"configured": configured, **counts})
 
 
+# ── Restoring from the off-site bucket ──────────────────────────────────────
+
+
+async def _saved_target():
+    """(target, None) for the saved destination, or (None, error response)."""
+    from ..services import backup_destination as dest
+    from ..services.backup_s3 import S3Error
+
+    async with get_session() as session:
+        row = await ImportSettings.load(session)
+        try:
+            target = dest.saved_target(row)
+        except S3Error as exc:
+            return None, _bad("destination_unreadable", detail=str(exc))
+    if target is None:
+        return None, _bad("no_destination",
+                          detail="set up an off-site destination before restoring from it")
+    return target, None
+
+
+async def _busy_reason() -> str | None:
+    from ..tasks.backup import backup_busy
+
+    async with get_session() as session:
+        return await session.run_sync(backup_busy)
+
+
+def restore_token(ts: str) -> str:
+    """What the person types to confirm replacing the database with the dump
+    taken at `ts`."""
+    return f"restore-offsite-{ts}"
+
+
+@system_backup_bp.route("/offsite/dumps", methods=["GET"])
+async def list_offsite_dumps():
+    """The database dumps in the off-site bucket, newest first, each with
+    the confirmation a restore of it asks for."""
+    import asyncio
+
+    from ..services import backup_restore
+    from ..services.backup_s3 import S3Error
+
+    target, err = await _saved_target()
+    if err:
+        return err
+    try:
+        dumps = await asyncio.to_thread(backup_restore.list_dumps, target)
+    except S3Error as exc:
+        return _bad("destination_unreachable", detail=str(exc), status=502)
+    return jsonify({"dumps": [
+        {"key": d.key, "ts": d.ts, "size_bytes": d.size_bytes, "confirm": restore_token(d.ts)}
+        for d in dumps
+    ]})
+
+
+@system_backup_bp.route("/offsite/restore", methods=["POST"])
+async def restore_from_offsite():
+    """Replace this install's database with a dump from the bucket, then
+    fetch the library's files back. Body: {key, confirm}, where confirm is
+    the dump's token from /offsite/dumps. Refused while a backup or another
+    restore is going."""
+    body = await request.get_json(silent=True) or {}
+    key = body.get("key")
+    if not isinstance(key, str) or not key.startswith("database/"):
+        return _bad("invalid_key", detail="key must be a dump's key from /offsite/dumps")
+    ts = key.removeprefix("database/fabledcurator-").removesuffix(".dump")
+    expected = restore_token(ts)
+    if body.get("confirm") != expected:
+        return _bad("confirm_mismatch", detail=f"confirm must equal {expected!r}",
+                    expected=expected)
+    _, err = await _saved_target()
+    if err:
+        return err
+    busy = await _busy_reason()
+    if busy:
+        return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
+    from ..tasks.backup import restore_offsite_task
+    restore_offsite_task.delay(dump_key=key)
+    return jsonify({"status": "dispatched"}), 202
+
+
+@system_backup_bp.route("/offsite/fetch-files", methods=["POST"])
+async def fetch_missing_files():
+    """Fetch every library file that is missing from disk back from the
+    bucket, leaving the database as it is."""
+    _, err = await _saved_target()
+    if err:
+        return err
+    busy = await _busy_reason()
+    if busy:
+        return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
+    from ..tasks.backup import restore_offsite_task
+    restore_offsite_task.delay(files_only=True)
+    return jsonify({"status": "dispatched"}), 202
+
+
 @system_backup_bp.route("/runs", methods=["GET"])
 async def list_runs():
     try: