feat: restore a new install from the off-site bucket, database first, then every file

Settings → Maintenance → Backups gains "Restore from the bucket": pick one of
the bucket's dated dumps, type its confirmation, and FabledCurator saves the
current database to the bucket, replaces it with the dump, migrates it to this
version, then fetches every image and attachment back to the path its row
records. Each file is checked against its sha256 on arrival and moved into
place only then; a file already on disk is left alone, so "Fetch missing
files" repairs a library that lost files without touching the database.

- services/backup_restore.py: list/find/fetch dumps, migrate, keep this
  install's destination (the dump's sealed secret can't be opened here),
  library_files work list, restore_file (one fetch per sha, copies for
  duplicate paths, refuses paths outside /images).
- restore_offsite_task: the database phase, then 45-minute file chunks with a
  heartbeat; a failed load leaves an empty migrated database and says why.
  The dump's own run row is marked ok, other runs then going "Interrupted".
- backup_s3.get_file: sha/size check on arrival, ObjectMissing for a 404,
  download pacing.
- backup_service: every dump leaves the credential table's rows out;
  restore_db reports pg_restore's last line, stops at the first error, ends
  other connections and waits at most 60 s for locks.
- Backups and orphan repair wait while a restore runs; the stall sweep judges
  restores by heartbeat; restore runs are pruned to 20.
- CLI: python -m backend.app.scripts.restore list|database|files.
- README: backup and restore section; logins are in no backup.
- Tests: a round trip through an in-process S3 bucket (tests/fake_s3.py) with
  real pg_dump/pg_restore, plus file, run, API and get_file cases.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_api_system_backup.py

@@ -496,3 +496,91 @@ async def test_an_upload_limit_that_isnt_a_whole_number_is_refused(client, _seal
     )
     assert resp.status_code == 400
     assert "upload_limit_mbps" in (await resp.get_json())["detail"]
+
+
+# ── restoring from the off-site bucket ──────────────────────────────────────
+
+
+@pytest.fixture
+def _saved_bucket(monkeypatch):
+    from backend.app.services import backup_destination
+    from backend.app.services.backup_s3 import S3Target
+
+    target = S3Target("http://archives.example:8081", "curator", "AK", "SK")
+    monkeypatch.setattr(backup_destination, "saved_target", lambda settings: target)
+    dispatched = []
+    monkeypatch.setattr("backend.app.tasks.backup.restore_offsite_task.delay",
+                        lambda **kw: dispatched.append(kw))
+    return dispatched
+
+
+_DUMP = "database/fabledcurator-20261107T134258Z.dump"
+
+
+@pytest.mark.asyncio
+async def test_the_bucket_dumps_are_listed_with_their_confirmation(
+    client, _saved_bucket, monkeypatch,
+):
+    from backend.app.services import backup_restore
+
+    monkeypatch.setattr(backup_restore, "list_dumps", lambda target: [
+        backup_restore.Dump(_DUMP, "20261107T134258Z", 3222868924),
+    ])
+    resp = await client.get("/api/system/backup/offsite/dumps")
+    assert resp.status_code == 200
+    assert (await resp.get_json())["dumps"] == [{
+        "key": _DUMP, "ts": "20261107T134258Z", "size_bytes": 3222868924,
+        "confirm": "restore-offsite-20261107T134258Z",
+    }]
+
+
+@pytest.mark.asyncio
+async def test_a_restore_needs_the_dump_typed_back(client, _saved_bucket):
+    resp = await client.post("/api/system/backup/offsite/restore",
+                             json={"key": _DUMP, "confirm": "yes"})
+    assert resp.status_code == 400
+    assert (await resp.get_json())["expected"] == "restore-offsite-20261107T134258Z"
+    assert _saved_bucket == []
+
+    resp = await client.post("/api/system/backup/offsite/restore", json={
+        "key": _DUMP, "confirm": "restore-offsite-20261107T134258Z"})
+    assert resp.status_code == 202
+    assert _saved_bucket == [{"dump_key": _DUMP}]
+
+
+@pytest.mark.asyncio
+async def test_a_restore_waits_while_a_backup_runs(client, _saved_bucket, db_sync):
+    db_sync.add(BackupRun(kind="db", status="running", triggered_by="nightly",
+                          started_at=datetime.now(UTC), manifest={}))
+    db_sync.commit()
+    resp = await client.post("/api/system/backup/offsite/restore", json={
+        "key": _DUMP, "confirm": "restore-offsite-20261107T134258Z"})
+    assert resp.status_code == 409
+    assert "db backup is running" in (await resp.get_json())["detail"]
+    resp = await client.post("/api/system/backup/offsite/fetch-files", json={})
+    assert resp.status_code == 409
+    assert _saved_bucket == []
+
+
+@pytest.mark.asyncio
+async def test_fetching_missing_files_leaves_the_database(client, _saved_bucket):
+    resp = await client.post("/api/system/backup/offsite/fetch-files", json={})
+    assert resp.status_code == 202
+    assert _saved_bucket == [{"files_only": True}]
+
+
+@pytest.mark.asyncio
+async def test_restoring_needs_a_destination(client):
+    resp = await client.get("/api/system/backup/offsite/dumps")
+    assert resp.status_code == 400
+    assert (await resp.get_json())["error"] == "no_destination"
+
+
+@pytest.mark.asyncio
+async def test_restore_runs_are_listed_as_their_own_kind(client, db_sync):
+    db_sync.add(BackupRun(kind="restore", status="restoring", triggered_by="restore",
+                          started_at=datetime.now(UTC), manifest={"phase": "files"}))
+    db_sync.commit()
+    resp = await client.get("/api/system/backup/runs?kind=restore")
+    runs = (await resp.get_json())["runs"]
+    assert [r["status"] for r in runs] == ["restoring"]