feat: restore a new install from the off-site bucket, database first, then every file

Settings → Maintenance → Backups gains "Restore from the bucket": pick one of
the bucket's dated dumps, type its confirmation, and FabledCurator saves the
current database to the bucket, replaces it with the dump, migrates it to this
version, then fetches every image and attachment back to the path its row
records. Each file is checked against its sha256 on arrival and moved into
place only then; a file already on disk is left alone, so "Fetch missing
files" repairs a library that lost files without touching the database.

- services/backup_restore.py: list/find/fetch dumps, migrate, keep this
  install's destination (the dump's sealed secret can't be opened here),
  library_files work list, restore_file (one fetch per sha, copies for
  duplicate paths, refuses paths outside /images).
- restore_offsite_task: the database phase, then 45-minute file chunks with a
  heartbeat; a failed load leaves an empty migrated database and says why.
  The dump's own run row is marked ok, other runs then going "Interrupted".
- backup_s3.get_file: sha/size check on arrival, ObjectMissing for a 404,
  download pacing.
- backup_service: every dump leaves the credential table's rows out;
  restore_db reports pg_restore's last line, stops at the first error, ends
  other connections and waits at most 60 s for locks.
- Backups and orphan repair wait while a restore runs; the stall sweep judges
  restores by heartbeat; restore runs are pruned to 20.
- CLI: python -m backend.app.scripts.restore list|database|files.
- README: backup and restore section; logins are in no backup.
- Tests: a round trip through an in-process S3 bucket (tests/fake_s3.py) with
  real pg_dump/pg_restore, plus file, run, API and get_file cases.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/tasks/admin.py

@@ -74,6 +74,12 @@ def prune_missing_file_records_task(self) -> dict:
     thumbs: list[str] = []
     last_id = 0
     with SessionLocal() as session:
+        # While a restore is fetching the library back, files are missing
+        # because they haven't arrived yet, not because they are gone.
+        from .backup import restore_going
+        if restore_going(session):
+            return {"checked": 0, "missing": 0, "deleted": 0,
+                    "skipped": "a restore is fetching the library's files"}
         while True:
             rows = session.execute(
                 select(ImageRecord.id, ImageRecord.path, ImageRecord.thumbnail_path)

backend/app/tasks/backup.py

@@ -9,17 +9,24 @@ from __future__ import annotations
 
 import logging
 import time
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
 from pathlib import Path
 
 from celery.exceptions import SoftTimeLimitExceeded
-from sqlalchemy import select
+from sqlalchemy import select, text, update
 from sqlalchemy.exc import DBAPIError, OperationalError
 
 from ..celery_app import celery
 from ..config import get_config
 from ..models import BackupRun, ImportSettings
-from ..services import backup_destination, backup_media, backup_s3, backup_service
+from ..services import (
+    backup_destination,
+    backup_media,
+    backup_restore,
+    backup_s3,
+    backup_service,
+)
+from ._sync_engine import get_sync_engine
 from ._sync_engine import sync_session_factory as _sync_session_factory
 
 log = logging.getLogger(__name__)
@@ -60,6 +67,9 @@ def backup_db_task(self, *, tag: str | None = None,
     cfg = get_config()
     now = datetime.now(UTC)
     with SessionLocal() as session:
+        # A restore replaces the database this would dump.
+        if restore_going(session):
+            return {"skipped": "a restore is going"}
         row = BackupRun(
             kind="db", status="running", tag=tag,
             triggered_by=triggered_by, started_at=now, manifest={},
@@ -269,8 +279,9 @@ def prune_backups() -> dict:
     """Daily Beat. Per-kind retention from ImportSettings.
 
     Returns {"db_deleted": N, "images_deleted": M, "media_deleted": L,
-    "files_unlinked": K}. Media runs leave no files here; only their history
-    is trimmed, to the latest MEDIA_RUNS_KEPT.
+    "restore_deleted": R, "files_unlinked": K}. Media and restore runs leave
+    no files here; only their history is trimmed, to the latest
+    MEDIA_RUNS_KEPT and RESTORE_RUNS_KEPT.
     Tagged rows (tag IS NOT NULL) are never pruned.
     Status='running' / 'restoring' rows are never pruned — the
     recover_stalled_backup_runs sweep flips truly-stuck ones to
@@ -279,19 +290,21 @@ def prune_backups() -> dict:
     Audit 2026-06-02 added the dedicated sweep.)
     """
     SessionLocal = _sync_session_factory()
-    counts = {"db_deleted": 0, "images_deleted": 0, "media_deleted": 0, "files_unlinked": 0}
+    counts = {"db_deleted": 0, "images_deleted": 0, "media_deleted": 0,
+              "restore_deleted": 0, "files_unlinked": 0}
     with SessionLocal() as session:
         s = ImportSettings.load_sync(session)
         for kind, keep in (
             ("db", s.backup_db_keep_last_n),
             ("images", s.backup_images_keep_last_n),
             ("media", MEDIA_RUNS_KEPT),
+            ("restore", RESTORE_RUNS_KEPT),
         ):
             candidates = session.execute(
                 select(BackupRun)
                 .where(BackupRun.kind == kind)
                 .where(BackupRun.tag.is_(None))
-                .where(BackupRun.status.in_(["ok", "error"]))
+                .where(BackupRun.status.in_(["ok", "error", "restored"]))
                 .order_by(BackupRun.started_at.desc())
                 .offset(keep)
             ).scalars().all()
@@ -385,6 +398,9 @@ def backup_media_task(self, *, run_id: int | None = None,
             ).scalar_one_or_none()
             if going is not None:
                 return {"backup_run_id": going, "skipped": "a media backup is already running"}
+            # During a restore the library's files are still arriving.
+            if restore_going(session):
+                return {"skipped": "a restore is going"}
             row = BackupRun(kind="media", status="running", triggered_by=triggered_by,
                             started_at=now, last_progress_at=now, size_bytes=0,
                             manifest={"sent": 0, "failed": 0, "failures": [], "cursor": ""})
@@ -494,3 +510,284 @@ def reconcile_media_backup() -> dict:
         if going is not None:
             return {"skipped": "a media backup is running"}
         return backup_media.reconcile(session, target)
+
+
+# ── Restoring from the off-site bucket ───────────────────────────────────────
+
+# The file phase fetches for this long per task, then queues its next chunk,
+# like a media run.
+RESTORE_CHUNK_SECONDS = 45 * 60
+RESTORE_RUNS_KEPT = 20
+RESTORE_FAILURES_LISTED = 20
+# A run over files already on disk notes its place this often, not per file.
+_RESTORE_NOTE_SECONDS = 5
+_RESTORE_COUNTS = ("restored", "copied", "present", "missing", "failed")
+
+
+def backup_busy(session) -> str | None:
+    """Why a backup or a restore can't start right now, or None."""
+    row = session.execute(
+        select(BackupRun.kind).where(BackupRun.status.in_(["running", "restoring"])).limit(1)
+    ).first()
+    if row is None:
+        return None
+    return "a restore is going" if row.kind == "restore" else f"a {row.kind} backup is running"
+
+
+def restore_going(session) -> bool:
+    return session.execute(
+        select(BackupRun.id).where(BackupRun.kind == "restore")
+        .where(BackupRun.status == "restoring").limit(1)
+    ).first() is not None
+
+
+def _library_is_empty(session) -> bool:
+    return session.execute(text(
+        "SELECT NOT EXISTS (SELECT 1 FROM image_record) "
+        "AND NOT EXISTS (SELECT 1 FROM post_attachment)"
+    )).scalar()
+
+
+def _note(session, row: BackupRun, m: dict) -> None:
+    row.manifest = dict(m)
+    row.last_progress_at = datetime.now(UTC)
+    session.commit()
+
+
+@celery.task(
+    name="backend.app.tasks.backup.restore_offsite_task",
+    bind=True,
+    max_retries=0,  # a half-applied restore is never retried on its own
+    soft_time_limit=21600, time_limit=23400,
+)
+def restore_offsite_task(self, *, dump_key: str | None = None, files_only: bool = False,
+                         run_id: int | None = None) -> dict:
+    """Restore this install from the off-site bucket, one chunk at a time.
+
+    Started without `run_id`, it replaces the database with the dump at
+    `dump_key` (the newest when None), then fetches every library file the
+    restored rows name. With `files_only`, the database stays and only the
+    missing files are fetched, which also repairs a library that lost files.
+    Each chunk queues the next; the run's row counts files restored from the
+    bucket, copied from a duplicate already on disk, already present, missing
+    from the bucket, and failed. Returns {'backup_run_id': N, ...}."""
+    SessionLocal = _sync_session_factory()
+    if run_id is not None:
+        return _restore_files(SessionLocal, run_id)
+
+    now = datetime.now(UTC)
+    with SessionLocal() as session:
+        busy = backup_busy(session)
+        if busy:
+            return {"skipped": busy}
+        settings = ImportSettings.load_sync(session)
+        limit_mbps = settings.backup_upload_limit_mbps
+        row = BackupRun(kind="restore", status="restoring", triggered_by="restore",
+                        started_at=now, last_progress_at=now, size_bytes=0,
+                        manifest={"phase": "files" if files_only else "database",
+                                  "cursor": "", "failures": [],
+                                  **dict.fromkeys(_RESTORE_COUNTS, 0)})
+        session.add(row)
+        session.commit()
+        run_id = row.id
+        try:
+            target = backup_destination.saved_target(settings)
+            if target is None:
+                raise backup_s3.S3Error("No off-site destination is set up.")
+        except backup_s3.S3Error as exc:
+            _mark_failed(session, row, exc)
+            raise
+        row.manifest = {**row.manifest, "offsite": _offsite_manifest(target)}
+        session.commit()
+
+    if files_only:
+        return _restore_files(SessionLocal, run_id)
+    run_id = _restore_database(SessionLocal, run_id, target, limit_mbps, dump_key)
+    # The files follow as chunks of their own, on the worker.
+    restore_offsite_task.delay(run_id=run_id)
+    return {"backup_run_id": run_id, "continues": True}
+
+
+def _restore_database(SessionLocal, run_id: int, target, limit_mbps, dump_key) -> int:
+    """Replace the database with a dump from the bucket. Returns the run's id
+    in the restored database, where its row is written again."""
+    cfg = get_config()
+    pacer = backup_s3.Pacer(limit_mbps)
+    staged = None
+    try:
+        dump = backup_restore.find_dump(target, dump_key)
+        with SessionLocal() as session:
+            row = session.get(BackupRun, run_id)
+            m = dict(row.manifest)
+            m["dump"] = {"key": dump.key, "ts": dump.ts, "size_bytes": dump.size_bytes}
+            # The database about to be replaced goes to the bucket first, so
+            # the restore can itself be undone. An empty one isn't worth a
+            # dump, and would sit in the list as the newest.
+            if not _library_is_empty(session):
+                session.commit()
+                saved = backup_service.backup_db_offsite(
+                    db_url=cfg.database_url_sync, target=target, limit_mbps=limit_mbps)
+                m["saved_before"] = saved["key"]
+            _note(session, row, m)
+            started_at, manifest = row.started_at, dict(m)
+        staged = backup_restore.fetch_dump(target, dump, IMAGES_ROOT / "_backups", pacer)
+    except (SoftTimeLimitExceeded, Exception) as exc:
+        if staged is not None:
+            staged.unlink(missing_ok=True)
+        with SessionLocal() as session:
+            row = session.get(BackupRun, run_id)
+            if row is not None:
+                _mark_failed(session, row, exc)
+        raise
+
+    # From here on the database is being replaced; every pooled connection
+    # still points at the tables being dropped.
+    get_sync_engine().dispose()
+    try:
+        backup_service.restore_db(db_url=cfg.database_url_sync, sql_path=staged)
+        backup_restore.migrate()
+        backup_service.end_other_connections(db_url=cfg.database_url_sync)
+    except (SoftTimeLimitExceeded, Exception) as exc:
+        _leave_empty_install(SessionLocal, cfg, target, limit_mbps, started_at, manifest, exc)
+        raise
+    finally:
+        staged.unlink(missing_ok=True)
+    get_sync_engine().dispose()
+
+    with SessionLocal() as session:
+        backup_restore.keep_destination(session, target, limit_mbps)
+        # The dump holds its own run as still going: it is the dump just
+        # restored, so it finished. Any other run going then is over.
+        dump_at = datetime.strptime(manifest["dump"]["ts"], "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC)
+        own = session.execute(
+            select(BackupRun).where(BackupRun.kind == "db", BackupRun.status == "running")
+            # The name's time is truncated to the second, after the row began.
+            .where(BackupRun.started_at.between(dump_at - timedelta(minutes=2),
+                                                dump_at + timedelta(seconds=1)))
+            .order_by(BackupRun.started_at.desc()).limit(1)
+        ).scalar_one_or_none()
+        if own is not None:
+            own.status = "ok"
+            own.manifest = {"ts": manifest["dump"]["ts"], "offsite": {
+                **manifest["offsite"], "key": manifest["dump"]["key"]}}
+            own.size_bytes = manifest["dump"]["size_bytes"]
+        session.execute(
+            update(BackupRun).where(BackupRun.status.in_(["running", "restoring"]))
+            .values(status="error", finished_at=datetime.now(UTC),
+                    error="Interrupted: this run was going when the restored dump was taken.")
+        )
+        row = BackupRun(kind="restore", status="restoring", triggered_by="restore",
+                        started_at=started_at, last_progress_at=datetime.now(UTC),
+                        size_bytes=0, manifest={**manifest, "phase": "files"})
+        session.add(row)
+        session.commit()
+        return row.id
+
+
+def _leave_empty_install(SessionLocal, cfg, target, limit_mbps, started_at, manifest,
+                         exc: BaseException) -> None:
+    """After a failed load, put back an empty database at this version, so the
+    install starts and can try again, and record why it stopped."""
+    try:
+        backup_service.wipe_db(db_url=cfg.database_url_sync)
+        backup_restore.migrate()
+        backup_service.end_other_connections(db_url=cfg.database_url_sync)
+        get_sync_engine().dispose()
+        with SessionLocal() as session:
+            backup_restore.keep_destination(session, target, limit_mbps)
+            session.add(BackupRun(
+                kind="restore", status="error", triggered_by="restore",
+                started_at=started_at, finished_at=datetime.now(UTC), manifest=manifest,
+                error=(f"The restore stopped while loading the database: {exc}. "
+                       "The database is empty again; choose a dump and restore again.")[:2000],
+            ))
+            session.commit()
+    except Exception:  # noqa: BLE001 — the original failure is the one raised
+        log.exception("restore: the database couldn't be reset after a failed load")
+
+
+def _restore_files(SessionLocal, run_id: int) -> dict:
+    """One chunk of a restore's file phase; queues the next while files remain."""
+    with SessionLocal() as session:
+        row = session.get(BackupRun, run_id)
+        if row is None or row.status != "restoring":
+            return {"backup_run_id": run_id, "skipped": "the restore is no longer going"}
+        settings = ImportSettings.load_sync(session)
+        limit_mbps = settings.backup_upload_limit_mbps
+        try:
+            target = backup_destination.saved_target(settings)
+            if target is None:
+                raise backup_s3.S3Error("No off-site destination is set up.")
+        except backup_s3.S3Error as exc:
+            _mark_failed(session, row, exc)
+            raise
+    try:
+        more = _fetch_files_chunk(SessionLocal, run_id, target, backup_s3.Pacer(limit_mbps))
+    except (SoftTimeLimitExceeded, Exception) as exc:
+        with SessionLocal() as session:
+            row = session.get(BackupRun, run_id)
+            if row is not None:
+                _mark_failed(session, row, exc)
+        raise
+
+    if more:
+        restore_offsite_task.delay(run_id=run_id)
+        return {"backup_run_id": run_id, "continues": True}
+    with SessionLocal() as session:
+        row = session.get(BackupRun, run_id)
+        m = dict(row.manifest or {})
+        row.status = "restored"
+        row.finished_at = datetime.now(UTC)
+        missed = m.get("missing", 0) + m.get("failed", 0)
+        if missed:
+            first = (m.get("failures") or [{}])[0]
+            row.error = (f"{m.get('missing', 0)} file(s) aren't in the bucket and "
+                         f"{m.get('failed', 0)} couldn't be restored. "
+                         f"First: {first.get('path')}: {first.get('error')}")[:2000]
+        session.commit()
+    # Thumbnails aren't backed up: they are made again from the restored files.
+    from .thumbnail import backfill_thumbnails
+    backfill_thumbnails.delay()
+    return {"backup_run_id": run_id, "continues": False}
+
+
+def _fetch_files_chunk(SessionLocal, run_id: int, target, pacer) -> bool:
+    """Put library files back until the chunk's time is up or none are left.
+    Returns True when files remain for another chunk."""
+    tid = backup_media.target_id(target)
+    deadline = time.monotonic() + RESTORE_CHUNK_SECONDS
+    with SessionLocal() as session:
+        row = session.get(BackupRun, run_id)
+        m = dict(row.manifest or {})
+        cursor = m.get("cursor") or ""
+        noted = time.monotonic()
+        while True:
+            batch = backup_restore.library_files(session, after=cursor)
+            keys = backup_restore.sent_keys(session, tid, [f.sha256 for f in batch])
+            # End the read before any download starts.
+            session.commit()
+            if not batch:
+                m["cursor"] = cursor
+                _note(session, row, m)
+                return False
+            for f in batch:
+                if time.monotonic() > deadline:
+                    m["cursor"] = cursor
+                    _note(session, row, m)
+                    return True
+                try:
+                    outcome = backup_restore.restore_file(
+                        target, f, keys.get(f.sha256), IMAGES_ROOT, pacer)
+                except (backup_s3.S3Error, OSError) as exc:
+                    outcome = "missing" if isinstance(exc, backup_s3.ObjectMissing) else "failed"
+                    if len(m.get("failures") or []) < RESTORE_FAILURES_LISTED:
+                        m["failures"] = [*(m.get("failures") or []),
+                                         {"path": f.paths[0], "error": str(exc)[:300]}]
+                m[outcome] = m.get(outcome, 0) + 1
+                if outcome in ("restored", "copied"):
+                    row.size_bytes = (row.size_bytes or 0) + f.size_bytes
+                cursor = f.sha256
+                m["cursor"] = cursor
+                if outcome != "present" or time.monotonic() - noted > _RESTORE_NOTE_SECONDS:
+                    _note(session, row, m)
+                    noted = time.monotonic()

backend/app/tasks/maintenance.py

@@ -176,6 +176,8 @@ TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
     # One media-backup chunk stops starting files after 45 min but may finish
     # a large file in flight, up to its 6.5 h hard limit.
     "backend.app.tasks.backup.backup_media_task": 420,
+    # A restore's database load, or one chunk of its file phase, likewise.
+    "backend.app.tasks.backup.restore_offsite_task": 420,
     # Library audit scans the full library — 2h hard limit.
     "backend.app.tasks.library_audit.scan_library_for_rule": 130,
     # External file-host fetches (mega/gdrive/film packs) can run to the task's
@@ -845,17 +847,17 @@ def recover_stalled_backup_runs() -> int:
             update(BackupRun)
             .where(BackupRun.status.in_(["running", "restoring"]))
             # db backups/restores are fast (40-min window); images run hours (7h).
-            # A media run spans many chunks and hours on purpose, so its window
-            # runs from its last progress, not from its start.
+            # A media run or an off-site restore spans many chunks and hours on
+            # purpose, so its window runs from its last progress, not its start.
             .where(
                 or_(
                     and_(BackupRun.kind == "db", BackupRun.started_at < db_cutoff),
                     and_(
-                        BackupRun.kind == "media",
+                        BackupRun.kind.in_(["media", "restore"]),
                         func.coalesce(BackupRun.last_progress_at, BackupRun.started_at)
                         < slow_cutoff,
                     ),
-                    and_(BackupRun.kind.not_in(["db", "media"]),
+                    and_(BackupRun.kind.not_in(["db", "media", "restore"]),
                          BackupRun.started_at < slow_cutoff),
                 )
             )