feat: restore a new install from the off-site bucket, database first, then every file

Settings → Maintenance → Backups gains "Restore from the bucket": pick one of
the bucket's dated dumps, type its confirmation, and FabledCurator saves the
current database to the bucket, replaces it with the dump, migrates it to this
version, then fetches every image and attachment back to the path its row
records. Each file is checked against its sha256 on arrival and moved into
place only then; a file already on disk is left alone, so "Fetch missing
files" repairs a library that lost files without touching the database.

- services/backup_restore.py: list/find/fetch dumps, migrate, keep this
  install's destination (the dump's sealed secret can't be opened here),
  library_files work list, restore_file (one fetch per sha, copies for
  duplicate paths, refuses paths outside /images).
- restore_offsite_task: the database phase, then 45-minute file chunks with a
  heartbeat; a failed load leaves an empty migrated database and says why.
  The dump's own run row is marked ok, other runs then going "Interrupted".
- backup_s3.get_file: sha/size check on arrival, ObjectMissing for a 404,
  download pacing.
- backup_service: every dump leaves the credential table's rows out;
  restore_db reports pg_restore's last line, stops at the first error, ends
  other connections and waits at most 60 s for locks.
- Backups and orphan repair wait while a restore runs; the stall sweep judges
  restores by heartbeat; restore runs are pruned to 20.
- CLI: python -m backend.app.scripts.restore list|database|files.
- README: backup and restore section; logins are in no backup.
- Tests: a round trip through an in-process S3 bucket (tests/fake_s3.py) with
  real pg_dump/pg_restore, plus file, run, API and get_file cases.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/services/backup_restore.py

@@ -0,0 +1,191 @@
+"""Restoring from the off-site bucket: the database from one of its dated
+dumps, then every library file the restored rows name, fetched back to the
+path each row records.
+
+The database comes first because it is the map: each image and attachment row
+holds its file's path and sha256, which is also the file's name in the bucket
+(`media/ab/cd/<sha256><ext>`). A file is checked against that sha256 on
+arrival and only then moved into place, so a restore never leaves a wrong or
+half-written file in the library. A file already on disk at the right size is
+left alone, which makes a restore safe to run again and lets it bring back
+only what a damaged library is missing.
+"""
+from __future__ import annotations
+
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+
+from sqlalchemy import select, text
+
+from ..models import BackupBlob, ImportSettings
+from . import backup_destination
+from .backup_media import _LIBRARY, blob_key
+from .backup_s3 import Pacer, S3Error, S3Target, get_file, list_objects
+
+DUMP_PREFIX = "database/"
+_DUMP_NAME = re.compile(r"^database/fabledcurator-(\d{8}T\d{6}Z)\.dump$")
+_ALEMBIC_INI = Path(__file__).resolve().parents[3] / "alembic.ini"
+_MIGRATE_TIMEOUT_S = 1800
+
+
+@dataclass(frozen=True)
+class Dump:
+    key: str
+    ts: str           # 20261107T134258Z, UTC
+    size_bytes: int
+
+
+def list_dumps(target: S3Target) -> list[Dump]:
+    """Every database dump in the bucket, newest first."""
+    dumps, token = [], None
+    while True:
+        page = list_objects(target, DUMP_PREFIX, 1000, continuation=token)
+        for o in page.objects:
+            if m := _DUMP_NAME.match(o.key):
+                dumps.append(Dump(o.key, m.group(1), o.size))
+        token = page.next_token
+        if not token:
+            break
+    return sorted(dumps, key=lambda d: d.ts, reverse=True)
+
+
+def find_dump(target: S3Target, key: str | None) -> Dump:
+    """The dump at `key`, or the newest when `key` is None or "latest"."""
+    dumps = list_dumps(target)
+    if not dumps:
+        raise S3Error(f"The {target.bucket} bucket holds no database dumps.")
+    if key in (None, "", "latest"):
+        return dumps[0]
+    for d in dumps:
+        if d.key == key:
+            return d
+    raise S3Error(f"{key} is not among the dumps in the {target.bucket} bucket.")
+
+
+def fetch_dump(target: S3Target, dump: Dump, into: Path, pacer: Pacer | None) -> Path:
+    """Download a dump into the folder `into`, checked against its listed
+    size. pg_restore reads it from there."""
+    into.mkdir(parents=True, exist_ok=True)
+    path = into / Path(dump.key).name
+    get_file(target, dump.key, path, size=dump.size_bytes, pacer=pacer)
+    return path
+
+
+def migrate() -> None:
+    """Bring a restored database up to this install's schema. A dump taken by
+    an older FabledCurator gains the migrations it predates; one from a newer
+    FabledCurator fails here, with alembic's reason, because this install
+    doesn't know its schema."""
+    with tempfile.TemporaryFile() as err:
+        proc = subprocess.run(
+            [sys.executable, "-m", "alembic", "-c", str(_ALEMBIC_INI), "upgrade", "head"],
+            cwd=_ALEMBIC_INI.parent, stdout=subprocess.DEVNULL, stderr=err,
+            timeout=_MIGRATE_TIMEOUT_S,
+        )
+        if proc.returncode != 0:
+            err.seek(0)
+            lines = err.read().decode(errors="replace").strip().splitlines()
+            raise S3Error("The restored database couldn't be brought up to this "
+                          f"version: {lines[-1] if lines else 'alembic failed'}")
+
+
+def keep_destination(session, target: S3Target, limit_mbps: int | None) -> None:
+    """Write this install's destination into the restored settings. The
+    dump's own copy of the secret is sealed with the old server's credential
+    key, which never leaves that server, so it can't be opened here."""
+    settings = ImportSettings.load_sync(session)
+    settings.backup_s3_endpoint = target.endpoint
+    settings.backup_s3_bucket = target.bucket
+    settings.backup_s3_access_key = target.access_key
+    settings.backup_s3_region = target.region
+    settings.backup_s3_secret_sealed = backup_destination.seal(target.secret_key)
+    settings.backup_upload_limit_mbps = limit_mbps
+    session.commit()
+
+
+# ── files ───────────────────────────────────────────────────────────────────
+
+
+@dataclass(frozen=True)
+class LibraryFile:
+    sha256: str
+    paths: tuple[str, ...]     # every row's path for this sha, sorted
+    size_bytes: int
+
+
+def library_files(session, *, after: str = "", limit: int = 200) -> list[LibraryFile]:
+    """The next `limit` distinct files the library names, in sha order after
+    `after`, each with every path a row gives it."""
+    rows = session.execute(text(f"""
+        SELECT lib.sha256, array_agg(DISTINCT lib.path ORDER BY lib.path) AS paths,
+               max(lib.size_bytes) AS size_bytes
+        FROM ({_LIBRARY}) AS lib
+        WHERE lib.sha256 > :after
+        GROUP BY lib.sha256
+        ORDER BY lib.sha256
+        LIMIT :limit
+    """), {"after": after, "limit": limit}).all()
+    return [LibraryFile(r.sha256, tuple(r.paths), r.size_bytes) for r in rows]
+
+
+def sent_keys(session, tid: str, shas: list[str]) -> dict[str, str]:
+    """The bucket key recorded for each sha that was sent to this destination."""
+    if not shas:
+        return {}
+    return dict(session.execute(
+        select(BackupBlob.sha256, BackupBlob.key)
+        .where(BackupBlob.target == tid, BackupBlob.sha256.in_(shas))
+    ).all())
+
+
+def _resolve(path: str, images_root: Path) -> Path:
+    """Where a row's path lands on this server. A path outside the library
+    folder is refused: a restore writes only inside it."""
+    p = Path(path)
+    if not p.is_absolute():
+        p = images_root / p
+    root = Path(*images_root.parts)
+    if ".." in p.parts or not (p == root or root in p.parents):
+        raise S3Error(f"{path} is outside {images_root}, so it isn't restored there.")
+    return p
+
+
+def _present(p: Path, size: int) -> bool:
+    try:
+        return p.is_file() and p.stat().st_size == size
+    except OSError:
+        return False
+
+
+def restore_file(target: S3Target, f: LibraryFile, key: str | None,
+                 images_root: Path, pacer: Pacer | None) -> str:
+    """Put one file back at every path the library gives it. Returns
+    "present" (nothing was missing), "copied" (a path already on disk
+    supplied the others) or "restored" (fetched from the bucket). Raises
+    ObjectMissing when the bucket doesn't hold it, S3Error or OSError
+    otherwise."""
+    places = [_resolve(p, images_root) for p in f.paths]
+    have = [p for p in places if _present(p, f.size_bytes)]
+    need = [p for p in places if p not in have]
+    if not need:
+        return "present"
+    outcome = "copied"
+    if not have:
+        first = need.pop(0)
+        first.parent.mkdir(parents=True, exist_ok=True)
+        get_file(target, key or blob_key(f.sha256, f.paths[0]), first,
+                 sha256=f.sha256, size=f.size_bytes, pacer=pacer)
+        have.append(first)
+        outcome = "restored"
+    for p in need:
+        p.parent.mkdir(parents=True, exist_ok=True)
+        partial = p.with_name(p.name + ".partial")
+        shutil.copyfile(have[0], partial)
+        partial.replace(p)
+    return outcome
+

backend/app/services/backup_s3.py

@@ -220,29 +220,55 @@ def put_file(target: S3Target, key: str, path: Path, *,
                       f"{_says(response.status, response.reason, content)}")
 
 
+class ObjectMissing(S3Error):
+    """The bucket holds nothing at the key asked for."""
+
+
 def get_file(target: S3Target, key: str, path: Path, *,
+             sha256: str | None = None, size: int | None = None,
+             pacer: Pacer | None = None,
              now: datetime | None = None) -> None:
     """Download one object to `path`, streamed to disk. The bytes land under a
     `.partial` name and are renamed into place when complete, so a broken
-    transfer never leaves a truncated file at `path`."""
+    transfer never leaves a truncated file at `path`. With `sha256` (and
+    `size`) given, the bytes are checked on arrival and a copy that doesn't
+    match is thrown away and reported. Raises ObjectMissing when the bucket
+    has no such object."""
     url = target.url(key)
     headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
                    target.region, now or datetime.now(UTC))
     partial = path.with_name(path.name + ".partial")
+    digest = hashlib.sha256()
+    got = 0
     conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS)
     try:
         conn.request("GET", _request_target(url), headers=headers)
         response = conn.getresponse()
+        if response.status == 404:
+            response.read()
+            raise ObjectMissing(f"{key} is not in the {target.bucket} bucket.")
         if response.status != 200:
             raise S3Error(f"Reading {key} refused: "
                           f"{_says(response.status, response.reason, response.read())}")
         with open(partial, "wb") as f:
             while chunk := response.read(CHUNK):
+                if pacer:
+                    pacer.take(len(chunk))
+                digest.update(chunk)
+                got += len(chunk)
                 f.write(chunk)
+        if size is not None and got != size:
+            raise S3Error(f"{key} arrived as {got} bytes, not the {size} expected.")
+        if sha256 and digest.hexdigest() != sha256:
+            raise S3Error(f"{key} arrived with a different fingerprint "
+                          f"({digest.hexdigest()[:12]}…), so it wasn't kept.")
         os.replace(partial, path)
     except (OSError, http.client.HTTPException) as exc:
         partial.unlink(missing_ok=True)
         raise _reach_error(target, exc) from None
+    except S3Error:
+        partial.unlink(missing_ok=True)
+        raise
     finally:
         conn.close()
 
@@ -304,15 +330,17 @@ _MAX_RETRY_AFTER = 300
 
 
 class Pacer:
-    """Holds the bytes a job sends to an average rate, across every request
-    it makes, so a backup leaves room on the link for everything else."""
+    """Holds the bytes a job moves to an average rate, across every request
+    it makes, so a backup or a restore leaves room on the link for everything
+    else."""
 
     def __init__(self, mbps: int | None):
         self.rate = mbps * 1_000_000 / 8 if mbps else None   # bytes per second
         self.sent = 0
         self.started = time.monotonic()
 
-    def _wait(self, n: int) -> None:
+    def take(self, n: int) -> None:
+        """Count `n` bytes moved, pausing while the job is ahead of its rate."""
         if self.rate:
             self.sent += n
             ahead = self.sent / self.rate - (time.monotonic() - self.started)
@@ -323,12 +351,12 @@ class Pacer:
         view = memoryview(data)
         for at in range(0, len(view), CHUNK):
             piece = view[at:at + CHUNK]
-            self._wait(len(piece))
+            self.take(len(piece))
             yield piece
 
     def file_chunks(self, f) -> Iterator[bytes]:
         while piece := f.read(CHUNK):
-            self._wait(len(piece))
+            self.take(len(piece))
             yield piece
 
 

backend/app/services/backup_service.py

@@ -65,6 +65,14 @@ _IMAGES_SUBPROCESS_TIMEOUT_S = 21000       # ~5.8 hr, under the 6-hr images soft
 _KILL_REAP_GRACE_S = 10
 
 
+# Every database dump, local or off-site: custom format (-Fc, compressed and
+# restored with pg_restore), without ownership or grants so it loads into any
+# install. The `credential` table's rows (account logins, sealed with the key
+# in /images/secrets) stay out: a restored install keeps the table, empty, and
+# you sign in to each account again.
+_PG_DUMP = ["pg_dump", "--no-owner", "--no-acl", "-Fc", "--exclude-table-data=credential"]
+
+
 def _run_bounded(cmd: list[str], timeout: int) -> None:
     """subprocess.run(check=True, timeout) whose reaper can't itself hang.
 
@@ -156,10 +164,7 @@ def backup_db(
     tmp_path = Path(tmp_name)
     try:
         _run_bounded(
-            _gently([
-                "pg_dump", "--no-owner", "--no-acl", "-Fc",
-                "-f", str(tmp_path), _libpq_url(db_url),
-            ]),
+            _gently([*_PG_DUMP, "-f", str(tmp_path), _libpq_url(db_url)]),
             _DB_SUBPROCESS_TIMEOUT_S,
         )
         shutil.move(str(tmp_path), str(sql_path))
@@ -217,7 +222,7 @@ def backup_db_offsite(*, db_url: str, target, limit_mbps: int | None = None) ->
     deadline = time.monotonic() + _DB_SUBPROCESS_TIMEOUT_S
     with tempfile.TemporaryFile() as err:
         proc = subprocess.Popen(
-            _gently(["pg_dump", "--no-owner", "--no-acl", "-Fc", _libpq_url(db_url)]),
+            _gently([*_PG_DUMP, _libpq_url(db_url)]),
             stdout=subprocess.PIPE, stderr=err,
         )
         upload = None
@@ -289,22 +294,69 @@ def backup_images(
     }
 
 
-def restore_db(*, db_url: str, sql_path: Path) -> None:
-    """Wipe public schema, then load from the custom-format dump. Raises on
-    subprocess failure; partial-restore state is the caller's concern."""
-    libpq = _libpq_url(db_url)
-    subprocess.run(
+class RestoreFailed(Exception):
+    """psql or pg_restore exited with an error; the message is its last line."""
+
+
+# A restore waits this long for other connections to let go of the tables it
+# replaces, then gives up with Postgres' own reason instead of queueing every
+# other query behind it.
+_RESTORE_LOCK_TIMEOUT = "60s"
+# pg_restore of a large library (vector tables, rebuilt indexes) runs well past
+# a dump's length; it is bounded on its own.
+_RESTORE_SUBPROCESS_TIMEOUT_S = 3 * 3600
+
+
+def _run_reporting(cmd: list[str], timeout: int, env: dict | None = None) -> None:
+    """Run `cmd`; raise RestoreFailed with the last line it printed when it
+    exits non-zero."""
+    with tempfile.TemporaryFile() as err:
+        proc = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=err, timeout=timeout,
+                              env=env)
+        if proc.returncode != 0:
+            raise RestoreFailed(_last_line(err) or f"{cmd[0]} exited with {proc.returncode}")
+
+
+# Every other connection to this database: the web server's, the other
+# workers'. Ended before the tables are dropped, so none holds a lock the drop
+# waits on, and again once the restore is in, so none keeps statements it
+# prepared against the old tables. Each reconnects on its next query.
+_END_OTHER_CONNECTIONS = (
+    "SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
+    "WHERE datname = current_database() AND pid <> pg_backend_pid();"
+)
+
+
+def end_other_connections(*, db_url: str) -> None:
+    _run_reporting(["psql", _libpq_url(db_url), "-v", "ON_ERROR_STOP=1",
+                    "-c", _END_OTHER_CONNECTIONS], 60)
+
+
+def wipe_db(*, db_url: str) -> None:
+    """Drop and recreate the public schema: an empty database, ready for a
+    dump or for `alembic upgrade head`."""
+    _run_reporting(
         [
-            "psql", libpq, "-c",
-            "DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;",
+            "psql", _libpq_url(db_url), "-v", "ON_ERROR_STOP=1", "-c",
+            f"{_END_OTHER_CONNECTIONS} DROP SCHEMA IF EXISTS public CASCADE; "
+            "CREATE SCHEMA public;",
         ],
-        capture_output=True, check=True, timeout=120,
+        120,
+        # Set for the session, not as a SET inside the same -c string.
+        env={**os.environ, "PGOPTIONS": f"-c lock_timeout={_RESTORE_LOCK_TIMEOUT}"},
     )
+
+
+def restore_db(*, db_url: str, sql_path: Path) -> None:
+    """Wipe the public schema, then load the custom-format dump. pg_restore
+    stops at the first error, which RestoreFailed carries; the database is
+    then partly loaded, and the caller decides what to put back."""
+    wipe_db(db_url=db_url)
     # Custom-format (-Fc) dumps are restored with pg_restore, not psql.
-    subprocess.run(
-        ["pg_restore", "--no-owner", "--no-acl", "-d", libpq, str(sql_path)],
-        capture_output=True, check=True,
-        timeout=_DB_SUBPROCESS_TIMEOUT_S,
+    _run_reporting(
+        _gently(["pg_restore", "--no-owner", "--no-acl", "--exit-on-error",
+                 "-d", _libpq_url(db_url), str(sql_path)]),
+        _RESTORE_SUBPROCESS_TIMEOUT_S,
     )