feat: restore a new install from the off-site bucket, database first, then every file

Settings → Maintenance → Backups gains "Restore from the bucket": pick one of
the bucket's dated dumps, type its confirmation, and FabledCurator saves the
current database to the bucket, replaces it with the dump, migrates it to this
version, then fetches every image and attachment back to the path its row
records. Each file is checked against its sha256 on arrival and moved into
place only then; a file already on disk is left alone, so "Fetch missing
files" repairs a library that lost files without touching the database.

- services/backup_restore.py: list/find/fetch dumps, migrate, keep this
  install's destination (the dump's sealed secret can't be opened here),
  library_files work list, restore_file (one fetch per sha, copies for
  duplicate paths, refuses paths outside /images).
- restore_offsite_task: the database phase, then 45-minute file chunks with a
  heartbeat; a failed load leaves an empty migrated database and says why.
  The dump's own run row is marked ok, other runs then going "Interrupted".
- backup_s3.get_file: sha/size check on arrival, ObjectMissing for a 404,
  download pacing.
- backup_service: every dump leaves the credential table's rows out;
  restore_db reports pg_restore's last line, stops at the first error, ends
  other connections and waits at most 60 s for locks.
- Backups and orphan repair wait while a restore runs; the stall sweep judges
  restores by heartbeat; restore runs are pruned to 20.
- CLI: python -m backend.app.scripts.restore list|database|files.
- README: backup and restore section; logins are in no backup.
- Tests: a round trip through an in-process S3 bucket (tests/fake_s3.py) with
  real pg_dump/pg_restore, plus file, run, API and get_file cases.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/services/backup_service.py

@@ -65,6 +65,14 @@ _IMAGES_SUBPROCESS_TIMEOUT_S = 21000       # ~5.8 hr, under the 6-hr images soft
 _KILL_REAP_GRACE_S = 10
 
 
+# Every database dump, local or off-site: custom format (-Fc, compressed and
+# restored with pg_restore), without ownership or grants so it loads into any
+# install. The `credential` table's rows (account logins, sealed with the key
+# in /images/secrets) stay out: a restored install keeps the table, empty, and
+# you sign in to each account again.
+_PG_DUMP = ["pg_dump", "--no-owner", "--no-acl", "-Fc", "--exclude-table-data=credential"]
+
+
 def _run_bounded(cmd: list[str], timeout: int) -> None:
     """subprocess.run(check=True, timeout) whose reaper can't itself hang.
 
@@ -156,10 +164,7 @@ def backup_db(
     tmp_path = Path(tmp_name)
     try:
         _run_bounded(
-            _gently([
-                "pg_dump", "--no-owner", "--no-acl", "-Fc",
-                "-f", str(tmp_path), _libpq_url(db_url),
-            ]),
+            _gently([*_PG_DUMP, "-f", str(tmp_path), _libpq_url(db_url)]),
             _DB_SUBPROCESS_TIMEOUT_S,
         )
         shutil.move(str(tmp_path), str(sql_path))
@@ -217,7 +222,7 @@ def backup_db_offsite(*, db_url: str, target, limit_mbps: int | None = None) ->
     deadline = time.monotonic() + _DB_SUBPROCESS_TIMEOUT_S
     with tempfile.TemporaryFile() as err:
         proc = subprocess.Popen(
-            _gently(["pg_dump", "--no-owner", "--no-acl", "-Fc", _libpq_url(db_url)]),
+            _gently([*_PG_DUMP, _libpq_url(db_url)]),
             stdout=subprocess.PIPE, stderr=err,
         )
         upload = None
@@ -289,22 +294,69 @@ def backup_images(
     }
 
 
-def restore_db(*, db_url: str, sql_path: Path) -> None:
-    """Wipe public schema, then load from the custom-format dump. Raises on
-    subprocess failure; partial-restore state is the caller's concern."""
-    libpq = _libpq_url(db_url)
-    subprocess.run(
+class RestoreFailed(Exception):
+    """psql or pg_restore exited with an error; the message is its last line."""
+
+
+# A restore waits this long for other connections to let go of the tables it
+# replaces, then gives up with Postgres' own reason instead of queueing every
+# other query behind it.
+_RESTORE_LOCK_TIMEOUT = "60s"
+# pg_restore of a large library (vector tables, rebuilt indexes) runs well past
+# a dump's length; it is bounded on its own.
+_RESTORE_SUBPROCESS_TIMEOUT_S = 3 * 3600
+
+
+def _run_reporting(cmd: list[str], timeout: int, env: dict | None = None) -> None:
+    """Run `cmd`; raise RestoreFailed with the last line it printed when it
+    exits non-zero."""
+    with tempfile.TemporaryFile() as err:
+        proc = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=err, timeout=timeout,
+                              env=env)
+        if proc.returncode != 0:
+            raise RestoreFailed(_last_line(err) or f"{cmd[0]} exited with {proc.returncode}")
+
+
+# Every other connection to this database: the web server's, the other
+# workers'. Ended before the tables are dropped, so none holds a lock the drop
+# waits on, and again once the restore is in, so none keeps statements it
+# prepared against the old tables. Each reconnects on its next query.
+_END_OTHER_CONNECTIONS = (
+    "SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
+    "WHERE datname = current_database() AND pid <> pg_backend_pid();"
+)
+
+
+def end_other_connections(*, db_url: str) -> None:
+    _run_reporting(["psql", _libpq_url(db_url), "-v", "ON_ERROR_STOP=1",
+                    "-c", _END_OTHER_CONNECTIONS], 60)
+
+
+def wipe_db(*, db_url: str) -> None:
+    """Drop and recreate the public schema: an empty database, ready for a
+    dump or for `alembic upgrade head`."""
+    _run_reporting(
         [
-            "psql", libpq, "-c",
-            "DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;",
+            "psql", _libpq_url(db_url), "-v", "ON_ERROR_STOP=1", "-c",
+            f"{_END_OTHER_CONNECTIONS} DROP SCHEMA IF EXISTS public CASCADE; "
+            "CREATE SCHEMA public;",
         ],
-        capture_output=True, check=True, timeout=120,
+        120,
+        # Set for the session, not as a SET inside the same -c string.
+        env={**os.environ, "PGOPTIONS": f"-c lock_timeout={_RESTORE_LOCK_TIMEOUT}"},
     )
+
+
+def restore_db(*, db_url: str, sql_path: Path) -> None:
+    """Wipe the public schema, then load the custom-format dump. pg_restore
+    stops at the first error, which RestoreFailed carries; the database is
+    then partly loaded, and the caller decides what to put back."""
+    wipe_db(db_url=db_url)
     # Custom-format (-Fc) dumps are restored with pg_restore, not psql.
-    subprocess.run(
-        ["pg_restore", "--no-owner", "--no-acl", "-d", libpq, str(sql_path)],
-        capture_output=True, check=True,
-        timeout=_DB_SUBPROCESS_TIMEOUT_S,
+    _run_reporting(
+        _gently(["pg_restore", "--no-owner", "--no-acl", "--exit-on-error",
+                 "-d", _libpq_url(db_url), str(sql_path)]),
+        _RESTORE_SUBPROCESS_TIMEOUT_S,
     )