feat: restore a new install from the off-site bucket, database first, then every file

Settings → Maintenance → Backups gains "Restore from the bucket": pick one of
the bucket's dated dumps, type its confirmation, and FabledCurator saves the
current database to the bucket, replaces it with the dump, migrates it to this
version, then fetches every image and attachment back to the path its row
records. Each file is checked against its sha256 on arrival and moved into
place only then; a file already on disk is left alone, so "Fetch missing
files" repairs a library that lost files without touching the database.

- services/backup_restore.py: list/find/fetch dumps, migrate, keep this
  install's destination (the dump's sealed secret can't be opened here),
  library_files work list, restore_file (one fetch per sha, copies for
  duplicate paths, refuses paths outside /images).
- restore_offsite_task: the database phase, then 45-minute file chunks with a
  heartbeat; a failed load leaves an empty migrated database and says why.
  The dump's own run row is marked ok, other runs then going "Interrupted".
- backup_s3.get_file: sha/size check on arrival, ObjectMissing for a 404,
  download pacing.
- backup_service: every dump leaves the credential table's rows out;
  restore_db reports pg_restore's last line, stops at the first error, ends
  other connections and waits at most 60 s for locks.
- Backups and orphan repair wait while a restore runs; the stall sweep judges
  restores by heartbeat; restore runs are pruned to 20.
- CLI: python -m backend.app.scripts.restore list|database|files.
- README: backup and restore section; logins are in no backup.
- Tests: a round trip through an in-process S3 bucket (tests/fake_s3.py) with
  real pg_dump/pg_restore, plus file, run, API and get_file cases.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/services/backup_s3.py

@@ -220,29 +220,55 @@ def put_file(target: S3Target, key: str, path: Path, *,
                       f"{_says(response.status, response.reason, content)}")
 
 
+class ObjectMissing(S3Error):
+    """The bucket holds nothing at the key asked for."""
+
+
 def get_file(target: S3Target, key: str, path: Path, *,
+             sha256: str | None = None, size: int | None = None,
+             pacer: Pacer | None = None,
              now: datetime | None = None) -> None:
     """Download one object to `path`, streamed to disk. The bytes land under a
     `.partial` name and are renamed into place when complete, so a broken
-    transfer never leaves a truncated file at `path`."""
+    transfer never leaves a truncated file at `path`. With `sha256` (and
+    `size`) given, the bytes are checked on arrival and a copy that doesn't
+    match is thrown away and reported. Raises ObjectMissing when the bucket
+    has no such object."""
     url = target.url(key)
     headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
                    target.region, now or datetime.now(UTC))
     partial = path.with_name(path.name + ".partial")
+    digest = hashlib.sha256()
+    got = 0
     conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS)
     try:
         conn.request("GET", _request_target(url), headers=headers)
         response = conn.getresponse()
+        if response.status == 404:
+            response.read()
+            raise ObjectMissing(f"{key} is not in the {target.bucket} bucket.")
         if response.status != 200:
             raise S3Error(f"Reading {key} refused: "
                           f"{_says(response.status, response.reason, response.read())}")
         with open(partial, "wb") as f:
             while chunk := response.read(CHUNK):
+                if pacer:
+                    pacer.take(len(chunk))
+                digest.update(chunk)
+                got += len(chunk)
                 f.write(chunk)
+        if size is not None and got != size:
+            raise S3Error(f"{key} arrived as {got} bytes, not the {size} expected.")
+        if sha256 and digest.hexdigest() != sha256:
+            raise S3Error(f"{key} arrived with a different fingerprint "
+                          f"({digest.hexdigest()[:12]}…), so it wasn't kept.")
         os.replace(partial, path)
     except (OSError, http.client.HTTPException) as exc:
         partial.unlink(missing_ok=True)
         raise _reach_error(target, exc) from None
+    except S3Error:
+        partial.unlink(missing_ok=True)
+        raise
     finally:
         conn.close()
 
@@ -304,15 +330,17 @@ _MAX_RETRY_AFTER = 300
 
 
 class Pacer:
-    """Holds the bytes a job sends to an average rate, across every request
-    it makes, so a backup leaves room on the link for everything else."""
+    """Holds the bytes a job moves to an average rate, across every request
+    it makes, so a backup or a restore leaves room on the link for everything
+    else."""
 
     def __init__(self, mbps: int | None):
         self.rate = mbps * 1_000_000 / 8 if mbps else None   # bytes per second
         self.sent = 0
         self.started = time.monotonic()
 
-    def _wait(self, n: int) -> None:
+    def take(self, n: int) -> None:
+        """Count `n` bytes moved, pausing while the job is ahead of its rate."""
         if self.rate:
             self.sent += n
             ahead = self.sent / self.rate - (time.monotonic() - self.started)
@@ -323,12 +351,12 @@ class Pacer:
         view = memoryview(data)
         for at in range(0, len(view), CHUNK):
             piece = view[at:at + CHUNK]
-            self._wait(len(piece))
+            self.take(len(piece))
             yield piece
 
     def file_chunks(self, f) -> Iterator[bytes]:
         while piece := f.read(CHUNK):
-            self._wait(len(piece))
+            self.take(len(piece))
             yield piece