feat: images and attachments go to the off-site bucket once each, by fingerprint

With an off-site destination set up, the library's files are sent to the
bucket's media/ folder as media/ab/cd/<sha256><ext>, beside the database
dumps under database/. Each file goes once: a new backup_blob table records
what each destination holds (migration 0121), and a run's work list is every
image and attachment sha with no row yet, so runs are incremental and a new
bucket starts its copy from the beginning.

- The bucket checks each file against the sha recorded at import; a file
  over 1 GiB goes in parts and is checked before the upload is completed.
  A file missing or changed on disk is reported, not stored.
- A run is chunked: each 45-minute chunk commits after every file and queues
  the next, so other long-maintenance jobs get turns. One run at a time.
  The stall sweep judges a media run by its progress heartbeat
  (backup_run.last_progress_at), not its start.
- A file that can't be sent is listed on the run (status ok, with the error
  line) and tried again by the next run. A refused put for an object
  already in an append-only bucket counts as sent.
- Nightly, with a destination set, runs the dump and then the media.
- Weekly reconcile lists media/ and re-sends what the bucket lost, and
  adopts what it already holds (a restored database sends nothing twice).
- Uploads use the bandwidth limit; read transactions end before transfers.
- Backups card: with a destination, an "Images and attachments" section
  shows coverage (files and bytes in the bucket), "Send new files now", and
  the media runs. Without one, the local tarball section is unchanged.
- Media run history keeps the latest 60 rows.

Task #6879, milestone 586.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/services/backup_media.py

@@ -0,0 +1,202 @@
+"""The off-site copy of the library's files: every image and attachment, sent
+once each to the backup bucket as a content-addressed object.
+
+Layout in the bucket, beside the database dumps under `database/`:
+
+    media/ab/cd/<sha256><ext>
+
+`ab` and `cd` are the first four hex digits of the file's sha256, which keeps
+any one prefix small, and the extension keeps the files readable in the bucket
+without FabledCurator. The sha256 is the one recorded at import, so a file
+uploads exactly once however many rows or posts point at it, and a run is
+incremental: its work list is every sha with no `backup_blob` row for the
+current destination. A file removed from the library stays in the bucket until
+the bucket's own retention removes it; FabledCurator never deletes there.
+
+The bucket checks every object against that recorded sha256, so a file whose
+bytes changed on disk since import is refused and reported, never stored under
+the wrong name.
+"""
+from __future__ import annotations
+
+import re
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from sqlalchemy import delete, select, text
+from sqlalchemy.dialects.postgresql import insert
+
+from ..models import BackupBlob
+from .backup_s3 import (
+    Pacer,
+    S3Error,
+    S3Target,
+    list_objects,
+    object_present,
+    put_file,
+    put_file_in_parts,
+)
+
+PREFIX = "media/"
+# Files above this size go in parts (S3 refuses a single PUT over 5 GiB, and a
+# part that fails is resent alone instead of the whole file).
+MULTIPART_OVER = 1 << 30
+# The bucket is listed this many keys at a time during a reconcile, with a
+# short rest between pages.
+LIST_PAGE = 1000
+LIST_REST_SECONDS = 0.2
+_EXT = re.compile(r"^\.[a-z0-9]{1,10}$")
+
+
+@dataclass(frozen=True)
+class Blob:
+    sha256: str
+    path: str
+    size_bytes: int
+    mime: str | None
+
+
+def destination_id(endpoint: str, bucket: str) -> str:
+    """How backup_blob names a destination: one copy per endpoint and bucket."""
+    return f"{endpoint.rstrip('/')}/{bucket}"
+
+
+def target_id(target: S3Target) -> str:
+    return destination_id(target.endpoint, target.bucket)
+
+
+def blob_key(sha256: str, path: str) -> str:
+    """The bucket key for a file: `media/ab/cd/<sha256><ext>`. An extension
+    that isn't short and plain is left off, so a key is always safe."""
+    ext = Path(path).suffix.lower()
+    return f"{PREFIX}{sha256[:2]}/{sha256[2:4]}/{sha256}{ext if _EXT.match(ext) else ''}"
+
+
+# Every library file once per sha256: images and attachments together. An
+# attachment's sha can repeat across posts, and an image extracted from an
+# archive is its own file with its own sha.
+_LIBRARY = """
+    SELECT sha256, path, size_bytes, mime FROM image_record
+    UNION ALL
+    SELECT sha256, path, size_bytes, mime FROM post_attachment
+"""
+
+
+def pending(session, tid: str, *, after: str = "", limit: int = 500) -> list[Blob]:
+    """The next `limit` files not yet in the bucket, in sha order after
+    `after`, so a run walks the library once even when some files fail."""
+    rows = session.execute(text(f"""
+        SELECT DISTINCT ON (lib.sha256) lib.sha256, lib.path, lib.size_bytes, lib.mime
+        FROM ({_LIBRARY}) AS lib
+        WHERE lib.sha256 > :after
+          AND NOT EXISTS (
+              SELECT 1 FROM backup_blob b WHERE b.target = :tid AND b.sha256 = lib.sha256
+          )
+        ORDER BY lib.sha256, lib.path
+        LIMIT :limit
+    """), {"after": after, "tid": tid, "limit": limit}).all()
+    return [Blob(r.sha256, r.path, r.size_bytes, r.mime) for r in rows]
+
+
+def coverage(session, tid: str | None) -> dict:
+    """How much of the library is in the bucket: files and bytes, sent and
+    total, each sha counted once."""
+    row = session.execute(text(f"""
+        WITH lib AS (
+            SELECT DISTINCT ON (sha256) sha256, size_bytes FROM ({_LIBRARY}) AS u
+            ORDER BY sha256
+        )
+        SELECT count(*) AS files_total,
+               coalesce(sum(lib.size_bytes), 0) AS bytes_total,
+               count(b.sha256) AS files_sent,
+               coalesce(sum(lib.size_bytes) FILTER (WHERE b.sha256 IS NOT NULL), 0)
+                   AS bytes_sent
+        FROM lib
+        LEFT JOIN backup_blob b ON b.sha256 = lib.sha256 AND b.target = :tid
+    """), {"tid": tid or ""}).one()
+    return {
+        "files_total": row.files_total, "bytes_total": int(row.bytes_total),
+        "files_sent": row.files_sent, "bytes_sent": int(row.bytes_sent),
+    }
+
+
+def send(target: S3Target, blob: Blob, images_root: Path, pacer: Pacer | None) -> str:
+    """Upload one file to its key and return the key. Raises S3Error (or
+    OSError) with the reason when it can't be sent."""
+    path = Path(blob.path)
+    if not path.is_absolute():
+        path = images_root / path
+    if not path.is_file():
+        raise S3Error(f"{path} is not on disk.")
+    size = path.stat().st_size
+    if size != blob.size_bytes:
+        raise S3Error(f"{path} is {size} bytes on disk but {blob.size_bytes} when imported.")
+    key = blob_key(blob.sha256, blob.path)
+    content_type = blob.mime or "application/octet-stream"
+    try:
+        if size > MULTIPART_OVER:
+            put_file_in_parts(target, key, path, sha256=blob.sha256,
+                              content_type=content_type, pacer=pacer)
+        else:
+            put_file(target, key, path, content_type=content_type,
+                     payload_sha256=blob.sha256, pacer=pacer)
+    except S3Error:
+        if object_present(target, key, size):
+            return key          # already there from an earlier run
+        raise
+    return key
+
+
+def record(session, tid: str, blobs: list[tuple[Blob, str]]) -> None:
+    """Note files as sent: (blob, key) pairs. A row already there is kept."""
+    if blobs:
+        session.execute(insert(BackupBlob).values([
+            {"target": tid, "sha256": b.sha256, "key": key, "size_bytes": b.size_bytes}
+            for b, key in blobs
+        ]).on_conflict_do_nothing())
+
+
+def reconcile(session, target: S3Target) -> dict:
+    """Check backup_blob against what the bucket actually holds under
+    `media/`. A row whose object is missing or the wrong size is dropped, so
+    the next run sends that file again; an object the bucket holds for a
+    library file with no row (a restored database, a lost commit) is recorded,
+    so it isn't sent twice. Returns the counts."""
+    tid = target_id(target)
+    held: dict[str, int] = {}
+    token = None
+    while True:
+        page = list_objects(target, PREFIX, LIST_PAGE, continuation=token)
+        held.update((o.key, o.size) for o in page.objects)
+        token = page.next_token
+        if not token:
+            break
+        time.sleep(LIST_REST_SECONDS)
+
+    rows = session.execute(
+        select(BackupBlob.sha256, BackupBlob.key, BackupBlob.size_bytes)
+        .where(BackupBlob.target == tid)
+    ).all()
+    gone = [r.sha256 for r in rows if held.get(r.key) != r.size_bytes]
+    for at in range(0, len(gone), 1000):
+        session.execute(delete(BackupBlob).where(
+            BackupBlob.target == tid, BackupBlob.sha256.in_(gone[at:at + 1000]),
+        ))
+    kept = {r.sha256 for r in rows} - set(gone)
+
+    found = []
+    for r in session.execute(text(f"""
+        SELECT DISTINCT ON (sha256) sha256, path, size_bytes, mime
+        FROM ({_LIBRARY}) AS lib ORDER BY sha256, path
+    """)).all():
+        if r.sha256 in kept:
+            continue
+        blob = Blob(r.sha256, r.path, r.size_bytes, r.mime)
+        key = blob_key(blob.sha256, blob.path)
+        if held.get(key) == blob.size_bytes:
+            found.append((blob, key))
+    for at in range(0, len(found), 1000):
+        record(session, tid, found[at:at + 1000])
+    session.commit()
+    return {"listed": len(held), "dropped": len(gone), "adopted": len(found)}

backend/app/services/backup_s3.py

@@ -166,6 +166,7 @@ def _says(status: int, reason: str, content: bytes) -> str:
         "InvalidAccessKeyId": "the access key isn't known there",
         "NoSuchBucket": "there is no such bucket",
         "AccessDenied": "the key isn't allowed to do that",
+        "XAmzContentSHA256Mismatch": "the file's bytes don't match its recorded checksum",
     }
     said = hints.get(code) or message or reason
     if not code and 400 <= status < 500:
@@ -190,10 +191,13 @@ def file_sha256(path: Path) -> str:
 def put_file(target: S3Target, key: str, path: Path, *,
              content_type: str = "application/octet-stream",
              payload_sha256: str | None = None,
+             pacer: Pacer | None = None,
              now: datetime | None = None) -> None:
     """Upload one file as one object, streamed from disk. `payload_sha256` is
     the file's sha256 when the caller already knows it (the library stores
-    every file's), which saves reading the file twice."""
+    every file's), which saves reading the file twice; the bucket checks the
+    bytes against it, so a file that changed on disk is refused rather than
+    stored under the wrong name."""
     digest = payload_sha256 or file_sha256(path)
     size = path.stat().st_size
     url = target.url(key)
@@ -203,7 +207,8 @@ def put_file(target: S3Target, key: str, path: Path, *,
     conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS)
     try:
         with open(path, "rb") as f:
-            conn.request("PUT", _request_target(url), body=f, headers=headers)
+            conn.request("PUT", _request_target(url),
+                         body=pacer.file_chunks(f) if pacer else f, headers=headers)
             response = conn.getresponse()
             content = response.read()
     except (OSError, http.client.HTTPException) as exc:
@@ -307,15 +312,23 @@ class Pacer:
         self.sent = 0
         self.started = time.monotonic()
 
+    def _wait(self, n: int) -> None:
+        if self.rate:
+            self.sent += n
+            ahead = self.sent / self.rate - (time.monotonic() - self.started)
+            if ahead > 0:
+                time.sleep(ahead)
+
     def chunks(self, data: bytes) -> Iterator[memoryview]:
         view = memoryview(data)
         for at in range(0, len(view), CHUNK):
             piece = view[at:at + CHUNK]
-            if self.rate:
-                self.sent += len(piece)
-                ahead = self.sent / self.rate - (time.monotonic() - self.started)
-                if ahead > 0:
-                    time.sleep(ahead)
+            self._wait(len(piece))
+            yield piece
+
+    def file_chunks(self, f) -> Iterator[bytes]:
+        while piece := f.read(CHUNK):
+            self._wait(len(piece))
             yield piece
 
 
@@ -455,3 +468,38 @@ class Upload:
             _send(self.target, "DELETE", self._url())
         except Exception:  # noqa: BLE001 — best effort, see above
             pass
+
+
+def put_file_in_parts(target: S3Target, key: str, path: Path, *, sha256: str,
+                      content_type: str = "application/octet-stream",
+                      pacer: Pacer | None = None) -> int:
+    """Upload a large file as a multipart object, read from disk a part at a
+    time. The whole file's sha256 is computed on the way and must equal
+    `sha256` before the upload is completed; otherwise it is aborted, so a file
+    that changed on disk is never stored under its old name. Returns the
+    number of bytes sent."""
+    upload = Upload(target, key, content_type=content_type, pacer=pacer)
+    digest = hashlib.sha256()
+    try:
+        with open(path, "rb") as f:
+            while part := f.read(PART_SIZE):
+                digest.update(part)
+                upload.add(part)
+        if digest.hexdigest() != sha256:
+            raise S3Error(f"{path} no longer matches its recorded checksum; not stored.")
+        upload.complete()
+    except BaseException:
+        upload.abort()
+        raise
+    return upload.size
+
+
+def object_present(target: S3Target, key: str, size: int) -> bool:
+    """Whether `key` is already in the bucket at `size` bytes. Used when a put
+    is refused: an append-only bucket refuses to replace an object that is
+    already there, which for a content-addressed key means the work is done."""
+    try:
+        page = list_objects(target, prefix=key, limit=1)
+    except S3Error:
+        return False
+    return any(o.key == key and o.size == size for o in page.objects)