feat: with an off-site destination set up, database backups stream to it

The nightly database backup (and "Run DB backup now") now sends pg_dump's
output straight to the off-site bucket as a multipart upload, with no copy
staged on this server. The object is named database/fabledcurator-<UTC
time>.dump and appears only once pg_dump has exited cleanly and every part
has landed; a failed dump aborts the upload and records pg_dump's last line.
Without a destination, dumps are written to /images/_backups as before.

- backup_s3: Upload (start / add part / complete / abort), with a busy or
  silent bucket rested and asked again (15 s, 60 s, Retry-After obeyed), and
  a Pacer that holds uploads to the new bandwidth limit.
- Settings: "Upload limit (Mbit/s)" on the destination card, 200 by default,
  0 for none (migration 0120).
- pg_dump runs under nice (and ionice where installed), locally too.
- Backups card says where dumps go; the runs table gains "Saved to" and
  shows a failed run's error. Off-site rows have no restore/tag/delete here:
  restoring from the bucket is a later step (#6880).

Task #6878, milestone 586.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/services/backup_s3.py

@@ -1,6 +1,7 @@
 """A small S3 client for sending backups off the box: AWS Signature Version 4,
 path-style addressing (endpoint/bucket/key), and the calls a backup needs —
-put an object, read one back, list a prefix.
+put an object (whole, or in parts when its size isn't known up front), read
+one back, list a prefix.
 
 FabledCurator puts, lists and reads; it never deletes. The bucket keeps the
 history and thins it on its own policy, so a key that may only add and read
@@ -21,7 +22,9 @@ import http.client
 import os
 import socket
 import ssl
+import time
 import xml.etree.ElementTree as ET
+from collections.abc import Iterator
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from pathlib import Path
@@ -280,3 +283,175 @@ def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *,
         objects.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0),
                                     fields.get("LastModified", "")))
     return ListPage(objects, next_token)
+
+
+# ── Multipart: one object sent in parts ─────────────────────────────────────
+
+# Each part is held in memory while it is sent, so this bounds the memory a
+# streamed upload uses. S3 allows 10,000 parts, which at this size is 640 GiB
+# for one object, and every part but the last must be at least 5 MiB.
+PART_SIZE = 64 * CHUNK
+# A part the bucket couldn't take (it didn't answer, or said it was busy) is
+# sent again after a rest that grows each time: 15 s, then 60 s.
+PART_RETRY_WAITS = (15, 60)
+_BUSY = frozenset({429, 500, 502, 503, 504})
+_MAX_RETRY_AFTER = 300
+
+
+class Pacer:
+    """Holds the bytes a job sends to an average rate, across every request
+    it makes, so a backup leaves room on the link for everything else."""
+
+    def __init__(self, mbps: int | None):
+        self.rate = mbps * 1_000_000 / 8 if mbps else None   # bytes per second
+        self.sent = 0
+        self.started = time.monotonic()
+
+    def chunks(self, data: bytes) -> Iterator[memoryview]:
+        view = memoryview(data)
+        for at in range(0, len(view), CHUNK):
+            piece = view[at:at + CHUNK]
+            if self.rate:
+                self.sent += len(piece)
+                ahead = self.sent / self.rate - (time.monotonic() - self.started)
+                if ahead > 0:
+                    time.sleep(ahead)
+            yield piece
+
+
+def _send(target: S3Target, method: str, url: str, body: bytes = b"",
+          headers: dict[str, str] | None = None, *, timeout: float = TIMEOUT_SECONDS,
+          pacer: Pacer | None = None,
+          now: datetime | None = None) -> tuple[int, str, http.client.HTTPMessage, bytes]:
+    """Sign and send one request with an in-memory body, paced when a pacer
+    is given. Returns (status, reason, headers, content)."""
+    digest = hashlib.sha256(body).hexdigest()
+    signed = sign(method, url, {"Content-Length": str(len(body)), **(headers or {})},
+                  digest, target.access_key, target.secret_key, target.region,
+                  now or datetime.now(UTC))
+    conn = _connect(target, url, timeout)
+    try:
+        conn.request(method, _request_target(url),
+                     body=pacer.chunks(body) if pacer and body else body, headers=signed)
+        response = conn.getresponse()
+        content = response.read()
+    except (OSError, http.client.HTTPException) as exc:
+        raise _reach_error(target, exc) from None
+    finally:
+        conn.close()
+    return response.status, response.reason, response.headers, content
+
+
+def _xml_field(content: bytes, name: str) -> str | None:
+    try:
+        root = ET.fromstring(content)
+    except ET.ParseError:
+        return None
+    for item in root.iter():
+        if item.tag.rsplit("}", 1)[-1] == name:
+            return item.text
+    return None
+
+
+def _is_error_body(content: bytes) -> bool:
+    try:
+        return bool(content) and ET.fromstring(content).tag.rsplit("}", 1)[-1] == "Error"
+    except ET.ParseError:
+        return False
+
+
+def _retry_after(headers: http.client.HTTPMessage | None, default: int) -> int:
+    try:
+        asked = int((headers or {}).get("Retry-After") or default)
+    except ValueError:
+        asked = default
+    return max(0, min(asked, _MAX_RETRY_AFTER))
+
+
+class Upload:
+    """One object written as a series of parts, for data whose size isn't known
+    when it starts, such as a database dump streamed from pg_dump.
+
+    The object appears in the bucket only when `complete()` succeeds. Until
+    then the parts are invisible, so a stream that fails partway leaves
+    nothing behind that looks like a finished backup. `abort()` asks the
+    bucket to drop the parts at once; if the abort is refused or never
+    arrives, S3 services expire an unfinished upload on their own (Fabled
+    Archives after a day)."""
+
+    def __init__(self, target: S3Target, key: str, *,
+                 content_type: str = "application/octet-stream",
+                 pacer: Pacer | None = None):
+        self.target = target
+        self.key = key
+        self.size = 0
+        self.pacer = pacer
+        self._etags: list[str] = []
+        status, reason, _, content = _send(
+            target, "POST", f"{target.url(key)}?uploads", headers={"Content-Type": content_type},
+        )
+        upload_id = _xml_field(content, "UploadId") if status == 200 else None
+        if not upload_id:
+            raise S3Error(f"Starting the upload of {key} refused: "
+                          f"{_says(status, reason, content)}")
+        self.upload_id = upload_id
+
+    def _url(self, **params: str) -> str:
+        query = urlencode({**params, "uploadId": self.upload_id}, quote_via=quote)
+        return f"{self.target.url(self.key)}?{query}"
+
+    @property
+    def parts(self) -> int:
+        return len(self._etags)
+
+    def add(self, data: bytes) -> None:
+        """Send the next part, resting and sending it again when the bucket
+        doesn't answer or says it is busy."""
+        n = len(self._etags) + 1
+        for wait in (*PART_RETRY_WAITS, None):
+            headers = None
+            try:
+                status, reason, headers, content = _send(
+                    self.target, "PUT", self._url(partNumber=str(n)), data,
+                    timeout=TRANSFER_TIMEOUT_SECONDS, pacer=self.pacer,
+                )
+            except S3Error as exc:
+                if wait is None:
+                    raise S3Error(f"Part {n} of {self.key}: {exc}") from None
+                time.sleep(wait)
+                continue
+            etag = headers.get("ETag") if status == 200 else None
+            if etag:
+                self._etags.append(etag)
+                self.size += len(data)
+                return
+            if status not in _BUSY or wait is None:
+                raise S3Error(f"Part {n} of {self.key} refused: "
+                              f"{_says(status, reason, content)}")
+            time.sleep(_retry_after(headers, wait))
+
+    def complete(self) -> None:
+        """Join the parts into the object. S3 can refuse a completion with an
+        error body under a 200 status, so the body is read too."""
+        if not self._etags:
+            raise S3Error(f"Nothing was sent for {self.key}.")
+        body = "<CompleteMultipartUpload>" + "".join(
+            f"<Part><PartNumber>{n}</PartNumber><ETag>{etag}</ETag></Part>"
+            for n, etag in enumerate(self._etags, start=1)
+        ) + "</CompleteMultipartUpload>"
+        status, reason, _, content = _send(
+            self.target, "POST", self._url(), body.encode(),
+            headers={"Content-Type": "application/xml"}, timeout=TRANSFER_TIMEOUT_SECONDS,
+        )
+        if status != 200 or _is_error_body(content):
+            raise S3Error(f"Finishing the upload of {self.key} refused: "
+                          f"{_says(status, reason, content)}")
+
+    def abort(self) -> None:
+        """Drop the parts sent so far. Never raises: it runs while another
+        failure is already being reported, and the bucket expires the parts on
+        its own if this doesn't reach it."""
+        try:
+            _send(self.target, "DELETE", self._url())
+        except Exception:  # noqa: BLE001 — best effort, see above
+            pass