feat: with an off-site destination set up, database backups stream to it

The nightly database backup (and "Run DB backup now") now sends pg_dump's
output straight to the off-site bucket as a multipart upload, with no copy
staged on this server. The object is named database/fabledcurator-<UTC
time>.dump and appears only once pg_dump has exited cleanly and every part
has landed; a failed dump aborts the upload and records pg_dump's last line.
Without a destination, dumps are written to /images/_backups as before.

- backup_s3: Upload (start / add part / complete / abort), with a busy or
  silent bucket rested and asked again (15 s, 60 s, Retry-After obeyed), and
  a Pacer that holds uploads to the new bandwidth limit.
- Settings: "Upload limit (Mbit/s)" on the destination card, 200 by default,
  0 for none (migration 0120).
- pg_dump runs under nice (and ionice where installed), locally too.
- Backups card says where dumps go; the runs table gains "Saved to" and
  shows a failed run's error. Off-site rows have no restore/tag/delete here:
  restoring from the bucket is a later step (#6880).

Task #6878, milestone 586.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_backup_s3.py

@@ -1,6 +1,7 @@
 """The backup destination's S3 client (services/backup_s3.py): signatures match
 AWS's published Signature Version 4 examples, and the calls send and read what
 S3 expects, over a real HTTP connection to a local stand-in server."""
+import hashlib
 import socket
 import threading
 from datetime import UTC, datetime
@@ -55,7 +56,7 @@ def test_paths_are_path_style_and_keys_are_escaped():
 
 class _Server:
     """A local HTTP server that records each request and answers with
-    `reply(handler) -> (status, body)`."""
+    `reply(handler) -> (status, body)` or `(status, body, headers)`."""
 
     def __init__(self, reply):
         self.seen = []
@@ -69,13 +70,15 @@ class _Server:
                     "headers": {k.lower(): v for k, v in self.headers.items()},
                     "body": self.rfile.read(length) if length else b"",
                 })
-                status, body = reply(self)
+                status, body, *extra = reply(self)
                 self.send_response(status)
+                for name, value in (extra[0] if extra else {}).items():
+                    self.send_header(name, value)
                 self.send_header("Content-Length", str(len(body)))
                 self.end_headers()
                 self.wfile.write(body)
 
-            do_GET = do_PUT = _serve
+            do_GET = do_PUT = do_POST = do_DELETE = _serve
 
             def log_message(self, *args):
                 pass
@@ -207,3 +210,115 @@ def test_a_refused_get_says_so_and_writes_nothing(tmp_path):
         s3.get_file(srv.target(), "b.dump", out)
     assert not out.exists()
     assert not (tmp_path / "b.dump.partial").exists()
+
+
+# ── Multipart ───────────────────────────────────────────────────────────────
+
+_STARTED = b"<InitiateMultipartUploadResult><UploadId>up1</UploadId></InitiateMultipartUploadResult>"
+
+
+def _bucket(parts_reply=None, complete=(200, b"<CompleteMultipartUploadResult/>")):
+    """A stand-in bucket for one multipart upload: starts it, takes parts (each
+    answered by `parts_reply(n)`, an ETag by default) and completes it."""
+    def reply(h):
+        if h.command == "POST" and h.path.endswith("?uploads"):
+            return 200, _STARTED
+        if h.command == "PUT":
+            n = int(h.path.split("partNumber=")[1].split("&")[0])
+            return parts_reply(n) if parts_reply else (200, b"", {"ETag": f'"e{n}"'})
+        if h.command == "POST":
+            return complete
+        return 204, b""
+    return reply
+
+
+def test_an_upload_sends_its_parts_then_joins_them_in_order():
+    with _Server(_bucket()) as srv:
+        up = s3.Upload(srv.target(), "database/x.dump")
+        up.add(b"first,")
+        up.add(b"second")
+        up.complete()
+    start, one, two, done = srv.seen
+    assert (start["method"], start["path"]) == ("POST", "/curator/database/x.dump?uploads")
+    assert one["path"] == "/curator/database/x.dump?partNumber=1&uploadId=up1"
+    assert two["body"] == b"second"
+    assert two["headers"]["x-amz-content-sha256"] == hashlib.sha256(b"second").hexdigest()
+    assert done["path"] == "/curator/database/x.dump?uploadId=up1"
+    assert done["body"].index(b'"e1"') < done["body"].index(b'"e2"')
+    assert (up.parts, up.size) == (2, len(b"first,second"))
+
+
+def test_a_refused_start_says_why():
+    with _Server(lambda h: (403, b"<Error><Code>AccessDenied</Code></Error>")) as srv:
+        with pytest.raises(s3.S3Error, match="the key isn't allowed"):
+            s3.Upload(srv.target(), "database/x.dump")
+
+
+def test_a_completion_refused_under_a_200_is_a_failure():
+    error = (200, b"<Error><Code>InternalError</Code><Message>disk full</Message></Error>")
+    with _Server(_bucket(complete=error)) as srv:
+        up = s3.Upload(srv.target(), "database/x.dump")
+        up.add(b"data")
+        with pytest.raises(s3.S3Error, match="Finishing the upload"):
+            up.complete()
+
+
+def test_nothing_sent_is_not_completed():
+    with _Server(_bucket()) as srv:
+        up = s3.Upload(srv.target(), "database/x.dump")
+        with pytest.raises(s3.S3Error, match="Nothing was sent"):
+            up.complete()
+
+
+def test_a_busy_bucket_gets_the_part_again_after_a_rest(monkeypatch):
+    rests = []
+    monkeypatch.setattr(s3.time, "sleep", rests.append)
+    answers = iter([(503, b"", {"Retry-After": "7"}), (200, b"", {"ETag": '"e1"'})])
+    with _Server(_bucket(parts_reply=lambda n: next(answers))) as srv:
+        up = s3.Upload(srv.target(), "database/x.dump")
+        up.add(b"data")
+    assert rests == [7]
+    assert up.parts == 1
+
+
+def test_a_part_refused_outright_is_not_sent_again(monkeypatch):
+    monkeypatch.setattr(s3.time, "sleep", lambda s: pytest.fail("rested on a refusal"))
+    refused = (403, b"<Error><Code>AccessDenied</Code></Error>")
+    with _Server(_bucket(parts_reply=lambda n: refused)) as srv:
+        up = s3.Upload(srv.target(), "database/x.dump")
+        with pytest.raises(s3.S3Error, match="Part 1"):
+            up.add(b"data")
+
+
+def test_a_part_that_never_lands_gives_up_after_the_rests(monkeypatch):
+    rests = []
+    monkeypatch.setattr(s3.time, "sleep", rests.append)
+    with _Server(_bucket(parts_reply=lambda n: (503, b""))) as srv:
+        up = s3.Upload(srv.target(), "database/x.dump")
+        with pytest.raises(s3.S3Error, match="Part 1"):
+            up.add(b"data")
+    assert rests == list(s3.PART_RETRY_WAITS)
+
+
+def test_abort_asks_the_bucket_to_drop_the_parts_and_never_raises():
+    with _Server(_bucket()) as srv:
+        up = s3.Upload(srv.target(), "database/x.dump")
+        up.abort()
+        assert srv.seen[-1]["method"] == "DELETE"
+        assert srv.seen[-1]["path"] == "/curator/database/x.dump?uploadId=up1"
+    up.abort()  # the server is gone now: still no exception
+
+
+def test_the_pacer_holds_bytes_to_the_limit(monkeypatch):
+    clock = [0.0]
+    monkeypatch.setattr(s3.time, "monotonic", lambda: clock[0])
+    monkeypatch.setattr(s3.time, "sleep", lambda secs: clock.__setitem__(0, clock[0] + secs))
+    pacer = s3.Pacer(8)                     # 8 Mbit/s = 1,000,000 bytes/s
+    data = b"x" * (3 * s3.CHUNK)
+    assert b"".join(pacer.chunks(data)) == data
+    assert clock[0] == pytest.approx(len(data) / 1_000_000)
+
+
+def test_no_limit_never_waits(monkeypatch):
+    monkeypatch.setattr(s3.time, "sleep", lambda secs: pytest.fail("waited without a limit"))
+    assert b"".join(s3.Pacer(0).chunks(b"y" * (2 * s3.CHUNK + 1))) == b"y" * (2 * s3.CHUNK + 1)