feat: Settings can point backups at an off-site S3 bucket, with a Test destination check

Settings → Maintenance → Storage gains "Off-site backup destination": the
address (with its port), bucket, region, access key and secret key of an S3
bucket such as Fabled Archives. The secret key is sealed with the credential
key, never returned by the API, and kept when the field is left blank. Test
destination lists the bucket with the fields as typed and writes nothing, and
names the cause when it fails ("nothing is listening there (check the port)",
"the secret key is wrong", "answered, but not as S3").

services/backup_s3 is a standard-library SigV4 client (put, get, paged list)
ported from Grimoire's; it is held to AWS's published signing examples and
exercised against a local HTTP server. Migration 0118 adds the columns; no
destination is set up until someone saves one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_api_system_backup.py

@@ -322,3 +322,136 @@ async def test_patch_settings_rejects_keep_n_below_min(client):
         json={"backup_db_keep_last_n": 0},
     )
     assert resp.status_code == 400
+
+
+# --- /destination ---------------------------------------------------
+
+
+@pytest.fixture
+def _sealing_key(tmp_path, monkeypatch):
+    from backend.app.services import backup_destination
+    monkeypatch.setattr(backup_destination, "KEY_PATH", tmp_path / "credential_key.b64")
+
+
+_DEST = {
+    "endpoint": "http://archives.example:8081/",
+    "bucket": "curator",
+    "access_key": "AKEXAMPLE",
+    "region": "",
+    "secret_key": "s3cr3t-example",
+}
+
+
+@pytest.mark.asyncio
+async def test_destination_saves_and_never_returns_the_secret(client, db_sync, _sealing_key):
+    from backend.app.models import ImportSettings
+
+    resp = await client.patch("/api/system/backup/destination", json=_DEST)
+    assert resp.status_code == 200
+    body = await resp.get_json()
+    assert body == {
+        "endpoint": "http://archives.example:8081", "bucket": "curator",
+        "access_key": "AKEXAMPLE", "region": "us-east-1",
+        "secret_set": True, "configured": True,
+    }
+    assert "s3cr3t-example" not in str(await (await client.get(
+        "/api/system/backup/destination")).get_json())
+    row = db_sync.execute(select(ImportSettings).where(ImportSettings.id == 1)).scalar_one()
+    db_sync.refresh(row)
+    assert row.backup_s3_secret_sealed and "s3cr3t-example" not in row.backup_s3_secret_sealed
+
+    from backend.app.services import backup_destination
+    assert backup_destination.unseal(row.backup_s3_secret_sealed) == "s3cr3t-example"
+
+
+@pytest.mark.asyncio
+async def test_destination_blank_secret_keeps_the_saved_one(client, db_sync, _sealing_key):
+    from backend.app.models import ImportSettings
+    from backend.app.services import backup_destination
+
+    await client.patch("/api/system/backup/destination", json=_DEST)
+    resp = await client.patch(
+        "/api/system/backup/destination", json={**_DEST, "bucket": "other", "secret_key": ""},
+    )
+    assert (await resp.get_json())["bucket"] == "other"
+    row = db_sync.execute(select(ImportSettings).where(ImportSettings.id == 1)).scalar_one()
+    db_sync.refresh(row)
+    assert backup_destination.unseal(row.backup_s3_secret_sealed) == "s3cr3t-example"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("patch, word", [
+    ({"endpoint": "archives.example:8081"}, "http://"),
+    ({"endpoint": "ftp://archives.example"}, "http://"),
+    ({"bucket": "curator/sub"}, "bucket"),
+    ({"access_key": 5}, "access_key"),
+])
+async def test_destination_rejects_what_cant_be_an_s3_target(client, _sealing_key, patch, word):
+    resp = await client.patch("/api/system/backup/destination", json={**_DEST, **patch})
+    assert resp.status_code == 400
+    assert word in (await resp.get_json())["detail"]
+
+
+@pytest.mark.asyncio
+async def test_destination_remove_forgets_everything(client, _sealing_key):
+    await client.patch("/api/system/backup/destination", json=_DEST)
+    resp = await client.delete("/api/system/backup/destination")
+    assert await resp.get_json() == {
+        "endpoint": "", "bucket": "", "access_key": "", "region": "us-east-1",
+        "secret_set": False, "configured": False,
+    }
+
+
+@pytest.mark.asyncio
+async def test_destination_test_uses_the_form_and_the_saved_secret(
+    client, _sealing_key, monkeypatch,
+):
+    from backend.app.services import backup_destination
+
+    seen = []
+
+    def fake_check(target):
+        seen.append(target)
+        return {"ok": True, "message": "Reached it."}
+
+    monkeypatch.setattr(backup_destination, "check_target", fake_check)
+    await client.patch("/api/system/backup/destination", json=_DEST)
+    resp = await client.post(
+        "/api/system/backup/destination/test",
+        json={**_DEST, "bucket": "typed-not-saved", "secret_key": ""},
+    )
+    assert await resp.get_json() == {"ok": True, "message": "Reached it."}
+    assert seen[0].bucket == "typed-not-saved"
+    assert seen[0].secret_key == "s3cr3t-example"
+    assert seen[0].endpoint == "http://archives.example:8081"
+
+
+@pytest.mark.asyncio
+async def test_destination_test_names_what_is_missing(client, _sealing_key):
+    await client.delete("/api/system/backup/destination")
+    resp = await client.post(
+        "/api/system/backup/destination/test",
+        json={"endpoint": "http://archives.example:8081", "bucket": "curator"},
+    )
+    body = await resp.get_json()
+    assert body["ok"] is False
+    assert body["message"] == "Not tested: fill in the access key, secret key."
+
+
+@pytest.mark.asyncio
+async def test_destination_test_reports_an_unreachable_bucket_as_a_result(
+    client, _sealing_key, monkeypatch,
+):
+    from backend.app.services import backup_destination
+    from backend.app.services.backup_s3 import S3Error
+
+    def refuse(target, **kw):
+        raise S3Error("Couldn't reach http://archives.example:8081: nothing is listening there.")
+
+    monkeypatch.setattr(backup_destination, "list_objects", refuse)
+    resp = await client.post("/api/system/backup/destination/test", json=_DEST)
+    assert resp.status_code == 200
+    assert await resp.get_json() == {
+        "ok": False,
+        "message": "Couldn't reach http://archives.example:8081: nothing is listening there.",
+    }

tests/test_backup_s3.py

@@ -0,0 +1,209 @@
+"""The backup destination's S3 client (services/backup_s3.py): signatures match
+AWS's published Signature Version 4 examples, and the calls send and read what
+S3 expects, over a real HTTP connection to a local stand-in server."""
+import socket
+import threading
+from datetime import UTC, datetime
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+import pytest
+
+from backend.app.services import backup_s3 as s3
+
+# AWS's example credentials and moment ("Signature Calculations for the
+# Authorization Header", Amazon S3 API reference).
+KEY_ID = "AKIAIOSFODNN7EXAMPLE"
+SECRET = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+AT = datetime(2013, 5, 24, tzinfo=UTC)
+
+
+def _signature(headers):
+    return headers["Authorization"].rsplit("Signature=", 1)[1]
+
+
+def test_aws_example_get_object():
+    h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt",
+                {"Range": "bytes=0-9"}, s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
+    assert "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date," in h["Authorization"]
+    assert "Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request" in h[
+        "Authorization"]
+    assert _signature(h) == "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
+
+
+def test_aws_example_list_objects():
+    h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", {},
+                s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
+    assert _signature(h) == "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"
+
+
+def test_a_changed_secret_changes_the_signature():
+    url = "https://examplebucket.s3.amazonaws.com/test.txt"
+    a = s3.sign("GET", url, {}, s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
+    b = s3.sign("GET", url, {}, s3.EMPTY_SHA256, KEY_ID, SECRET + "x", "us-east-1", AT)
+    assert _signature(a) != _signature(b)
+
+
+def test_paths_are_path_style_and_keys_are_escaped():
+    t = s3.S3Target("http://archives.example:8081/s3", "curator", KEY_ID, SECRET)
+    assert t.url("media/ab/cd/abcd.jpg") == "http://archives.example:8081/s3/curator/media/ab/cd/abcd.jpg"
+    assert t.url() == "http://archives.example:8081/s3/curator/"
+    assert t.url("a b+c.jpg") == "http://archives.example:8081/s3/curator/a%20b%2Bc.jpg"
+
+
+# ── Calls, against a local stand-in ─────────────────────────────────────────
+
+
+class _Server:
+    """A local HTTP server that records each request and answers with
+    `reply(handler) -> (status, body)`."""
+
+    def __init__(self, reply):
+        self.seen = []
+        outer = self
+
+        class Handler(BaseHTTPRequestHandler):
+            def _serve(self):
+                length = int(self.headers.get("Content-Length") or 0)
+                outer.seen.append({
+                    "method": self.command, "path": self.path,
+                    "headers": {k.lower(): v for k, v in self.headers.items()},
+                    "body": self.rfile.read(length) if length else b"",
+                })
+                status, body = reply(self)
+                self.send_response(status)
+                self.send_header("Content-Length", str(len(body)))
+                self.end_headers()
+                self.wfile.write(body)
+
+            do_GET = do_PUT = _serve
+
+            def log_message(self, *args):
+                pass
+
+        self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+        self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
+
+    def __enter__(self):
+        self.thread.start()
+        return self
+
+    def __exit__(self, *exc):
+        self.httpd.shutdown()
+        self.httpd.server_close()
+
+    def target(self, bucket="curator"):
+        host, port = self.httpd.server_address
+        return s3.S3Target(f"http://{host}:{port}", bucket, KEY_ID, SECRET)
+
+
+def test_put_streams_the_file_signed(tmp_path):
+    f = tmp_path / "abcd.jpg"
+    f.write_bytes(b"x" * 5000)
+    with _Server(lambda h: (200, b"")) as srv:
+        s3.put_file(srv.target(), "media/ab/cd/abcd.jpg", f, content_type="image/jpeg")
+    req = srv.seen[0]
+    assert req["method"] == "PUT"
+    assert req["path"] == "/curator/media/ab/cd/abcd.jpg"
+    assert req["body"] == b"x" * 5000
+    assert req["headers"]["x-amz-content-sha256"] == s3.file_sha256(f)
+    assert req["headers"]["content-type"] == "image/jpeg"
+    assert req["headers"]["authorization"].startswith(
+        "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/")
+
+
+def test_put_signs_with_a_known_sha_without_rereading(tmp_path, monkeypatch):
+    f = tmp_path / "b.bin"
+    f.write_bytes(b"y")
+    monkeypatch.setattr(s3, "file_sha256", lambda p: pytest.fail("read the file twice"))
+    with _Server(lambda h: (200, b"")) as srv:
+        s3.put_file(srv.target(), "b.bin", f, payload_sha256="a" * 64)
+    assert srv.seen[0]["headers"]["x-amz-content-sha256"] == "a" * 64
+
+
+def test_a_refusal_says_why(tmp_path):
+    f = tmp_path / "b.bin"
+    f.write_bytes(b"x")
+    body = b"<Error><Code>SignatureDoesNotMatch</Code><Message>no</Message></Error>"
+    with _Server(lambda h: (403, body)) as srv, pytest.raises(
+        s3.S3Error, match="HTTP 403 SignatureDoesNotMatch: the secret key is wrong",
+    ):
+        s3.put_file(srv.target(), "b.bin", f)
+
+
+def test_a_web_server_at_the_address_is_named_as_not_s3():
+    """An address without the S3 port lands on whatever serves the default
+    port, and its plain 404 says nothing about S3."""
+    page = b"<html><body><h1>404 Not Found</h1></body></html>"
+    with _Server(lambda h: (404, page)) as srv, pytest.raises(
+        s3.S3Error,
+        match=r"HTTP 404: the address answered, but not as S3 — check the scheme and the port",
+    ):
+        s3.list_objects(srv.target())
+
+
+def test_a_server_error_without_a_body_is_not_blamed_on_the_port():
+    with _Server(lambda h: (503, b"")) as srv, pytest.raises(
+        s3.S3Error, match="HTTP 503: Service Unavailable",
+    ) as err:
+        s3.list_objects(srv.target())
+    assert "port" not in str(err.value)
+
+
+def test_nothing_listening_says_check_the_port():
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        port = sock.getsockname()[1]
+    target = s3.S3Target(f"http://127.0.0.1:{port}", "curator", KEY_ID, SECRET)
+    with pytest.raises(s3.S3Error, match="nothing is listening there"):
+        s3.list_objects(target)
+
+
+def test_list_reads_contents_and_the_next_page_token():
+    xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Name>curator</Name><KeyCount>2</KeyCount><IsTruncated>true</IsTruncated>
+  <NextContinuationToken>tok-2</NextContinuationToken>
+  <Contents><Key>database/fabledcurator-20261105T030000Z.dump</Key><Size>1200</Size>
+    <LastModified>2026-11-05T03:00:01.000Z</LastModified></Contents>
+  <Contents><Key>database/fabledcurator-20261106T030000Z.dump</Key><Size>1300</Size>
+    <LastModified>2026-11-06T03:00:01.000Z</LastModified></Contents>
+</ListBucketResult>"""
+    with _Server(lambda h: (200, xml)) as srv:
+        page = s3.list_objects(srv.target(), "database/", continuation="tok-1")
+    path = srv.seen[0]["path"]
+    assert path.startswith("/curator/?")
+    assert "list-type=2" in path and "prefix=database%2F" in path
+    assert "continuation-token=tok-1" in path
+    assert page.next_token == "tok-2"
+    assert page.objects == [
+        s3.StoredObject("database/fabledcurator-20261105T030000Z.dump", 1200,
+                        "2026-11-05T03:00:01.000Z"),
+        s3.StoredObject("database/fabledcurator-20261106T030000Z.dump", 1300,
+                        "2026-11-06T03:00:01.000Z"),
+    ]
+
+
+def test_a_listing_that_isnt_xml_is_named():
+    with _Server(lambda h: (200, b"hello")) as srv, pytest.raises(
+        s3.S3Error, match="isn't S3's XML",
+    ):
+        s3.list_objects(srv.target())
+
+
+def test_get_streams_the_object_to_disk(tmp_path):
+    with _Server(lambda h: (200, b"y" * 3000)) as srv:
+        out = tmp_path / "b.dump"
+        s3.get_file(srv.target(), "database/b.dump", out)
+    assert srv.seen[0]["path"] == "/curator/database/b.dump"
+    assert srv.seen[0]["headers"]["x-amz-content-sha256"] == s3.EMPTY_SHA256
+    assert out.read_bytes() == b"y" * 3000
+    assert not (tmp_path / "b.dump.partial").exists()
+
+
+def test_a_refused_get_says_so_and_writes_nothing(tmp_path):
+    body = b"<Error><Code>AccessDenied</Code></Error>"
+    out = tmp_path / "b.dump"
+    with _Server(lambda h: (403, body)) as srv, pytest.raises(s3.S3Error, match="b.dump"):
+        s3.get_file(srv.target(), "b.dump", out)
+    assert not out.exists()
+    assert not (tmp_path / "b.dump.partial").exists()