"""The backup destination's S3 client (services/backup_s3.py): signatures match AWS's published Signature Version 4 examples, and the calls send and read what S3 expects, over a real HTTP connection to a local stand-in server.""" import hashlib import socket import threading from datetime import UTC, datetime from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import pytest from backend.app.services import backup_s3 as s3 # AWS's example credentials and moment ("Signature Calculations for the # Authorization Header", Amazon S3 API reference). KEY_ID = "AKIAIOSFODNN7EXAMPLE" SECRET = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" AT = datetime(2013, 5, 24, tzinfo=UTC) def _signature(headers): return headers["Authorization"].rsplit("Signature=", 1)[1] def test_aws_example_get_object(): h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {"Range": "bytes=0-9"}, s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT) assert "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date," in h["Authorization"] assert "Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request" in h[ "Authorization"] assert _signature(h) == "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41" def test_aws_example_list_objects(): h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", {}, s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT) assert _signature(h) == "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7" def test_a_changed_secret_changes_the_signature(): url = "https://examplebucket.s3.amazonaws.com/test.txt" a = s3.sign("GET", url, {}, s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT) b = s3.sign("GET", url, {}, s3.EMPTY_SHA256, KEY_ID, SECRET + "x", "us-east-1", AT) assert _signature(a) != _signature(b) def test_paths_are_path_style_and_keys_are_escaped(): t = s3.S3Target("http://archives.example:8081/s3", "curator", KEY_ID, SECRET) assert t.url("media/ab/cd/abcd.jpg") == "http://archives.example:8081/s3/curator/media/ab/cd/abcd.jpg" assert t.url() == "http://archives.example:8081/s3/curator/" assert t.url("a b+c.jpg") == "http://archives.example:8081/s3/curator/a%20b%2Bc.jpg" # ── Calls, against a local stand-in ───────────────────────────────────────── class _Server: """A local HTTP server that records each request and answers with `reply(handler) -> (status, body)` or `(status, body, headers)`.""" def __init__(self, reply): self.seen = [] outer = self class Handler(BaseHTTPRequestHandler): def _serve(self): length = int(self.headers.get("Content-Length") or 0) outer.seen.append({ "method": self.command, "path": self.path, "headers": {k.lower(): v for k, v in self.headers.items()}, "body": self.rfile.read(length) if length else b"", }) status, body, *extra = reply(self) self.send_response(status) for name, value in (extra[0] if extra else {}).items(): self.send_header(name, value) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) do_GET = do_PUT = do_POST = do_DELETE = _serve def log_message(self, *args): pass self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True) def __enter__(self): self.thread.start() return self def __exit__(self, *exc): self.httpd.shutdown() self.httpd.server_close() def target(self, bucket="curator"): host, port = self.httpd.server_address return s3.S3Target(f"http://{host}:{port}", bucket, KEY_ID, SECRET) def test_put_streams_the_file_signed(tmp_path): f = tmp_path / "abcd.jpg" f.write_bytes(b"x" * 5000) with _Server(lambda h: (200, b"")) as srv: s3.put_file(srv.target(), "media/ab/cd/abcd.jpg", f, content_type="image/jpeg") req = srv.seen[0] assert req["method"] == "PUT" assert req["path"] == "/curator/media/ab/cd/abcd.jpg" assert req["body"] == b"x" * 5000 assert req["headers"]["x-amz-content-sha256"] == s3.file_sha256(f) assert req["headers"]["content-type"] == "image/jpeg" assert req["headers"]["authorization"].startswith( "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/") def test_put_signs_with_a_known_sha_without_rereading(tmp_path, monkeypatch): f = tmp_path / "b.bin" f.write_bytes(b"y") monkeypatch.setattr(s3, "file_sha256", lambda p: pytest.fail("read the file twice")) with _Server(lambda h: (200, b"")) as srv: s3.put_file(srv.target(), "b.bin", f, payload_sha256="a" * 64) assert srv.seen[0]["headers"]["x-amz-content-sha256"] == "a" * 64 def test_a_refusal_says_why(tmp_path): f = tmp_path / "b.bin" f.write_bytes(b"x") body = b"SignatureDoesNotMatchno" with _Server(lambda h: (403, body)) as srv, pytest.raises( s3.S3Error, match="HTTP 403 SignatureDoesNotMatch: the secret key is wrong", ): s3.put_file(srv.target(), "b.bin", f) def test_a_web_server_at_the_address_is_named_as_not_s3(): """An address without the S3 port lands on whatever serves the default port, and its plain 404 says nothing about S3.""" page = b"

404 Not Found

" with _Server(lambda h: (404, page)) as srv, pytest.raises( s3.S3Error, match=r"HTTP 404: the address answered, but not as S3 — check the scheme and the port", ): s3.list_objects(srv.target()) def test_a_server_error_without_a_body_is_not_blamed_on_the_port(): with _Server(lambda h: (503, b"")) as srv, pytest.raises( s3.S3Error, match="HTTP 503: Service Unavailable", ) as err: s3.list_objects(srv.target()) assert "port" not in str(err.value) def test_nothing_listening_says_check_the_port(): with socket.socket() as sock: sock.bind(("127.0.0.1", 0)) port = sock.getsockname()[1] target = s3.S3Target(f"http://127.0.0.1:{port}", "curator", KEY_ID, SECRET) with pytest.raises(s3.S3Error, match="nothing is listening there"): s3.list_objects(target) def test_list_reads_contents_and_the_next_page_token(): xml = b""" curator2true tok-2 database/fabledcurator-20261105T030000Z.dump1200 2026-11-05T03:00:01.000Z database/fabledcurator-20261106T030000Z.dump1300 2026-11-06T03:00:01.000Z """ with _Server(lambda h: (200, xml)) as srv: page = s3.list_objects(srv.target(), "database/", continuation="tok-1") path = srv.seen[0]["path"] assert path.startswith("/curator/?") assert "list-type=2" in path and "prefix=database%2F" in path assert "continuation-token=tok-1" in path assert page.next_token == "tok-2" assert page.objects == [ s3.StoredObject("database/fabledcurator-20261105T030000Z.dump", 1200, "2026-11-05T03:00:01.000Z"), s3.StoredObject("database/fabledcurator-20261106T030000Z.dump", 1300, "2026-11-06T03:00:01.000Z"), ] def test_a_listing_that_isnt_xml_is_named(): with _Server(lambda h: (200, b"hello")) as srv, pytest.raises( s3.S3Error, match="isn't S3's XML", ): s3.list_objects(srv.target()) def test_get_streams_the_object_to_disk(tmp_path): with _Server(lambda h: (200, b"y" * 3000)) as srv: out = tmp_path / "b.dump" s3.get_file(srv.target(), "database/b.dump", out) assert srv.seen[0]["path"] == "/curator/database/b.dump" assert srv.seen[0]["headers"]["x-amz-content-sha256"] == s3.EMPTY_SHA256 assert out.read_bytes() == b"y" * 3000 assert not (tmp_path / "b.dump.partial").exists() def test_a_refused_get_says_so_and_writes_nothing(tmp_path): body = b"AccessDenied" out = tmp_path / "b.dump" with _Server(lambda h: (403, body)) as srv, pytest.raises(s3.S3Error, match="b.dump"): s3.get_file(srv.target(), "b.dump", out) assert not out.exists() assert not (tmp_path / "b.dump.partial").exists() # ── Multipart ─────────────────────────────────────────────────────────────── _STARTED = b"up1" def _bucket(parts_reply=None, complete=(200, b"")): """A stand-in bucket for one multipart upload: starts it, takes parts (each answered by `parts_reply(n)`, an ETag by default) and completes it.""" def reply(h): if h.command == "POST" and h.path.endswith("?uploads"): return 200, _STARTED if h.command == "PUT": n = int(h.path.split("partNumber=")[1].split("&")[0]) return parts_reply(n) if parts_reply else (200, b"", {"ETag": f'"e{n}"'}) if h.command == "POST": return complete return 204, b"" return reply def test_an_upload_sends_its_parts_then_joins_them_in_order(): with _Server(_bucket()) as srv: up = s3.Upload(srv.target(), "database/x.dump") up.add(b"first,") up.add(b"second") up.complete() start, one, two, done = srv.seen assert (start["method"], start["path"]) == ("POST", "/curator/database/x.dump?uploads") assert one["path"] == "/curator/database/x.dump?partNumber=1&uploadId=up1" assert two["body"] == b"second" assert two["headers"]["x-amz-content-sha256"] == hashlib.sha256(b"second").hexdigest() assert done["path"] == "/curator/database/x.dump?uploadId=up1" assert done["body"].index(b'"e1"') < done["body"].index(b'"e2"') assert (up.parts, up.size) == (2, len(b"first,second")) def test_a_refused_start_says_why(): with _Server(lambda h: (403, b"AccessDenied")) as srv: with pytest.raises(s3.S3Error, match="the key isn't allowed"): s3.Upload(srv.target(), "database/x.dump") def test_a_completion_refused_under_a_200_is_a_failure(): error = (200, b"InternalErrordisk full") with _Server(_bucket(complete=error)) as srv: up = s3.Upload(srv.target(), "database/x.dump") up.add(b"data") with pytest.raises(s3.S3Error, match="Finishing the upload"): up.complete() def test_nothing_sent_is_not_completed(): with _Server(_bucket()) as srv: up = s3.Upload(srv.target(), "database/x.dump") with pytest.raises(s3.S3Error, match="Nothing was sent"): up.complete() def test_a_busy_bucket_gets_the_part_again_after_a_rest(monkeypatch): rests = [] monkeypatch.setattr(s3.time, "sleep", rests.append) answers = iter([(503, b"", {"Retry-After": "7"}), (200, b"", {"ETag": '"e1"'})]) with _Server(_bucket(parts_reply=lambda n: next(answers))) as srv: up = s3.Upload(srv.target(), "database/x.dump") up.add(b"data") assert rests == [7] assert up.parts == 1 def test_a_part_refused_outright_is_not_sent_again(monkeypatch): monkeypatch.setattr(s3.time, "sleep", lambda s: pytest.fail("rested on a refusal")) refused = (403, b"AccessDenied") with _Server(_bucket(parts_reply=lambda n: refused)) as srv: up = s3.Upload(srv.target(), "database/x.dump") with pytest.raises(s3.S3Error, match="Part 1"): up.add(b"data") def test_a_part_that_never_lands_gives_up_after_the_rests(monkeypatch): rests = [] monkeypatch.setattr(s3.time, "sleep", rests.append) with _Server(_bucket(parts_reply=lambda n: (503, b""))) as srv: up = s3.Upload(srv.target(), "database/x.dump") with pytest.raises(s3.S3Error, match="Part 1"): up.add(b"data") assert rests == list(s3.PART_RETRY_WAITS) def test_abort_asks_the_bucket_to_drop_the_parts_and_never_raises(): with _Server(_bucket()) as srv: up = s3.Upload(srv.target(), "database/x.dump") up.abort() assert srv.seen[-1]["method"] == "DELETE" assert srv.seen[-1]["path"] == "/curator/database/x.dump?uploadId=up1" up.abort() # the server is gone now: still no exception def test_the_pacer_holds_bytes_to_the_limit(monkeypatch): clock = [0.0] monkeypatch.setattr(s3.time, "monotonic", lambda: clock[0]) monkeypatch.setattr(s3.time, "sleep", lambda secs: clock.__setitem__(0, clock[0] + secs)) pacer = s3.Pacer(8) # 8 Mbit/s = 1,000,000 bytes/s data = b"x" * (3 * s3.CHUNK) assert b"".join(pacer.chunks(data)) == data assert clock[0] == pytest.approx(len(data) / 1_000_000) def test_no_limit_never_waits(monkeypatch): monkeypatch.setattr(s3.time, "sleep", lambda secs: pytest.fail("waited without a limit")) assert b"".join(s3.Pacer(0).chunks(b"y" * (2 * s3.CHUNK + 1))) == b"y" * (2 * s3.CHUNK + 1)