"""A small S3 client for sending backups off the box: AWS Signature Version 4, path-style addressing (endpoint/bucket/key), and the calls a backup needs — put an object, read one back, list a prefix. FabledCurator puts, lists and reads; it never deletes. The bucket keeps the history and thins it on its own policy, so a key that may only add and read objects is enough, and is the safer key to hand a service: a stolen one cannot wipe the history it writes into. Built on the standard library's http.client rather than boto3: three calls don't justify a large dependency, and signing is a pure function the tests hold to AWS's own published examples. Every call is synchronous, because the uploads run in Celery workers; the API runs the Test destination call in a thread. """ from __future__ import annotations import hashlib import hmac import http.client import os import socket import ssl import xml.etree.ElementTree as ET from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit EMPTY_SHA256 = hashlib.sha256(b"").hexdigest() TIMEOUT_SECONDS = 30.0 # A single object can be a multi-gigabyte video or archive; a slow link still # gets this long between bytes before the call gives up. TRANSFER_TIMEOUT_SECONDS = 15 * 60 CHUNK = 1 << 20 class S3Error(Exception): """Why a call to the bucket failed, said to the person.""" @dataclass(frozen=True) class S3Target: endpoint: str # http(s)://host[:port][/prefix], no bucket bucket: str access_key: str secret_key: str region: str = "us-east-1" verify_tls: bool = True def url(self, key: str = "") -> str: base = f"{self.endpoint.rstrip('/')}/{quote(self.bucket, safe='')}" return f"{base}/{quote(key, safe='/-_.~')}" if key else f"{base}/" @dataclass(frozen=True) class StoredObject: key: str size: int last_modified: str @dataclass(frozen=True) class ListPage: objects: list[StoredObject] next_token: str | None # pass back as `continuation` for the next page # ── Signing ────────────────────────────────────────────────────────────────── def _encode(text: str, keep_slash: bool = False) -> str: return quote(text, safe="-_.~" + ("/" if keep_slash else "")) def _hmac(key: bytes, msg: str) -> bytes: return hmac.new(key, msg.encode(), hashlib.sha256).digest() def sign(method: str, url: str, headers: dict[str, str], payload_sha256: str, access_key: str, secret_key: str, region: str, at: datetime, service: str = "s3") -> dict[str, str]: """The request's headers with Host, x-amz-date, x-amz-content-sha256 and Authorization added. Every header given is signed, Host always.""" u = urlsplit(url) stamp = at.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ") day = stamp[:8] out = dict(headers) out["Host"] = u.netloc out["x-amz-date"] = stamp out["x-amz-content-sha256"] = payload_sha256 signed = {k.lower(): " ".join(str(v).split()) for k, v in out.items()} names = sorted(signed) query = sorted( (_encode(k), _encode(v)) for k, v in parse_qsl(u.query, keep_blank_values=True) ) canonical = "\n".join([ method.upper(), _encode(unquote(u.path) or "/", keep_slash=True), "&".join(f"{k}={v}" for k, v in query), "".join(f"{n}:{signed[n]}\n" for n in names), ";".join(names), payload_sha256, ]) scope = f"{day}/{region}/{service}/aws4_request" to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope, hashlib.sha256(canonical.encode()).hexdigest()]) key = _hmac(_hmac(_hmac(_hmac(f"AWS4{secret_key}".encode(), day), region), service), "aws4_request") signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest() out["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, " f"SignedHeaders={';'.join(names)}, Signature={signature}") return out # ── Transport ──────────────────────────────────────────────────────────────── def _connect(target: S3Target, url: str, timeout: float) -> http.client.HTTPConnection: u = urlsplit(url) if u.scheme == "https": ctx = ssl.create_default_context() if not target.verify_tls: ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE return http.client.HTTPSConnection( u.hostname, u.port, timeout=timeout, context=ctx, blocksize=CHUNK, ) if u.scheme != "http": raise S3Error(f"{target.endpoint} isn't an http:// or https:// address.") return http.client.HTTPConnection(u.hostname, u.port, timeout=timeout, blocksize=CHUNK) def _request_target(url: str) -> str: u = urlsplit(url) return (u.path or "/") + (f"?{u.query}" if u.query else "") def _reach_error(target: S3Target, exc: Exception) -> S3Error: if isinstance(exc, TimeoutError): return S3Error(f"{target.endpoint} didn't answer in time.") if isinstance(exc, socket.gaierror): return S3Error(f"Couldn't reach {target.endpoint}: the host name doesn't resolve.") if isinstance(exc, ConnectionRefusedError): return S3Error( f"Couldn't reach {target.endpoint}: nothing is listening there " "(check the port)." ) if isinstance(exc, ssl.SSLError): return S3Error(f"Couldn't reach {target.endpoint}: TLS failed ({exc.reason}).") return S3Error(f"Couldn't reach {target.endpoint}: {exc}.") def _says(status: int, reason: str, content: bytes) -> str: """An S3 error response in words: its Code and Message when it has them.""" try: root = ET.fromstring(content) if content else None code = (root.findtext("Code") or "") if root is not None else "" message = (root.findtext("Message") or "") if root is not None else "" except ET.ParseError: code, message = "", "" hints = { "SignatureDoesNotMatch": "the secret key is wrong", "InvalidAccessKeyId": "the access key isn't known there", "NoSuchBucket": "there is no such bucket", "AccessDenied": "the key isn't allowed to do that", } said = hints.get(code) or message or reason if not code and 400 <= status < 500: # S3 refuses with an body. A 4xx without one came from # something else at that address — most often a web server on the # default port, when the S3 service listens on a port of its own. said = ("the address answered, but not as S3 — check the scheme and " "the port (an S3 service usually listens on its own port)") return f"HTTP {status}{f' {code}' if code else ''}: {said}." # ── Calls ──────────────────────────────────────────────────────────────────── def file_sha256(path: Path) -> str: h = hashlib.sha256() with open(path, "rb") as f: while chunk := f.read(CHUNK): h.update(chunk) return h.hexdigest() def put_file(target: S3Target, key: str, path: Path, *, content_type: str = "application/octet-stream", payload_sha256: str | None = None, now: datetime | None = None) -> None: """Upload one file as one object, streamed from disk. `payload_sha256` is the file's sha256 when the caller already knows it (the library stores every file's), which saves reading the file twice.""" digest = payload_sha256 or file_sha256(path) size = path.stat().st_size url = target.url(key) headers = sign("PUT", url, {"Content-Length": str(size), "Content-Type": content_type}, digest, target.access_key, target.secret_key, target.region, now or datetime.now(UTC)) conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS) try: with open(path, "rb") as f: conn.request("PUT", _request_target(url), body=f, headers=headers) response = conn.getresponse() content = response.read() except (OSError, http.client.HTTPException) as exc: raise _reach_error(target, exc) from None finally: conn.close() if response.status // 100 != 2: raise S3Error(f"Upload of {key} refused: " f"{_says(response.status, response.reason, content)}") def get_file(target: S3Target, key: str, path: Path, *, now: datetime | None = None) -> None: """Download one object to `path`, streamed to disk. The bytes land under a `.partial` name and are renamed into place when complete, so a broken transfer never leaves a truncated file at `path`.""" url = target.url(key) headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key, target.region, now or datetime.now(UTC)) partial = path.with_name(path.name + ".partial") conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS) try: conn.request("GET", _request_target(url), headers=headers) response = conn.getresponse() if response.status != 200: raise S3Error(f"Reading {key} refused: " f"{_says(response.status, response.reason, response.read())}") with open(partial, "wb") as f: while chunk := response.read(CHUNK): f.write(chunk) os.replace(partial, path) except (OSError, http.client.HTTPException) as exc: partial.unlink(missing_ok=True) raise _reach_error(target, exc) from None finally: conn.close() def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *, continuation: str | None = None, now: datetime | None = None) -> ListPage: """One page (up to `limit`) of the objects under `prefix`, ListObjectsV2.""" params = {"list-type": "2", "max-keys": str(limit)} if prefix: params["prefix"] = prefix if continuation: params["continuation-token"] = continuation url = f"{target.url()}?{urlencode(params, quote_via=quote)}" headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key, target.region, now or datetime.now(UTC)) conn = _connect(target, url, TIMEOUT_SECONDS) try: conn.request("GET", _request_target(url), headers=headers) response = conn.getresponse() content = response.read() except (OSError, http.client.HTTPException) as exc: raise _reach_error(target, exc) from None finally: conn.close() if response.status != 200: raise S3Error(f"Listing {target.bucket} refused: " f"{_says(response.status, response.reason, content)}") try: root = ET.fromstring(content) except ET.ParseError: raise S3Error( f"{target.endpoint} answered a listing with something that isn't S3's XML." ) from None objects, next_token = [], None for item in root.iter(): tag = item.tag.rsplit("}", 1)[-1] if tag == "NextContinuationToken": next_token = item.text or None if tag != "Contents": continue fields = {child.tag.rsplit("}", 1)[-1]: (child.text or "") for child in item} objects.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0), fields.get("LastModified", ""))) return ListPage(objects, next_token)