FabledCurator/backend/app/services/backup_s3.py

"""A small S3 client for sending backups off the box: AWS Signature Version 4,
path-style addressing (endpoint/bucket/key), and the calls a backup needs —
put an object, read one back, list a prefix.

FabledCurator puts, lists and reads; it never deletes. The bucket keeps the
history and thins it on its own policy, so a key that may only add and read
objects is enough, and is the safer key to hand a service: a stolen one cannot
wipe the history it writes into.

Built on the standard library's http.client rather than boto3: three calls
don't justify a large dependency, and signing is a pure function the tests
hold to AWS's own published examples. Every call is synchronous, because the
uploads run in Celery workers; the API runs the Test destination call in a
thread.
"""
from __future__ import annotations

import hashlib
import hmac
import http.client
import os
import socket
import ssl
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit

EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
TIMEOUT_SECONDS = 30.0
# A single object can be a multi-gigabyte video or archive; a slow link still
# gets this long between bytes before the call gives up.
TRANSFER_TIMEOUT_SECONDS = 15 * 60
CHUNK = 1 << 20


class S3Error(Exception):
    """Why a call to the bucket failed, said to the person."""


@dataclass(frozen=True)
class S3Target:
    endpoint: str          # http(s)://host[:port][/prefix], no bucket
    bucket: str
    access_key: str
    secret_key: str
    region: str = "us-east-1"
    verify_tls: bool = True

    def url(self, key: str = "") -> str:
        base = f"{self.endpoint.rstrip('/')}/{quote(self.bucket, safe='')}"
        return f"{base}/{quote(key, safe='/-_.~')}" if key else f"{base}/"


@dataclass(frozen=True)
class StoredObject:
    key: str
    size: int
    last_modified: str


@dataclass(frozen=True)
class ListPage:
    objects: list[StoredObject]
    next_token: str | None     # pass back as `continuation` for the next page


# ── Signing ──────────────────────────────────────────────────────────────────

def _encode(text: str, keep_slash: bool = False) -> str:
    return quote(text, safe="-_.~" + ("/" if keep_slash else ""))


def _hmac(key: bytes, msg: str) -> bytes:
    return hmac.new(key, msg.encode(), hashlib.sha256).digest()


def sign(method: str, url: str, headers: dict[str, str], payload_sha256: str,
         access_key: str, secret_key: str, region: str, at: datetime,
         service: str = "s3") -> dict[str, str]:
    """The request's headers with Host, x-amz-date, x-amz-content-sha256 and
    Authorization added. Every header given is signed, Host always."""
    u = urlsplit(url)
    stamp = at.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ")
    day = stamp[:8]
    out = dict(headers)
    out["Host"] = u.netloc
    out["x-amz-date"] = stamp
    out["x-amz-content-sha256"] = payload_sha256
    signed = {k.lower(): " ".join(str(v).split()) for k, v in out.items()}
    names = sorted(signed)

    query = sorted(
        (_encode(k), _encode(v)) for k, v in parse_qsl(u.query, keep_blank_values=True)
    )
    canonical = "\n".join([
        method.upper(),
        _encode(unquote(u.path) or "/", keep_slash=True),
        "&".join(f"{k}={v}" for k, v in query),
        "".join(f"{n}:{signed[n]}\n" for n in names),
        ";".join(names),
        payload_sha256,
    ])
    scope = f"{day}/{region}/{service}/aws4_request"
    to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope,
                         hashlib.sha256(canonical.encode()).hexdigest()])
    key = _hmac(_hmac(_hmac(_hmac(f"AWS4{secret_key}".encode(), day), region),
                      service), "aws4_request")
    signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
    out["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, "
                            f"SignedHeaders={';'.join(names)}, Signature={signature}")
    return out


# ── Transport ────────────────────────────────────────────────────────────────

def _connect(target: S3Target, url: str, timeout: float) -> http.client.HTTPConnection:
    u = urlsplit(url)
    if u.scheme == "https":
        ctx = ssl.create_default_context()
        if not target.verify_tls:
            ctx.check_hostname = False
            ctx.verify_mode = ssl.CERT_NONE
        return http.client.HTTPSConnection(
            u.hostname, u.port, timeout=timeout, context=ctx, blocksize=CHUNK,
        )
    if u.scheme != "http":
        raise S3Error(f"{target.endpoint} isn't an http:// or https:// address.")
    return http.client.HTTPConnection(u.hostname, u.port, timeout=timeout, blocksize=CHUNK)


def _request_target(url: str) -> str:
    u = urlsplit(url)
    return (u.path or "/") + (f"?{u.query}" if u.query else "")


def _reach_error(target: S3Target, exc: Exception) -> S3Error:
    if isinstance(exc, TimeoutError):
        return S3Error(f"{target.endpoint} didn't answer in time.")
    if isinstance(exc, socket.gaierror):
        return S3Error(f"Couldn't reach {target.endpoint}: the host name doesn't resolve.")
    if isinstance(exc, ConnectionRefusedError):
        return S3Error(
            f"Couldn't reach {target.endpoint}: nothing is listening there "
            "(check the port)."
        )
    if isinstance(exc, ssl.SSLError):
        return S3Error(f"Couldn't reach {target.endpoint}: TLS failed ({exc.reason}).")
    return S3Error(f"Couldn't reach {target.endpoint}: {exc}.")


def _says(status: int, reason: str, content: bytes) -> str:
    """An S3 error response in words: its Code and Message when it has them."""
    try:
        root = ET.fromstring(content) if content else None
        code = (root.findtext("Code") or "") if root is not None else ""
        message = (root.findtext("Message") or "") if root is not None else ""
    except ET.ParseError:
        code, message = "", ""
    hints = {
        "SignatureDoesNotMatch": "the secret key is wrong",
        "InvalidAccessKeyId": "the access key isn't known there",
        "NoSuchBucket": "there is no such bucket",
        "AccessDenied": "the key isn't allowed to do that",
    }
    said = hints.get(code) or message or reason
    if not code and 400 <= status < 500:
        # S3 refuses with an <Error><Code> body. A 4xx without one came from
        # something else at that address — most often a web server on the
        # default port, when the S3 service listens on a port of its own.
        said = ("the address answered, but not as S3 — check the scheme and "
                "the port (an S3 service usually listens on its own port)")
    return f"HTTP {status}{f' {code}' if code else ''}: {said}."


# ── Calls ────────────────────────────────────────────────────────────────────

def file_sha256(path: Path) -> str:
    h = hashlib.sha256()
    with open(path, "rb") as f:
        while chunk := f.read(CHUNK):
            h.update(chunk)
    return h.hexdigest()


def put_file(target: S3Target, key: str, path: Path, *,
             content_type: str = "application/octet-stream",
             payload_sha256: str | None = None,
             now: datetime | None = None) -> None:
    """Upload one file as one object, streamed from disk. `payload_sha256` is
    the file's sha256 when the caller already knows it (the library stores
    every file's), which saves reading the file twice."""
    digest = payload_sha256 or file_sha256(path)
    size = path.stat().st_size
    url = target.url(key)
    headers = sign("PUT", url, {"Content-Length": str(size), "Content-Type": content_type},
                   digest, target.access_key, target.secret_key, target.region,
                   now or datetime.now(UTC))
    conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS)
    try:
        with open(path, "rb") as f:
            conn.request("PUT", _request_target(url), body=f, headers=headers)
            response = conn.getresponse()
            content = response.read()
    except (OSError, http.client.HTTPException) as exc:
        raise _reach_error(target, exc) from None
    finally:
        conn.close()
    if response.status // 100 != 2:
        raise S3Error(f"Upload of {key} refused: "
                      f"{_says(response.status, response.reason, content)}")


def get_file(target: S3Target, key: str, path: Path, *,
             now: datetime | None = None) -> None:
    """Download one object to `path`, streamed to disk. The bytes land under a
    `.partial` name and are renamed into place when complete, so a broken
    transfer never leaves a truncated file at `path`."""
    url = target.url(key)
    headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
                   target.region, now or datetime.now(UTC))
    partial = path.with_name(path.name + ".partial")
    conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS)
    try:
        conn.request("GET", _request_target(url), headers=headers)
        response = conn.getresponse()
        if response.status != 200:
            raise S3Error(f"Reading {key} refused: "
                          f"{_says(response.status, response.reason, response.read())}")
        with open(partial, "wb") as f:
            while chunk := response.read(CHUNK):
                f.write(chunk)
        os.replace(partial, path)
    except (OSError, http.client.HTTPException) as exc:
        partial.unlink(missing_ok=True)
        raise _reach_error(target, exc) from None
    finally:
        conn.close()


def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *,
                 continuation: str | None = None,
                 now: datetime | None = None) -> ListPage:
    """One page (up to `limit`) of the objects under `prefix`, ListObjectsV2."""
    params = {"list-type": "2", "max-keys": str(limit)}
    if prefix:
        params["prefix"] = prefix
    if continuation:
        params["continuation-token"] = continuation
    url = f"{target.url()}?{urlencode(params, quote_via=quote)}"
    headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
                   target.region, now or datetime.now(UTC))
    conn = _connect(target, url, TIMEOUT_SECONDS)
    try:
        conn.request("GET", _request_target(url), headers=headers)
        response = conn.getresponse()
        content = response.read()
    except (OSError, http.client.HTTPException) as exc:
        raise _reach_error(target, exc) from None
    finally:
        conn.close()
    if response.status != 200:
        raise S3Error(f"Listing {target.bucket} refused: "
                      f"{_says(response.status, response.reason, content)}")
    try:
        root = ET.fromstring(content)
    except ET.ParseError:
        raise S3Error(
            f"{target.endpoint} answered a listing with something that isn't S3's XML."
        ) from None
    objects, next_token = [], None
    for item in root.iter():
        tag = item.tag.rsplit("}", 1)[-1]
        if tag == "NextContinuationToken":
            next_token = item.text or None
        if tag != "Contents":
            continue
        fields = {child.tag.rsplit("}", 1)[-1]: (child.text or "") for child in item}
        objects.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0),
                                    fields.get("LastModified", "")))
    return ListPage(objects, next_token)