diff --git a/pyproject.toml b/pyproject.toml index ccb3309d..0ff5483a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,10 @@ dependencies = [ "asyncpg>=0.29", "alembic>=1.13", "httpx>=0.27", + # Seals the backup destination's S3 secret at rest (services/secrets.py). + # Already in the image through pyjwt; named here because Scribe now + # imports it directly. + "cryptography", "hypercorn>=0.17", "bcrypt>=4.0", "aiosmtplib>=3.0", diff --git a/src/scribe/services/backup.py b/src/scribe/services/backup.py index f4c04420..a57ab811 100644 --- a/src/scribe/services/backup.py +++ b/src/scribe/services/backup.py @@ -772,8 +772,18 @@ def _rule_version_rows(rows) -> list[dict]: ] +# Setting KEYS no backup carries. The backup destination's S3 secret is sealed +# under SECRET_KEY, but a backup is exactly the thing copied off-site — it must +# not carry the key that writes to the place it is copied to. A restore leaves +# the destination configured and asks for the secret again. +_SETTINGS_NOT_EXPORTED = frozenset({"backup_s3_secret"}) + + def _setting_rows(rows) -> list[dict]: - return [{"user_id": s.user_id, "key": s.key, "value": s.value} for s in rows] + return [ + {"user_id": s.user_id, "key": s.key, "value": s.value} + for s in rows if s.key not in _SETTINGS_NOT_EXPORTED + ] def _rulebook_rows(rows) -> list[dict]: diff --git a/src/scribe/services/backup_destination.py b/src/scribe/services/backup_destination.py new file mode 100644 index 00000000..f339a981 --- /dev/null +++ b/src/scribe/services/backup_destination.py @@ -0,0 +1,235 @@ +"""Where the scheduled backup goes, and getting a backup file there. + +Two kinds of destination, both instance-wide and set by an admin: + +- **folder**: a directory inside the container, usually a mounted share. A + backup service that watches the share (or a person) takes it from there. +- **s3**: any S3-compatible bucket, addressed by path. Scribe only puts and + lists, so an append-only key is enough. + +Either way Scribe ADDS one dated file per run and never removes one: keeping +and thinning the history is the destination's job (services/backup_file.py +says why the file is shaped the way it is for that). + +The settings live as admin settings: `backup_config` holds everything but the +S3 secret as JSON, and `backup_s3_secret` holds the secret sealed under the +instance's SECRET_KEY (services/secrets.py). The secret is also kept out of +every backup (backup._SETTINGS_NOT_EXPORTED), so a backup copied off-site +does not carry the key that writes to that very place. +""" +from __future__ import annotations + +import asyncio +import json +import os +import tempfile +from dataclasses import asdict, dataclass, fields, replace +from datetime import datetime +from pathlib import Path + +from scribe.services import backup_file +from scribe.services.backup_s3 import S3Error, S3Target, list_objects, put_file +from scribe.services.secrets import seal, unseal +from scribe.services.settings import get_admin_setting, set_admin_setting + +CONFIG_SETTING = "backup_config" +SECRET_SETTING = "backup_s3_secret" +DESTINATION_KINDS = ("folder", "s3") +DEFAULT_FOLDER = "/backups" +# 02:00 UTC: before the 03:30 trash purge and the 04:00 DB maintenance, so the +# backup never contends with a VACUUM, and the night's deletions are still +# recoverable from the backup taken just before them. +DEFAULT_HOUR = 2 + + +class DestinationError(Exception): + """Why the destination can't take a backup, said to the person.""" + + +@dataclass(frozen=True) +class BackupConfig: + enabled: bool = False + hour: int = DEFAULT_HOUR + destination: str = "folder" + folder_path: str = DEFAULT_FOLDER + s3_endpoint: str = "" + s3_bucket: str = "" + s3_access_key: str = "" + s3_region: str = "us-east-1" + s3_verify_tls: bool = True + + @classmethod + def from_dict(cls, raw: dict) -> "BackupConfig": + """Lenient: unknown keys are ignored and bad values fall back to the + default, so an edited or older stored config still loads.""" + known = {f.name: f for f in fields(cls)} + values = {} + for name, value in (raw or {}).items(): + f = known.get(name) + if f is None: + continue + default = f.default + if isinstance(default, bool): + values[name] = value if isinstance(value, bool) else default + elif isinstance(default, int): + try: + values[name] = int(value) + except (TypeError, ValueError): + values[name] = default + else: + values[name] = str(value).strip() if value is not None else default + config = cls(**values) + if not 0 <= config.hour <= 23: + config = replace(config, hour=DEFAULT_HOUR) + if config.destination not in DESTINATION_KINDS: + config = replace(config, destination="folder") + return config + + def to_dict(self) -> dict: + return asdict(self) + + def problems(self, *, has_secret: bool) -> list[str]: + """What stops this config from sending a backup, in words. Empty when + it is complete (which says nothing about whether the place answers — + check_destination does that).""" + out: list[str] = [] + if self.destination == "folder": + if not self.folder_path.startswith("/"): + out.append("The folder must be an absolute path inside the container.") + else: + if not self.s3_endpoint.startswith(("http://", "https://")): + out.append("The S3 address must start with http:// or https://.") + if not self.s3_bucket: + out.append("The bucket name is missing.") + if not self.s3_access_key: + out.append("The access key is missing.") + if not has_secret: + out.append("The secret key is missing.") + return out + + +async def load_config() -> BackupConfig: + raw = await get_admin_setting(CONFIG_SETTING, "") + try: + data = json.loads(raw) if raw else {} + except ValueError: + data = {} + return BackupConfig.from_dict(data if isinstance(data, dict) else {}) + + +async def save_config(config: BackupConfig) -> None: + await set_admin_setting(CONFIG_SETTING, json.dumps(config.to_dict(), sort_keys=True)) + + +async def load_secret() -> tuple[str | None, bool]: + """(the S3 secret or None, whether one is stored). + + Stored but None means it was sealed under another SECRET_KEY — after a + restore onto a different instance, say — and has to be entered again. + """ + stored = await get_admin_setting(SECRET_SETTING, "") + return (unseal(stored), bool(stored)) + + +async def save_secret(secret: str) -> None: + await set_admin_setting(SECRET_SETTING, seal(secret) if secret else "") + + +def s3_target(config: BackupConfig, secret: str) -> S3Target: + return S3Target( + endpoint=config.s3_endpoint, bucket=config.s3_bucket, + access_key=config.s3_access_key, secret_key=secret, + region=config.s3_region or "us-east-1", verify_tls=config.s3_verify_tls, + ) + + +@dataclass(frozen=True) +class Delivered: + name: str + size_bytes: int + where: str + + +async def deliver( + payload: dict, config: BackupConfig, secret: str | None, at: datetime, + *, app_version: str, transport=None, +) -> Delivered: + """Write `payload` as the dated backup file and put it at the destination. + + The file is written off the event loop: serialising a full export is + seconds of CPU on a large corpus, and the app keeps serving meanwhile. + """ + problems = config.problems(has_secret=bool(secret)) + if problems: + raise DestinationError(" ".join(problems)) + + if config.destination == "folder": + folder = Path(config.folder_path) + if not folder.is_dir(): + raise DestinationError( + f"{folder} isn't there. Mount the backup share at that path, " + "or choose the folder it is mounted at." + ) + try: + path = await asyncio.to_thread( + backup_file.write_archive, payload, folder, at, app_version=app_version, + ) + except PermissionError: + raise DestinationError(f"Scribe isn't allowed to write in {folder}.") from None + return Delivered(path.name, path.stat().st_size, str(folder)) + + target = s3_target(config, secret or "") + with tempfile.TemporaryDirectory(prefix="scribe-backup-") as staging: + path = await asyncio.to_thread( + backup_file.write_archive, payload, Path(staging), at, app_version=app_version, + ) + size = path.stat().st_size + try: + await put_file(target, path.name, path, transport=transport) + except S3Error as exc: + raise DestinationError(str(exc)) from None + return Delivered(path.name, size, target.url()) + + +async def check_destination( + config: BackupConfig, secret: str | None, *, transport=None, +) -> str: + """Ask the destination whether it would take a backup, without sending + one. Returns what was found, in words; raises DestinationError otherwise. + + The folder check writes and removes a dot-file, which a folder watcher + ignores. The S3 check only lists: an append-only key could add a test + object and then never remove it. + """ + problems = config.problems(has_secret=bool(secret)) + if problems: + raise DestinationError(" ".join(problems)) + + if config.destination == "folder": + folder = Path(config.folder_path) + if not folder.is_dir(): + raise DestinationError(f"{folder} isn't there, or isn't a folder.") + probe = folder / ".scribe-write-check" + try: + await asyncio.to_thread(probe.write_bytes, b"") + await asyncio.to_thread(probe.unlink) + except PermissionError: + raise DestinationError(f"Scribe isn't allowed to write in {folder}.") from None + except OSError as exc: + raise DestinationError(f"Writing in {folder} failed: {exc.strerror or exc}.") from None + count = await asyncio.to_thread(_count_backups, folder) + return f"{folder} is writable and holds {count} Scribe backup(s)." + + target = s3_target(config, secret or "") + try: + objects = await list_objects(target, backup_file.NAME_PREFIX, transport=transport) + except S3Error as exc: + raise DestinationError(str(exc)) from None + return f"Bucket {config.s3_bucket} answered and holds {len(objects)} Scribe backup(s)." + + +def _count_backups(folder: Path) -> int: + return sum( + 1 for name in os.listdir(folder) + if name.startswith(backup_file.NAME_PREFIX) and name.endswith(backup_file.NAME_SUFFIX) + ) diff --git a/src/scribe/services/backup_s3.py b/src/scribe/services/backup_s3.py new file mode 100644 index 00000000..2531aea7 --- /dev/null +++ b/src/scribe/services/backup_s3.py @@ -0,0 +1,199 @@ +"""A small S3 client for sending backups: AWS Signature Version 4 over httpx, +path-style addressing (endpoint/bucket/key), and the two calls a backup needs. + +Ported from FabledSteward's `steward/backup/s3.py`, which is tested against +the same AWS examples as tests/test_services_backup_s3.py. + +Scribe puts and lists; it never deletes. The bucket keeps the history and +thins it on its own policy, so a key that may only add objects is enough — +and is the safer key to hand a service, because a stolen one cannot wipe the +history it writes into. + +Written by hand rather than with boto3: two calls don't justify a large +dependency, and signing is a pure function tests can hold to AWS's own +published examples. +""" +from __future__ import annotations + +import asyncio +import hashlib +import hmac +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import quote + +import httpx + +EMPTY_SHA256 = hashlib.sha256(b"").hexdigest() +TIMEOUT_SECONDS = 30.0 +# A backup is tens of megabytes; a slow link still gets this long to send one. +UPLOAD_TIMEOUT_SECONDS = 15 * 60 +CHUNK = 1 << 20 + + +class S3Error(Exception): + """Why a call to the bucket failed, said to the person.""" + + +@dataclass(frozen=True) +class S3Target: + endpoint: str # http(s)://host[:port][/prefix], no bucket + bucket: str + access_key: str + secret_key: str + region: str = "us-east-1" + verify_tls: bool = True + + def url(self, key: str = "") -> str: + base = f"{self.endpoint.rstrip('/')}/{self.bucket}" + return f"{base}/{key}" if key else f"{base}/" + + +@dataclass(frozen=True) +class StoredObject: + key: str + size: int + last_modified: str + + +# ── Signing ────────────────────────────────────────────────────────────────── + +def _encode(text: str, keep_slash: bool = False) -> str: + return quote(text, safe="-_.~" + ("/" if keep_slash else "")) + + +def _hmac(key: bytes, msg: str) -> bytes: + return hmac.new(key, msg.encode(), hashlib.sha256).digest() + + +def sign(method: str, url: str, headers: dict[str, str], payload_sha256: str, + access_key: str, secret_key: str, region: str, at: datetime, + service: str = "s3") -> dict[str, str]: + """The request's headers with x-amz-date, x-amz-content-sha256 and + Authorization added. Every header given is signed, Host always.""" + u = httpx.URL(url) + stamp = at.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + day = stamp[:8] + out = dict(headers) + out["x-amz-date"] = stamp + out["x-amz-content-sha256"] = payload_sha256 + signed = {k.lower(): " ".join(str(v).split()) for k, v in out.items()} + signed["host"] = u.netloc.decode() + names = sorted(signed) + + query = sorted((_encode(k), _encode(v)) for k, v in u.params.multi_items()) + canonical = "\n".join([ + method.upper(), + _encode(u.path or "/", keep_slash=True), + "&".join(f"{k}={v}" for k, v in query), + "".join(f"{n}:{signed[n]}\n" for n in names), + ";".join(names), + payload_sha256, + ]) + scope = f"{day}/{region}/{service}/aws4_request" + to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope, + hashlib.sha256(canonical.encode()).hexdigest()]) + key = _hmac(_hmac(_hmac(_hmac(f"AWS4{secret_key}".encode(), day), region), + service), "aws4_request") + signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest() + out["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, " + f"SignedHeaders={';'.join(names)}, Signature={signature}") + return out + + +# ── Calls ──────────────────────────────────────────────────────────────────── + +def _says(response: httpx.Response) -> str: + """An S3 error response in words: its Code and Message when it has them.""" + try: + root = ET.fromstring(response.content) + code = root.findtext("Code") or "" + message = root.findtext("Message") or "" + except ET.ParseError: + code, message = "", "" + hints = { + "SignatureDoesNotMatch": "the secret key is wrong", + "InvalidAccessKeyId": "the access key isn't known there", + "NoSuchBucket": "there is no such bucket", + "AccessDenied": "the key isn't allowed to do that", + } + said = hints.get(code) or message or response.reason_phrase + return f"HTTP {response.status_code}{f' {code}' if code else ''}: {said}." + + +def _reach_error(target: S3Target, exc: httpx.HTTPError) -> S3Error: + if isinstance(exc, httpx.TimeoutException): + return S3Error(f"{target.endpoint} didn't answer in time.") + return S3Error(f"Couldn't reach {target.endpoint}: {exc}.") + + +def file_sha256(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + while chunk := f.read(CHUNK): + h.update(chunk) + return h.hexdigest() + + +async def put_file(target: S3Target, key: str, path: Path, *, + transport: httpx.AsyncBaseTransport | None = None, + now: datetime | None = None) -> None: + """Upload one file as one object, streamed from disk.""" + digest = await asyncio.to_thread(file_sha256, path) + size = path.stat().st_size + url = target.url(key) + headers = sign("PUT", url, {"Content-Length": str(size), + "Content-Type": "application/x-tar"}, + digest, target.access_key, target.secret_key, target.region, + now or datetime.now(timezone.utc)) + + async def body(): + with open(path, "rb") as f: + while chunk := f.read(CHUNK): + yield chunk + + try: + async with httpx.AsyncClient(verify=target.verify_tls, + timeout=UPLOAD_TIMEOUT_SECONDS, + transport=transport) as client: + response = await client.put(url, headers=headers, content=body()) + except httpx.HTTPError as exc: + raise _reach_error(target, exc) from None + if response.status_code // 100 != 2: + raise S3Error(f"Upload of {key} refused: {_says(response)}") + + +async def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *, + transport: httpx.AsyncBaseTransport | None = None, + now: datetime | None = None) -> list[StoredObject]: + """Up to `limit` objects under `prefix` (ListObjectsV2, one page).""" + params = {"list-type": "2", "max-keys": str(limit)} + if prefix: + params["prefix"] = prefix + url = str(httpx.URL(target.url(), params=params)) + headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key, + target.region, now or datetime.now(timezone.utc)) + try: + async with httpx.AsyncClient(verify=target.verify_tls, timeout=TIMEOUT_SECONDS, + transport=transport) as client: + response = await client.get(url, headers=headers) + except httpx.HTTPError as exc: + raise _reach_error(target, exc) from None + if response.status_code != 200: + raise S3Error(f"Listing {target.bucket} refused: {_says(response)}") + try: + root = ET.fromstring(response.content) + except ET.ParseError: + raise S3Error( + f"{target.endpoint} answered a listing with something that isn't S3's XML." + ) from None + out = [] + for item in root.iter(): + if item.tag.rsplit("}", 1)[-1] != "Contents": + continue + fields = {child.tag.rsplit("}", 1)[-1]: (child.text or "") for child in item} + out.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0), + fields.get("LastModified", ""))) + return out diff --git a/src/scribe/services/secrets.py b/src/scribe/services/secrets.py new file mode 100644 index 00000000..91ced9ee --- /dev/null +++ b/src/scribe/services/secrets.py @@ -0,0 +1,44 @@ +"""Sealing a credential Scribe stores so the database alone cannot read it. + +Fernet (AES-128-CBC with an HMAC) under a key derived from the instance's +SECRET_KEY. A copy of the database, or of a backup made from it, then holds +only ciphertext: reading the credential also needs the SECRET_KEY, which lives +in the deployment's environment rather than in the data. + +The derivation is labelled and versioned, so this key is never the same bytes +as anything else made from SECRET_KEY (the session-cookie signing uses it +raw). An instance that changes its SECRET_KEY can no longer open what it +sealed before: `unseal` answers None, and the caller asks for the credential +again rather than failing on it. +""" +from __future__ import annotations + +import base64 +import hashlib + +from cryptography.fernet import Fernet, InvalidToken + +from scribe.config import Config + +_LABEL = b"scribe.secrets.v1\x00" + + +def _fernet() -> Fernet: + digest = hashlib.sha256(_LABEL + Config.SECRET_KEY.encode()).digest() + return Fernet(base64.urlsafe_b64encode(digest)) + + +def seal(plain: str) -> str: + """Encrypt `plain` for storage; the result is ASCII.""" + return _fernet().encrypt(plain.encode()).decode() + + +def unseal(token: str) -> str | None: + """Decrypt a value `seal` produced, or None when it can't be opened — + sealed under a different SECRET_KEY, or not a sealed value at all.""" + if not token: + return None + try: + return _fernet().decrypt(token.encode()).decode() + except (InvalidToken, ValueError): + return None diff --git a/tests/test_services_backup_destination.py b/tests/test_services_backup_destination.py new file mode 100644 index 00000000..9bc4da5f --- /dev/null +++ b/tests/test_services_backup_destination.py @@ -0,0 +1,179 @@ +"""The backup destination (services/backup_destination.py) and the sealing it +keeps the S3 secret under (services/secrets.py). + +No database: the admin settings are patched, the folder is a tmp_path, and the +bucket is an httpx.MockTransport. What these pin is the contract a person +meets in Settings — a config that is incomplete says what is missing, a place +that can't take a backup says why, and the secret never leaves sealed. +""" +import asyncio +import io +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import patch + +import httpx +import pytest + +from scribe.services import backup, backup_destination, secrets +from scribe.services.backup_destination import ( + BackupConfig, DestinationError, check_destination, deliver, +) +from scribe.services.backup_file import read_archive + +AT = datetime(2026, 10, 15, 2, 0, 0, tzinfo=timezone.utc) +PAYLOAD = { + "version": 25, "scope": "full", "exported_at": AT.isoformat(), + "users": [{"id": 1, "username": "ada"}], "notes": [], +} +S3 = BackupConfig( + destination="s3", s3_endpoint="http://archives:8081", s3_bucket="scribe", + s3_access_key="AKIA", s3_region="us-east-1", +) + + +# ── Sealing ────────────────────────────────────────────────────────────────── + +def test_a_sealed_secret_opens_and_is_not_the_plain_text(): + token = secrets.seal("wJalrXUtnFEMI/K7MDENG") + assert "wJalrXUtnFEMI" not in token + assert secrets.unseal(token) == "wJalrXUtnFEMI/K7MDENG" + + +def test_another_secret_key_cannot_open_it(): + token = secrets.seal("s3cret") + with patch.object(secrets.Config, "SECRET_KEY", "a-different-instance"): + assert secrets.unseal(token) is None + + +def test_unseal_of_nothing_or_garbage_is_none(): + assert secrets.unseal("") is None + assert secrets.unseal("not a fernet token") is None + + +def test_the_secret_setting_never_reaches_a_backup(): + rows = [SimpleNamespace(user_id=1, key=k, value="v") for k in ( + "backup_config", backup_destination.SECRET_SETTING, "theme", + )] + keys = [r["key"] for r in backup._setting_rows(rows)] + assert keys == ["backup_config", "theme"] + + +def test_load_secret_tells_a_missing_secret_from_one_it_cannot_open(): + async def stored(value): + async def get(key, default=""): + return value + with patch.object(backup_destination, "get_admin_setting", get): + return await backup_destination.load_secret() + + assert asyncio.run(stored("")) == (None, False) + assert asyncio.run(stored(secrets.seal("k"))) == ("k", True) + assert asyncio.run(stored("sealed-elsewhere")) == (None, True) + + +# ── Config ─────────────────────────────────────────────────────────────────── + +def test_config_round_trips_and_defaults_to_off(): + assert BackupConfig().enabled is False + assert BackupConfig.from_dict(S3.to_dict()) == S3 + + +def test_a_damaged_config_still_loads(): + config = BackupConfig.from_dict({ + "hour": "25", "destination": "ftp", "enabled": "yes", "unknown": 1, + "s3_bucket": " scribe ", + }) + assert config.hour == backup_destination.DEFAULT_HOUR + assert config.destination == "folder" + assert config.enabled is False + assert config.s3_bucket == "scribe" + + +def test_an_incomplete_s3_config_names_what_is_missing(): + said = BackupConfig(destination="s3").problems(has_secret=False) + assert said == [ + "The S3 address must start with http:// or https://.", + "The bucket name is missing.", + "The access key is missing.", + "The secret key is missing.", + ] + assert S3.problems(has_secret=True) == [] + + +def test_a_relative_folder_is_refused(): + assert BackupConfig(folder_path="backups").problems(has_secret=False) + + +# ── Folder ─────────────────────────────────────────────────────────────────── + +def test_deliver_to_a_folder_adds_one_dated_file(tmp_path): + config = BackupConfig(folder_path=str(tmp_path)) + got = asyncio.run(deliver(PAYLOAD, config, None, AT, app_version="v")) + assert got.name == "scribe-20261015T020000Z.tar" + assert [p.name for p in tmp_path.iterdir()] == [got.name] + assert got.size_bytes == (tmp_path / got.name).stat().st_size + with open(tmp_path / got.name, "rb") as f: + assert read_archive(f) == PAYLOAD + + +def test_a_folder_that_isnt_mounted_says_so(tmp_path): + config = BackupConfig(folder_path=str(tmp_path / "not-mounted")) + with pytest.raises(DestinationError, match="isn't there"): + asyncio.run(deliver(PAYLOAD, config, None, AT, app_version="v")) + + +def test_checking_a_folder_leaves_nothing_behind(tmp_path): + (tmp_path / "scribe-20261014T020000Z.tar").write_bytes(b"x") + config = BackupConfig(folder_path=str(tmp_path)) + said = asyncio.run(check_destination(config, None)) + assert "holds 1 Scribe backup" in said + assert [p.name for p in tmp_path.iterdir()] == ["scribe-20261014T020000Z.tar"] + + +# ── S3 ─────────────────────────────────────────────────────────────────────── + +def test_deliver_to_s3_puts_the_dated_file_and_keeps_no_copy(): + seen = {} + + def handler(request: httpx.Request): + seen["method"], seen["url"] = request.method, str(request.url) + seen["body"] = request.read() + return httpx.Response(200) + + got = asyncio.run(deliver(PAYLOAD, S3, "secret", AT, app_version="v", + transport=httpx.MockTransport(handler))) + assert seen["method"] == "PUT" + assert seen["url"] == "http://archives:8081/scribe/scribe-20261015T020000Z.tar" + assert read_archive(io.BytesIO(seen["body"])) == PAYLOAD + assert got.size_bytes == len(seen["body"]) + + +def test_a_refused_upload_is_a_destination_error_in_words(): + body = b"AccessDeniedno" + transport = httpx.MockTransport(lambda r: httpx.Response(403, content=body)) + with pytest.raises(DestinationError, match="the key isn't allowed to do that"): + asyncio.run(deliver(PAYLOAD, S3, "secret", AT, app_version="v", transport=transport)) + + +def test_s3_without_its_secret_sends_nothing(): + def handler(request): + raise AssertionError("nothing should be sent") + + with pytest.raises(DestinationError, match="secret key is missing"): + asyncio.run(deliver(PAYLOAD, S3, None, AT, app_version="v", + transport=httpx.MockTransport(handler))) + + +def test_checking_s3_only_lists(): + xml = (b'' + b"scribe-20261014T020000Z.tar9" + b"x") + methods = [] + + def handler(request): + methods.append(request.method) + return httpx.Response(200, content=xml) + + said = asyncio.run(check_destination(S3, "secret", transport=httpx.MockTransport(handler))) + assert methods == ["GET"] + assert "holds 1 Scribe backup" in said diff --git a/tests/test_services_backup_s3.py b/tests/test_services_backup_s3.py new file mode 100644 index 00000000..c9ab0855 --- /dev/null +++ b/tests/test_services_backup_s3.py @@ -0,0 +1,112 @@ +"""The backup destination's S3 client (services/backup_s3.py): signatures match AWS's +published Signature Version 4 examples, and the calls read S3's answers.""" +import asyncio +from datetime import datetime, timezone + +import httpx +import pytest + +from scribe.services import backup_s3 as s3 + +# AWS's example credentials and moment ("Signature Calculations for the +# Authorization Header", Amazon S3 API reference). +KEY_ID = "AKIAIOSFODNN7EXAMPLE" +SECRET = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +AT = datetime(2013, 5, 24, tzinfo=timezone.utc) + + +def _signature(headers): + return headers["Authorization"].rsplit("Signature=", 1)[1] + + +def test_aws_example_get_object(): + h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {"Range": "bytes=0-9"}, + s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT) + assert "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date," in h["Authorization"] + assert "Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request" in h["Authorization"] + assert _signature(h) == "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41" + + +def test_aws_example_list_objects(): + h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", {}, + s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT) + assert _signature(h) == "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7" + + +def test_a_changed_secret_changes_the_signature(): + a = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {}, + s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT) + b = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {}, + s3.EMPTY_SHA256, KEY_ID, SECRET + "x", "us-east-1", AT) + assert _signature(a) != _signature(b) + + +TARGET = s3.S3Target("http://archives:8081/s3", "scribe", KEY_ID, SECRET) + + +def test_paths_are_path_style(): + assert TARGET.url("scribe-20261012T000000Z.tar") == "http://archives:8081/s3/scribe/scribe-20261012T000000Z.tar" + assert TARGET.url() == "http://archives:8081/s3/scribe/" + + +def test_put_sends_the_file_signed(tmp_path): + f = tmp_path / "b.tar" + f.write_bytes(b"x" * 5000) + seen = {} + + def handler(request: httpx.Request): + seen["method"], seen["url"] = request.method, str(request.url) + seen["body"] = request.read() + seen["auth"] = request.headers["authorization"] + seen["sha"] = request.headers["x-amz-content-sha256"] + return httpx.Response(200) + + asyncio.run(s3.put_file(TARGET, "b.tar", f, transport=httpx.MockTransport(handler))) + assert seen["method"] == "PUT" + assert seen["url"] == "http://archives:8081/s3/scribe/b.tar" + assert seen["body"] == b"x" * 5000 + assert seen["sha"] == s3.file_sha256(f) + assert seen["auth"].startswith("AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/") + + +def test_a_refusal_says_why(tmp_path): + f = tmp_path / "b.tar" + f.write_bytes(b"x") + body = b"SignatureDoesNotMatchno" + transport = httpx.MockTransport(lambda r: httpx.Response(403, content=body)) + with pytest.raises(s3.S3Error, match="HTTP 403 SignatureDoesNotMatch: the secret key is wrong"): + asyncio.run(s3.put_file(TARGET, "b.tar", f, transport=transport)) + + +def test_unreachable_says_so(tmp_path): + f = tmp_path / "b.tar" + f.write_bytes(b"x") + + def handler(request): + raise httpx.ConnectError("refused", request=request) + + with pytest.raises(s3.S3Error, match="Couldn't reach http://archives:8081/s3"): + asyncio.run(s3.put_file(TARGET, "b.tar", f, transport=httpx.MockTransport(handler))) + + +def test_list_reads_contents(): + xml = b""" + + scribe2 + scribe-20261011T000000Z.tar1200 + 2026-10-11T00:00:01.000Z + scribe-20261012T000000Z.tar1300 + 2026-10-12T00:00:01.000Z +""" + seen = {} + + def handler(request): + seen["params"] = dict(request.url.params) + return httpx.Response(200, content=xml) + + got = asyncio.run(s3.list_objects(TARGET, "scribe-", transport=httpx.MockTransport(handler))) + assert seen["params"] == {"list-type": "2", "max-keys": "1000", "prefix": "scribe-"} + assert got == [ + s3.StoredObject("scribe-20261011T000000Z.tar", 1200, "2026-10-11T00:00:01.000Z"), + s3.StoredObject("scribe-20261012T000000Z.tar", 1300, "2026-10-12T00:00:01.000Z"), + ] diff --git a/uv.lock b/uv.lock index a28edb65..d9a168b4 100644 --- a/uv.lock +++ b/uv.lock @@ -1508,6 +1508,7 @@ dependencies = [ { name = "asyncpg" }, { name = "bcrypt" }, { name = "caldav" }, + { name = "cryptography" }, { name = "fastembed" }, { name = "httpx" }, { name = "hypercorn" }, @@ -1533,6 +1534,7 @@ requires-dist = [ { name = "asyncpg", specifier = ">=0.29" }, { name = "bcrypt", specifier = ">=4.0" }, { name = "caldav", specifier = ">=1.3" }, + { name = "cryptography" }, { name = "fastembed", specifier = ">=0.4" }, { name = "httpx", specifier = ">=0.27" }, { name = "hypercorn", specifier = ">=0.17" },