feat(523): backup destinations - a folder or an S3 bucket, with the S3 secret sealed and kept out of backups

services/backup_s3.py is FabledSteward's SigV4 client ported (put + list,
never delete, so an append-only key is enough), with its tests against
AWS's published signatures. services/backup_destination.py holds the
admin config (backup_config JSON) and delivers a dated tar to a mounted
folder or a bucket, and checks either without sending a backup.
services/secrets.py seals the S3 secret with Fernet under a key derived
from SECRET_KEY; backup._SETTINGS_NOT_EXPORTED keeps it out of every
backup. cryptography becomes a direct dependency (already in the image
through pyjwt).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/scribe/services/backup.py

@@ -772,8 +772,18 @@ def _rule_version_rows(rows) -> list[dict]:
     ]
 
 
+# Setting KEYS no backup carries. The backup destination's S3 secret is sealed
+# under SECRET_KEY, but a backup is exactly the thing copied off-site — it must
+# not carry the key that writes to the place it is copied to. A restore leaves
+# the destination configured and asks for the secret again.
+_SETTINGS_NOT_EXPORTED = frozenset({"backup_s3_secret"})
+
+
 def _setting_rows(rows) -> list[dict]:
-    return [{"user_id": s.user_id, "key": s.key, "value": s.value} for s in rows]
+    return [
+        {"user_id": s.user_id, "key": s.key, "value": s.value}
+        for s in rows if s.key not in _SETTINGS_NOT_EXPORTED
+    ]
 
 
 def _rulebook_rows(rows) -> list[dict]:

src/scribe/services/backup_destination.py

@@ -0,0 +1,235 @@
+"""Where the scheduled backup goes, and getting a backup file there.
+
+Two kinds of destination, both instance-wide and set by an admin:
+
+- **folder**: a directory inside the container, usually a mounted share. A
+  backup service that watches the share (or a person) takes it from there.
+- **s3**: any S3-compatible bucket, addressed by path. Scribe only puts and
+  lists, so an append-only key is enough.
+
+Either way Scribe ADDS one dated file per run and never removes one: keeping
+and thinning the history is the destination's job (services/backup_file.py
+says why the file is shaped the way it is for that).
+
+The settings live as admin settings: `backup_config` holds everything but the
+S3 secret as JSON, and `backup_s3_secret` holds the secret sealed under the
+instance's SECRET_KEY (services/secrets.py). The secret is also kept out of
+every backup (backup._SETTINGS_NOT_EXPORTED), so a backup copied off-site
+does not carry the key that writes to that very place.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import tempfile
+from dataclasses import asdict, dataclass, fields, replace
+from datetime import datetime
+from pathlib import Path
+
+from scribe.services import backup_file
+from scribe.services.backup_s3 import S3Error, S3Target, list_objects, put_file
+from scribe.services.secrets import seal, unseal
+from scribe.services.settings import get_admin_setting, set_admin_setting
+
+CONFIG_SETTING = "backup_config"
+SECRET_SETTING = "backup_s3_secret"
+DESTINATION_KINDS = ("folder", "s3")
+DEFAULT_FOLDER = "/backups"
+# 02:00 UTC: before the 03:30 trash purge and the 04:00 DB maintenance, so the
+# backup never contends with a VACUUM, and the night's deletions are still
+# recoverable from the backup taken just before them.
+DEFAULT_HOUR = 2
+
+
+class DestinationError(Exception):
+    """Why the destination can't take a backup, said to the person."""
+
+
+@dataclass(frozen=True)
+class BackupConfig:
+    enabled: bool = False
+    hour: int = DEFAULT_HOUR
+    destination: str = "folder"
+    folder_path: str = DEFAULT_FOLDER
+    s3_endpoint: str = ""
+    s3_bucket: str = ""
+    s3_access_key: str = ""
+    s3_region: str = "us-east-1"
+    s3_verify_tls: bool = True
+
+    @classmethod
+    def from_dict(cls, raw: dict) -> "BackupConfig":
+        """Lenient: unknown keys are ignored and bad values fall back to the
+        default, so an edited or older stored config still loads."""
+        known = {f.name: f for f in fields(cls)}
+        values = {}
+        for name, value in (raw or {}).items():
+            f = known.get(name)
+            if f is None:
+                continue
+            default = f.default
+            if isinstance(default, bool):
+                values[name] = value if isinstance(value, bool) else default
+            elif isinstance(default, int):
+                try:
+                    values[name] = int(value)
+                except (TypeError, ValueError):
+                    values[name] = default
+            else:
+                values[name] = str(value).strip() if value is not None else default
+        config = cls(**values)
+        if not 0 <= config.hour <= 23:
+            config = replace(config, hour=DEFAULT_HOUR)
+        if config.destination not in DESTINATION_KINDS:
+            config = replace(config, destination="folder")
+        return config
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+    def problems(self, *, has_secret: bool) -> list[str]:
+        """What stops this config from sending a backup, in words. Empty when
+        it is complete (which says nothing about whether the place answers —
+        check_destination does that)."""
+        out: list[str] = []
+        if self.destination == "folder":
+            if not self.folder_path.startswith("/"):
+                out.append("The folder must be an absolute path inside the container.")
+        else:
+            if not self.s3_endpoint.startswith(("http://", "https://")):
+                out.append("The S3 address must start with http:// or https://.")
+            if not self.s3_bucket:
+                out.append("The bucket name is missing.")
+            if not self.s3_access_key:
+                out.append("The access key is missing.")
+            if not has_secret:
+                out.append("The secret key is missing.")
+        return out
+
+
+async def load_config() -> BackupConfig:
+    raw = await get_admin_setting(CONFIG_SETTING, "")
+    try:
+        data = json.loads(raw) if raw else {}
+    except ValueError:
+        data = {}
+    return BackupConfig.from_dict(data if isinstance(data, dict) else {})
+
+
+async def save_config(config: BackupConfig) -> None:
+    await set_admin_setting(CONFIG_SETTING, json.dumps(config.to_dict(), sort_keys=True))
+
+
+async def load_secret() -> tuple[str | None, bool]:
+    """(the S3 secret or None, whether one is stored).
+
+    Stored but None means it was sealed under another SECRET_KEY — after a
+    restore onto a different instance, say — and has to be entered again.
+    """
+    stored = await get_admin_setting(SECRET_SETTING, "")
+    return (unseal(stored), bool(stored))
+
+
+async def save_secret(secret: str) -> None:
+    await set_admin_setting(SECRET_SETTING, seal(secret) if secret else "")
+
+
+def s3_target(config: BackupConfig, secret: str) -> S3Target:
+    return S3Target(
+        endpoint=config.s3_endpoint, bucket=config.s3_bucket,
+        access_key=config.s3_access_key, secret_key=secret,
+        region=config.s3_region or "us-east-1", verify_tls=config.s3_verify_tls,
+    )
+
+
+@dataclass(frozen=True)
+class Delivered:
+    name: str
+    size_bytes: int
+    where: str
+
+
+async def deliver(
+    payload: dict, config: BackupConfig, secret: str | None, at: datetime,
+    *, app_version: str, transport=None,
+) -> Delivered:
+    """Write `payload` as the dated backup file and put it at the destination.
+
+    The file is written off the event loop: serialising a full export is
+    seconds of CPU on a large corpus, and the app keeps serving meanwhile.
+    """
+    problems = config.problems(has_secret=bool(secret))
+    if problems:
+        raise DestinationError(" ".join(problems))
+
+    if config.destination == "folder":
+        folder = Path(config.folder_path)
+        if not folder.is_dir():
+            raise DestinationError(
+                f"{folder} isn't there. Mount the backup share at that path, "
+                "or choose the folder it is mounted at."
+            )
+        try:
+            path = await asyncio.to_thread(
+                backup_file.write_archive, payload, folder, at, app_version=app_version,
+            )
+        except PermissionError:
+            raise DestinationError(f"Scribe isn't allowed to write in {folder}.") from None
+        return Delivered(path.name, path.stat().st_size, str(folder))
+
+    target = s3_target(config, secret or "")
+    with tempfile.TemporaryDirectory(prefix="scribe-backup-") as staging:
+        path = await asyncio.to_thread(
+            backup_file.write_archive, payload, Path(staging), at, app_version=app_version,
+        )
+        size = path.stat().st_size
+        try:
+            await put_file(target, path.name, path, transport=transport)
+        except S3Error as exc:
+            raise DestinationError(str(exc)) from None
+    return Delivered(path.name, size, target.url())
+
+
+async def check_destination(
+    config: BackupConfig, secret: str | None, *, transport=None,
+) -> str:
+    """Ask the destination whether it would take a backup, without sending
+    one. Returns what was found, in words; raises DestinationError otherwise.
+
+    The folder check writes and removes a dot-file, which a folder watcher
+    ignores. The S3 check only lists: an append-only key could add a test
+    object and then never remove it.
+    """
+    problems = config.problems(has_secret=bool(secret))
+    if problems:
+        raise DestinationError(" ".join(problems))
+
+    if config.destination == "folder":
+        folder = Path(config.folder_path)
+        if not folder.is_dir():
+            raise DestinationError(f"{folder} isn't there, or isn't a folder.")
+        probe = folder / ".scribe-write-check"
+        try:
+            await asyncio.to_thread(probe.write_bytes, b"")
+            await asyncio.to_thread(probe.unlink)
+        except PermissionError:
+            raise DestinationError(f"Scribe isn't allowed to write in {folder}.") from None
+        except OSError as exc:
+            raise DestinationError(f"Writing in {folder} failed: {exc.strerror or exc}.") from None
+        count = await asyncio.to_thread(_count_backups, folder)
+        return f"{folder} is writable and holds {count} Scribe backup(s)."
+
+    target = s3_target(config, secret or "")
+    try:
+        objects = await list_objects(target, backup_file.NAME_PREFIX, transport=transport)
+    except S3Error as exc:
+        raise DestinationError(str(exc)) from None
+    return f"Bucket {config.s3_bucket} answered and holds {len(objects)} Scribe backup(s)."
+
+
+def _count_backups(folder: Path) -> int:
+    return sum(
+        1 for name in os.listdir(folder)
+        if name.startswith(backup_file.NAME_PREFIX) and name.endswith(backup_file.NAME_SUFFIX)
+    )

src/scribe/services/backup_s3.py

@@ -0,0 +1,199 @@
+"""A small S3 client for sending backups: AWS Signature Version 4 over httpx,
+path-style addressing (endpoint/bucket/key), and the two calls a backup needs.
+
+Ported from FabledSteward's `steward/backup/s3.py`, which is tested against
+the same AWS examples as tests/test_services_backup_s3.py.
+
+Scribe puts and lists; it never deletes. The bucket keeps the history and
+thins it on its own policy, so a key that may only add objects is enough —
+and is the safer key to hand a service, because a stolen one cannot wipe the
+history it writes into.
+
+Written by hand rather than with boto3: two calls don't justify a large
+dependency, and signing is a pure function tests can hold to AWS's own
+published examples.
+"""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import hmac
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import quote
+
+import httpx
+
+EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
+TIMEOUT_SECONDS = 30.0
+# A backup is tens of megabytes; a slow link still gets this long to send one.
+UPLOAD_TIMEOUT_SECONDS = 15 * 60
+CHUNK = 1 << 20
+
+
+class S3Error(Exception):
+    """Why a call to the bucket failed, said to the person."""
+
+
+@dataclass(frozen=True)
+class S3Target:
+    endpoint: str          # http(s)://host[:port][/prefix], no bucket
+    bucket: str
+    access_key: str
+    secret_key: str
+    region: str = "us-east-1"
+    verify_tls: bool = True
+
+    def url(self, key: str = "") -> str:
+        base = f"{self.endpoint.rstrip('/')}/{self.bucket}"
+        return f"{base}/{key}" if key else f"{base}/"
+
+
+@dataclass(frozen=True)
+class StoredObject:
+    key: str
+    size: int
+    last_modified: str
+
+
+# ── Signing ──────────────────────────────────────────────────────────────────
+
+def _encode(text: str, keep_slash: bool = False) -> str:
+    return quote(text, safe="-_.~" + ("/" if keep_slash else ""))
+
+
+def _hmac(key: bytes, msg: str) -> bytes:
+    return hmac.new(key, msg.encode(), hashlib.sha256).digest()
+
+
+def sign(method: str, url: str, headers: dict[str, str], payload_sha256: str,
+         access_key: str, secret_key: str, region: str, at: datetime,
+         service: str = "s3") -> dict[str, str]:
+    """The request's headers with x-amz-date, x-amz-content-sha256 and
+    Authorization added. Every header given is signed, Host always."""
+    u = httpx.URL(url)
+    stamp = at.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    day = stamp[:8]
+    out = dict(headers)
+    out["x-amz-date"] = stamp
+    out["x-amz-content-sha256"] = payload_sha256
+    signed = {k.lower(): " ".join(str(v).split()) for k, v in out.items()}
+    signed["host"] = u.netloc.decode()
+    names = sorted(signed)
+
+    query = sorted((_encode(k), _encode(v)) for k, v in u.params.multi_items())
+    canonical = "\n".join([
+        method.upper(),
+        _encode(u.path or "/", keep_slash=True),
+        "&".join(f"{k}={v}" for k, v in query),
+        "".join(f"{n}:{signed[n]}\n" for n in names),
+        ";".join(names),
+        payload_sha256,
+    ])
+    scope = f"{day}/{region}/{service}/aws4_request"
+    to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope,
+                         hashlib.sha256(canonical.encode()).hexdigest()])
+    key = _hmac(_hmac(_hmac(_hmac(f"AWS4{secret_key}".encode(), day), region),
+                      service), "aws4_request")
+    signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
+    out["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, "
+                            f"SignedHeaders={';'.join(names)}, Signature={signature}")
+    return out
+
+
+# ── Calls ────────────────────────────────────────────────────────────────────
+
+def _says(response: httpx.Response) -> str:
+    """An S3 error response in words: its Code and Message when it has them."""
+    try:
+        root = ET.fromstring(response.content)
+        code = root.findtext("Code") or ""
+        message = root.findtext("Message") or ""
+    except ET.ParseError:
+        code, message = "", ""
+    hints = {
+        "SignatureDoesNotMatch": "the secret key is wrong",
+        "InvalidAccessKeyId": "the access key isn't known there",
+        "NoSuchBucket": "there is no such bucket",
+        "AccessDenied": "the key isn't allowed to do that",
+    }
+    said = hints.get(code) or message or response.reason_phrase
+    return f"HTTP {response.status_code}{f' {code}' if code else ''}: {said}."
+
+
+def _reach_error(target: S3Target, exc: httpx.HTTPError) -> S3Error:
+    if isinstance(exc, httpx.TimeoutException):
+        return S3Error(f"{target.endpoint} didn't answer in time.")
+    return S3Error(f"Couldn't reach {target.endpoint}: {exc}.")
+
+
+def file_sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        while chunk := f.read(CHUNK):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+async def put_file(target: S3Target, key: str, path: Path, *,
+                   transport: httpx.AsyncBaseTransport | None = None,
+                   now: datetime | None = None) -> None:
+    """Upload one file as one object, streamed from disk."""
+    digest = await asyncio.to_thread(file_sha256, path)
+    size = path.stat().st_size
+    url = target.url(key)
+    headers = sign("PUT", url, {"Content-Length": str(size),
+                                "Content-Type": "application/x-tar"},
+                   digest, target.access_key, target.secret_key, target.region,
+                   now or datetime.now(timezone.utc))
+
+    async def body():
+        with open(path, "rb") as f:
+            while chunk := f.read(CHUNK):
+                yield chunk
+
+    try:
+        async with httpx.AsyncClient(verify=target.verify_tls,
+                                     timeout=UPLOAD_TIMEOUT_SECONDS,
+                                     transport=transport) as client:
+            response = await client.put(url, headers=headers, content=body())
+    except httpx.HTTPError as exc:
+        raise _reach_error(target, exc) from None
+    if response.status_code // 100 != 2:
+        raise S3Error(f"Upload of {key} refused: {_says(response)}")
+
+
+async def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *,
+                       transport: httpx.AsyncBaseTransport | None = None,
+                       now: datetime | None = None) -> list[StoredObject]:
+    """Up to `limit` objects under `prefix` (ListObjectsV2, one page)."""
+    params = {"list-type": "2", "max-keys": str(limit)}
+    if prefix:
+        params["prefix"] = prefix
+    url = str(httpx.URL(target.url(), params=params))
+    headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
+                   target.region, now or datetime.now(timezone.utc))
+    try:
+        async with httpx.AsyncClient(verify=target.verify_tls, timeout=TIMEOUT_SECONDS,
+                                     transport=transport) as client:
+            response = await client.get(url, headers=headers)
+    except httpx.HTTPError as exc:
+        raise _reach_error(target, exc) from None
+    if response.status_code != 200:
+        raise S3Error(f"Listing {target.bucket} refused: {_says(response)}")
+    try:
+        root = ET.fromstring(response.content)
+    except ET.ParseError:
+        raise S3Error(
+            f"{target.endpoint} answered a listing with something that isn't S3's XML."
+        ) from None
+    out = []
+    for item in root.iter():
+        if item.tag.rsplit("}", 1)[-1] != "Contents":
+            continue
+        fields = {child.tag.rsplit("}", 1)[-1]: (child.text or "") for child in item}
+        out.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0),
+                                fields.get("LastModified", "")))
+    return out

src/scribe/services/secrets.py

@@ -0,0 +1,44 @@
+"""Sealing a credential Scribe stores so the database alone cannot read it.
+
+Fernet (AES-128-CBC with an HMAC) under a key derived from the instance's
+SECRET_KEY. A copy of the database, or of a backup made from it, then holds
+only ciphertext: reading the credential also needs the SECRET_KEY, which lives
+in the deployment's environment rather than in the data.
+
+The derivation is labelled and versioned, so this key is never the same bytes
+as anything else made from SECRET_KEY (the session-cookie signing uses it
+raw). An instance that changes its SECRET_KEY can no longer open what it
+sealed before: `unseal` answers None, and the caller asks for the credential
+again rather than failing on it.
+"""
+from __future__ import annotations
+
+import base64
+import hashlib
+
+from cryptography.fernet import Fernet, InvalidToken
+
+from scribe.config import Config
+
+_LABEL = b"scribe.secrets.v1\x00"
+
+
+def _fernet() -> Fernet:
+    digest = hashlib.sha256(_LABEL + Config.SECRET_KEY.encode()).digest()
+    return Fernet(base64.urlsafe_b64encode(digest))
+
+
+def seal(plain: str) -> str:
+    """Encrypt `plain` for storage; the result is ASCII."""
+    return _fernet().encrypt(plain.encode()).decode()
+
+
+def unseal(token: str) -> str | None:
+    """Decrypt a value `seal` produced, or None when it can't be opened —
+    sealed under a different SECRET_KEY, or not a sealed value at all."""
+    if not token:
+        return None
+    try:
+        return _fernet().decrypt(token.encode()).decode()
+    except (InvalidToken, ValueError):
+        return None