feat(523): backup destinations - a folder or an S3 bucket, with the S3 secret sealed and kept out of backups
CI & Build / Python lint (push) Successful in 2s
CI & Build / Plugin hooks (push) Successful in 15s
CI & Build / TypeScript typecheck (push) Successful in 53s
CI & Build / integration (push) Successful in 1m8s
CI & Build / Python tests (push) Successful in 1m53s
CI & Build / Build & push image (push) Successful in 36s
CI & Build / Python lint (push) Successful in 2s
CI & Build / Plugin hooks (push) Successful in 15s
CI & Build / TypeScript typecheck (push) Successful in 53s
CI & Build / integration (push) Successful in 1m8s
CI & Build / Python tests (push) Successful in 1m53s
CI & Build / Build & push image (push) Successful in 36s
services/backup_s3.py is FabledSteward's SigV4 client ported (put + list, never delete, so an append-only key is enough), with its tests against AWS's published signatures. services/backup_destination.py holds the admin config (backup_config JSON) and delivers a dated tar to a mounted folder or a bucket, and checks either without sending a backup. services/secrets.py seals the S3 secret with Fernet under a key derived from SECRET_KEY; backup._SETTINGS_NOT_EXPORTED keeps it out of every backup. cryptography becomes a direct dependency (already in the image through pyjwt). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -13,6 +13,10 @@ dependencies = [
|
|||||||
"asyncpg>=0.29",
|
"asyncpg>=0.29",
|
||||||
"alembic>=1.13",
|
"alembic>=1.13",
|
||||||
"httpx>=0.27",
|
"httpx>=0.27",
|
||||||
|
# Seals the backup destination's S3 secret at rest (services/secrets.py).
|
||||||
|
# Already in the image through pyjwt; named here because Scribe now
|
||||||
|
# imports it directly.
|
||||||
|
"cryptography",
|
||||||
"hypercorn>=0.17",
|
"hypercorn>=0.17",
|
||||||
"bcrypt>=4.0",
|
"bcrypt>=4.0",
|
||||||
"aiosmtplib>=3.0",
|
"aiosmtplib>=3.0",
|
||||||
|
|||||||
@@ -772,8 +772,18 @@ def _rule_version_rows(rows) -> list[dict]:
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# Setting KEYS no backup carries. The backup destination's S3 secret is sealed
|
||||||
|
# under SECRET_KEY, but a backup is exactly the thing copied off-site — it must
|
||||||
|
# not carry the key that writes to the place it is copied to. A restore leaves
|
||||||
|
# the destination configured and asks for the secret again.
|
||||||
|
_SETTINGS_NOT_EXPORTED = frozenset({"backup_s3_secret"})
|
||||||
|
|
||||||
|
|
||||||
def _setting_rows(rows) -> list[dict]:
|
def _setting_rows(rows) -> list[dict]:
|
||||||
return [{"user_id": s.user_id, "key": s.key, "value": s.value} for s in rows]
|
return [
|
||||||
|
{"user_id": s.user_id, "key": s.key, "value": s.value}
|
||||||
|
for s in rows if s.key not in _SETTINGS_NOT_EXPORTED
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def _rulebook_rows(rows) -> list[dict]:
|
def _rulebook_rows(rows) -> list[dict]:
|
||||||
|
|||||||
@@ -0,0 +1,235 @@
|
|||||||
|
"""Where the scheduled backup goes, and getting a backup file there.
|
||||||
|
|
||||||
|
Two kinds of destination, both instance-wide and set by an admin:
|
||||||
|
|
||||||
|
- **folder**: a directory inside the container, usually a mounted share. A
|
||||||
|
backup service that watches the share (or a person) takes it from there.
|
||||||
|
- **s3**: any S3-compatible bucket, addressed by path. Scribe only puts and
|
||||||
|
lists, so an append-only key is enough.
|
||||||
|
|
||||||
|
Either way Scribe ADDS one dated file per run and never removes one: keeping
|
||||||
|
and thinning the history is the destination's job (services/backup_file.py
|
||||||
|
says why the file is shaped the way it is for that).
|
||||||
|
|
||||||
|
The settings live as admin settings: `backup_config` holds everything but the
|
||||||
|
S3 secret as JSON, and `backup_s3_secret` holds the secret sealed under the
|
||||||
|
instance's SECRET_KEY (services/secrets.py). The secret is also kept out of
|
||||||
|
every backup (backup._SETTINGS_NOT_EXPORTED), so a backup copied off-site
|
||||||
|
does not carry the key that writes to that very place.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from dataclasses import asdict, dataclass, fields, replace
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from scribe.services import backup_file
|
||||||
|
from scribe.services.backup_s3 import S3Error, S3Target, list_objects, put_file
|
||||||
|
from scribe.services.secrets import seal, unseal
|
||||||
|
from scribe.services.settings import get_admin_setting, set_admin_setting
|
||||||
|
|
||||||
|
CONFIG_SETTING = "backup_config"
|
||||||
|
SECRET_SETTING = "backup_s3_secret"
|
||||||
|
DESTINATION_KINDS = ("folder", "s3")
|
||||||
|
DEFAULT_FOLDER = "/backups"
|
||||||
|
# 02:00 UTC: before the 03:30 trash purge and the 04:00 DB maintenance, so the
|
||||||
|
# backup never contends with a VACUUM, and the night's deletions are still
|
||||||
|
# recoverable from the backup taken just before them.
|
||||||
|
DEFAULT_HOUR = 2
|
||||||
|
|
||||||
|
|
||||||
|
class DestinationError(Exception):
|
||||||
|
"""Why the destination can't take a backup, said to the person."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class BackupConfig:
|
||||||
|
enabled: bool = False
|
||||||
|
hour: int = DEFAULT_HOUR
|
||||||
|
destination: str = "folder"
|
||||||
|
folder_path: str = DEFAULT_FOLDER
|
||||||
|
s3_endpoint: str = ""
|
||||||
|
s3_bucket: str = ""
|
||||||
|
s3_access_key: str = ""
|
||||||
|
s3_region: str = "us-east-1"
|
||||||
|
s3_verify_tls: bool = True
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, raw: dict) -> "BackupConfig":
|
||||||
|
"""Lenient: unknown keys are ignored and bad values fall back to the
|
||||||
|
default, so an edited or older stored config still loads."""
|
||||||
|
known = {f.name: f for f in fields(cls)}
|
||||||
|
values = {}
|
||||||
|
for name, value in (raw or {}).items():
|
||||||
|
f = known.get(name)
|
||||||
|
if f is None:
|
||||||
|
continue
|
||||||
|
default = f.default
|
||||||
|
if isinstance(default, bool):
|
||||||
|
values[name] = value if isinstance(value, bool) else default
|
||||||
|
elif isinstance(default, int):
|
||||||
|
try:
|
||||||
|
values[name] = int(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
values[name] = default
|
||||||
|
else:
|
||||||
|
values[name] = str(value).strip() if value is not None else default
|
||||||
|
config = cls(**values)
|
||||||
|
if not 0 <= config.hour <= 23:
|
||||||
|
config = replace(config, hour=DEFAULT_HOUR)
|
||||||
|
if config.destination not in DESTINATION_KINDS:
|
||||||
|
config = replace(config, destination="folder")
|
||||||
|
return config
|
||||||
|
|
||||||
|
def to_dict(self) -> dict:
|
||||||
|
return asdict(self)
|
||||||
|
|
||||||
|
def problems(self, *, has_secret: bool) -> list[str]:
|
||||||
|
"""What stops this config from sending a backup, in words. Empty when
|
||||||
|
it is complete (which says nothing about whether the place answers —
|
||||||
|
check_destination does that)."""
|
||||||
|
out: list[str] = []
|
||||||
|
if self.destination == "folder":
|
||||||
|
if not self.folder_path.startswith("/"):
|
||||||
|
out.append("The folder must be an absolute path inside the container.")
|
||||||
|
else:
|
||||||
|
if not self.s3_endpoint.startswith(("http://", "https://")):
|
||||||
|
out.append("The S3 address must start with http:// or https://.")
|
||||||
|
if not self.s3_bucket:
|
||||||
|
out.append("The bucket name is missing.")
|
||||||
|
if not self.s3_access_key:
|
||||||
|
out.append("The access key is missing.")
|
||||||
|
if not has_secret:
|
||||||
|
out.append("The secret key is missing.")
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
async def load_config() -> BackupConfig:
|
||||||
|
raw = await get_admin_setting(CONFIG_SETTING, "")
|
||||||
|
try:
|
||||||
|
data = json.loads(raw) if raw else {}
|
||||||
|
except ValueError:
|
||||||
|
data = {}
|
||||||
|
return BackupConfig.from_dict(data if isinstance(data, dict) else {})
|
||||||
|
|
||||||
|
|
||||||
|
async def save_config(config: BackupConfig) -> None:
|
||||||
|
await set_admin_setting(CONFIG_SETTING, json.dumps(config.to_dict(), sort_keys=True))
|
||||||
|
|
||||||
|
|
||||||
|
async def load_secret() -> tuple[str | None, bool]:
|
||||||
|
"""(the S3 secret or None, whether one is stored).
|
||||||
|
|
||||||
|
Stored but None means it was sealed under another SECRET_KEY — after a
|
||||||
|
restore onto a different instance, say — and has to be entered again.
|
||||||
|
"""
|
||||||
|
stored = await get_admin_setting(SECRET_SETTING, "")
|
||||||
|
return (unseal(stored), bool(stored))
|
||||||
|
|
||||||
|
|
||||||
|
async def save_secret(secret: str) -> None:
|
||||||
|
await set_admin_setting(SECRET_SETTING, seal(secret) if secret else "")
|
||||||
|
|
||||||
|
|
||||||
|
def s3_target(config: BackupConfig, secret: str) -> S3Target:
|
||||||
|
return S3Target(
|
||||||
|
endpoint=config.s3_endpoint, bucket=config.s3_bucket,
|
||||||
|
access_key=config.s3_access_key, secret_key=secret,
|
||||||
|
region=config.s3_region or "us-east-1", verify_tls=config.s3_verify_tls,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Delivered:
|
||||||
|
name: str
|
||||||
|
size_bytes: int
|
||||||
|
where: str
|
||||||
|
|
||||||
|
|
||||||
|
async def deliver(
|
||||||
|
payload: dict, config: BackupConfig, secret: str | None, at: datetime,
|
||||||
|
*, app_version: str, transport=None,
|
||||||
|
) -> Delivered:
|
||||||
|
"""Write `payload` as the dated backup file and put it at the destination.
|
||||||
|
|
||||||
|
The file is written off the event loop: serialising a full export is
|
||||||
|
seconds of CPU on a large corpus, and the app keeps serving meanwhile.
|
||||||
|
"""
|
||||||
|
problems = config.problems(has_secret=bool(secret))
|
||||||
|
if problems:
|
||||||
|
raise DestinationError(" ".join(problems))
|
||||||
|
|
||||||
|
if config.destination == "folder":
|
||||||
|
folder = Path(config.folder_path)
|
||||||
|
if not folder.is_dir():
|
||||||
|
raise DestinationError(
|
||||||
|
f"{folder} isn't there. Mount the backup share at that path, "
|
||||||
|
"or choose the folder it is mounted at."
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
path = await asyncio.to_thread(
|
||||||
|
backup_file.write_archive, payload, folder, at, app_version=app_version,
|
||||||
|
)
|
||||||
|
except PermissionError:
|
||||||
|
raise DestinationError(f"Scribe isn't allowed to write in {folder}.") from None
|
||||||
|
return Delivered(path.name, path.stat().st_size, str(folder))
|
||||||
|
|
||||||
|
target = s3_target(config, secret or "")
|
||||||
|
with tempfile.TemporaryDirectory(prefix="scribe-backup-") as staging:
|
||||||
|
path = await asyncio.to_thread(
|
||||||
|
backup_file.write_archive, payload, Path(staging), at, app_version=app_version,
|
||||||
|
)
|
||||||
|
size = path.stat().st_size
|
||||||
|
try:
|
||||||
|
await put_file(target, path.name, path, transport=transport)
|
||||||
|
except S3Error as exc:
|
||||||
|
raise DestinationError(str(exc)) from None
|
||||||
|
return Delivered(path.name, size, target.url())
|
||||||
|
|
||||||
|
|
||||||
|
async def check_destination(
|
||||||
|
config: BackupConfig, secret: str | None, *, transport=None,
|
||||||
|
) -> str:
|
||||||
|
"""Ask the destination whether it would take a backup, without sending
|
||||||
|
one. Returns what was found, in words; raises DestinationError otherwise.
|
||||||
|
|
||||||
|
The folder check writes and removes a dot-file, which a folder watcher
|
||||||
|
ignores. The S3 check only lists: an append-only key could add a test
|
||||||
|
object and then never remove it.
|
||||||
|
"""
|
||||||
|
problems = config.problems(has_secret=bool(secret))
|
||||||
|
if problems:
|
||||||
|
raise DestinationError(" ".join(problems))
|
||||||
|
|
||||||
|
if config.destination == "folder":
|
||||||
|
folder = Path(config.folder_path)
|
||||||
|
if not folder.is_dir():
|
||||||
|
raise DestinationError(f"{folder} isn't there, or isn't a folder.")
|
||||||
|
probe = folder / ".scribe-write-check"
|
||||||
|
try:
|
||||||
|
await asyncio.to_thread(probe.write_bytes, b"")
|
||||||
|
await asyncio.to_thread(probe.unlink)
|
||||||
|
except PermissionError:
|
||||||
|
raise DestinationError(f"Scribe isn't allowed to write in {folder}.") from None
|
||||||
|
except OSError as exc:
|
||||||
|
raise DestinationError(f"Writing in {folder} failed: {exc.strerror or exc}.") from None
|
||||||
|
count = await asyncio.to_thread(_count_backups, folder)
|
||||||
|
return f"{folder} is writable and holds {count} Scribe backup(s)."
|
||||||
|
|
||||||
|
target = s3_target(config, secret or "")
|
||||||
|
try:
|
||||||
|
objects = await list_objects(target, backup_file.NAME_PREFIX, transport=transport)
|
||||||
|
except S3Error as exc:
|
||||||
|
raise DestinationError(str(exc)) from None
|
||||||
|
return f"Bucket {config.s3_bucket} answered and holds {len(objects)} Scribe backup(s)."
|
||||||
|
|
||||||
|
|
||||||
|
def _count_backups(folder: Path) -> int:
|
||||||
|
return sum(
|
||||||
|
1 for name in os.listdir(folder)
|
||||||
|
if name.startswith(backup_file.NAME_PREFIX) and name.endswith(backup_file.NAME_SUFFIX)
|
||||||
|
)
|
||||||
@@ -0,0 +1,199 @@
|
|||||||
|
"""A small S3 client for sending backups: AWS Signature Version 4 over httpx,
|
||||||
|
path-style addressing (endpoint/bucket/key), and the two calls a backup needs.
|
||||||
|
|
||||||
|
Ported from FabledSteward's `steward/backup/s3.py`, which is tested against
|
||||||
|
the same AWS examples as tests/test_services_backup_s3.py.
|
||||||
|
|
||||||
|
Scribe puts and lists; it never deletes. The bucket keeps the history and
|
||||||
|
thins it on its own policy, so a key that may only add objects is enough —
|
||||||
|
and is the safer key to hand a service, because a stolen one cannot wipe the
|
||||||
|
history it writes into.
|
||||||
|
|
||||||
|
Written by hand rather than with boto3: two calls don't justify a large
|
||||||
|
dependency, and signing is a pure function tests can hold to AWS's own
|
||||||
|
published examples.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
|
||||||
|
TIMEOUT_SECONDS = 30.0
|
||||||
|
# A backup is tens of megabytes; a slow link still gets this long to send one.
|
||||||
|
UPLOAD_TIMEOUT_SECONDS = 15 * 60
|
||||||
|
CHUNK = 1 << 20
|
||||||
|
|
||||||
|
|
||||||
|
class S3Error(Exception):
|
||||||
|
"""Why a call to the bucket failed, said to the person."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class S3Target:
|
||||||
|
endpoint: str # http(s)://host[:port][/prefix], no bucket
|
||||||
|
bucket: str
|
||||||
|
access_key: str
|
||||||
|
secret_key: str
|
||||||
|
region: str = "us-east-1"
|
||||||
|
verify_tls: bool = True
|
||||||
|
|
||||||
|
def url(self, key: str = "") -> str:
|
||||||
|
base = f"{self.endpoint.rstrip('/')}/{self.bucket}"
|
||||||
|
return f"{base}/{key}" if key else f"{base}/"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class StoredObject:
|
||||||
|
key: str
|
||||||
|
size: int
|
||||||
|
last_modified: str
|
||||||
|
|
||||||
|
|
||||||
|
# ── Signing ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _encode(text: str, keep_slash: bool = False) -> str:
|
||||||
|
return quote(text, safe="-_.~" + ("/" if keep_slash else ""))
|
||||||
|
|
||||||
|
|
||||||
|
def _hmac(key: bytes, msg: str) -> bytes:
|
||||||
|
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
|
||||||
|
|
||||||
|
|
||||||
|
def sign(method: str, url: str, headers: dict[str, str], payload_sha256: str,
|
||||||
|
access_key: str, secret_key: str, region: str, at: datetime,
|
||||||
|
service: str = "s3") -> dict[str, str]:
|
||||||
|
"""The request's headers with x-amz-date, x-amz-content-sha256 and
|
||||||
|
Authorization added. Every header given is signed, Host always."""
|
||||||
|
u = httpx.URL(url)
|
||||||
|
stamp = at.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||||
|
day = stamp[:8]
|
||||||
|
out = dict(headers)
|
||||||
|
out["x-amz-date"] = stamp
|
||||||
|
out["x-amz-content-sha256"] = payload_sha256
|
||||||
|
signed = {k.lower(): " ".join(str(v).split()) for k, v in out.items()}
|
||||||
|
signed["host"] = u.netloc.decode()
|
||||||
|
names = sorted(signed)
|
||||||
|
|
||||||
|
query = sorted((_encode(k), _encode(v)) for k, v in u.params.multi_items())
|
||||||
|
canonical = "\n".join([
|
||||||
|
method.upper(),
|
||||||
|
_encode(u.path or "/", keep_slash=True),
|
||||||
|
"&".join(f"{k}={v}" for k, v in query),
|
||||||
|
"".join(f"{n}:{signed[n]}\n" for n in names),
|
||||||
|
";".join(names),
|
||||||
|
payload_sha256,
|
||||||
|
])
|
||||||
|
scope = f"{day}/{region}/{service}/aws4_request"
|
||||||
|
to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope,
|
||||||
|
hashlib.sha256(canonical.encode()).hexdigest()])
|
||||||
|
key = _hmac(_hmac(_hmac(_hmac(f"AWS4{secret_key}".encode(), day), region),
|
||||||
|
service), "aws4_request")
|
||||||
|
signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
|
||||||
|
out["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, "
|
||||||
|
f"SignedHeaders={';'.join(names)}, Signature={signature}")
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# ── Calls ────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _says(response: httpx.Response) -> str:
|
||||||
|
"""An S3 error response in words: its Code and Message when it has them."""
|
||||||
|
try:
|
||||||
|
root = ET.fromstring(response.content)
|
||||||
|
code = root.findtext("Code") or ""
|
||||||
|
message = root.findtext("Message") or ""
|
||||||
|
except ET.ParseError:
|
||||||
|
code, message = "", ""
|
||||||
|
hints = {
|
||||||
|
"SignatureDoesNotMatch": "the secret key is wrong",
|
||||||
|
"InvalidAccessKeyId": "the access key isn't known there",
|
||||||
|
"NoSuchBucket": "there is no such bucket",
|
||||||
|
"AccessDenied": "the key isn't allowed to do that",
|
||||||
|
}
|
||||||
|
said = hints.get(code) or message or response.reason_phrase
|
||||||
|
return f"HTTP {response.status_code}{f' {code}' if code else ''}: {said}."
|
||||||
|
|
||||||
|
|
||||||
|
def _reach_error(target: S3Target, exc: httpx.HTTPError) -> S3Error:
|
||||||
|
if isinstance(exc, httpx.TimeoutException):
|
||||||
|
return S3Error(f"{target.endpoint} didn't answer in time.")
|
||||||
|
return S3Error(f"Couldn't reach {target.endpoint}: {exc}.")
|
||||||
|
|
||||||
|
|
||||||
|
def file_sha256(path: Path) -> str:
|
||||||
|
h = hashlib.sha256()
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
while chunk := f.read(CHUNK):
|
||||||
|
h.update(chunk)
|
||||||
|
return h.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
async def put_file(target: S3Target, key: str, path: Path, *,
|
||||||
|
transport: httpx.AsyncBaseTransport | None = None,
|
||||||
|
now: datetime | None = None) -> None:
|
||||||
|
"""Upload one file as one object, streamed from disk."""
|
||||||
|
digest = await asyncio.to_thread(file_sha256, path)
|
||||||
|
size = path.stat().st_size
|
||||||
|
url = target.url(key)
|
||||||
|
headers = sign("PUT", url, {"Content-Length": str(size),
|
||||||
|
"Content-Type": "application/x-tar"},
|
||||||
|
digest, target.access_key, target.secret_key, target.region,
|
||||||
|
now or datetime.now(timezone.utc))
|
||||||
|
|
||||||
|
async def body():
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
while chunk := f.read(CHUNK):
|
||||||
|
yield chunk
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(verify=target.verify_tls,
|
||||||
|
timeout=UPLOAD_TIMEOUT_SECONDS,
|
||||||
|
transport=transport) as client:
|
||||||
|
response = await client.put(url, headers=headers, content=body())
|
||||||
|
except httpx.HTTPError as exc:
|
||||||
|
raise _reach_error(target, exc) from None
|
||||||
|
if response.status_code // 100 != 2:
|
||||||
|
raise S3Error(f"Upload of {key} refused: {_says(response)}")
|
||||||
|
|
||||||
|
|
||||||
|
async def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *,
|
||||||
|
transport: httpx.AsyncBaseTransport | None = None,
|
||||||
|
now: datetime | None = None) -> list[StoredObject]:
|
||||||
|
"""Up to `limit` objects under `prefix` (ListObjectsV2, one page)."""
|
||||||
|
params = {"list-type": "2", "max-keys": str(limit)}
|
||||||
|
if prefix:
|
||||||
|
params["prefix"] = prefix
|
||||||
|
url = str(httpx.URL(target.url(), params=params))
|
||||||
|
headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
|
||||||
|
target.region, now or datetime.now(timezone.utc))
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(verify=target.verify_tls, timeout=TIMEOUT_SECONDS,
|
||||||
|
transport=transport) as client:
|
||||||
|
response = await client.get(url, headers=headers)
|
||||||
|
except httpx.HTTPError as exc:
|
||||||
|
raise _reach_error(target, exc) from None
|
||||||
|
if response.status_code != 200:
|
||||||
|
raise S3Error(f"Listing {target.bucket} refused: {_says(response)}")
|
||||||
|
try:
|
||||||
|
root = ET.fromstring(response.content)
|
||||||
|
except ET.ParseError:
|
||||||
|
raise S3Error(
|
||||||
|
f"{target.endpoint} answered a listing with something that isn't S3's XML."
|
||||||
|
) from None
|
||||||
|
out = []
|
||||||
|
for item in root.iter():
|
||||||
|
if item.tag.rsplit("}", 1)[-1] != "Contents":
|
||||||
|
continue
|
||||||
|
fields = {child.tag.rsplit("}", 1)[-1]: (child.text or "") for child in item}
|
||||||
|
out.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0),
|
||||||
|
fields.get("LastModified", "")))
|
||||||
|
return out
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
"""Sealing a credential Scribe stores so the database alone cannot read it.
|
||||||
|
|
||||||
|
Fernet (AES-128-CBC with an HMAC) under a key derived from the instance's
|
||||||
|
SECRET_KEY. A copy of the database, or of a backup made from it, then holds
|
||||||
|
only ciphertext: reading the credential also needs the SECRET_KEY, which lives
|
||||||
|
in the deployment's environment rather than in the data.
|
||||||
|
|
||||||
|
The derivation is labelled and versioned, so this key is never the same bytes
|
||||||
|
as anything else made from SECRET_KEY (the session-cookie signing uses it
|
||||||
|
raw). An instance that changes its SECRET_KEY can no longer open what it
|
||||||
|
sealed before: `unseal` answers None, and the caller asks for the credential
|
||||||
|
again rather than failing on it.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
from cryptography.fernet import Fernet, InvalidToken
|
||||||
|
|
||||||
|
from scribe.config import Config
|
||||||
|
|
||||||
|
_LABEL = b"scribe.secrets.v1\x00"
|
||||||
|
|
||||||
|
|
||||||
|
def _fernet() -> Fernet:
|
||||||
|
digest = hashlib.sha256(_LABEL + Config.SECRET_KEY.encode()).digest()
|
||||||
|
return Fernet(base64.urlsafe_b64encode(digest))
|
||||||
|
|
||||||
|
|
||||||
|
def seal(plain: str) -> str:
|
||||||
|
"""Encrypt `plain` for storage; the result is ASCII."""
|
||||||
|
return _fernet().encrypt(plain.encode()).decode()
|
||||||
|
|
||||||
|
|
||||||
|
def unseal(token: str) -> str | None:
|
||||||
|
"""Decrypt a value `seal` produced, or None when it can't be opened —
|
||||||
|
sealed under a different SECRET_KEY, or not a sealed value at all."""
|
||||||
|
if not token:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return _fernet().decrypt(token.encode()).decode()
|
||||||
|
except (InvalidToken, ValueError):
|
||||||
|
return None
|
||||||
@@ -0,0 +1,179 @@
|
|||||||
|
"""The backup destination (services/backup_destination.py) and the sealing it
|
||||||
|
keeps the S3 secret under (services/secrets.py).
|
||||||
|
|
||||||
|
No database: the admin settings are patched, the folder is a tmp_path, and the
|
||||||
|
bucket is an httpx.MockTransport. What these pin is the contract a person
|
||||||
|
meets in Settings — a config that is incomplete says what is missing, a place
|
||||||
|
that can't take a backup says why, and the secret never leaves sealed.
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
import io
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from scribe.services import backup, backup_destination, secrets
|
||||||
|
from scribe.services.backup_destination import (
|
||||||
|
BackupConfig, DestinationError, check_destination, deliver,
|
||||||
|
)
|
||||||
|
from scribe.services.backup_file import read_archive
|
||||||
|
|
||||||
|
AT = datetime(2026, 10, 15, 2, 0, 0, tzinfo=timezone.utc)
|
||||||
|
PAYLOAD = {
|
||||||
|
"version": 25, "scope": "full", "exported_at": AT.isoformat(),
|
||||||
|
"users": [{"id": 1, "username": "ada"}], "notes": [],
|
||||||
|
}
|
||||||
|
S3 = BackupConfig(
|
||||||
|
destination="s3", s3_endpoint="http://archives:8081", s3_bucket="scribe",
|
||||||
|
s3_access_key="AKIA", s3_region="us-east-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Sealing ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_a_sealed_secret_opens_and_is_not_the_plain_text():
|
||||||
|
token = secrets.seal("wJalrXUtnFEMI/K7MDENG")
|
||||||
|
assert "wJalrXUtnFEMI" not in token
|
||||||
|
assert secrets.unseal(token) == "wJalrXUtnFEMI/K7MDENG"
|
||||||
|
|
||||||
|
|
||||||
|
def test_another_secret_key_cannot_open_it():
|
||||||
|
token = secrets.seal("s3cret")
|
||||||
|
with patch.object(secrets.Config, "SECRET_KEY", "a-different-instance"):
|
||||||
|
assert secrets.unseal(token) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_unseal_of_nothing_or_garbage_is_none():
|
||||||
|
assert secrets.unseal("") is None
|
||||||
|
assert secrets.unseal("not a fernet token") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_secret_setting_never_reaches_a_backup():
|
||||||
|
rows = [SimpleNamespace(user_id=1, key=k, value="v") for k in (
|
||||||
|
"backup_config", backup_destination.SECRET_SETTING, "theme",
|
||||||
|
)]
|
||||||
|
keys = [r["key"] for r in backup._setting_rows(rows)]
|
||||||
|
assert keys == ["backup_config", "theme"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_secret_tells_a_missing_secret_from_one_it_cannot_open():
|
||||||
|
async def stored(value):
|
||||||
|
async def get(key, default=""):
|
||||||
|
return value
|
||||||
|
with patch.object(backup_destination, "get_admin_setting", get):
|
||||||
|
return await backup_destination.load_secret()
|
||||||
|
|
||||||
|
assert asyncio.run(stored("")) == (None, False)
|
||||||
|
assert asyncio.run(stored(secrets.seal("k"))) == ("k", True)
|
||||||
|
assert asyncio.run(stored("sealed-elsewhere")) == (None, True)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Config ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_config_round_trips_and_defaults_to_off():
|
||||||
|
assert BackupConfig().enabled is False
|
||||||
|
assert BackupConfig.from_dict(S3.to_dict()) == S3
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_damaged_config_still_loads():
|
||||||
|
config = BackupConfig.from_dict({
|
||||||
|
"hour": "25", "destination": "ftp", "enabled": "yes", "unknown": 1,
|
||||||
|
"s3_bucket": " scribe ",
|
||||||
|
})
|
||||||
|
assert config.hour == backup_destination.DEFAULT_HOUR
|
||||||
|
assert config.destination == "folder"
|
||||||
|
assert config.enabled is False
|
||||||
|
assert config.s3_bucket == "scribe"
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_incomplete_s3_config_names_what_is_missing():
|
||||||
|
said = BackupConfig(destination="s3").problems(has_secret=False)
|
||||||
|
assert said == [
|
||||||
|
"The S3 address must start with http:// or https://.",
|
||||||
|
"The bucket name is missing.",
|
||||||
|
"The access key is missing.",
|
||||||
|
"The secret key is missing.",
|
||||||
|
]
|
||||||
|
assert S3.problems(has_secret=True) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_relative_folder_is_refused():
|
||||||
|
assert BackupConfig(folder_path="backups").problems(has_secret=False)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Folder ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_deliver_to_a_folder_adds_one_dated_file(tmp_path):
|
||||||
|
config = BackupConfig(folder_path=str(tmp_path))
|
||||||
|
got = asyncio.run(deliver(PAYLOAD, config, None, AT, app_version="v"))
|
||||||
|
assert got.name == "scribe-20261015T020000Z.tar"
|
||||||
|
assert [p.name for p in tmp_path.iterdir()] == [got.name]
|
||||||
|
assert got.size_bytes == (tmp_path / got.name).stat().st_size
|
||||||
|
with open(tmp_path / got.name, "rb") as f:
|
||||||
|
assert read_archive(f) == PAYLOAD
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_folder_that_isnt_mounted_says_so(tmp_path):
|
||||||
|
config = BackupConfig(folder_path=str(tmp_path / "not-mounted"))
|
||||||
|
with pytest.raises(DestinationError, match="isn't there"):
|
||||||
|
asyncio.run(deliver(PAYLOAD, config, None, AT, app_version="v"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_checking_a_folder_leaves_nothing_behind(tmp_path):
|
||||||
|
(tmp_path / "scribe-20261014T020000Z.tar").write_bytes(b"x")
|
||||||
|
config = BackupConfig(folder_path=str(tmp_path))
|
||||||
|
said = asyncio.run(check_destination(config, None))
|
||||||
|
assert "holds 1 Scribe backup" in said
|
||||||
|
assert [p.name for p in tmp_path.iterdir()] == ["scribe-20261014T020000Z.tar"]
|
||||||
|
|
||||||
|
|
||||||
|
# ── S3 ───────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_deliver_to_s3_puts_the_dated_file_and_keeps_no_copy():
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def handler(request: httpx.Request):
|
||||||
|
seen["method"], seen["url"] = request.method, str(request.url)
|
||||||
|
seen["body"] = request.read()
|
||||||
|
return httpx.Response(200)
|
||||||
|
|
||||||
|
got = asyncio.run(deliver(PAYLOAD, S3, "secret", AT, app_version="v",
|
||||||
|
transport=httpx.MockTransport(handler)))
|
||||||
|
assert seen["method"] == "PUT"
|
||||||
|
assert seen["url"] == "http://archives:8081/scribe/scribe-20261015T020000Z.tar"
|
||||||
|
assert read_archive(io.BytesIO(seen["body"])) == PAYLOAD
|
||||||
|
assert got.size_bytes == len(seen["body"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_refused_upload_is_a_destination_error_in_words():
|
||||||
|
body = b"<Error><Code>AccessDenied</Code><Message>no</Message></Error>"
|
||||||
|
transport = httpx.MockTransport(lambda r: httpx.Response(403, content=body))
|
||||||
|
with pytest.raises(DestinationError, match="the key isn't allowed to do that"):
|
||||||
|
asyncio.run(deliver(PAYLOAD, S3, "secret", AT, app_version="v", transport=transport))
|
||||||
|
|
||||||
|
|
||||||
|
def test_s3_without_its_secret_sends_nothing():
|
||||||
|
def handler(request):
|
||||||
|
raise AssertionError("nothing should be sent")
|
||||||
|
|
||||||
|
with pytest.raises(DestinationError, match="secret key is missing"):
|
||||||
|
asyncio.run(deliver(PAYLOAD, S3, None, AT, app_version="v",
|
||||||
|
transport=httpx.MockTransport(handler)))
|
||||||
|
|
||||||
|
|
||||||
|
def test_checking_s3_only_lists():
|
||||||
|
xml = (b'<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
|
||||||
|
b"<Contents><Key>scribe-20261014T020000Z.tar</Key><Size>9</Size>"
|
||||||
|
b"<LastModified>x</LastModified></Contents></ListBucketResult>")
|
||||||
|
methods = []
|
||||||
|
|
||||||
|
def handler(request):
|
||||||
|
methods.append(request.method)
|
||||||
|
return httpx.Response(200, content=xml)
|
||||||
|
|
||||||
|
said = asyncio.run(check_destination(S3, "secret", transport=httpx.MockTransport(handler)))
|
||||||
|
assert methods == ["GET"]
|
||||||
|
assert "holds 1 Scribe backup" in said
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
"""The backup destination's S3 client (services/backup_s3.py): signatures match AWS's
|
||||||
|
published Signature Version 4 examples, and the calls read S3's answers."""
|
||||||
|
import asyncio
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from scribe.services import backup_s3 as s3
|
||||||
|
|
||||||
|
# AWS's example credentials and moment ("Signature Calculations for the
|
||||||
|
# Authorization Header", Amazon S3 API reference).
|
||||||
|
KEY_ID = "AKIAIOSFODNN7EXAMPLE"
|
||||||
|
SECRET = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||||
|
AT = datetime(2013, 5, 24, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _signature(headers):
|
||||||
|
return headers["Authorization"].rsplit("Signature=", 1)[1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_aws_example_get_object():
|
||||||
|
h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {"Range": "bytes=0-9"},
|
||||||
|
s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
|
||||||
|
assert "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date," in h["Authorization"]
|
||||||
|
assert "Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request" in h["Authorization"]
|
||||||
|
assert _signature(h) == "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
|
||||||
|
|
||||||
|
|
||||||
|
def test_aws_example_list_objects():
|
||||||
|
h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", {},
|
||||||
|
s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
|
||||||
|
assert _signature(h) == "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_changed_secret_changes_the_signature():
|
||||||
|
a = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {},
|
||||||
|
s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
|
||||||
|
b = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {},
|
||||||
|
s3.EMPTY_SHA256, KEY_ID, SECRET + "x", "us-east-1", AT)
|
||||||
|
assert _signature(a) != _signature(b)
|
||||||
|
|
||||||
|
|
||||||
|
TARGET = s3.S3Target("http://archives:8081/s3", "scribe", KEY_ID, SECRET)
|
||||||
|
|
||||||
|
|
||||||
|
def test_paths_are_path_style():
|
||||||
|
assert TARGET.url("scribe-20261012T000000Z.tar") == "http://archives:8081/s3/scribe/scribe-20261012T000000Z.tar"
|
||||||
|
assert TARGET.url() == "http://archives:8081/s3/scribe/"
|
||||||
|
|
||||||
|
|
||||||
|
def test_put_sends_the_file_signed(tmp_path):
|
||||||
|
f = tmp_path / "b.tar"
|
||||||
|
f.write_bytes(b"x" * 5000)
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def handler(request: httpx.Request):
|
||||||
|
seen["method"], seen["url"] = request.method, str(request.url)
|
||||||
|
seen["body"] = request.read()
|
||||||
|
seen["auth"] = request.headers["authorization"]
|
||||||
|
seen["sha"] = request.headers["x-amz-content-sha256"]
|
||||||
|
return httpx.Response(200)
|
||||||
|
|
||||||
|
asyncio.run(s3.put_file(TARGET, "b.tar", f, transport=httpx.MockTransport(handler)))
|
||||||
|
assert seen["method"] == "PUT"
|
||||||
|
assert seen["url"] == "http://archives:8081/s3/scribe/b.tar"
|
||||||
|
assert seen["body"] == b"x" * 5000
|
||||||
|
assert seen["sha"] == s3.file_sha256(f)
|
||||||
|
assert seen["auth"].startswith("AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_refusal_says_why(tmp_path):
|
||||||
|
f = tmp_path / "b.tar"
|
||||||
|
f.write_bytes(b"x")
|
||||||
|
body = b"<Error><Code>SignatureDoesNotMatch</Code><Message>no</Message></Error>"
|
||||||
|
transport = httpx.MockTransport(lambda r: httpx.Response(403, content=body))
|
||||||
|
with pytest.raises(s3.S3Error, match="HTTP 403 SignatureDoesNotMatch: the secret key is wrong"):
|
||||||
|
asyncio.run(s3.put_file(TARGET, "b.tar", f, transport=transport))
|
||||||
|
|
||||||
|
|
||||||
|
def test_unreachable_says_so(tmp_path):
|
||||||
|
f = tmp_path / "b.tar"
|
||||||
|
f.write_bytes(b"x")
|
||||||
|
|
||||||
|
def handler(request):
|
||||||
|
raise httpx.ConnectError("refused", request=request)
|
||||||
|
|
||||||
|
with pytest.raises(s3.S3Error, match="Couldn't reach http://archives:8081/s3"):
|
||||||
|
asyncio.run(s3.put_file(TARGET, "b.tar", f, transport=httpx.MockTransport(handler)))
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_reads_contents():
|
||||||
|
xml = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||||
|
<Name>scribe</Name><KeyCount>2</KeyCount>
|
||||||
|
<Contents><Key>scribe-20261011T000000Z.tar</Key><Size>1200</Size>
|
||||||
|
<LastModified>2026-10-11T00:00:01.000Z</LastModified></Contents>
|
||||||
|
<Contents><Key>scribe-20261012T000000Z.tar</Key><Size>1300</Size>
|
||||||
|
<LastModified>2026-10-12T00:00:01.000Z</LastModified></Contents>
|
||||||
|
</ListBucketResult>"""
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def handler(request):
|
||||||
|
seen["params"] = dict(request.url.params)
|
||||||
|
return httpx.Response(200, content=xml)
|
||||||
|
|
||||||
|
got = asyncio.run(s3.list_objects(TARGET, "scribe-", transport=httpx.MockTransport(handler)))
|
||||||
|
assert seen["params"] == {"list-type": "2", "max-keys": "1000", "prefix": "scribe-"}
|
||||||
|
assert got == [
|
||||||
|
s3.StoredObject("scribe-20261011T000000Z.tar", 1200, "2026-10-11T00:00:01.000Z"),
|
||||||
|
s3.StoredObject("scribe-20261012T000000Z.tar", 1300, "2026-10-12T00:00:01.000Z"),
|
||||||
|
]
|
||||||
@@ -1508,6 +1508,7 @@ dependencies = [
|
|||||||
{ name = "asyncpg" },
|
{ name = "asyncpg" },
|
||||||
{ name = "bcrypt" },
|
{ name = "bcrypt" },
|
||||||
{ name = "caldav" },
|
{ name = "caldav" },
|
||||||
|
{ name = "cryptography" },
|
||||||
{ name = "fastembed" },
|
{ name = "fastembed" },
|
||||||
{ name = "httpx" },
|
{ name = "httpx" },
|
||||||
{ name = "hypercorn" },
|
{ name = "hypercorn" },
|
||||||
@@ -1533,6 +1534,7 @@ requires-dist = [
|
|||||||
{ name = "asyncpg", specifier = ">=0.29" },
|
{ name = "asyncpg", specifier = ">=0.29" },
|
||||||
{ name = "bcrypt", specifier = ">=4.0" },
|
{ name = "bcrypt", specifier = ">=4.0" },
|
||||||
{ name = "caldav", specifier = ">=1.3" },
|
{ name = "caldav", specifier = ">=1.3" },
|
||||||
|
{ name = "cryptography" },
|
||||||
{ name = "fastembed", specifier = ">=0.4" },
|
{ name = "fastembed", specifier = ">=0.4" },
|
||||||
{ name = "httpx", specifier = ">=0.27" },
|
{ name = "httpx", specifier = ">=0.27" },
|
||||||
{ name = "hypercorn", specifier = ">=0.17" },
|
{ name = "hypercorn", specifier = ">=0.17" },
|
||||||
|
|||||||
Reference in New Issue
Block a user