From 4477d7d14fd3e47400ba056ac237c32fae0b2fce Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 6 Nov 2026 16:04:57 -0500 Subject: [PATCH] feat: Settings can point backups at an off-site S3 bucket, with a Test destination check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settings → Maintenance → Storage gains "Off-site backup destination": the address (with its port), bucket, region, access key and secret key of an S3 bucket such as Fabled Archives. The secret key is sealed with the credential key, never returned by the API, and kept when the field is left blank. Test destination lists the bucket with the fields as typed and writes nothing, and names the cause when it fails ("nothing is listening there (check the port)", "the secret key is wrong", "answered, but not as S3"). services/backup_s3 is a standard-library SigV4 client (put, get, paged list) ported from Grimoire's; it is held to AWS's published signing examples and exercised against a local HTTP server. Migration 0118 adds the columns; no destination is set up until someone saves one. Co-Authored-By: Claude Opus 5.5 --- .../versions/0118_backup_s3_destination.py | 43 +++ backend/app/api/system_backup.py | 103 +++++++ backend/app/models/import_settings.py | 11 + backend/app/services/backup_destination.py | 137 +++++++++ backend/app/services/backup_s3.py | 282 ++++++++++++++++++ .../settings/BackupDestinationCard.vue | 198 ++++++++++++ .../components/settings/MaintenancePanel.vue | 6 +- frontend/src/stores/backup.js | 25 ++ tests/test_api_system_backup.py | 133 +++++++++ tests/test_backup_s3.py | 209 +++++++++++++ 10 files changed, 1146 insertions(+), 1 deletion(-) create mode 100644 alembic/versions/0118_backup_s3_destination.py create mode 100644 backend/app/services/backup_destination.py create mode 100644 backend/app/services/backup_s3.py create mode 100644 frontend/src/components/settings/BackupDestinationCard.vue create mode 100644 tests/test_backup_s3.py diff --git a/alembic/versions/0118_backup_s3_destination.py b/alembic/versions/0118_backup_s3_destination.py new file mode 100644 index 0000000..e2d13a5 --- /dev/null +++ b/alembic/versions/0118_backup_s3_destination.py @@ -0,0 +1,43 @@ +"""import_settings: the off-site backup destination (an S3 bucket). + +Address, bucket, access key and region are plain settings; the secret key is +stored sealed with the credential key. All empty means no destination is set +up, which is every existing install's state. + +Revision ID: 0118 +Revises: 0117 +Create Date: 2026-11-06 + +""" +import sqlalchemy as sa +from alembic import op + +revision = "0118" +down_revision = "0117" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("import_settings", sa.Column("backup_s3_endpoint", sa.Text(), nullable=True)) + op.add_column( + "import_settings", sa.Column("backup_s3_bucket", sa.String(255), nullable=True), + ) + op.add_column("import_settings", sa.Column("backup_s3_access_key", sa.Text(), nullable=True)) + op.add_column( + "import_settings", sa.Column("backup_s3_secret_sealed", sa.Text(), nullable=True), + ) + op.add_column( + "import_settings", + sa.Column( + "backup_s3_region", sa.String(64), nullable=False, server_default="us-east-1", + ), + ) + + +def downgrade() -> None: + for column in ( + "backup_s3_region", "backup_s3_secret_sealed", "backup_s3_access_key", + "backup_s3_bucket", "backup_s3_endpoint", + ): + op.drop_column("import_settings", column) diff --git a/backend/app/api/system_backup.py b/backend/app/api/system_backup.py index a0f9ef2..a2b84c9 100644 --- a/backend/app/api/system_backup.py +++ b/backend/app/api/system_backup.py @@ -236,6 +236,109 @@ async def get_settings(): }) +@system_backup_bp.route("/destination", methods=["GET"]) +async def get_destination(): + """The off-site destination as the settings screen shows it. The secret + key is never returned, only whether one is saved.""" + from ..services import backup_destination as dest + + async with get_session() as session: + row = await ImportSettings.load(session) + return jsonify(dest.describe(row)) + + +@system_backup_bp.route("/destination", methods=["PATCH"]) +async def patch_destination(): + """Save the destination. Body {endpoint, bucket, access_key, region, + secret_key}; an empty field clears it. A missing or empty secret_key + keeps the saved one, so the screen can save the other fields without the + secret being typed again.""" + from ..services import backup_destination as dest + + body = await request.get_json(silent=True) + if not isinstance(body, dict): + return _bad("invalid_body", detail="body must be a JSON object") + fields, why = dest.clean_fields(body) + if fields is None: + return _bad("invalid_destination", detail=why) + secret = body.get("secret_key") + if secret is not None and not isinstance(secret, str): + return _bad("invalid_destination", detail="secret_key must be text") + + async with get_session() as session: + row = await ImportSettings.load(session) + row.backup_s3_endpoint = fields.endpoint + row.backup_s3_bucket = fields.bucket + row.backup_s3_access_key = fields.access_key + row.backup_s3_region = fields.region + if secret and secret.strip(): + row.backup_s3_secret_sealed = dest.seal(secret.strip()) + await session.commit() + return jsonify(dest.describe(row)) + + +@system_backup_bp.route("/destination", methods=["DELETE"]) +async def delete_destination(): + """Forget the destination, secret included. What was already sent stays in + the bucket: FabledCurator never deletes there.""" + from ..services import backup_destination as dest + + async with get_session() as session: + row = await ImportSettings.load(session) + row.backup_s3_endpoint = None + row.backup_s3_bucket = None + row.backup_s3_access_key = None + row.backup_s3_secret_sealed = None + row.backup_s3_region = dest.DEFAULT_REGION + await session.commit() + return jsonify(dest.describe(row)) + + +@system_backup_bp.route("/destination/test", methods=["POST"]) +async def try_destination(): + """Reach the bucket with the fields in the body (the form as typed, saved + or not); a missing secret_key uses the saved one. Lists the bucket and + writes nothing. Answers {ok, message} either way: a destination that + can't be reached is a result to show, not an API error.""" + import asyncio + + from ..services import backup_destination as dest + from ..services.backup_s3 import S3Error + from ..services.credential_crypto import InvalidCredentialBlob + + body = await request.get_json(silent=True) or {} + if not isinstance(body, dict): + return _bad("invalid_body", detail="body must be a JSON object") + fields, why = dest.clean_fields(body) + if fields is None: + return jsonify({"ok": False, "message": f"Not tested: {why}."}) + secret = body.get("secret_key") + secret = secret.strip() if isinstance(secret, str) else "" + if not secret: + async with get_session() as session: + row = await ImportSettings.load(session) + sealed = row.backup_s3_secret_sealed + if sealed: + try: + secret = dest.unseal(sealed) + except InvalidCredentialBlob: + return jsonify({"ok": False, "message": ( + "Not tested: the saved secret key can't be opened on this server. " + "Enter the secret key again." + )}) + target = dest.target_for(fields, secret or None) + if target is None: + missing = [name for name, v in ( + ("address", fields.endpoint), ("bucket", fields.bucket), + ("access key", fields.access_key), ("secret key", secret), + ) if not v] + return jsonify({"ok": False, "message": f"Not tested: fill in the {', '.join(missing)}."}) + try: + return jsonify(await asyncio.to_thread(dest.check_target, target)) + except S3Error as exc: + return jsonify({"ok": False, "message": str(exc)}) + + @system_backup_bp.route("/settings", methods=["PATCH"]) async def patch_settings(): body = await request.get_json(silent=True) diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py index c9aac02..2f3efea 100644 --- a/backend/app/models/import_settings.py +++ b/backend/app/models/import_settings.py @@ -9,6 +9,7 @@ from sqlalchemy import ( CheckConstraint, Float, Integer, + String, Text, select, text, @@ -100,6 +101,16 @@ class ImportSettings(Base): Integer, nullable=False, default=3, server_default="3", ) + # Off-site backup destination: an S3 bucket (services/backup_destination). + # The secret key is stored sealed with the credential key, never in clear. + backup_s3_endpoint: Mapped[str | None] = mapped_column(Text, nullable=True) + backup_s3_bucket: Mapped[str | None] = mapped_column(String(255), nullable=True) + backup_s3_access_key: Mapped[str | None] = mapped_column(Text, nullable=True) + backup_s3_secret_sealed: Mapped[str | None] = mapped_column(Text, nullable=True) + backup_s3_region: Mapped[str] = mapped_column( + String(64), nullable=False, default="us-east-1", + server_default="us-east-1", + ) # FC-6.3 series continuation matcher. enabled gates the rescan; threshold is # the weighted-score cut-off (0..1) above which a pending suggestion is made. diff --git a/backend/app/services/backup_destination.py b/backend/app/services/backup_destination.py new file mode 100644 index 0000000..fb4d261 --- /dev/null +++ b/backend/app/services/backup_destination.py @@ -0,0 +1,137 @@ +"""The off-site backup destination: an S3 bucket (Fabled Archives, or any S3 +service) that the database dumps and the media blobs are sent to. + +The address, bucket, access key and region are plain settings. The secret key +is sealed at rest with the credential key (`/images/secrets`), the same key +that seals account logins, and no API ever returns it: the settings answer +only whether one is set. Because the credential key is kept out of every +backup, a database dump carries the secret only as ciphertext nobody can open +without the original server. +""" +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlsplit + +from .backup_s3 import S3Error, S3Target, list_objects +from .credential_crypto import CredentialCrypto, InvalidCredentialBlob + +KEY_PATH = Path("/images/secrets/credential_key.b64") +DEFAULT_REGION = "us-east-1" +_TEST_LISTING = 5 + + +def _crypto() -> CredentialCrypto: + return CredentialCrypto(KEY_PATH) + + +def seal(secret: str) -> str: + return _crypto().encrypt(secret).decode("ascii") + + +def unseal(sealed: str) -> str: + return _crypto().decrypt(sealed.encode("ascii")) + + +@dataclass(frozen=True) +class DestinationFields: + endpoint: str | None + bucket: str | None + access_key: str | None + region: str + + +def clean_fields(body: dict) -> tuple[DestinationFields | None, str | None]: + """The destination fields from a request body, trimmed, with empty ones as + None; or (None, why) when the address can't be an S3 endpoint.""" + def text(name: str) -> str | None: + v = body.get(name) + if v is None: + return None + if not isinstance(v, str): + raise TypeError(name) + return v.strip() or None + + try: + endpoint = text("endpoint") + bucket = text("bucket") + access_key = text("access_key") + region = text("region") or DEFAULT_REGION + except TypeError as exc: + return None, f"{exc.args[0]} must be text" + if endpoint is not None: + u = urlsplit(endpoint) + if u.scheme not in ("http", "https") or not u.hostname: + return None, ("the address must start with http:// or https:// and name " + "a host, e.g. http://archives.example:8081") + if u.query or u.fragment: + return None, "the address can't carry a query or a fragment" + endpoint = endpoint.rstrip("/") + if bucket is not None and "/" in bucket: + return None, "the bucket is a name, not a path: put any path in the address" + return DestinationFields(endpoint, bucket, access_key, region), None + + +def target_for(fields: DestinationFields, secret: str | None) -> S3Target | None: + """The S3 target, or None while any part of it is missing.""" + if not (fields.endpoint and fields.bucket and fields.access_key and secret): + return None + return S3Target(fields.endpoint, fields.bucket, fields.access_key, secret, + region=fields.region) + + +def saved_target(settings) -> S3Target | None: + """The destination saved in ImportSettings, or None when none is set up. + Raises S3Error when the secret can't be opened (the credential key + changed since it was saved).""" + sealed = settings.backup_s3_secret_sealed + if not sealed: + return None + try: + secret = unseal(sealed) + except InvalidCredentialBlob: + raise S3Error( + "The saved secret key can't be opened on this server (the credential " + "key changed since it was saved). Enter the secret key again." + ) from None + return target_for(fields_of(settings), secret) + + +def fields_of(settings) -> DestinationFields: + return DestinationFields( + settings.backup_s3_endpoint, settings.backup_s3_bucket, + settings.backup_s3_access_key, settings.backup_s3_region or DEFAULT_REGION, + ) + + +def describe(settings) -> dict: + """The destination as the settings screen shows it: never the secret.""" + f = fields_of(settings) + return { + "endpoint": f.endpoint or "", + "bucket": f.bucket or "", + "access_key": f.access_key or "", + "region": f.region, + "secret_set": bool(settings.backup_s3_secret_sealed), + "configured": bool( + f.endpoint and f.bucket and f.access_key and settings.backup_s3_secret_sealed + ), + } + + +def check_target(target: S3Target) -> dict: + """Reach the bucket by listing it. A listing proves the address, the + bucket and both keys at once, and writes nothing, because an Archives key + is allowed to add objects but a test object could never be removed.""" + try: + page = list_objects(target, limit=_TEST_LISTING) + except S3Error as exc: + return {"ok": False, "message": str(exc)} + if not page.objects: + said = f"Reached {target.bucket} at {target.endpoint}. The bucket is empty." + else: + more = "+" if page.next_token else "" + said = (f"Reached {target.bucket} at {target.endpoint}. It holds " + f"{len(page.objects)}{more} object(s), e.g. {page.objects[0].key}.") + return {"ok": True, "message": said} diff --git a/backend/app/services/backup_s3.py b/backend/app/services/backup_s3.py new file mode 100644 index 0000000..7ed2bb4 --- /dev/null +++ b/backend/app/services/backup_s3.py @@ -0,0 +1,282 @@ +"""A small S3 client for sending backups off the box: AWS Signature Version 4, +path-style addressing (endpoint/bucket/key), and the calls a backup needs — +put an object, read one back, list a prefix. + +FabledCurator puts, lists and reads; it never deletes. The bucket keeps the +history and thins it on its own policy, so a key that may only add and read +objects is enough, and is the safer key to hand a service: a stolen one cannot +wipe the history it writes into. + +Built on the standard library's http.client rather than boto3: three calls +don't justify a large dependency, and signing is a pure function the tests +hold to AWS's own published examples. Every call is synchronous, because the +uploads run in Celery workers; the API runs the Test destination call in a +thread. +""" +from __future__ import annotations + +import hashlib +import hmac +import http.client +import os +import socket +import ssl +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit + +EMPTY_SHA256 = hashlib.sha256(b"").hexdigest() +TIMEOUT_SECONDS = 30.0 +# A single object can be a multi-gigabyte video or archive; a slow link still +# gets this long between bytes before the call gives up. +TRANSFER_TIMEOUT_SECONDS = 15 * 60 +CHUNK = 1 << 20 + + +class S3Error(Exception): + """Why a call to the bucket failed, said to the person.""" + + +@dataclass(frozen=True) +class S3Target: + endpoint: str # http(s)://host[:port][/prefix], no bucket + bucket: str + access_key: str + secret_key: str + region: str = "us-east-1" + verify_tls: bool = True + + def url(self, key: str = "") -> str: + base = f"{self.endpoint.rstrip('/')}/{quote(self.bucket, safe='')}" + return f"{base}/{quote(key, safe='/-_.~')}" if key else f"{base}/" + + +@dataclass(frozen=True) +class StoredObject: + key: str + size: int + last_modified: str + + +@dataclass(frozen=True) +class ListPage: + objects: list[StoredObject] + next_token: str | None # pass back as `continuation` for the next page + + +# ── Signing ────────────────────────────────────────────────────────────────── + +def _encode(text: str, keep_slash: bool = False) -> str: + return quote(text, safe="-_.~" + ("/" if keep_slash else "")) + + +def _hmac(key: bytes, msg: str) -> bytes: + return hmac.new(key, msg.encode(), hashlib.sha256).digest() + + +def sign(method: str, url: str, headers: dict[str, str], payload_sha256: str, + access_key: str, secret_key: str, region: str, at: datetime, + service: str = "s3") -> dict[str, str]: + """The request's headers with Host, x-amz-date, x-amz-content-sha256 and + Authorization added. Every header given is signed, Host always.""" + u = urlsplit(url) + stamp = at.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ") + day = stamp[:8] + out = dict(headers) + out["Host"] = u.netloc + out["x-amz-date"] = stamp + out["x-amz-content-sha256"] = payload_sha256 + signed = {k.lower(): " ".join(str(v).split()) for k, v in out.items()} + names = sorted(signed) + + query = sorted( + (_encode(k), _encode(v)) for k, v in parse_qsl(u.query, keep_blank_values=True) + ) + canonical = "\n".join([ + method.upper(), + _encode(unquote(u.path) or "/", keep_slash=True), + "&".join(f"{k}={v}" for k, v in query), + "".join(f"{n}:{signed[n]}\n" for n in names), + ";".join(names), + payload_sha256, + ]) + scope = f"{day}/{region}/{service}/aws4_request" + to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope, + hashlib.sha256(canonical.encode()).hexdigest()]) + key = _hmac(_hmac(_hmac(_hmac(f"AWS4{secret_key}".encode(), day), region), + service), "aws4_request") + signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest() + out["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, " + f"SignedHeaders={';'.join(names)}, Signature={signature}") + return out + + +# ── Transport ──────────────────────────────────────────────────────────────── + +def _connect(target: S3Target, url: str, timeout: float) -> http.client.HTTPConnection: + u = urlsplit(url) + if u.scheme == "https": + ctx = ssl.create_default_context() + if not target.verify_tls: + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return http.client.HTTPSConnection( + u.hostname, u.port, timeout=timeout, context=ctx, blocksize=CHUNK, + ) + if u.scheme != "http": + raise S3Error(f"{target.endpoint} isn't an http:// or https:// address.") + return http.client.HTTPConnection(u.hostname, u.port, timeout=timeout, blocksize=CHUNK) + + +def _request_target(url: str) -> str: + u = urlsplit(url) + return (u.path or "/") + (f"?{u.query}" if u.query else "") + + +def _reach_error(target: S3Target, exc: Exception) -> S3Error: + if isinstance(exc, TimeoutError): + return S3Error(f"{target.endpoint} didn't answer in time.") + if isinstance(exc, socket.gaierror): + return S3Error(f"Couldn't reach {target.endpoint}: the host name doesn't resolve.") + if isinstance(exc, ConnectionRefusedError): + return S3Error( + f"Couldn't reach {target.endpoint}: nothing is listening there " + "(check the port)." + ) + if isinstance(exc, ssl.SSLError): + return S3Error(f"Couldn't reach {target.endpoint}: TLS failed ({exc.reason}).") + return S3Error(f"Couldn't reach {target.endpoint}: {exc}.") + + +def _says(status: int, reason: str, content: bytes) -> str: + """An S3 error response in words: its Code and Message when it has them.""" + try: + root = ET.fromstring(content) if content else None + code = (root.findtext("Code") or "") if root is not None else "" + message = (root.findtext("Message") or "") if root is not None else "" + except ET.ParseError: + code, message = "", "" + hints = { + "SignatureDoesNotMatch": "the secret key is wrong", + "InvalidAccessKeyId": "the access key isn't known there", + "NoSuchBucket": "there is no such bucket", + "AccessDenied": "the key isn't allowed to do that", + } + said = hints.get(code) or message or reason + if not code and 400 <= status < 500: + # S3 refuses with an body. A 4xx without one came from + # something else at that address — most often a web server on the + # default port, when the S3 service listens on a port of its own. + said = ("the address answered, but not as S3 — check the scheme and " + "the port (an S3 service usually listens on its own port)") + return f"HTTP {status}{f' {code}' if code else ''}: {said}." + + +# ── Calls ──────────────────────────────────────────────────────────────────── + +def file_sha256(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + while chunk := f.read(CHUNK): + h.update(chunk) + return h.hexdigest() + + +def put_file(target: S3Target, key: str, path: Path, *, + content_type: str = "application/octet-stream", + payload_sha256: str | None = None, + now: datetime | None = None) -> None: + """Upload one file as one object, streamed from disk. `payload_sha256` is + the file's sha256 when the caller already knows it (the library stores + every file's), which saves reading the file twice.""" + digest = payload_sha256 or file_sha256(path) + size = path.stat().st_size + url = target.url(key) + headers = sign("PUT", url, {"Content-Length": str(size), "Content-Type": content_type}, + digest, target.access_key, target.secret_key, target.region, + now or datetime.now(UTC)) + conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS) + try: + with open(path, "rb") as f: + conn.request("PUT", _request_target(url), body=f, headers=headers) + response = conn.getresponse() + content = response.read() + except (OSError, http.client.HTTPException) as exc: + raise _reach_error(target, exc) from None + finally: + conn.close() + if response.status // 100 != 2: + raise S3Error(f"Upload of {key} refused: " + f"{_says(response.status, response.reason, content)}") + + +def get_file(target: S3Target, key: str, path: Path, *, + now: datetime | None = None) -> None: + """Download one object to `path`, streamed to disk. The bytes land under a + `.partial` name and are renamed into place when complete, so a broken + transfer never leaves a truncated file at `path`.""" + url = target.url(key) + headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key, + target.region, now or datetime.now(UTC)) + partial = path.with_name(path.name + ".partial") + conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS) + try: + conn.request("GET", _request_target(url), headers=headers) + response = conn.getresponse() + if response.status != 200: + raise S3Error(f"Reading {key} refused: " + f"{_says(response.status, response.reason, response.read())}") + with open(partial, "wb") as f: + while chunk := response.read(CHUNK): + f.write(chunk) + os.replace(partial, path) + except (OSError, http.client.HTTPException) as exc: + partial.unlink(missing_ok=True) + raise _reach_error(target, exc) from None + finally: + conn.close() + + +def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *, + continuation: str | None = None, + now: datetime | None = None) -> ListPage: + """One page (up to `limit`) of the objects under `prefix`, ListObjectsV2.""" + params = {"list-type": "2", "max-keys": str(limit)} + if prefix: + params["prefix"] = prefix + if continuation: + params["continuation-token"] = continuation + url = f"{target.url()}?{urlencode(params, quote_via=quote)}" + headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key, + target.region, now or datetime.now(UTC)) + conn = _connect(target, url, TIMEOUT_SECONDS) + try: + conn.request("GET", _request_target(url), headers=headers) + response = conn.getresponse() + content = response.read() + except (OSError, http.client.HTTPException) as exc: + raise _reach_error(target, exc) from None + finally: + conn.close() + if response.status != 200: + raise S3Error(f"Listing {target.bucket} refused: " + f"{_says(response.status, response.reason, content)}") + try: + root = ET.fromstring(content) + except ET.ParseError: + raise S3Error( + f"{target.endpoint} answered a listing with something that isn't S3's XML." + ) from None + objects, next_token = [], None + for item in root.iter(): + tag = item.tag.rsplit("}", 1)[-1] + if tag == "NextContinuationToken": + next_token = item.text or None + if tag != "Contents": + continue + fields = {child.tag.rsplit("}", 1)[-1]: (child.text or "") for child in item} + objects.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0), + fields.get("LastModified", ""))) + return ListPage(objects, next_token) diff --git a/frontend/src/components/settings/BackupDestinationCard.vue b/frontend/src/components/settings/BackupDestinationCard.vue new file mode 100644 index 0000000..8b1a13d --- /dev/null +++ b/frontend/src/components/settings/BackupDestinationCard.vue @@ -0,0 +1,198 @@ + + + + + + diff --git a/frontend/src/components/settings/MaintenancePanel.vue b/frontend/src/components/settings/MaintenancePanel.vue index 3931384..db3620c 100644 --- a/frontend/src/components/settings/MaintenancePanel.vue +++ b/frontend/src/components/settings/MaintenancePanel.vue @@ -63,8 +63,11 @@

Storage

-

Database + image backups and restore.

+

+ Database + image backups and restore, and where backups are sent off this server. +

+
@@ -93,6 +96,7 @@ import PostAssociationsCard from './PostAssociationsCard.vue' import GpuAgentCard from './GpuAgentCard.vue' import AliasTable from './AliasTable.vue' import BackupCard from './BackupCard.vue' +import BackupDestinationCard from './BackupDestinationCard.vue' import { useSystemActivityStore } from '../../stores/systemActivity.js' // Poll queue depths so each task tile's QueueStatusBar shows live pending diff --git a/frontend/src/stores/backup.js b/frontend/src/stores/backup.js index 1bc5c85..af099a3 100644 --- a/frontend/src/stores/backup.js +++ b/frontend/src/stores/backup.js @@ -101,9 +101,34 @@ export const useBackupStore = defineStore('backup', () => { } } + // Off-site destination (an S3 bucket). The secret key goes up but never + // comes back: the server answers `secret_set` instead. + const destination = ref(null) + + async function loadDestination() { + destination.value = await api.get('/api/system/backup/destination') + return destination.value + } + + async function saveDestination(fields) { + destination.value = await api.patch('/api/system/backup/destination', { body: fields }) + return destination.value + } + + async function removeDestination() { + destination.value = await api.delete('/api/system/backup/destination') + return destination.value + } + + // Resolves to {ok, message}; an unreachable bucket is a result, not a throw. + async function testDestination(fields) { + return api.post('/api/system/backup/destination/test', { body: fields }) + } + return { dbRuns, imagesRuns, settings, loading, lastError, loadRuns, triggerBackup, restore, deleteRun, setTag, loadSettings, patchSettings, + destination, loadDestination, saveDestination, removeDestination, testDestination, } }) diff --git a/tests/test_api_system_backup.py b/tests/test_api_system_backup.py index 0759011..d0d82a4 100644 --- a/tests/test_api_system_backup.py +++ b/tests/test_api_system_backup.py @@ -322,3 +322,136 @@ async def test_patch_settings_rejects_keep_n_below_min(client): json={"backup_db_keep_last_n": 0}, ) assert resp.status_code == 400 + + +# --- /destination --------------------------------------------------- + + +@pytest.fixture +def _sealing_key(tmp_path, monkeypatch): + from backend.app.services import backup_destination + monkeypatch.setattr(backup_destination, "KEY_PATH", tmp_path / "credential_key.b64") + + +_DEST = { + "endpoint": "http://archives.example:8081/", + "bucket": "curator", + "access_key": "AKEXAMPLE", + "region": "", + "secret_key": "s3cr3t-example", +} + + +@pytest.mark.asyncio +async def test_destination_saves_and_never_returns_the_secret(client, db_sync, _sealing_key): + from backend.app.models import ImportSettings + + resp = await client.patch("/api/system/backup/destination", json=_DEST) + assert resp.status_code == 200 + body = await resp.get_json() + assert body == { + "endpoint": "http://archives.example:8081", "bucket": "curator", + "access_key": "AKEXAMPLE", "region": "us-east-1", + "secret_set": True, "configured": True, + } + assert "s3cr3t-example" not in str(await (await client.get( + "/api/system/backup/destination")).get_json()) + row = db_sync.execute(select(ImportSettings).where(ImportSettings.id == 1)).scalar_one() + db_sync.refresh(row) + assert row.backup_s3_secret_sealed and "s3cr3t-example" not in row.backup_s3_secret_sealed + + from backend.app.services import backup_destination + assert backup_destination.unseal(row.backup_s3_secret_sealed) == "s3cr3t-example" + + +@pytest.mark.asyncio +async def test_destination_blank_secret_keeps_the_saved_one(client, db_sync, _sealing_key): + from backend.app.models import ImportSettings + from backend.app.services import backup_destination + + await client.patch("/api/system/backup/destination", json=_DEST) + resp = await client.patch( + "/api/system/backup/destination", json={**_DEST, "bucket": "other", "secret_key": ""}, + ) + assert (await resp.get_json())["bucket"] == "other" + row = db_sync.execute(select(ImportSettings).where(ImportSettings.id == 1)).scalar_one() + db_sync.refresh(row) + assert backup_destination.unseal(row.backup_s3_secret_sealed) == "s3cr3t-example" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("patch, word", [ + ({"endpoint": "archives.example:8081"}, "http://"), + ({"endpoint": "ftp://archives.example"}, "http://"), + ({"bucket": "curator/sub"}, "bucket"), + ({"access_key": 5}, "access_key"), +]) +async def test_destination_rejects_what_cant_be_an_s3_target(client, _sealing_key, patch, word): + resp = await client.patch("/api/system/backup/destination", json={**_DEST, **patch}) + assert resp.status_code == 400 + assert word in (await resp.get_json())["detail"] + + +@pytest.mark.asyncio +async def test_destination_remove_forgets_everything(client, _sealing_key): + await client.patch("/api/system/backup/destination", json=_DEST) + resp = await client.delete("/api/system/backup/destination") + assert await resp.get_json() == { + "endpoint": "", "bucket": "", "access_key": "", "region": "us-east-1", + "secret_set": False, "configured": False, + } + + +@pytest.mark.asyncio +async def test_destination_test_uses_the_form_and_the_saved_secret( + client, _sealing_key, monkeypatch, +): + from backend.app.services import backup_destination + + seen = [] + + def fake_check(target): + seen.append(target) + return {"ok": True, "message": "Reached it."} + + monkeypatch.setattr(backup_destination, "check_target", fake_check) + await client.patch("/api/system/backup/destination", json=_DEST) + resp = await client.post( + "/api/system/backup/destination/test", + json={**_DEST, "bucket": "typed-not-saved", "secret_key": ""}, + ) + assert await resp.get_json() == {"ok": True, "message": "Reached it."} + assert seen[0].bucket == "typed-not-saved" + assert seen[0].secret_key == "s3cr3t-example" + assert seen[0].endpoint == "http://archives.example:8081" + + +@pytest.mark.asyncio +async def test_destination_test_names_what_is_missing(client, _sealing_key): + await client.delete("/api/system/backup/destination") + resp = await client.post( + "/api/system/backup/destination/test", + json={"endpoint": "http://archives.example:8081", "bucket": "curator"}, + ) + body = await resp.get_json() + assert body["ok"] is False + assert body["message"] == "Not tested: fill in the access key, secret key." + + +@pytest.mark.asyncio +async def test_destination_test_reports_an_unreachable_bucket_as_a_result( + client, _sealing_key, monkeypatch, +): + from backend.app.services import backup_destination + from backend.app.services.backup_s3 import S3Error + + def refuse(target, **kw): + raise S3Error("Couldn't reach http://archives.example:8081: nothing is listening there.") + + monkeypatch.setattr(backup_destination, "list_objects", refuse) + resp = await client.post("/api/system/backup/destination/test", json=_DEST) + assert resp.status_code == 200 + assert await resp.get_json() == { + "ok": False, + "message": "Couldn't reach http://archives.example:8081: nothing is listening there.", + } diff --git a/tests/test_backup_s3.py b/tests/test_backup_s3.py new file mode 100644 index 0000000..d62ccfd --- /dev/null +++ b/tests/test_backup_s3.py @@ -0,0 +1,209 @@ +"""The backup destination's S3 client (services/backup_s3.py): signatures match +AWS's published Signature Version 4 examples, and the calls send and read what +S3 expects, over a real HTTP connection to a local stand-in server.""" +import socket +import threading +from datetime import UTC, datetime +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from backend.app.services import backup_s3 as s3 + +# AWS's example credentials and moment ("Signature Calculations for the +# Authorization Header", Amazon S3 API reference). +KEY_ID = "AKIAIOSFODNN7EXAMPLE" +SECRET = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +AT = datetime(2013, 5, 24, tzinfo=UTC) + + +def _signature(headers): + return headers["Authorization"].rsplit("Signature=", 1)[1] + + +def test_aws_example_get_object(): + h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", + {"Range": "bytes=0-9"}, s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT) + assert "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date," in h["Authorization"] + assert "Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request" in h[ + "Authorization"] + assert _signature(h) == "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41" + + +def test_aws_example_list_objects(): + h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", {}, + s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT) + assert _signature(h) == "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7" + + +def test_a_changed_secret_changes_the_signature(): + url = "https://examplebucket.s3.amazonaws.com/test.txt" + a = s3.sign("GET", url, {}, s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT) + b = s3.sign("GET", url, {}, s3.EMPTY_SHA256, KEY_ID, SECRET + "x", "us-east-1", AT) + assert _signature(a) != _signature(b) + + +def test_paths_are_path_style_and_keys_are_escaped(): + t = s3.S3Target("http://archives.example:8081/s3", "curator", KEY_ID, SECRET) + assert t.url("media/ab/cd/abcd.jpg") == "http://archives.example:8081/s3/curator/media/ab/cd/abcd.jpg" + assert t.url() == "http://archives.example:8081/s3/curator/" + assert t.url("a b+c.jpg") == "http://archives.example:8081/s3/curator/a%20b%2Bc.jpg" + + +# ── Calls, against a local stand-in ───────────────────────────────────────── + + +class _Server: + """A local HTTP server that records each request and answers with + `reply(handler) -> (status, body)`.""" + + def __init__(self, reply): + self.seen = [] + outer = self + + class Handler(BaseHTTPRequestHandler): + def _serve(self): + length = int(self.headers.get("Content-Length") or 0) + outer.seen.append({ + "method": self.command, "path": self.path, + "headers": {k.lower(): v for k, v in self.headers.items()}, + "body": self.rfile.read(length) if length else b"", + }) + status, body = reply(self) + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + do_GET = do_PUT = _serve + + def log_message(self, *args): + pass + + self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True) + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, *exc): + self.httpd.shutdown() + self.httpd.server_close() + + def target(self, bucket="curator"): + host, port = self.httpd.server_address + return s3.S3Target(f"http://{host}:{port}", bucket, KEY_ID, SECRET) + + +def test_put_streams_the_file_signed(tmp_path): + f = tmp_path / "abcd.jpg" + f.write_bytes(b"x" * 5000) + with _Server(lambda h: (200, b"")) as srv: + s3.put_file(srv.target(), "media/ab/cd/abcd.jpg", f, content_type="image/jpeg") + req = srv.seen[0] + assert req["method"] == "PUT" + assert req["path"] == "/curator/media/ab/cd/abcd.jpg" + assert req["body"] == b"x" * 5000 + assert req["headers"]["x-amz-content-sha256"] == s3.file_sha256(f) + assert req["headers"]["content-type"] == "image/jpeg" + assert req["headers"]["authorization"].startswith( + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/") + + +def test_put_signs_with_a_known_sha_without_rereading(tmp_path, monkeypatch): + f = tmp_path / "b.bin" + f.write_bytes(b"y") + monkeypatch.setattr(s3, "file_sha256", lambda p: pytest.fail("read the file twice")) + with _Server(lambda h: (200, b"")) as srv: + s3.put_file(srv.target(), "b.bin", f, payload_sha256="a" * 64) + assert srv.seen[0]["headers"]["x-amz-content-sha256"] == "a" * 64 + + +def test_a_refusal_says_why(tmp_path): + f = tmp_path / "b.bin" + f.write_bytes(b"x") + body = b"SignatureDoesNotMatchno" + with _Server(lambda h: (403, body)) as srv, pytest.raises( + s3.S3Error, match="HTTP 403 SignatureDoesNotMatch: the secret key is wrong", + ): + s3.put_file(srv.target(), "b.bin", f) + + +def test_a_web_server_at_the_address_is_named_as_not_s3(): + """An address without the S3 port lands on whatever serves the default + port, and its plain 404 says nothing about S3.""" + page = b"

404 Not Found

" + with _Server(lambda h: (404, page)) as srv, pytest.raises( + s3.S3Error, + match=r"HTTP 404: the address answered, but not as S3 — check the scheme and the port", + ): + s3.list_objects(srv.target()) + + +def test_a_server_error_without_a_body_is_not_blamed_on_the_port(): + with _Server(lambda h: (503, b"")) as srv, pytest.raises( + s3.S3Error, match="HTTP 503: Service Unavailable", + ) as err: + s3.list_objects(srv.target()) + assert "port" not in str(err.value) + + +def test_nothing_listening_says_check_the_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + target = s3.S3Target(f"http://127.0.0.1:{port}", "curator", KEY_ID, SECRET) + with pytest.raises(s3.S3Error, match="nothing is listening there"): + s3.list_objects(target) + + +def test_list_reads_contents_and_the_next_page_token(): + xml = b""" + + curator2true + tok-2 + database/fabledcurator-20261105T030000Z.dump1200 + 2026-11-05T03:00:01.000Z + database/fabledcurator-20261106T030000Z.dump1300 + 2026-11-06T03:00:01.000Z +""" + with _Server(lambda h: (200, xml)) as srv: + page = s3.list_objects(srv.target(), "database/", continuation="tok-1") + path = srv.seen[0]["path"] + assert path.startswith("/curator/?") + assert "list-type=2" in path and "prefix=database%2F" in path + assert "continuation-token=tok-1" in path + assert page.next_token == "tok-2" + assert page.objects == [ + s3.StoredObject("database/fabledcurator-20261105T030000Z.dump", 1200, + "2026-11-05T03:00:01.000Z"), + s3.StoredObject("database/fabledcurator-20261106T030000Z.dump", 1300, + "2026-11-06T03:00:01.000Z"), + ] + + +def test_a_listing_that_isnt_xml_is_named(): + with _Server(lambda h: (200, b"hello")) as srv, pytest.raises( + s3.S3Error, match="isn't S3's XML", + ): + s3.list_objects(srv.target()) + + +def test_get_streams_the_object_to_disk(tmp_path): + with _Server(lambda h: (200, b"y" * 3000)) as srv: + out = tmp_path / "b.dump" + s3.get_file(srv.target(), "database/b.dump", out) + assert srv.seen[0]["path"] == "/curator/database/b.dump" + assert srv.seen[0]["headers"]["x-amz-content-sha256"] == s3.EMPTY_SHA256 + assert out.read_bytes() == b"y" * 3000 + assert not (tmp_path / "b.dump.partial").exists() + + +def test_a_refused_get_says_so_and_writes_nothing(tmp_path): + body = b"AccessDenied" + out = tmp_path / "b.dump" + with _Server(lambda h: (403, body)) as srv, pytest.raises(s3.S3Error, match="b.dump"): + s3.get_file(srv.target(), "b.dump", out) + assert not out.exists() + assert not (tmp_path / "b.dump.partial").exists()