feat: Settings can point backups at an off-site S3 bucket, with a Test destination check
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 3s
CI and images / extension-test (push) Successful in 20s
CI and images / frontend-build (push) Successful in 27s
CI and images / backend-lint-and-test (push) Successful in 40s
CI and images / integration (push) Failing after 19s
CI and images / sign-extension (push) Skipped
CI and images / build-web (push) Skipped
CI and images / smoke-web (push) Skipped
CI and images / promote (push) Skipped
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / build-agent (push) Skipped

Settings → Maintenance → Storage gains "Off-site backup destination": the
address (with its port), bucket, region, access key and secret key of an S3
bucket such as Fabled Archives. The secret key is sealed with the credential
key, never returned by the API, and kept when the field is left blank. Test
destination lists the bucket with the fields as typed and writes nothing, and
names the cause when it fails ("nothing is listening there (check the port)",
"the secret key is wrong", "answered, but not as S3").

services/backup_s3 is a standard-library SigV4 client (put, get, paged list)
ported from Grimoire's; it is held to AWS's published signing examples and
exercised against a local HTTP server. Migration 0118 adds the columns; no
destination is set up until someone saves one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-06 16:04:57 -05:00
co-authored by Claude Opus 5.5
parent 315399c4e5
commit 4477d7d14f
10 changed files with 1146 additions and 1 deletions
@@ -0,0 +1,43 @@
"""import_settings: the off-site backup destination (an S3 bucket).
Address, bucket, access key and region are plain settings; the secret key is
stored sealed with the credential key. All empty means no destination is set
up, which is every existing install's state.
Revision ID: 0118
Revises: 0117
Create Date: 2026-11-06
"""
import sqlalchemy as sa
from alembic import op
revision = "0118"
down_revision = "0117"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("import_settings", sa.Column("backup_s3_endpoint", sa.Text(), nullable=True))
op.add_column(
"import_settings", sa.Column("backup_s3_bucket", sa.String(255), nullable=True),
)
op.add_column("import_settings", sa.Column("backup_s3_access_key", sa.Text(), nullable=True))
op.add_column(
"import_settings", sa.Column("backup_s3_secret_sealed", sa.Text(), nullable=True),
)
op.add_column(
"import_settings",
sa.Column(
"backup_s3_region", sa.String(64), nullable=False, server_default="us-east-1",
),
)
def downgrade() -> None:
for column in (
"backup_s3_region", "backup_s3_secret_sealed", "backup_s3_access_key",
"backup_s3_bucket", "backup_s3_endpoint",
):
op.drop_column("import_settings", column)
+103
View File
@@ -236,6 +236,109 @@ async def get_settings():
})
@system_backup_bp.route("/destination", methods=["GET"])
async def get_destination():
"""The off-site destination as the settings screen shows it. The secret
key is never returned, only whether one is saved."""
from ..services import backup_destination as dest
async with get_session() as session:
row = await ImportSettings.load(session)
return jsonify(dest.describe(row))
@system_backup_bp.route("/destination", methods=["PATCH"])
async def patch_destination():
"""Save the destination. Body {endpoint, bucket, access_key, region,
secret_key}; an empty field clears it. A missing or empty secret_key
keeps the saved one, so the screen can save the other fields without the
secret being typed again."""
from ..services import backup_destination as dest
body = await request.get_json(silent=True)
if not isinstance(body, dict):
return _bad("invalid_body", detail="body must be a JSON object")
fields, why = dest.clean_fields(body)
if fields is None:
return _bad("invalid_destination", detail=why)
secret = body.get("secret_key")
if secret is not None and not isinstance(secret, str):
return _bad("invalid_destination", detail="secret_key must be text")
async with get_session() as session:
row = await ImportSettings.load(session)
row.backup_s3_endpoint = fields.endpoint
row.backup_s3_bucket = fields.bucket
row.backup_s3_access_key = fields.access_key
row.backup_s3_region = fields.region
if secret and secret.strip():
row.backup_s3_secret_sealed = dest.seal(secret.strip())
await session.commit()
return jsonify(dest.describe(row))
@system_backup_bp.route("/destination", methods=["DELETE"])
async def delete_destination():
"""Forget the destination, secret included. What was already sent stays in
the bucket: FabledCurator never deletes there."""
from ..services import backup_destination as dest
async with get_session() as session:
row = await ImportSettings.load(session)
row.backup_s3_endpoint = None
row.backup_s3_bucket = None
row.backup_s3_access_key = None
row.backup_s3_secret_sealed = None
row.backup_s3_region = dest.DEFAULT_REGION
await session.commit()
return jsonify(dest.describe(row))
@system_backup_bp.route("/destination/test", methods=["POST"])
async def try_destination():
"""Reach the bucket with the fields in the body (the form as typed, saved
or not); a missing secret_key uses the saved one. Lists the bucket and
writes nothing. Answers {ok, message} either way: a destination that
can't be reached is a result to show, not an API error."""
import asyncio
from ..services import backup_destination as dest
from ..services.backup_s3 import S3Error
from ..services.credential_crypto import InvalidCredentialBlob
body = await request.get_json(silent=True) or {}
if not isinstance(body, dict):
return _bad("invalid_body", detail="body must be a JSON object")
fields, why = dest.clean_fields(body)
if fields is None:
return jsonify({"ok": False, "message": f"Not tested: {why}."})
secret = body.get("secret_key")
secret = secret.strip() if isinstance(secret, str) else ""
if not secret:
async with get_session() as session:
row = await ImportSettings.load(session)
sealed = row.backup_s3_secret_sealed
if sealed:
try:
secret = dest.unseal(sealed)
except InvalidCredentialBlob:
return jsonify({"ok": False, "message": (
"Not tested: the saved secret key can't be opened on this server. "
"Enter the secret key again."
)})
target = dest.target_for(fields, secret or None)
if target is None:
missing = [name for name, v in (
("address", fields.endpoint), ("bucket", fields.bucket),
("access key", fields.access_key), ("secret key", secret),
) if not v]
return jsonify({"ok": False, "message": f"Not tested: fill in the {', '.join(missing)}."})
try:
return jsonify(await asyncio.to_thread(dest.check_target, target))
except S3Error as exc:
return jsonify({"ok": False, "message": str(exc)})
@system_backup_bp.route("/settings", methods=["PATCH"])
async def patch_settings():
body = await request.get_json(silent=True)
+11
View File
@@ -9,6 +9,7 @@ from sqlalchemy import (
CheckConstraint,
Float,
Integer,
String,
Text,
select,
text,
@@ -100,6 +101,16 @@ class ImportSettings(Base):
Integer, nullable=False, default=3,
server_default="3",
)
# Off-site backup destination: an S3 bucket (services/backup_destination).
# The secret key is stored sealed with the credential key, never in clear.
backup_s3_endpoint: Mapped[str | None] = mapped_column(Text, nullable=True)
backup_s3_bucket: Mapped[str | None] = mapped_column(String(255), nullable=True)
backup_s3_access_key: Mapped[str | None] = mapped_column(Text, nullable=True)
backup_s3_secret_sealed: Mapped[str | None] = mapped_column(Text, nullable=True)
backup_s3_region: Mapped[str] = mapped_column(
String(64), nullable=False, default="us-east-1",
server_default="us-east-1",
)
# FC-6.3 series continuation matcher. enabled gates the rescan; threshold is
# the weighted-score cut-off (0..1) above which a pending suggestion is made.
+137
View File
@@ -0,0 +1,137 @@
"""The off-site backup destination: an S3 bucket (Fabled Archives, or any S3
service) that the database dumps and the media blobs are sent to.
The address, bucket, access key and region are plain settings. The secret key
is sealed at rest with the credential key (`/images/secrets`), the same key
that seals account logins, and no API ever returns it: the settings answer
only whether one is set. Because the credential key is kept out of every
backup, a database dump carries the secret only as ciphertext nobody can open
without the original server.
"""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import urlsplit
from .backup_s3 import S3Error, S3Target, list_objects
from .credential_crypto import CredentialCrypto, InvalidCredentialBlob
KEY_PATH = Path("/images/secrets/credential_key.b64")
DEFAULT_REGION = "us-east-1"
_TEST_LISTING = 5
def _crypto() -> CredentialCrypto:
return CredentialCrypto(KEY_PATH)
def seal(secret: str) -> str:
return _crypto().encrypt(secret).decode("ascii")
def unseal(sealed: str) -> str:
return _crypto().decrypt(sealed.encode("ascii"))
@dataclass(frozen=True)
class DestinationFields:
endpoint: str | None
bucket: str | None
access_key: str | None
region: str
def clean_fields(body: dict) -> tuple[DestinationFields | None, str | None]:
"""The destination fields from a request body, trimmed, with empty ones as
None; or (None, why) when the address can't be an S3 endpoint."""
def text(name: str) -> str | None:
v = body.get(name)
if v is None:
return None
if not isinstance(v, str):
raise TypeError(name)
return v.strip() or None
try:
endpoint = text("endpoint")
bucket = text("bucket")
access_key = text("access_key")
region = text("region") or DEFAULT_REGION
except TypeError as exc:
return None, f"{exc.args[0]} must be text"
if endpoint is not None:
u = urlsplit(endpoint)
if u.scheme not in ("http", "https") or not u.hostname:
return None, ("the address must start with http:// or https:// and name "
"a host, e.g. http://archives.example:8081")
if u.query or u.fragment:
return None, "the address can't carry a query or a fragment"
endpoint = endpoint.rstrip("/")
if bucket is not None and "/" in bucket:
return None, "the bucket is a name, not a path: put any path in the address"
return DestinationFields(endpoint, bucket, access_key, region), None
def target_for(fields: DestinationFields, secret: str | None) -> S3Target | None:
"""The S3 target, or None while any part of it is missing."""
if not (fields.endpoint and fields.bucket and fields.access_key and secret):
return None
return S3Target(fields.endpoint, fields.bucket, fields.access_key, secret,
region=fields.region)
def saved_target(settings) -> S3Target | None:
"""The destination saved in ImportSettings, or None when none is set up.
Raises S3Error when the secret can't be opened (the credential key
changed since it was saved)."""
sealed = settings.backup_s3_secret_sealed
if not sealed:
return None
try:
secret = unseal(sealed)
except InvalidCredentialBlob:
raise S3Error(
"The saved secret key can't be opened on this server (the credential "
"key changed since it was saved). Enter the secret key again."
) from None
return target_for(fields_of(settings), secret)
def fields_of(settings) -> DestinationFields:
return DestinationFields(
settings.backup_s3_endpoint, settings.backup_s3_bucket,
settings.backup_s3_access_key, settings.backup_s3_region or DEFAULT_REGION,
)
def describe(settings) -> dict:
"""The destination as the settings screen shows it: never the secret."""
f = fields_of(settings)
return {
"endpoint": f.endpoint or "",
"bucket": f.bucket or "",
"access_key": f.access_key or "",
"region": f.region,
"secret_set": bool(settings.backup_s3_secret_sealed),
"configured": bool(
f.endpoint and f.bucket and f.access_key and settings.backup_s3_secret_sealed
),
}
def check_target(target: S3Target) -> dict:
"""Reach the bucket by listing it. A listing proves the address, the
bucket and both keys at once, and writes nothing, because an Archives key
is allowed to add objects but a test object could never be removed."""
try:
page = list_objects(target, limit=_TEST_LISTING)
except S3Error as exc:
return {"ok": False, "message": str(exc)}
if not page.objects:
said = f"Reached {target.bucket} at {target.endpoint}. The bucket is empty."
else:
more = "+" if page.next_token else ""
said = (f"Reached {target.bucket} at {target.endpoint}. It holds "
f"{len(page.objects)}{more} object(s), e.g. {page.objects[0].key}.")
return {"ok": True, "message": said}
+282
View File
@@ -0,0 +1,282 @@
"""A small S3 client for sending backups off the box: AWS Signature Version 4,
path-style addressing (endpoint/bucket/key), and the calls a backup needs —
put an object, read one back, list a prefix.
FabledCurator puts, lists and reads; it never deletes. The bucket keeps the
history and thins it on its own policy, so a key that may only add and read
objects is enough, and is the safer key to hand a service: a stolen one cannot
wipe the history it writes into.
Built on the standard library's http.client rather than boto3: three calls
don't justify a large dependency, and signing is a pure function the tests
hold to AWS's own published examples. Every call is synchronous, because the
uploads run in Celery workers; the API runs the Test destination call in a
thread.
"""
from __future__ import annotations
import hashlib
import hmac
import http.client
import os
import socket
import ssl
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit
EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
TIMEOUT_SECONDS = 30.0
# A single object can be a multi-gigabyte video or archive; a slow link still
# gets this long between bytes before the call gives up.
TRANSFER_TIMEOUT_SECONDS = 15 * 60
CHUNK = 1 << 20
class S3Error(Exception):
"""Why a call to the bucket failed, said to the person."""
@dataclass(frozen=True)
class S3Target:
endpoint: str # http(s)://host[:port][/prefix], no bucket
bucket: str
access_key: str
secret_key: str
region: str = "us-east-1"
verify_tls: bool = True
def url(self, key: str = "") -> str:
base = f"{self.endpoint.rstrip('/')}/{quote(self.bucket, safe='')}"
return f"{base}/{quote(key, safe='/-_.~')}" if key else f"{base}/"
@dataclass(frozen=True)
class StoredObject:
key: str
size: int
last_modified: str
@dataclass(frozen=True)
class ListPage:
objects: list[StoredObject]
next_token: str | None # pass back as `continuation` for the next page
# ── Signing ──────────────────────────────────────────────────────────────────
def _encode(text: str, keep_slash: bool = False) -> str:
return quote(text, safe="-_.~" + ("/" if keep_slash else ""))
def _hmac(key: bytes, msg: str) -> bytes:
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
def sign(method: str, url: str, headers: dict[str, str], payload_sha256: str,
access_key: str, secret_key: str, region: str, at: datetime,
service: str = "s3") -> dict[str, str]:
"""The request's headers with Host, x-amz-date, x-amz-content-sha256 and
Authorization added. Every header given is signed, Host always."""
u = urlsplit(url)
stamp = at.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ")
day = stamp[:8]
out = dict(headers)
out["Host"] = u.netloc
out["x-amz-date"] = stamp
out["x-amz-content-sha256"] = payload_sha256
signed = {k.lower(): " ".join(str(v).split()) for k, v in out.items()}
names = sorted(signed)
query = sorted(
(_encode(k), _encode(v)) for k, v in parse_qsl(u.query, keep_blank_values=True)
)
canonical = "\n".join([
method.upper(),
_encode(unquote(u.path) or "/", keep_slash=True),
"&".join(f"{k}={v}" for k, v in query),
"".join(f"{n}:{signed[n]}\n" for n in names),
";".join(names),
payload_sha256,
])
scope = f"{day}/{region}/{service}/aws4_request"
to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope,
hashlib.sha256(canonical.encode()).hexdigest()])
key = _hmac(_hmac(_hmac(_hmac(f"AWS4{secret_key}".encode(), day), region),
service), "aws4_request")
signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
out["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, "
f"SignedHeaders={';'.join(names)}, Signature={signature}")
return out
# ── Transport ────────────────────────────────────────────────────────────────
def _connect(target: S3Target, url: str, timeout: float) -> http.client.HTTPConnection:
u = urlsplit(url)
if u.scheme == "https":
ctx = ssl.create_default_context()
if not target.verify_tls:
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
return http.client.HTTPSConnection(
u.hostname, u.port, timeout=timeout, context=ctx, blocksize=CHUNK,
)
if u.scheme != "http":
raise S3Error(f"{target.endpoint} isn't an http:// or https:// address.")
return http.client.HTTPConnection(u.hostname, u.port, timeout=timeout, blocksize=CHUNK)
def _request_target(url: str) -> str:
u = urlsplit(url)
return (u.path or "/") + (f"?{u.query}" if u.query else "")
def _reach_error(target: S3Target, exc: Exception) -> S3Error:
if isinstance(exc, TimeoutError):
return S3Error(f"{target.endpoint} didn't answer in time.")
if isinstance(exc, socket.gaierror):
return S3Error(f"Couldn't reach {target.endpoint}: the host name doesn't resolve.")
if isinstance(exc, ConnectionRefusedError):
return S3Error(
f"Couldn't reach {target.endpoint}: nothing is listening there "
"(check the port)."
)
if isinstance(exc, ssl.SSLError):
return S3Error(f"Couldn't reach {target.endpoint}: TLS failed ({exc.reason}).")
return S3Error(f"Couldn't reach {target.endpoint}: {exc}.")
def _says(status: int, reason: str, content: bytes) -> str:
"""An S3 error response in words: its Code and Message when it has them."""
try:
root = ET.fromstring(content) if content else None
code = (root.findtext("Code") or "") if root is not None else ""
message = (root.findtext("Message") or "") if root is not None else ""
except ET.ParseError:
code, message = "", ""
hints = {
"SignatureDoesNotMatch": "the secret key is wrong",
"InvalidAccessKeyId": "the access key isn't known there",
"NoSuchBucket": "there is no such bucket",
"AccessDenied": "the key isn't allowed to do that",
}
said = hints.get(code) or message or reason
if not code and 400 <= status < 500:
# S3 refuses with an <Error><Code> body. A 4xx without one came from
# something else at that address — most often a web server on the
# default port, when the S3 service listens on a port of its own.
said = ("the address answered, but not as S3 — check the scheme and "
"the port (an S3 service usually listens on its own port)")
return f"HTTP {status}{f' {code}' if code else ''}: {said}."
# ── Calls ────────────────────────────────────────────────────────────────────
def file_sha256(path: Path) -> str:
h = hashlib.sha256()
with open(path, "rb") as f:
while chunk := f.read(CHUNK):
h.update(chunk)
return h.hexdigest()
def put_file(target: S3Target, key: str, path: Path, *,
content_type: str = "application/octet-stream",
payload_sha256: str | None = None,
now: datetime | None = None) -> None:
"""Upload one file as one object, streamed from disk. `payload_sha256` is
the file's sha256 when the caller already knows it (the library stores
every file's), which saves reading the file twice."""
digest = payload_sha256 or file_sha256(path)
size = path.stat().st_size
url = target.url(key)
headers = sign("PUT", url, {"Content-Length": str(size), "Content-Type": content_type},
digest, target.access_key, target.secret_key, target.region,
now or datetime.now(UTC))
conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS)
try:
with open(path, "rb") as f:
conn.request("PUT", _request_target(url), body=f, headers=headers)
response = conn.getresponse()
content = response.read()
except (OSError, http.client.HTTPException) as exc:
raise _reach_error(target, exc) from None
finally:
conn.close()
if response.status // 100 != 2:
raise S3Error(f"Upload of {key} refused: "
f"{_says(response.status, response.reason, content)}")
def get_file(target: S3Target, key: str, path: Path, *,
now: datetime | None = None) -> None:
"""Download one object to `path`, streamed to disk. The bytes land under a
`.partial` name and are renamed into place when complete, so a broken
transfer never leaves a truncated file at `path`."""
url = target.url(key)
headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
target.region, now or datetime.now(UTC))
partial = path.with_name(path.name + ".partial")
conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS)
try:
conn.request("GET", _request_target(url), headers=headers)
response = conn.getresponse()
if response.status != 200:
raise S3Error(f"Reading {key} refused: "
f"{_says(response.status, response.reason, response.read())}")
with open(partial, "wb") as f:
while chunk := response.read(CHUNK):
f.write(chunk)
os.replace(partial, path)
except (OSError, http.client.HTTPException) as exc:
partial.unlink(missing_ok=True)
raise _reach_error(target, exc) from None
finally:
conn.close()
def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *,
continuation: str | None = None,
now: datetime | None = None) -> ListPage:
"""One page (up to `limit`) of the objects under `prefix`, ListObjectsV2."""
params = {"list-type": "2", "max-keys": str(limit)}
if prefix:
params["prefix"] = prefix
if continuation:
params["continuation-token"] = continuation
url = f"{target.url()}?{urlencode(params, quote_via=quote)}"
headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
target.region, now or datetime.now(UTC))
conn = _connect(target, url, TIMEOUT_SECONDS)
try:
conn.request("GET", _request_target(url), headers=headers)
response = conn.getresponse()
content = response.read()
except (OSError, http.client.HTTPException) as exc:
raise _reach_error(target, exc) from None
finally:
conn.close()
if response.status != 200:
raise S3Error(f"Listing {target.bucket} refused: "
f"{_says(response.status, response.reason, content)}")
try:
root = ET.fromstring(content)
except ET.ParseError:
raise S3Error(
f"{target.endpoint} answered a listing with something that isn't S3's XML."
) from None
objects, next_token = [], None
for item in root.iter():
tag = item.tag.rsplit("}", 1)[-1]
if tag == "NextContinuationToken":
next_token = item.text or None
if tag != "Contents":
continue
fields = {child.tag.rsplit("}", 1)[-1]: (child.text or "") for child in item}
objects.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0),
fields.get("LastModified", "")))
return ListPage(objects, next_token)
@@ -0,0 +1,198 @@
<!--
Off-site backup destination: an S3 bucket (Fabled Archives or any S3
service) that backups are sent to. The five fields only work together, so
they save as one with an explicit button rather than field by field. The
secret key is write-only: once saved it is never shown again, and leaving the
field blank keeps it. Test destination lists the bucket with what is typed
in the form (saved or not) and writes nothing.
-->
<template>
<MaintenanceTile
icon="mdi-cloud-upload-outline"
title="Off-site backup destination"
blurb="The S3 bucket (Fabled Archives or any S3 service) that backups are sent to."
>
<p class="fc-muted text-body-2 mb-3">
Backups sent here survive losing this server. FabledCurator only adds to the
bucket and never deletes from it, so a key that may add and read objects but
not delete them is enough, and is the safer key to use. Account logins are
never sent.
</p>
<v-alert
v-if="loadError" type="warning" variant="tonal" density="compact" class="mb-3"
>{{ loadError }}</v-alert>
<v-text-field
v-model="form.endpoint" label="Address"
placeholder="http://archives.example:8081" density="compact"
hint="Include the port the S3 service listens on." persistent-hint
class="mb-3" :disabled="busy"
/>
<div class="fc-dest-row mb-3">
<v-text-field
v-model="form.bucket" label="Bucket" density="compact" hide-details
:disabled="busy"
/>
<v-text-field
v-model="form.region" label="Region" density="compact" hide-details
class="fc-dest-region" :disabled="busy"
/>
</div>
<v-text-field
v-model="form.access_key" label="Access key" density="compact" hide-details
autocomplete="off" class="mb-3" :disabled="busy"
/>
<v-text-field
v-model="form.secret_key" label="Secret key" type="password"
autocomplete="new-password" density="compact"
:placeholder="secretSet ? 'Saved. Leave blank to keep it.' : ''"
:persistent-placeholder="secretSet"
:hint="secretSet ? 'A saved secret is never shown again.' : ''" persistent-hint
class="mb-3" :disabled="busy"
/>
<div class="d-flex align-center flex-wrap mb-2" style="gap: 8px;">
<v-btn
size="small" variant="flat" color="accent" rounded="pill"
prepend-icon="mdi-content-save" :loading="saving"
:disabled="busy || !dirty" @click="onSave"
>Save</v-btn>
<v-btn
size="small" variant="tonal" rounded="pill"
prepend-icon="mdi-lan-connect" :loading="testing"
:disabled="busy || !form.endpoint" @click="onTest"
>Test destination</v-btn>
<v-btn
v-if="configured" size="small" variant="text" rounded="pill" color="error"
:disabled="busy" @click="confirmRemove = true"
>Remove</v-btn>
<span v-if="savedNote" class="text-caption text-success">{{ savedNote }}</span>
</div>
<v-alert
v-if="testResult" :type="testResult.ok ? 'success' : 'error'"
variant="tonal" density="compact" class="mb-1"
>{{ testResult.message }}</v-alert>
<p v-if="!configured && !testResult" class="fc-muted text-caption mb-0">
No destination is set up, so nothing is sent off this server yet.
</p>
<v-dialog v-model="confirmRemove" max-width="440">
<v-card>
<v-card-title class="text-h6">Remove the destination?</v-card-title>
<v-card-text class="text-body-2">
FabledCurator forgets the address and both keys and stops sending
backups there. What was already sent stays in the bucket.
</v-card-text>
<v-card-actions>
<v-spacer />
<v-btn variant="text" @click="confirmRemove = false">Cancel</v-btn>
<v-btn color="error" variant="flat" :loading="saving" @click="onRemove">
Remove
</v-btn>
</v-card-actions>
</v-card>
</v-dialog>
</MaintenanceTile>
</template>
<script setup>
import { computed, onMounted, reactive, ref } from 'vue'
import { useBackupStore } from '../../stores/backup.js'
import MaintenanceTile from '../common/MaintenanceTile.vue'
const store = useBackupStore()
const FIELDS = ['endpoint', 'bucket', 'access_key', 'region']
const form = reactive({ endpoint: '', bucket: '', access_key: '', region: 'us-east-1', secret_key: '' })
const saved = ref(null)
const loadError = ref('')
const saving = ref(false)
const testing = ref(false)
const testResult = ref(null)
const savedNote = ref('')
const confirmRemove = ref(false)
const busy = computed(() => saving.value || testing.value)
const secretSet = computed(() => Boolean(saved.value?.secret_set))
const configured = computed(() => Boolean(saved.value?.configured))
const dirty = computed(() => (
!saved.value
|| form.secret_key.trim() !== ''
|| FIELDS.some((f) => (form[f] || '').trim() !== (saved.value[f] || ''))
))
function apply(dest) {
saved.value = dest
for (const f of FIELDS) form[f] = dest[f] || (f === 'region' ? 'us-east-1' : '')
form.secret_key = ''
}
function fields() {
const out = {}
for (const f of FIELDS) out[f] = form[f]
if (form.secret_key.trim()) out.secret_key = form.secret_key
return out
}
function messageOf(e) {
return e?.body?.detail || e?.message || 'The request failed.'
}
onMounted(async () => {
try {
apply(await store.loadDestination())
} catch (e) {
loadError.value = `Couldn't load the destination: ${messageOf(e)}`
}
})
async function onSave() {
saving.value = true
savedNote.value = ''
testResult.value = null
try {
apply(await store.saveDestination(fields()))
savedNote.value = 'Saved.'
} catch (e) {
testResult.value = { ok: false, message: `Not saved: ${messageOf(e)}` }
} finally {
saving.value = false
}
}
async function onTest() {
testing.value = true
savedNote.value = ''
testResult.value = null
try {
testResult.value = await store.testDestination(fields())
} catch (e) {
testResult.value = { ok: false, message: messageOf(e) }
} finally {
testing.value = false
}
}
async function onRemove() {
saving.value = true
try {
apply(await store.removeDestination())
testResult.value = null
savedNote.value = 'Removed.'
confirmRemove.value = false
} catch (e) {
testResult.value = { ok: false, message: `Not removed: ${messageOf(e)}` }
} finally {
saving.value = false
}
}
</script>
<style scoped>
.fc-dest-row { display: flex; gap: 12px; flex-wrap: wrap; }
.fc-dest-row > :first-child { flex: 1 1 200px; }
.fc-dest-region { flex: 0 1 160px; }
</style>
@@ -63,8 +63,11 @@
<section class="fc-section">
<h3 class="fc-section__title">Storage</h3>
<p class="fc-section__hint">Database + image backups and restore.</p>
<p class="fc-section__hint">
Database + image backups and restore, and where backups are sent off this server.
</p>
<div class="fc-tile-stack">
<BackupDestinationCard />
<BackupCard />
</div>
</section>
@@ -93,6 +96,7 @@ import PostAssociationsCard from './PostAssociationsCard.vue'
import GpuAgentCard from './GpuAgentCard.vue'
import AliasTable from './AliasTable.vue'
import BackupCard from './BackupCard.vue'
import BackupDestinationCard from './BackupDestinationCard.vue'
import { useSystemActivityStore } from '../../stores/systemActivity.js'
// Poll queue depths so each task tile's QueueStatusBar shows live pending
+25
View File
@@ -101,9 +101,34 @@ export const useBackupStore = defineStore('backup', () => {
}
}
// Off-site destination (an S3 bucket). The secret key goes up but never
// comes back: the server answers `secret_set` instead.
const destination = ref(null)
async function loadDestination() {
destination.value = await api.get('/api/system/backup/destination')
return destination.value
}
async function saveDestination(fields) {
destination.value = await api.patch('/api/system/backup/destination', { body: fields })
return destination.value
}
async function removeDestination() {
destination.value = await api.delete('/api/system/backup/destination')
return destination.value
}
// Resolves to {ok, message}; an unreachable bucket is a result, not a throw.
async function testDestination(fields) {
return api.post('/api/system/backup/destination/test', { body: fields })
}
return {
dbRuns, imagesRuns, settings, loading, lastError,
loadRuns, triggerBackup, restore, deleteRun, setTag,
loadSettings, patchSettings,
destination, loadDestination, saveDestination, removeDestination, testDestination,
}
})
+133
View File
@@ -322,3 +322,136 @@ async def test_patch_settings_rejects_keep_n_below_min(client):
json={"backup_db_keep_last_n": 0},
)
assert resp.status_code == 400
# --- /destination ---------------------------------------------------
@pytest.fixture
def _sealing_key(tmp_path, monkeypatch):
from backend.app.services import backup_destination
monkeypatch.setattr(backup_destination, "KEY_PATH", tmp_path / "credential_key.b64")
_DEST = {
"endpoint": "http://archives.example:8081/",
"bucket": "curator",
"access_key": "AKEXAMPLE",
"region": "",
"secret_key": "s3cr3t-example",
}
@pytest.mark.asyncio
async def test_destination_saves_and_never_returns_the_secret(client, db_sync, _sealing_key):
from backend.app.models import ImportSettings
resp = await client.patch("/api/system/backup/destination", json=_DEST)
assert resp.status_code == 200
body = await resp.get_json()
assert body == {
"endpoint": "http://archives.example:8081", "bucket": "curator",
"access_key": "AKEXAMPLE", "region": "us-east-1",
"secret_set": True, "configured": True,
}
assert "s3cr3t-example" not in str(await (await client.get(
"/api/system/backup/destination")).get_json())
row = db_sync.execute(select(ImportSettings).where(ImportSettings.id == 1)).scalar_one()
db_sync.refresh(row)
assert row.backup_s3_secret_sealed and "s3cr3t-example" not in row.backup_s3_secret_sealed
from backend.app.services import backup_destination
assert backup_destination.unseal(row.backup_s3_secret_sealed) == "s3cr3t-example"
@pytest.mark.asyncio
async def test_destination_blank_secret_keeps_the_saved_one(client, db_sync, _sealing_key):
from backend.app.models import ImportSettings
from backend.app.services import backup_destination
await client.patch("/api/system/backup/destination", json=_DEST)
resp = await client.patch(
"/api/system/backup/destination", json={**_DEST, "bucket": "other", "secret_key": ""},
)
assert (await resp.get_json())["bucket"] == "other"
row = db_sync.execute(select(ImportSettings).where(ImportSettings.id == 1)).scalar_one()
db_sync.refresh(row)
assert backup_destination.unseal(row.backup_s3_secret_sealed) == "s3cr3t-example"
@pytest.mark.asyncio
@pytest.mark.parametrize("patch, word", [
({"endpoint": "archives.example:8081"}, "http://"),
({"endpoint": "ftp://archives.example"}, "http://"),
({"bucket": "curator/sub"}, "bucket"),
({"access_key": 5}, "access_key"),
])
async def test_destination_rejects_what_cant_be_an_s3_target(client, _sealing_key, patch, word):
resp = await client.patch("/api/system/backup/destination", json={**_DEST, **patch})
assert resp.status_code == 400
assert word in (await resp.get_json())["detail"]
@pytest.mark.asyncio
async def test_destination_remove_forgets_everything(client, _sealing_key):
await client.patch("/api/system/backup/destination", json=_DEST)
resp = await client.delete("/api/system/backup/destination")
assert await resp.get_json() == {
"endpoint": "", "bucket": "", "access_key": "", "region": "us-east-1",
"secret_set": False, "configured": False,
}
@pytest.mark.asyncio
async def test_destination_test_uses_the_form_and_the_saved_secret(
client, _sealing_key, monkeypatch,
):
from backend.app.services import backup_destination
seen = []
def fake_check(target):
seen.append(target)
return {"ok": True, "message": "Reached it."}
monkeypatch.setattr(backup_destination, "check_target", fake_check)
await client.patch("/api/system/backup/destination", json=_DEST)
resp = await client.post(
"/api/system/backup/destination/test",
json={**_DEST, "bucket": "typed-not-saved", "secret_key": ""},
)
assert await resp.get_json() == {"ok": True, "message": "Reached it."}
assert seen[0].bucket == "typed-not-saved"
assert seen[0].secret_key == "s3cr3t-example"
assert seen[0].endpoint == "http://archives.example:8081"
@pytest.mark.asyncio
async def test_destination_test_names_what_is_missing(client, _sealing_key):
await client.delete("/api/system/backup/destination")
resp = await client.post(
"/api/system/backup/destination/test",
json={"endpoint": "http://archives.example:8081", "bucket": "curator"},
)
body = await resp.get_json()
assert body["ok"] is False
assert body["message"] == "Not tested: fill in the access key, secret key."
@pytest.mark.asyncio
async def test_destination_test_reports_an_unreachable_bucket_as_a_result(
client, _sealing_key, monkeypatch,
):
from backend.app.services import backup_destination
from backend.app.services.backup_s3 import S3Error
def refuse(target, **kw):
raise S3Error("Couldn't reach http://archives.example:8081: nothing is listening there.")
monkeypatch.setattr(backup_destination, "list_objects", refuse)
resp = await client.post("/api/system/backup/destination/test", json=_DEST)
assert resp.status_code == 200
assert await resp.get_json() == {
"ok": False,
"message": "Couldn't reach http://archives.example:8081: nothing is listening there.",
}
+209
View File
@@ -0,0 +1,209 @@
"""The backup destination's S3 client (services/backup_s3.py): signatures match
AWS's published Signature Version 4 examples, and the calls send and read what
S3 expects, over a real HTTP connection to a local stand-in server."""
import socket
import threading
from datetime import UTC, datetime
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pytest
from backend.app.services import backup_s3 as s3
# AWS's example credentials and moment ("Signature Calculations for the
# Authorization Header", Amazon S3 API reference).
KEY_ID = "AKIAIOSFODNN7EXAMPLE"
SECRET = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
AT = datetime(2013, 5, 24, tzinfo=UTC)
def _signature(headers):
return headers["Authorization"].rsplit("Signature=", 1)[1]
def test_aws_example_get_object():
h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt",
{"Range": "bytes=0-9"}, s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
assert "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date," in h["Authorization"]
assert "Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request" in h[
"Authorization"]
assert _signature(h) == "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
def test_aws_example_list_objects():
h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", {},
s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
assert _signature(h) == "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"
def test_a_changed_secret_changes_the_signature():
url = "https://examplebucket.s3.amazonaws.com/test.txt"
a = s3.sign("GET", url, {}, s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
b = s3.sign("GET", url, {}, s3.EMPTY_SHA256, KEY_ID, SECRET + "x", "us-east-1", AT)
assert _signature(a) != _signature(b)
def test_paths_are_path_style_and_keys_are_escaped():
t = s3.S3Target("http://archives.example:8081/s3", "curator", KEY_ID, SECRET)
assert t.url("media/ab/cd/abcd.jpg") == "http://archives.example:8081/s3/curator/media/ab/cd/abcd.jpg"
assert t.url() == "http://archives.example:8081/s3/curator/"
assert t.url("a b+c.jpg") == "http://archives.example:8081/s3/curator/a%20b%2Bc.jpg"
# ── Calls, against a local stand-in ─────────────────────────────────────────
class _Server:
"""A local HTTP server that records each request and answers with
`reply(handler) -> (status, body)`."""
def __init__(self, reply):
self.seen = []
outer = self
class Handler(BaseHTTPRequestHandler):
def _serve(self):
length = int(self.headers.get("Content-Length") or 0)
outer.seen.append({
"method": self.command, "path": self.path,
"headers": {k.lower(): v for k, v in self.headers.items()},
"body": self.rfile.read(length) if length else b"",
})
status, body = reply(self)
self.send_response(status)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
do_GET = do_PUT = _serve
def log_message(self, *args):
pass
self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
def __enter__(self):
self.thread.start()
return self
def __exit__(self, *exc):
self.httpd.shutdown()
self.httpd.server_close()
def target(self, bucket="curator"):
host, port = self.httpd.server_address
return s3.S3Target(f"http://{host}:{port}", bucket, KEY_ID, SECRET)
def test_put_streams_the_file_signed(tmp_path):
f = tmp_path / "abcd.jpg"
f.write_bytes(b"x" * 5000)
with _Server(lambda h: (200, b"")) as srv:
s3.put_file(srv.target(), "media/ab/cd/abcd.jpg", f, content_type="image/jpeg")
req = srv.seen[0]
assert req["method"] == "PUT"
assert req["path"] == "/curator/media/ab/cd/abcd.jpg"
assert req["body"] == b"x" * 5000
assert req["headers"]["x-amz-content-sha256"] == s3.file_sha256(f)
assert req["headers"]["content-type"] == "image/jpeg"
assert req["headers"]["authorization"].startswith(
"AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/")
def test_put_signs_with_a_known_sha_without_rereading(tmp_path, monkeypatch):
f = tmp_path / "b.bin"
f.write_bytes(b"y")
monkeypatch.setattr(s3, "file_sha256", lambda p: pytest.fail("read the file twice"))
with _Server(lambda h: (200, b"")) as srv:
s3.put_file(srv.target(), "b.bin", f, payload_sha256="a" * 64)
assert srv.seen[0]["headers"]["x-amz-content-sha256"] == "a" * 64
def test_a_refusal_says_why(tmp_path):
f = tmp_path / "b.bin"
f.write_bytes(b"x")
body = b"<Error><Code>SignatureDoesNotMatch</Code><Message>no</Message></Error>"
with _Server(lambda h: (403, body)) as srv, pytest.raises(
s3.S3Error, match="HTTP 403 SignatureDoesNotMatch: the secret key is wrong",
):
s3.put_file(srv.target(), "b.bin", f)
def test_a_web_server_at_the_address_is_named_as_not_s3():
"""An address without the S3 port lands on whatever serves the default
port, and its plain 404 says nothing about S3."""
page = b"<html><body><h1>404 Not Found</h1></body></html>"
with _Server(lambda h: (404, page)) as srv, pytest.raises(
s3.S3Error,
match=r"HTTP 404: the address answered, but not as S3 — check the scheme and the port",
):
s3.list_objects(srv.target())
def test_a_server_error_without_a_body_is_not_blamed_on_the_port():
with _Server(lambda h: (503, b"")) as srv, pytest.raises(
s3.S3Error, match="HTTP 503: Service Unavailable",
) as err:
s3.list_objects(srv.target())
assert "port" not in str(err.value)
def test_nothing_listening_says_check_the_port():
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
port = sock.getsockname()[1]
target = s3.S3Target(f"http://127.0.0.1:{port}", "curator", KEY_ID, SECRET)
with pytest.raises(s3.S3Error, match="nothing is listening there"):
s3.list_objects(target)
def test_list_reads_contents_and_the_next_page_token():
xml = b"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>curator</Name><KeyCount>2</KeyCount><IsTruncated>true</IsTruncated>
<NextContinuationToken>tok-2</NextContinuationToken>
<Contents><Key>database/fabledcurator-20261105T030000Z.dump</Key><Size>1200</Size>
<LastModified>2026-11-05T03:00:01.000Z</LastModified></Contents>
<Contents><Key>database/fabledcurator-20261106T030000Z.dump</Key><Size>1300</Size>
<LastModified>2026-11-06T03:00:01.000Z</LastModified></Contents>
</ListBucketResult>"""
with _Server(lambda h: (200, xml)) as srv:
page = s3.list_objects(srv.target(), "database/", continuation="tok-1")
path = srv.seen[0]["path"]
assert path.startswith("/curator/?")
assert "list-type=2" in path and "prefix=database%2F" in path
assert "continuation-token=tok-1" in path
assert page.next_token == "tok-2"
assert page.objects == [
s3.StoredObject("database/fabledcurator-20261105T030000Z.dump", 1200,
"2026-11-05T03:00:01.000Z"),
s3.StoredObject("database/fabledcurator-20261106T030000Z.dump", 1300,
"2026-11-06T03:00:01.000Z"),
]
def test_a_listing_that_isnt_xml_is_named():
with _Server(lambda h: (200, b"hello")) as srv, pytest.raises(
s3.S3Error, match="isn't S3's XML",
):
s3.list_objects(srv.target())
def test_get_streams_the_object_to_disk(tmp_path):
with _Server(lambda h: (200, b"y" * 3000)) as srv:
out = tmp_path / "b.dump"
s3.get_file(srv.target(), "database/b.dump", out)
assert srv.seen[0]["path"] == "/curator/database/b.dump"
assert srv.seen[0]["headers"]["x-amz-content-sha256"] == s3.EMPTY_SHA256
assert out.read_bytes() == b"y" * 3000
assert not (tmp_path / "b.dump.partial").exists()
def test_a_refused_get_says_so_and_writes_nothing(tmp_path):
body = b"<Error><Code>AccessDenied</Code></Error>"
out = tmp_path / "b.dump"
with _Server(lambda h: (403, body)) as srv, pytest.raises(s3.S3Error, match="b.dump"):
s3.get_file(srv.target(), "b.dump", out)
assert not out.exists()
assert not (tmp_path / "b.dump.partial").exists()