From 49799bd0b31f619a6dd7731ce57d9a25ffd21cb3 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 12 Oct 2026 07:23:58 -0400 Subject: [PATCH] =?UTF-8?q?feat(backup):=20daily=20backups=20to=20an=20S3?= =?UTF-8?q?=20bucket,=20Settings=20=E2=86=92=20Backups,=20and=20the=20guid?= =?UTF-8?q?e=20(#5673,=20#5675)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #5673 "S3 client and the backup job: settings, schedule, history, metrics": - steward/backup/s3.py: SigV4 over httpx, path-style; put and list only (an append-only key is enough; retention is the bucket's). Signing is held to AWS's published examples. - steward/backup/job.py: a 15-minute tick. The daily backup goes at the hour chosen on the reader's clock; a missed day is made up, and a failure is retried an hour later. History of the last 30 runs, audit (backup.made / backup.failed), and backup_age_secs / backup_bytes under "Steward itself", resource backup, for alert rules. - Settings → Backups: verdict first, then the bucket (secret sealed; a blank box keeps it), the schedule, Test the bucket, Back up now, and the recent runs. #5675 "Docs: backing up to Archives, and restoring": docs/core/backups.md. Also: the audit page now words the Prometheus target actions from #5668, which showed as codes, and a guard fails on any logged action without wording. Co-Authored-By: Claude Opus 5.5 --- README.md | 1 + docs/core/backups.md | 117 ++++++++++++ steward/alerts/service.py | 3 +- steward/app.py | 13 ++ steward/audit/service.py | 11 +- steward/backup/job.py | 239 +++++++++++++++++++++++- steward/backup/s3.py | 191 +++++++++++++++++++ steward/core/formatting.py | 2 + steward/core/settings.py | 15 ++ steward/settings/routes.py | 86 +++++++++ steward/templates/settings/_tabs.html | 1 + steward/templates/settings/backups.html | 138 ++++++++++++++ tests/core/test_audit_actions_worded.py | 38 ++++ tests/core/test_backup_job.py | 123 ++++++++++++ tests/core/test_backup_s3.py | 112 +++++++++++ tests/integration/test_backup_job.py | 169 +++++++++++++++++ 16 files changed, 1254 insertions(+), 5 deletions(-) create mode 100644 docs/core/backups.md create mode 100644 steward/backup/s3.py create mode 100644 steward/templates/settings/backups.html create mode 100644 tests/core/test_audit_actions_worded.py create mode 100644 tests/core/test_backup_job.py create mode 100644 tests/core/test_backup_s3.py create mode 100644 tests/integration/test_backup_job.py diff --git a/README.md b/README.md index 3a4e09f..1734c7e 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,7 @@ See `docs/plugins/` for a full plugin development guide. | `docs/core/monitors.md` | Ping and DNS monitors: probe logic, metrics emitted, data models | | `docs/core/alerting.md` | Alert rules, state machine, notification channels, template variables | | `docs/core/ansible.md` | Playbook sources, run lifecycle, SSE streaming | +| `docs/core/backups.md` | Steward's daily backup to an S3 bucket (Fabled Archives), what it holds, and `steward restore` | | `docs/plugins/overview.md` | Plugin system: how loading works, plugin.yaml schema, required exports | | `docs/plugins/writing-a-plugin.md` | Step-by-step plugin development guide | | `docs/plugins/traefik.md` | Traefik plugin: config, metrics, alert rule examples | diff --git a/docs/core/backups.md b/docs/core/backups.md new file mode 100644 index 0000000..223f810 --- /dev/null +++ b/docs/core/backups.md @@ -0,0 +1,117 @@ +# Backups: Steward backing itself up, and coming back + +Steward sends a backup of itself to an S3 bucket once a day. Fabled Archives +is the intended target. A backup is one file, and `steward restore` brings +Steward back from it into an empty database. + +## What a backup holds + +`steward-.tar`, containing: + +| Member | What it is | +|---|---| +| `steward.dump` | `pg_dump --format=custom` of Steward's database | +| `secret.key` | The key Steward's stored passwords and tokens are sealed with | +| `manifest.json` | Format, Steward version, migration heads, time, and the tables whose rows were left out | + +**Kept:** +- configuration: settings, users, API tokens, hosts, monitors, alert rules, maintenance windows, Ansible inventory and schedules, dashboards, SNMP devices, Prometheus targets, UniFi client names; +- the audit log; +- alert history. + +**Left out** (the tables come back empty and fill again as Steward runs): +- metrics and their rollups; +- monitor check results; +- Docker, Traefik and UniFi history; +- Ansible run records; +- the email queue; +- password-reset links. + +The list is `steward/backup/contents.py`. A test fails when a new table +appears in neither list, so whoever adds a table decides. + +**The secret key is in the file.** That makes a restore complete on its own. +It also means anyone holding a copy can read every password and key Steward +keeps. Keep the bucket private. + +## Setting it up with Fabled Archives + +1. In Archives, make a backup target for Steward with an S3 key. The default + append-only key is enough: Steward only uploads and never deletes. + How long backups are kept is set on the Archives side. +2. In Steward, open **Settings → Backups** and fill in: + - **Endpoint**: the S3 address Archives shows for the target, without the + bucket (for example `http://archives:8081`). Requests are path-style: + `endpoint/bucket/file`. + - **Bucket**: the target's bucket name. + - **Access key** and **Secret key**: from Archives. The secret is sealed at + rest; a blank box on save keeps the stored one. +3. Press **Test the bucket**. A key that may not list ("AccessDenied") still + counts as reached: an append-only key can't list, and uploads work anyway. +4. Tick **Back up every day**, pick the hour (on your clock), and save. + **Back up now** makes one straight away. + +A day missed while Steward was down is made up when it comes back. A failed +backup is tried again an hour later, never sooner. The last 30 runs are +listed on the page, and each one is in the audit log (`backup.made` / +`backup.failed`). + +### Alert when backups stop + +While backups are on, Steward records two metrics under the source +**Steward itself**, resource `backup`: + +| Metric | Meaning | +|---|---| +| `backup_age_secs` ("Newest backup's age") | Seconds since the last backup was sent, updated every 15 minutes | +| `backup_bytes` ("Backup size") | The size of the last backup sent | + +A rule such as `backup_age_secs > 93600` (26 hours) fires when a daily backup +has been missed. + +## Making a backup by hand + +```sh +docker exec -u app steward backup --to /tmp/backups +``` + +This writes the file and sends it nowhere. It needs the same database access +as Steward itself. + +## Restoring + +`steward restore` writes into an **empty** database. Pass `--wipe` to drop +everything in the database's public schema first. + +1. Stop Steward (scale its service to 0) so nothing writes during the restore. +2. Download the backup file from Archives. +3. Run the restore with the image, pointed at the database: + + ```sh + docker run --rm \ + -e STEWARD_DATABASE_URL=postgresql+asyncpg://steward:…@db:5432/steward \ + -v steward_data:/data -v "$PWD":/restore \ + git.fabledsword.com/bvandeusen/steward:latest \ + steward restore /restore/steward-20261012T030000Z.tar + ``` + + - The backup's secret key is written to `/data/secret.key`. + - If `STEWARD_SECRET_KEY` is set instead, it must equal the backup's key, + or the restore stops and says so. Read the key with + `tar -xOf secret.key`. +4. Start Steward. On start it runs any migrations the backup is behind on. + +**How the restore works.** pg_restore runs in its three sections: schema, +then data, then constraints and indexes. Between the data and the +constraints, a kept row that points at a left-out row is put right by its +foreign key's ON DELETE: +- an alert event's link to an Ansible run is cleared; +- a row whose foreign key would cascade or block is removed. + +pg_restore 17 writes each section as a script for psql. It runs against +Postgres 16 with one setting line dropped (`transaction_timeout`, which 16 +doesn't know). + +CI checks the whole path on every build. The built image backs up its own +database, restores into a second one, checks a setting came back and the +metrics table is empty, then starts Steward on it. diff --git a/steward/alerts/service.py b/steward/alerts/service.py index d98bb76..f4ffd7f 100644 --- a/steward/alerts/service.py +++ b/steward/alerts/service.py @@ -28,7 +28,8 @@ CORE_METRICS: dict[str, list[str]] = { "dns": ["is_up", "response_ms"], "http": ["is_up", "response_ms"], "steward": ["ingest_posts", "ingest_failed", "ingest_slowest_ms", "db_roundtrip_ms", - "unwatched_secs", "cleanup_secs", "open_fds", "open_fds_pct"], + "unwatched_secs", "cleanup_secs", "open_fds", "open_fds_pct", + "backup_age_secs", "backup_bytes"], } diff --git a/steward/app.py b/steward/app.py index 630f926..7829453 100644 --- a/steward/app.py +++ b/steward/app.py @@ -441,12 +441,25 @@ def _register_core_tasks(app: Quart) -> None: from .core.self_monitor import record_self_metrics await record_self_metrics(app) + async def run_backup_tick(): + from .backup.job import tick + await tick(app) + async def run_email_outbox(): from .core.outbox import send_due await send_due(app) + from .backup.job import TICK_SECONDS as BACKUP_TICK_S from .core.outbox import SENDER_TICK_S app._task_registry.extend([ + # Steward's own daily backup to its bucket (#5673): a tick that + # records the newest backup's age and makes one when it is due. + ScheduledTask( + name="backup", + coro_factory=run_backup_tick, + interval_seconds=BACKUP_TICK_S, + run_on_startup=False, + ), # Every email is a row until the mail server accepts it (#5506); this # sends what is due and retries what failed. On startup too, so what a # restart left unsent goes out at once. diff --git a/steward/audit/service.py b/steward/audit/service.py index 7366fc2..cad640d 100644 --- a/steward/audit/service.py +++ b/steward/audit/service.py @@ -187,6 +187,11 @@ ACTIONS = { "snmp.device_created": "added SNMP device", "snmp.device_updated": "changed SNMP device", "snmp.device_deleted": "deleted SNMP device", + "prometheus.target_created": "added Prometheus target", + "prometheus.target_updated": "changed Prometheus target", + "prometheus.target_deleted": "deleted Prometheus target", + "backup.made": "sent a backup", + "backup.failed": "tried to send a backup, which failed:", } # The filter's choices: an area, matched as a prefix of the action code. @@ -194,7 +199,8 @@ AREAS = (("auth.", "Sign-ins"), ("account.", "Accounts"), ("user.", "Users"), (" ("alert_rule.", "Alert rules"), ("maintenance_window.", "Maintenance"), ("ansible.run", "Playbook runs"), ("ansible.", "Ansible (all)"), ("settings.", "Settings"), ("plugin.", "Plugins"), ("api_token.", "AI access"), - ("docker.", "Docker"), ("snmp.", "SNMP")) + ("docker.", "Docker"), ("snmp.", "SNMP"), ("prometheus.", "Prometheus"), + ("backup.", "Backups")) def action_text(action: str) -> str: @@ -211,7 +217,8 @@ _LABELS = {"check": "dry run", "interval_seconds": "every", "interval_s": "every _METHODS = {"local": "password", "ldap": "LDAP", "oidc": "single sign-on"} _SECTIONS = {"general": "General", "thresholds": "Thresholds", "notifications": "Notifications", "ansible.managed_key": "Ansible: managed SSH key", - "ansible.credentials": "Ansible: credentials", "auth": "Authentication"} + "ansible.credentials": "Ansible: credentials", "auth": "Authentication", + "backups": "Backups"} def _plain(value) -> str: diff --git a/steward/backup/job.py b/steward/backup/job.py index 6cfaa9a..4ee93b8 100644 --- a/steward/backup/job.py +++ b/steward/backup/job.py @@ -1,10 +1,40 @@ -"""Steward's own backups as a scheduled job: make the file, send it, record -how it went (milestone 507).""" +"""Steward's own backups as a scheduled job: make the file, send it to the +bucket, record how it went (#5673). + +Once a day at the hour chosen in Settings → Backups, in the zone of whoever +saved it. A day missed while Steward was down is made up on the next tick; +a failed run is tried again an hour later, never sooner, so a target that is +down isn't pressed. Each run is kept in `backup.history` (newest first, the +last HISTORY_KEEP), audited, and leaves two metrics under source "steward", +resource "backup": backup_age_secs (since the last success, every tick while +backups are on, so an alert rule can watch it) and backup_bytes. +""" from __future__ import annotations +import asyncio +import logging +import tempfile +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path + from sqlalchemy import text from sqlalchemy.ext.asyncio import create_async_engine +from .dump import BackupError, make_backup +from .s3 import S3Error, S3Target, list_objects, put_file + +logger = logging.getLogger(__name__) + +TICK_SECONDS = 15 * 60 +RETRY_AFTER = timedelta(hours=1) +HISTORY_KEEP = 30 +PREFIX = "steward-" + +# One backup at a time, whether the schedule or a person started it. +_running = asyncio.Lock() +_background: set[asyncio.Task] = set() + async def migration_heads(database_url: str) -> list[str]: """The migration revisions the database is at, for the manifest.""" @@ -14,3 +44,208 @@ async def migration_heads(database_url: str) -> list[str]: return list((await conn.execute(text("SELECT version_num FROM alembic_version"))).scalars()) finally: await engine.dispose() + + +def target_from(settings: dict) -> S3Target | None: + """The bucket the settings name, or None while they're incomplete.""" + endpoint = (settings.get("backup.s3_endpoint") or "").strip() + bucket = (settings.get("backup.s3_bucket") or "").strip() + access = (settings.get("backup.s3_access_key") or "").strip() + secret = settings.get("backup.s3_secret_key") or "" + if not (endpoint and bucket and access and secret): + return None + return S3Target(endpoint, bucket, access, secret, + region=(settings.get("backup.s3_region") or "us-east-1").strip(), + verify_tls=bool(settings.get("backup.s3_verify_tls", True))) + + +def _moment(raw) -> datetime | None: + try: + return datetime.fromisoformat(raw) if raw else None + except (TypeError, ValueError): + return None + + +def last_slot(now: datetime, hour: int, tz) -> datetime: + """The latest scheduled moment at or before `now`: today's hour in `tz`, + or yesterday's when today's hasn't come yet.""" + local = now.astimezone(tz) + slot = local.replace(hour=hour, minute=0, second=0, microsecond=0) + if slot > local: + slot -= timedelta(days=1) + return slot.astimezone(timezone.utc) + + +def is_due(now: datetime, hour: int, tz, last_success: datetime | None, + last_attempt: datetime | None) -> bool: + """Due when no backup has succeeded since the latest slot, and the last + try (if it failed) was at least RETRY_AFTER ago.""" + slot = last_slot(now, hour, tz) + if last_success is not None and last_success >= slot: + return False + return last_attempt is None or now - last_attempt >= RETRY_AFTER + + +def history_with(history: list, entry: dict) -> list: + return ([entry] + list(history or []))[:HISTORY_KEEP] + + +async def _record(app, entry: dict, user_id: str | None, username: str) -> None: + from steward.core.alerts import MetricBatch + from steward.core.audit import log_audit + from steward.core.settings import get_setting, set_setting + + async with app.db_sessionmaker() as db: + async with db.begin(): + history = await get_setting(db, "backup.history") or [] + await set_setting(db, "backup.history", history_with(history, entry)) + await set_setting(db, "backup.last_attempt_at", entry["at"]) + if entry["ok"]: + await set_setting(db, "backup.last_success_at", entry["at"]) + batch = MetricBatch() + batch.add(source_module="steward", resource_name="backup", + metric_name="backup_bytes", value=float(entry["size"])) + batch.add(source_module="steward", resource_name="backup", + metric_name="backup_age_secs", value=0.0) + await batch.record(db) + await log_audit(app, user_id, username, "backup.made" if entry["ok"] else "backup.failed", + entity_type="backup", entity_id=entry.get("name") or None, + detail={k: entry[k] for k in ("name", "size", "secs", "error", "by") if entry.get(k)}) + + +async def run_backup(app, by: str = "schedule", user_id: str | None = None) -> dict: + """Make one backup and send it. Returns the history entry it recorded.""" + from steward.core.settings import get_all_settings + + async with _running: + started = time.monotonic() + at = datetime.now(timezone.utc) + entry = {"at": at.isoformat(), "ok": False, "by": by, "name": "", "size": 0, + "secs": 0.0, "error": ""} + async with app.db_sessionmaker() as db: + settings = await get_all_settings(db) + target = target_from(settings) + try: + if target is None: + raise BackupError("The bucket isn't set up: fill in the endpoint, bucket and keys.") + db_url = app.config["DATABASE_URL"] + heads = await migration_heads(db_url) + with tempfile.TemporaryDirectory(prefix="steward-backup-") as tmp: + made = await make_backup(db_url, app.config["SECRET_KEY"], heads, Path(tmp), now=at) + await put_file(target, made.name, made.path) + entry.update(ok=True, name=made.name, size=made.size) + except (BackupError, S3Error) as exc: + entry["error"] = str(exc) + except Exception as exc: # recorded and logged: a run never takes the scheduler down + logger.exception("Backup failed") + entry["error"] = f"Unexpected error: {exc}" + entry["secs"] = round(time.monotonic() - started, 1) + if entry["ok"]: + logger.info("Backup %s sent (%d bytes, %.1fs)", entry["name"], entry["size"], entry["secs"]) + else: + logger.warning("Backup failed: %s", entry["error"]) + await _record(app, entry, user_id, by if by != "schedule" else "") + return entry + + +def start_backup(app, by: str, user_id: str | None) -> bool: + """Start a backup in the background for a person; False if one is running.""" + if _running.locked(): + return False + task = asyncio.get_running_loop().create_task(run_backup(app, by=by, user_id=user_id)) + _background.add(task) + task.add_done_callback(_background.discard) + return True + + +def is_running() -> bool: + return _running.locked() + + +async def tick(app, now: datetime | None = None) -> None: + """The scheduled task: record how old the newest backup is, and make one + when it is due.""" + from steward.core.alerts import MetricBatch + from steward.core.formatting import zone + from steward.core.settings import get_all_settings + + now = now or datetime.now(timezone.utc) + async with app.db_sessionmaker() as db: + settings = await get_all_settings(db) + if not settings.get("backup.enabled"): + return + last_success = _moment(settings.get("backup.last_success_at")) + if last_success is not None: + async with app.db_sessionmaker() as db: + async with db.begin(): + batch = MetricBatch() + batch.add(source_module="steward", resource_name="backup", + metric_name="backup_age_secs", value=(now - last_success).total_seconds()) + await batch.record(db) + if _running.locked(): + return + if is_due(now, int(settings.get("backup.schedule_hour", 3)), zone(settings.get("backup.schedule_tz")), + last_success, _moment(settings.get("backup.last_attempt_at"))): + await run_backup(app) + + +async def test_target(target: S3Target) -> tuple[bool, str]: + """Settings → Backups' Test button: can Steward reach the bucket with + these keys? A listing refused as AccessDenied still proves the keys: + an append-only key may not list, and uploads anyway.""" + try: + found = await list_objects(target, PREFIX, limit=1000) + except S3Error as exc: + said = str(exc) + if "AccessDenied" in said: + return True, ("Reached the bucket and the keys were accepted. They may not list " + "what is there, which an append-only key can't; uploads still work.") + return False, said + if not found: + return True, f"Reached {target.bucket}; no Steward backups there yet." + newest = max(found, key=lambda o: o.key) + return True, f"Reached {target.bucket}: {len(found)} Steward backups, the newest {newest.key}." + + +# ── What Settings → Backups shows ──────────────────────────────────────────── + +def shown_hour(hour: int, saved_tz, view_tz, now: datetime | None = None) -> int: + """The schedule's hour as a reader in `view_tz` sees it today.""" + local = (now or datetime.now(timezone.utc)).astimezone(saved_tz) + return local.replace(hour=hour, minute=0, second=0, microsecond=0).astimezone(view_tz).hour + + +def history_rows(history: list) -> list[dict]: + """The stored history with its times read back as datetimes.""" + return [entry | {"at": _moment(entry.get("at"))} for entry in history or []] + + +def verdict(settings: dict, running: bool) -> dict: + """One line for the top of the page: {"tone": ok|warn|bad|off, "text", "moment"}. + `moment` is the time the line is about, shown as "ago" by the template, + which ends the sentence; a failure also carries `failed`, its `error` and + `good` (the newest success, or None).""" + rows = history_rows(settings.get("backup.history") or []) + last_ok = _moment(settings.get("backup.last_success_at")) + if running: + return {"tone": "ok", "text": "A backup is being made now", "moment": None} + if not settings.get("backup.enabled"): + if target_from(settings) is None: + return {"tone": "off", "text": "Backups are off, and no bucket is set up", "moment": None} + return {"tone": "off", "text": "Scheduled backups are off; Back up now still works", "moment": None} + if target_from(settings) is None: + return {"tone": "bad", "text": "Backups are on, but the bucket isn't set up", "moment": None} + if rows and not rows[0]["ok"]: + # `good`: when the newest backup that did go was sent (None: never). + return {"tone": "bad", "text": "The last backup failed", "moment": rows[0]["at"], + "error": rows[0]["error"], "good": last_ok, "failed": True} + if last_ok is None: + return {"tone": "warn", "text": "No backup has been sent yet; the first goes within 15 minutes", + "moment": None} + return {"tone": "ok", "text": f"The newest backup is {rows[0]['name']} ({_size(rows[0]['size'])}), sent", + "moment": last_ok} + + +def _size(n) -> str: + from steward.core.formatting import human_bytes + return human_bytes(n) diff --git a/steward/backup/s3.py b/steward/backup/s3.py new file mode 100644 index 0000000..c6cb151 --- /dev/null +++ b/steward/backup/s3.py @@ -0,0 +1,191 @@ +"""A small S3 client for sending backups: AWS Signature Version 4 over httpx, +path-style addressing (endpoint/bucket/key), the three calls a backup needs. + +Fabled Archives speaks S3 with path-style buckets and append-only keys by +default (System 180 ruling: backups go there with an append-only key), so +Steward only ever puts and lists; it never deletes. Retention is the +target's business. + +Written by hand rather than with boto3: three calls don't justify a large +dependency, and signing is a pure function tests can hold to AWS's own +published examples. +""" +from __future__ import annotations + +import hashlib +import hmac +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import quote + +import httpx + +EMPTY_SHA256 = hashlib.sha256(b"").hexdigest() +TIMEOUT_SECONDS = 30.0 +# A backup is megabytes; a slow link still gets this long to send one. +UPLOAD_TIMEOUT_SECONDS = 15 * 60 +CHUNK = 1 << 20 + + +class S3Error(Exception): + """Why a call to the bucket failed, said to the person.""" + + +@dataclass(frozen=True) +class S3Target: + endpoint: str # http(s)://host[:port], no bucket + bucket: str + access_key: str + secret_key: str + region: str = "us-east-1" + verify_tls: bool = True + + def url(self, key: str = "") -> str: + base = f"{self.endpoint.rstrip('/')}/{self.bucket}" + return f"{base}/{key}" if key else f"{base}/" + + +@dataclass(frozen=True) +class StoredObject: + key: str + size: int + last_modified: str + + +# ── Signing ────────────────────────────────────────────────────────────────── + +def _encode(text: str, keep_slash: bool = False) -> str: + return quote(text, safe="-_.~" + ("/" if keep_slash else "")) + + +def _hmac(key: bytes, msg: str) -> bytes: + return hmac.new(key, msg.encode(), hashlib.sha256).digest() + + +def sign(method: str, url: str, headers: dict[str, str], payload_sha256: str, + access_key: str, secret_key: str, region: str, at: datetime, + service: str = "s3") -> dict[str, str]: + """The request's headers with x-amz-date, x-amz-content-sha256 and + Authorization added. Every header given is signed, Host always.""" + u = httpx.URL(url) + stamp = at.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + day = stamp[:8] + out = dict(headers) + out["x-amz-date"] = stamp + out["x-amz-content-sha256"] = payload_sha256 + signed = {k.lower(): " ".join(str(v).split()) for k, v in out.items()} + signed["host"] = u.netloc.decode() + names = sorted(signed) + + query = sorted((_encode(k), _encode(v)) for k, v in u.params.multi_items()) + canonical = "\n".join([ + method.upper(), + _encode(u.path or "/", keep_slash=True), + "&".join(f"{k}={v}" for k, v in query), + "".join(f"{n}:{signed[n]}\n" for n in names), + ";".join(names), + payload_sha256, + ]) + scope = f"{day}/{region}/{service}/aws4_request" + to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope, + hashlib.sha256(canonical.encode()).hexdigest()]) + key = _hmac(_hmac(_hmac(_hmac(f"AWS4{secret_key}".encode(), day), region), service), "aws4_request") + signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest() + out["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, " + f"SignedHeaders={';'.join(names)}, Signature={signature}") + return out + + +# ── Calls ──────────────────────────────────────────────────────────────────── + +def _says(response: httpx.Response) -> str: + """An S3 error response in words: its Code and Message when it has them.""" + try: + root = ET.fromstring(response.content) + code = root.findtext("Code") or "" + message = root.findtext("Message") or "" + except ET.ParseError: + code, message = "", "" + hints = { + "SignatureDoesNotMatch": "the secret key is wrong", + "InvalidAccessKeyId": "the access key isn't known there", + "NoSuchBucket": "there is no such bucket", + "AccessDenied": "the key isn't allowed to do that", + } + said = hints.get(code) or message or response.reason_phrase + return f"HTTP {response.status_code}{f' {code}' if code else ''}: {said}." + + +def _reach_error(target: S3Target, exc: httpx.HTTPError) -> S3Error: + if isinstance(exc, httpx.TimeoutException): + return S3Error(f"{target.endpoint} didn't answer in time.") + return S3Error(f"Couldn't reach {target.endpoint}: {exc}.") + + +def file_sha256(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + while chunk := f.read(CHUNK): + h.update(chunk) + return h.hexdigest() + + +async def put_file(target: S3Target, key: str, path: Path, *, + transport: httpx.AsyncBaseTransport | None = None, + now: datetime | None = None) -> None: + """Upload one file as one object.""" + import asyncio + digest = await asyncio.to_thread(file_sha256, path) + size = path.stat().st_size + url = target.url(key) + headers = sign("PUT", url, {"Content-Length": str(size), "Content-Type": "application/x-tar"}, + digest, target.access_key, target.secret_key, target.region, + now or datetime.now(timezone.utc)) + + async def body(): + with open(path, "rb") as f: + while chunk := f.read(CHUNK): + yield chunk + + try: + async with httpx.AsyncClient(verify=target.verify_tls, timeout=UPLOAD_TIMEOUT_SECONDS, + transport=transport) as client: + response = await client.put(url, headers=headers, content=body()) + except httpx.HTTPError as exc: + raise _reach_error(target, exc) from None + if response.status_code // 100 != 2: + raise S3Error(f"Upload of {key} refused: {_says(response)}") + + +async def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *, + transport: httpx.AsyncBaseTransport | None = None, + now: datetime | None = None) -> list[StoredObject]: + """Up to `limit` objects under `prefix` (ListObjectsV2, one page).""" + params = {"list-type": "2", "max-keys": str(limit)} + if prefix: + params["prefix"] = prefix + url = str(httpx.URL(target.url(), params=params)) + headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key, + target.region, now or datetime.now(timezone.utc)) + try: + async with httpx.AsyncClient(verify=target.verify_tls, timeout=TIMEOUT_SECONDS, + transport=transport) as client: + response = await client.get(url, headers=headers) + except httpx.HTTPError as exc: + raise _reach_error(target, exc) from None + if response.status_code != 200: + raise S3Error(f"Listing {target.bucket} refused: {_says(response)}") + try: + root = ET.fromstring(response.content) + except ET.ParseError: + raise S3Error(f"{target.endpoint} answered a listing with something that isn't S3's XML.") from None + out = [] + for item in root.iter(): + if item.tag.rsplit("}", 1)[-1] != "Contents": + continue + fields = {child.tag.rsplit("}", 1)[-1]: (child.text or "") for child in item} + out.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0), + fields.get("LastModified", ""))) + return out diff --git a/steward/core/formatting.py b/steward/core/formatting.py index d1b81fe..8bc550b 100644 --- a/steward/core/formatting.py +++ b/steward/core/formatting.py @@ -286,6 +286,8 @@ METRICS: dict[str, tuple[str, str]] = { "db_roundtrip_ms": ("Database round trip", "a trivial query to Steward's database, connection included"), "unwatched_secs": ("Time not watching", "how long Steward wasn't recording before this minute; 0 when unbroken"), "cleanup_secs": ("Hourly cleanup took", "how long the last retention and rollup run took"), + "backup_age_secs": ("Newest backup's age", "time since Steward last sent a backup to its bucket"), + "backup_bytes": ("Backup size", "the size of the last backup sent"), "open_fds": ("Open files", "files Steward itself has open"), "open_fds_pct": ("Open files (% of limit)", "files Steward has open, against its limit"), "scrape_ms": ("Scrape time", "how long the target took to hand over its metrics"), diff --git a/steward/core/settings.py b/steward/core/settings.py index 5a90bc0..85bcad3 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -176,6 +176,20 @@ DEFAULTS: dict[str, Any] = { # so "Sunday 08:00" means 08:00 where they are (#5378). "reports.schedule_tz": "", "reports.last_sent_at": "", + # Steward's own backups (#5673), sent to an S3 bucket (Fabled Archives). + "backup.enabled": False, + "backup.s3_endpoint": "", + "backup.s3_bucket": "", + "backup.s3_region": "us-east-1", + "backup.s3_access_key": "", + "backup.s3_secret_key": "", + "backup.s3_verify_tls": True, + "backup.schedule_hour": 3, # an hour in backup.schedule_tz + "backup.schedule_tz": "", + "backup.last_success_at": "", + "backup.last_attempt_at": "", + # Newest first: {at, ok, by, name, size, secs, error}. + "backup.history": [], # What the last hourly cleanup did (#5469), written by core/cleanup.py for # Settings → System: {"at": ISO time, "secs": float, "removed": [[what, n]]}. "cleanup.last_run": {}, @@ -194,6 +208,7 @@ SECRET_KEYS: set[str] = { # A webhook URL carries its own credential (Discord, Slack and ntfy put the # token in the path). "webhook.url", + "backup.s3_secret_key", } diff --git a/steward/settings/routes.py b/steward/settings/routes.py index e9c0b70..ece099c 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -687,6 +687,92 @@ async def send_report_now(): return await _reports_page(settings, {"ok": ok, "message": message}) +# ── Backups ─────────────────────────────────────────────────────────────────── + +async def _backups_page(flash=None): + """Settings → Backups (#5673): how the newest backup went, the bucket and + the daily hour (in the reader's zone), and the runs before.""" + from steward.backup import job + from steward.core.formatting import zone + async with current_app.db_sessionmaker() as db: + settings = await get_all_settings(db) + hour = job.shown_hour(int(settings.get("backup.schedule_hour", 3)), + zone(settings.get("backup.schedule_tz")), zone(session.get("tz"))) + return await render_template( + "settings/backups.html", settings=settings, flash=flash, shown_hour=hour, + verdict=job.verdict(settings, job.is_running()), + history=job.history_rows(settings.get("backup.history")), + has_secret=bool(settings.get("backup.s3_secret_key"))) + + +@settings_bp.get("/backups/") +@require_role(UserRole.admin) +async def backups(): + return await _backups_page() + + +@settings_bp.post("/backups/") +@require_role(UserRole.admin) +async def save_backups(): + """The hour is what the reader picked on their own clock, so it is kept in + their zone. A blank secret key keeps the stored one.""" + form = await request.form + endpoint = form.get("backup.s3_endpoint", "").strip() + if endpoint and not endpoint.startswith(("http://", "https://")): + return await _backups_page({"ok": False, "message": + "The endpoint starts with http:// or https://, e.g. http://archives:8081."}) + try: + hour = int(form.get("backup.schedule_hour", 3)) + except ValueError: + hour = 3 + async with current_app.db_sessionmaker() as db: + async with db.begin(): + await set_setting(db, "backup.enabled", "backup.enabled" in form) + await set_setting(db, "backup.s3_endpoint", endpoint) + await set_setting(db, "backup.s3_bucket", form.get("backup.s3_bucket", "").strip()) + await set_setting(db, "backup.s3_region", form.get("backup.s3_region", "").strip() or "us-east-1") + await set_setting(db, "backup.s3_access_key", form.get("backup.s3_access_key", "").strip()) + secret = form.get("backup.s3_secret_key", "") + if secret.strip(): + await set_setting(db, "backup.s3_secret_key", secret.strip()) + elif "backup.clear_secret" in form: + await set_setting(db, "backup.s3_secret_key", "") + await set_setting(db, "backup.s3_verify_tls", "backup.s3_verify_tls" in form) + await set_setting(db, "backup.schedule_hour", min(max(hour, 0), 23)) + await set_setting(db, "backup.schedule_tz", session.get("tz") or "") + await log_audit(current_app, session.get("user_id"), session.get("username", ""), + "settings.saved", detail={"section": "backups"}) + return redirect(url_for("settings.backups")) + + +@settings_bp.post("/backups/test/") +@require_role(UserRole.admin) +async def test_backups(): + """Reach the saved bucket with the saved keys (the operator asked, so the + target is contacted).""" + from steward.backup import job + async with current_app.db_sessionmaker() as db: + settings = await get_all_settings(db) + target = job.target_from(settings) + if target is None: + return await _backups_page({"ok": False, "message": + "Save the endpoint, bucket and both keys first."}) + ok, message = await job.test_target(target) + return await _backups_page({"ok": ok, "message": message}) + + +@settings_bp.post("/backups/now/") +@require_role(UserRole.admin) +async def backup_now(): + from steward.backup import job + started = job.start_backup(current_app._get_current_object(), + by=session.get("username", "") or "an admin", + user_id=session.get("user_id")) + message = ("Making a backup now; it shows under Recent backups when it is sent." + if started else "A backup is already being made.") + return await _backups_page({"ok": started, "message": message}) + + def _build_plugin_cfg_from_form(plugin: dict, form) -> dict: """Extract config values for one plugin from a form submission.""" name = plugin["_dir"] diff --git a/steward/templates/settings/_tabs.html b/steward/templates/settings/_tabs.html index e2e2434..c2206f3 100644 --- a/steward/templates/settings/_tabs.html +++ b/steward/templates/settings/_tabs.html @@ -7,6 +7,7 @@ ("thresholds", "Thresholds & Retention", "/settings/thresholds/"), ("notifications", "Notifications", "/settings/notifications/"), ("reports", "Reports", "/settings/reports/"), + ("backups", "Backups", "/settings/backups/"), ("auth", "Authentication", "/settings/auth/"), ("users", "Users", "/settings/users/"), ("ansible", "Ansible", "/settings/ansible/"), diff --git a/steward/templates/settings/backups.html b/steward/templates/settings/backups.html new file mode 100644 index 0000000..1a6720c --- /dev/null +++ b/steward/templates/settings/backups.html @@ -0,0 +1,138 @@ +{# settings/backups.html — Steward's own backups (#5673): a verdict, the bucket + they go to (Fabled Archives over S3) and the daily hour, then the recent + runs. Restoring is `steward restore`; see docs/core/backups.md. #} +{% extends "base.html" %} +{% from "_macros.html" import crumbs %} +{% from "settings/_tabs.html" import settings_tabs %} +{% block title %}Settings — Backups — Steward{% endblock %} +{% block section_tabs %}{{ settings_tabs("backups") }}{% endblock %} +{% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("Backups", "")]) }}{% endblock %} +{% block content %} +{% set tone = {"ok": "tone-ok", "warn": "tone-warn", "bad": "tone-crit", "off": "tone-dim"}[verdict.tone] %} +

{{ verdict.text }} + {%- if verdict.moment %} {{ verdict.moment|ago }}{% endif %}. + {%- if verdict.failed %} {{ verdict.error }} + {% if verdict.good %}The newest good backup was sent {{ verdict.good|ago }}.{% else %}No backup has been sent yet.{% endif %} + {%- endif %}

+ +{% if flash %} +
+ {{ flash.message }} +
+{% endif %} + +
+
{{ csrf_field() }} +
+

Bucket

+

+ Each backup is one file, steward-<time>.tar: Steward's configuration, its audit log + and alert history, and its secret key. Metrics and check results are left out. Anyone holding a + copy can read every password and key Steward keeps, so keep the bucket private. +

+ +
+ + +

The S3 address, without the bucket. Requests are path-style: endpoint/bucket/file.

+
+
+
+ + +
+
+ + +
+
+
+
+ + +
+
+ + + {% if has_secret %} + + {% endif %} +
+
+

An append-only key is enough: Steward only uploads and never deletes. How long + backups are kept is the bucket's own setting.

+
+ + +
+ +

Schedule

+
+ + +
+
+ + +
+

A day missed while Steward was down is made up when it is back; a failed backup is + tried again an hour later. Watch it with an alert rule on + "Steward itself", resource backup, metric "Newest backup's age".

+ +
+
+
+ +
+
{{ csrf_field() }} + +
+
{{ csrf_field() }} + +
+

Both use the saved settings.

+
+
+ +

Recent backups

+{% if history %} +
+ + + + {% for r in history %} + + + + + + + + + {% endfor %} + +
WhenOutcomeFileSizeTookStarted by
{{ r.at|when if r.at else "—" }}{% if r.ok %}Sent{% else %}Failed +
{{ r.error }}
{% endif %}		{{ r.name or "—" }}{{ r.size|bytes if r.ok else "—" }}{{ r.secs|duration }}{{ "the schedule" if r.by == "schedule" else r.by }}
+
+{% else %} +

None yet.

+{% endif %} +

To bring Steward back from one, see + steward restore in the backups guide (docs/core/backups.md).

+{% endblock %} diff --git a/tests/core/test_audit_actions_worded.py b/tests/core/test_audit_actions_worded.py new file mode 100644 index 0000000..73f6804 --- /dev/null +++ b/tests/core/test_audit_actions_worded.py @@ -0,0 +1,38 @@ +"""Every action Steward writes to the audit log reads as words on the audit +page (steward/audit/service.py ACTIONS), not as its code.""" +import ast +import pathlib +import re + +import steward +from steward.audit.service import ACTIONS + +_REPO = pathlib.Path(steward.__file__).parent.parent +_CODE = re.compile(r"[a-z_]+(\.[a-z_]+)+") +# The writers: log_audit, and audit_request (imported as _audit in places). +_WRITERS = {"log_audit", "audit_request", "_audit"} + + +def logged_actions(source: str) -> set[str]: + """Literal action codes handed to a writer (positionally or as action=).""" + found = set() + for call in ast.walk(ast.parse(source)): + if not (isinstance(call, ast.Call) and getattr(call.func, "id", getattr(call.func, "attr", "")) in _WRITERS): + continue + args = list(call.args) + [k.value for k in call.keywords if k.arg == "action"] + found |= {a.value for a in args + if isinstance(a, ast.Constant) and isinstance(a.value, str) and _CODE.fullmatch(a.value)} + return found + + +def test_every_logged_action_is_worded(): + files = [p for root in ("steward", "plugins") for p in (_REPO / root).rglob("*.py") + if any(w in p.read_text() for w in _WRITERS)] + logged = set().union(*(logged_actions(p.read_text()) for p in files)) + assert "settings.saved" in logged and "backup.made" in logged # the walk finds them + assert logged - set(ACTIONS) == set(), "Word these in ACTIONS in steward/audit/service.py" + + +def test_the_guard_can_fail(): + code = 'async def f(app):\n await log_audit(app, None, "", "widget.exploded", entity_type="widget")\n' + assert logged_actions(code) - set(ACTIONS) == {"widget.exploded"} diff --git a/tests/core/test_backup_job.py b/tests/core/test_backup_job.py new file mode 100644 index 0000000..c03f16e --- /dev/null +++ b/tests/core/test_backup_job.py @@ -0,0 +1,123 @@ +"""Steward's backup job (steward/backup/job.py): when a backup is due, what +the Backups page says, and the Test button's words.""" +import asyncio +from datetime import datetime, timezone +from zoneinfo import ZoneInfo + +from quart import render_template, session + +from steward.backup import job +from steward.backup.s3 import S3Error, StoredObject +from steward.core.settings import DEFAULTS + +UTC = timezone.utc +NY = ZoneInfo("America/New_York") + + +def at(h, m=0, day=12): + return datetime(2026, 10, day, h, m, tzinfo=UTC) + + +def test_the_slot_is_todays_hour_once_it_has_come_else_yesterdays(): + assert job.last_slot(at(2), 3, UTC) == at(3, day=11) + assert job.last_slot(at(4), 3, UTC) == at(3) + # 03:00 in New York in October is 07:00 UTC. + assert job.last_slot(at(8), 3, NY) == at(7) + + +def test_due_once_per_slot_and_retried_after_an_hour(): + assert job.is_due(at(4), 3, UTC, last_success=None, last_attempt=None) + assert not job.is_due(at(4), 3, UTC, last_success=at(3, 1), last_attempt=at(3, 1)) + # Yesterday's success doesn't cover today's slot: a missed day is made up. + assert job.is_due(at(9), 3, UTC, last_success=at(3, 1, day=11), last_attempt=at(3, 1, day=11)) + # A failure at 03:01 waits an hour before the next try. + assert not job.is_due(at(3, 30), 3, UTC, last_success=None, last_attempt=at(3, 1)) + assert job.is_due(at(4, 2), 3, UTC, last_success=None, last_attempt=at(3, 1)) + + +def test_history_keeps_the_newest_thirty(): + history = [{"n": i} for i in range(30)] + kept = job.history_with(history, {"n": "new"}) + assert len(kept) == job.HISTORY_KEEP and kept[0] == {"n": "new"} and kept[-1] == {"n": 28} + + +def _settings(**over): + return {**DEFAULTS, "backup.s3_endpoint": "http://archives:8081", "backup.s3_bucket": "steward", + "backup.s3_access_key": "AK", "backup.s3_secret_key": "SK", **over} + + +def test_a_target_needs_every_field(): + t = job.target_from(_settings()) + assert t.endpoint == "http://archives:8081" and t.bucket == "steward" and t.region == "us-east-1" + assert job.target_from(_settings(**{"backup.s3_secret_key": ""})) is None + + +def test_the_hour_is_shown_on_the_readers_clock(): + assert job.shown_hour(3, NY, UTC, now=at(12)) == 7 + assert job.shown_hour(3, UTC, UTC, now=at(12)) == 3 + + +def test_the_verdict_names_what_matters_first(): + ok = {"at": at(3).isoformat(), "ok": True, "name": "steward-x.tar", "size": 2048, "error": ""} + bad = {"at": at(4).isoformat(), "ok": False, "name": "", "size": 0, "error": "HTTP 403: the key is wrong."} + assert job.verdict(DEFAULTS, False)["tone"] == "off" + assert job.verdict(_settings(**{"backup.enabled": True}), False)["tone"] == "warn" + assert job.verdict(_settings(**{"backup.enabled": True, "backup.s3_bucket": ""}), False)["tone"] == "bad" + good = job.verdict(_settings(**{"backup.enabled": True, "backup.history": [ok], + "backup.last_success_at": ok["at"]}), False) + assert good["tone"] == "ok" and "steward-x.tar (2.0 KiB)" in good["text"] and good["moment"] == at(3) + failed = job.verdict(_settings(**{"backup.enabled": True, "backup.history": [bad, ok], + "backup.last_success_at": ok["at"]}), False) + assert failed["tone"] == "bad" and failed["error"] == bad["error"] and failed["good"] == at(3) + assert job.verdict(DEFAULTS, True)["text"] == "A backup is being made now" + + +def test_test_button_reads_an_append_only_refusal_as_reached(monkeypatch): + target = job.target_from(_settings()) + + async def denied(*a, **k): + raise S3Error("Listing steward refused: HTTP 403 AccessDenied: the key isn't allowed to do that.") + + async def wrong(*a, **k): + raise S3Error("Listing steward refused: HTTP 403 SignatureDoesNotMatch: the secret key is wrong.") + + async def two(*a, **k): + return [StoredObject("steward-20261011T030000Z.tar", 1, ""), StoredObject("steward-20261012T030000Z.tar", 1, "")] + + monkeypatch.setattr(job, "list_objects", denied) + ok, said = asyncio.run(job.test_target(target)) + assert ok and "append-only" in said + monkeypatch.setattr(job, "list_objects", wrong) + ok, said = asyncio.run(job.test_target(target)) + assert not ok and "secret key is wrong" in said + monkeypatch.setattr(job, "list_objects", two) + ok, said = asyncio.run(job.test_target(target)) + assert ok and "2 Steward backups, the newest steward-20261012T030000Z.tar" in said + + +async def _render(app, **ctx): + async with app.test_request_context("/settings/backups/"): + session["user_role"] = "admin" + session["tz"] = "America/New_York" + return await render_template("settings/backups.html", **ctx) + + +async def test_the_page_leads_with_the_verdict_and_lists_runs_in_the_readers_zone(app): + rows = job.history_rows([ + {"at": at(4).isoformat(), "ok": False, "by": "schedule", "name": "", "size": 0, "secs": 1.2, + "error": "HTTP 403 SignatureDoesNotMatch: the secret key is wrong."}, + {"at": at(3).isoformat(), "ok": True, "by": "admin", "name": "steward-20261012T030000Z.tar", + "size": 2048, "secs": 3.4, "error": ""}, + ]) + settings = _settings(**{"backup.enabled": True}) + v = {"tone": "bad", "text": "The last backup failed", "moment": at(4), + "error": rows[0]["error"], "good": at(3), "failed": True} + page = await _render(app, settings=settings, flash=None, shown_hour=3, verdict=v, + history=rows, has_secret=True) + main = page[page.index("' in page diff --git a/tests/core/test_backup_s3.py b/tests/core/test_backup_s3.py new file mode 100644 index 0000000..4638723 --- /dev/null +++ b/tests/core/test_backup_s3.py @@ -0,0 +1,112 @@ +"""The backup's S3 client (steward/backup/s3.py): signatures match AWS's +published Signature Version 4 examples, and the calls read S3's answers.""" +import asyncio +from datetime import datetime, timezone + +import httpx +import pytest + +from steward.backup import s3 + +# AWS's example credentials and moment ("Signature Calculations for the +# Authorization Header", Amazon S3 API reference). +KEY_ID = "AKIAIOSFODNN7EXAMPLE" +SECRET = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +AT = datetime(2013, 5, 24, tzinfo=timezone.utc) + + +def _signature(headers): + return headers["Authorization"].rsplit("Signature=", 1)[1] + + +def test_aws_example_get_object(): + h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {"Range": "bytes=0-9"}, + s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT) + assert "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date," in h["Authorization"] + assert "Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request" in h["Authorization"] + assert _signature(h) == "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41" + + +def test_aws_example_list_objects(): + h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", {}, + s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT) + assert _signature(h) == "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7" + + +def test_a_changed_secret_changes_the_signature(): + a = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {}, + s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT) + b = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {}, + s3.EMPTY_SHA256, KEY_ID, SECRET + "x", "us-east-1", AT) + assert _signature(a) != _signature(b) + + +TARGET = s3.S3Target("http://archives:8081/s3", "steward", KEY_ID, SECRET) + + +def test_paths_are_path_style(): + assert TARGET.url("steward-20261012T000000Z.tar") == "http://archives:8081/s3/steward/steward-20261012T000000Z.tar" + assert TARGET.url() == "http://archives:8081/s3/steward/" + + +def test_put_sends_the_file_signed(tmp_path): + f = tmp_path / "b.tar" + f.write_bytes(b"x" * 5000) + seen = {} + + def handler(request: httpx.Request): + seen["method"], seen["url"] = request.method, str(request.url) + seen["body"] = request.read() + seen["auth"] = request.headers["authorization"] + seen["sha"] = request.headers["x-amz-content-sha256"] + return httpx.Response(200) + + asyncio.run(s3.put_file(TARGET, "b.tar", f, transport=httpx.MockTransport(handler))) + assert seen["method"] == "PUT" + assert seen["url"] == "http://archives:8081/s3/steward/b.tar" + assert seen["body"] == b"x" * 5000 + assert seen["sha"] == s3.file_sha256(f) + assert seen["auth"].startswith("AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/") + + +def test_a_refusal_says_why(tmp_path): + f = tmp_path / "b.tar" + f.write_bytes(b"x") + body = b"SignatureDoesNotMatchno" + transport = httpx.MockTransport(lambda r: httpx.Response(403, content=body)) + with pytest.raises(s3.S3Error, match="HTTP 403 SignatureDoesNotMatch: the secret key is wrong"): + asyncio.run(s3.put_file(TARGET, "b.tar", f, transport=transport)) + + +def test_unreachable_says_so(tmp_path): + f = tmp_path / "b.tar" + f.write_bytes(b"x") + + def handler(request): + raise httpx.ConnectError("refused", request=request) + + with pytest.raises(s3.S3Error, match="Couldn't reach http://archives:8081/s3"): + asyncio.run(s3.put_file(TARGET, "b.tar", f, transport=httpx.MockTransport(handler))) + + +def test_list_reads_contents(): + xml = b""" + + steward2 + steward-20261011T000000Z.tar1200 + 2026-10-11T00:00:01.000Z + steward-20261012T000000Z.tar1300 + 2026-10-12T00:00:01.000Z +""" + seen = {} + + def handler(request): + seen["params"] = dict(request.url.params) + return httpx.Response(200, content=xml) + + got = asyncio.run(s3.list_objects(TARGET, "steward-", transport=httpx.MockTransport(handler))) + assert seen["params"] == {"list-type": "2", "max-keys": "1000", "prefix": "steward-"} + assert got == [ + s3.StoredObject("steward-20261011T000000Z.tar", 1200, "2026-10-11T00:00:01.000Z"), + s3.StoredObject("steward-20261012T000000Z.tar", 1300, "2026-10-12T00:00:01.000Z"), + ] diff --git a/tests/integration/test_backup_job.py b/tests/integration/test_backup_job.py new file mode 100644 index 0000000..85bc58d --- /dev/null +++ b/tests/integration/test_backup_job.py @@ -0,0 +1,169 @@ +"""Integration: the backup job records each run against Postgres (#5673). +pg_dump isn't in this lane (the publish smoke step runs the real one), so the +dump and the upload are stubbed; what is real is everything Steward records: +history, the last-success time, metrics, audit, and the sealed secret key. +Requires STEWARD_DATABASE_URL.""" +from __future__ import annotations + +import asyncio +import os +import time +import uuid +from datetime import datetime, timedelta, timezone + +import pytest + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +async def _configure(app, **over): + from steward.core.settings import set_setting + values = {"backup.enabled": True, "backup.s3_endpoint": "http://archives:8081", + "backup.s3_bucket": "steward", "backup.s3_access_key": "AK", + "backup.s3_secret_key": "SK", "backup.history": [], + "backup.last_success_at": "", "backup.last_attempt_at": "", **over} + async with app.db_sessionmaker() as db: + async with db.begin(): + for k, v in values.items(): + await set_setting(db, k, v) + + +def _stub(monkeypatch, *, fail_upload: str = ""): + from steward.backup import job + from steward.backup.dump import Made + from steward.backup.s3 import S3Error + sent = [] + + async def make_backup(database_url, secret_key, heads, workdir, now=None): + path = workdir / "steward-stub.tar" + path.write_bytes(b"x" * 4096) + return Made(path, path.name, 4096, {"migration_heads": heads}) + + async def put_file(target, key, path, **kw): + if fail_upload: + raise S3Error(fail_upload) + sent.append((target.bucket, key, path.read_bytes())) + + monkeypatch.setattr(job, "make_backup", make_backup) + monkeypatch.setattr(job, "put_file", put_file) + return sent + + +async def _state(app): + from sqlalchemy import select + from steward.core.settings import get_all_settings + from steward.models.audit import AuditEvent + from steward.models.metrics import PluginMetricLatest + async with app.db_sessionmaker() as db: + settings = await get_all_settings(db) + metrics = dict((await db.execute(select(PluginMetricLatest.metric_name, PluginMetricLatest.value).where( + PluginMetricLatest.source_module == "steward", + PluginMetricLatest.resource_name == "backup"))).all()) + audits = list((await db.execute(select(AuditEvent.action).where( + AuditEvent.action.like("backup.%")).order_by(AuditEvent.timestamp.desc(), AuditEvent.id.desc()).limit(1))).scalars()) + return settings, metrics, audits + + +def test_a_sent_backup_is_recorded_and_a_failed_one_says_why(app, monkeypatch): + from steward.backup import job + + async def _go(): + await _configure(app) + sent = _stub(monkeypatch) + entry = await job.run_backup(app, by="admin-x") + ok_state = await _state(app) + + _stub(monkeypatch, fail_upload="Upload of steward-stub.tar refused: HTTP 403 AccessDenied.") + failed = await job.run_backup(app) + bad_state = await _state(app) + return sent, entry, ok_state, failed, bad_state + + sent, entry, (settings, metrics, audits), failed, (settings2, _, audits2) = asyncio.run(_go()) + assert sent == [("steward", "steward-stub.tar", b"x" * 4096)] + assert entry["ok"] and entry["by"] == "admin-x" and entry["size"] == 4096 + assert settings["backup.history"][0]["name"] == "steward-stub.tar" + assert settings["backup.last_success_at"] == entry["at"] + assert metrics["backup_bytes"] == 4096 and metrics["backup_age_secs"] == 0 + assert audits == ["backup.made"] + + assert not failed["ok"] and "AccessDenied" in failed["error"] + assert [h["ok"] for h in settings2["backup.history"]] == [False, True] + assert settings2["backup.last_success_at"] == entry["at"] # the good one still stands + assert settings2["backup.last_attempt_at"] == failed["at"] + assert audits2 == ["backup.failed"] + + +def test_the_tick_records_the_age_and_backs_up_when_due(app, monkeypatch): + from steward.backup import job + + async def _go(): + long_ago = (datetime.now(timezone.utc) - timedelta(days=2)).isoformat() + await _configure(app, **{"backup.last_success_at": long_ago, "backup.last_attempt_at": long_ago}) + sent = _stub(monkeypatch) + await job.tick(app) + settings, metrics, _ = await _state(app) + await job.tick(app) # not due again until the next slot + return sent, settings + + sent, settings = asyncio.run(_go()) + assert len(sent) == 1 + assert settings["backup.history"][0]["by"] == "schedule" + + +def test_the_tick_does_nothing_while_backups_are_off(app, monkeypatch): + from steward.backup import job + + async def _go(): + await _configure(app, **{"backup.enabled": False}) + sent = _stub(monkeypatch) + await job.tick(app) + return sent + + assert asyncio.run(_go()) == [] + + +def test_the_secret_key_is_sealed_and_a_blank_box_keeps_it(app): + from sqlalchemy import select + from steward.models.settings import AppSetting + from steward.models.users import User, UserRole + + async def _go(): + await _configure(app) + uid = str(uuid.uuid4()) + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add(User(id=uid, username=f"bk-{uid[:8]}", email=f"bk-{uid[:8]}@test", + password_hash="x", role=UserRole.admin, is_active=True)) + client = app.test_client() + async with client.session_transaction() as sess: + sess.update(user_id=uid, user_role="admin", username=f"bk-{uid[:8]}", _csrf="test-token", + login_at=time.time(), tz="UTC") + form = {"csrf_token": "test-token", "backup.enabled": "on", "backup.s3_endpoint": "http://archives:8081", + "backup.s3_bucket": "steward", "backup.s3_region": "", "backup.s3_access_key": "AK2", + "backup.s3_secret_key": "new-secret", "backup.schedule_hour": "4"} + r1 = await client.post("/settings/backups/", form=form) + async with app.db_sessionmaker() as db: + stored = (await db.execute(select(AppSetting.value_json).where( + AppSetting.key == "backup.s3_secret_key"))).scalar() + r2 = await client.post("/settings/backups/", form=form | {"backup.s3_secret_key": ""}) + page = await client.get("/settings/backups/") + from steward.core.settings import get_all_settings + async with app.db_sessionmaker() as db: + settings = await get_all_settings(db) + return r1.status_code, stored, r2.status_code, page.status_code, await page.get_data(as_text=True), settings + + s1, stored, s2, s3, page, settings = asyncio.run(_go()) + assert s1 == 302 and s2 == 302 and s3 == 200 + assert "enc:v1:" in stored and "new-secret" not in stored + assert settings["backup.s3_secret_key"] == "new-secret" + assert settings["backup.s3_access_key"] == "AK2" and settings["backup.s3_region"] == "us-east-1" + assert settings["backup.schedule_hour"] == 4 and settings["backup.schedule_tz"] == "UTC" + assert "new-secret" not in page and "kept; type to change" in page