feat(backup): daily backups to an S3 bucket, Settings → Backups, and the guide (#5673, #5675)

#5673 "S3 client and the backup job: settings, schedule, history, metrics":
- steward/backup/s3.py: SigV4 over httpx, path-style; put and list only
  (an append-only key is enough; retention is the bucket's). Signing is held
  to AWS's published examples.
- steward/backup/job.py: a 15-minute tick. The daily backup goes at the hour
  chosen on the reader's clock; a missed day is made up, and a failure is
  retried an hour later. History of the last 30 runs, audit
  (backup.made / backup.failed), and backup_age_secs / backup_bytes under
  "Steward itself", resource backup, for alert rules.
- Settings → Backups: verdict first, then the bucket (secret sealed; a
  blank box keeps it), the schedule, Test the bucket, Back up now, and the
  recent runs.

#5675 "Docs: backing up to Archives, and restoring": docs/core/backups.md.

Also: the audit page now words the Prometheus target actions from #5668,
which showed as codes, and a guard fails on any logged action without
wording.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_audit_actions_worded.py

@@ -0,0 +1,38 @@
+"""Every action Steward writes to the audit log reads as words on the audit
+page (steward/audit/service.py ACTIONS), not as its code."""
+import ast
+import pathlib
+import re
+
+import steward
+from steward.audit.service import ACTIONS
+
+_REPO = pathlib.Path(steward.__file__).parent.parent
+_CODE = re.compile(r"[a-z_]+(\.[a-z_]+)+")
+# The writers: log_audit, and audit_request (imported as _audit in places).
+_WRITERS = {"log_audit", "audit_request", "_audit"}
+
+
+def logged_actions(source: str) -> set[str]:
+    """Literal action codes handed to a writer (positionally or as action=)."""
+    found = set()
+    for call in ast.walk(ast.parse(source)):
+        if not (isinstance(call, ast.Call) and getattr(call.func, "id", getattr(call.func, "attr", "")) in _WRITERS):
+            continue
+        args = list(call.args) + [k.value for k in call.keywords if k.arg == "action"]
+        found |= {a.value for a in args
+                  if isinstance(a, ast.Constant) and isinstance(a.value, str) and _CODE.fullmatch(a.value)}
+    return found
+
+
+def test_every_logged_action_is_worded():
+    files = [p for root in ("steward", "plugins") for p in (_REPO / root).rglob("*.py")
+             if any(w in p.read_text() for w in _WRITERS)]
+    logged = set().union(*(logged_actions(p.read_text()) for p in files))
+    assert "settings.saved" in logged and "backup.made" in logged      # the walk finds them
+    assert logged - set(ACTIONS) == set(), "Word these in ACTIONS in steward/audit/service.py"
+
+
+def test_the_guard_can_fail():
+    code = 'async def f(app):\n    await log_audit(app, None, "", "widget.exploded", entity_type="widget")\n'
+    assert logged_actions(code) - set(ACTIONS) == {"widget.exploded"}

tests/core/test_backup_job.py

@@ -0,0 +1,123 @@
+"""Steward's backup job (steward/backup/job.py): when a backup is due, what
+the Backups page says, and the Test button's words."""
+import asyncio
+from datetime import datetime, timezone
+from zoneinfo import ZoneInfo
+
+from quart import render_template, session
+
+from steward.backup import job
+from steward.backup.s3 import S3Error, StoredObject
+from steward.core.settings import DEFAULTS
+
+UTC = timezone.utc
+NY = ZoneInfo("America/New_York")
+
+
+def at(h, m=0, day=12):
+    return datetime(2026, 10, day, h, m, tzinfo=UTC)
+
+
+def test_the_slot_is_todays_hour_once_it_has_come_else_yesterdays():
+    assert job.last_slot(at(2), 3, UTC) == at(3, day=11)
+    assert job.last_slot(at(4), 3, UTC) == at(3)
+    # 03:00 in New York in October is 07:00 UTC.
+    assert job.last_slot(at(8), 3, NY) == at(7)
+
+
+def test_due_once_per_slot_and_retried_after_an_hour():
+    assert job.is_due(at(4), 3, UTC, last_success=None, last_attempt=None)
+    assert not job.is_due(at(4), 3, UTC, last_success=at(3, 1), last_attempt=at(3, 1))
+    # Yesterday's success doesn't cover today's slot: a missed day is made up.
+    assert job.is_due(at(9), 3, UTC, last_success=at(3, 1, day=11), last_attempt=at(3, 1, day=11))
+    # A failure at 03:01 waits an hour before the next try.
+    assert not job.is_due(at(3, 30), 3, UTC, last_success=None, last_attempt=at(3, 1))
+    assert job.is_due(at(4, 2), 3, UTC, last_success=None, last_attempt=at(3, 1))
+
+
+def test_history_keeps_the_newest_thirty():
+    history = [{"n": i} for i in range(30)]
+    kept = job.history_with(history, {"n": "new"})
+    assert len(kept) == job.HISTORY_KEEP and kept[0] == {"n": "new"} and kept[-1] == {"n": 28}
+
+
+def _settings(**over):
+    return {**DEFAULTS, "backup.s3_endpoint": "http://archives:8081", "backup.s3_bucket": "steward",
+            "backup.s3_access_key": "AK", "backup.s3_secret_key": "SK", **over}
+
+
+def test_a_target_needs_every_field():
+    t = job.target_from(_settings())
+    assert t.endpoint == "http://archives:8081" and t.bucket == "steward" and t.region == "us-east-1"
+    assert job.target_from(_settings(**{"backup.s3_secret_key": ""})) is None
+
+
+def test_the_hour_is_shown_on_the_readers_clock():
+    assert job.shown_hour(3, NY, UTC, now=at(12)) == 7
+    assert job.shown_hour(3, UTC, UTC, now=at(12)) == 3
+
+
+def test_the_verdict_names_what_matters_first():
+    ok = {"at": at(3).isoformat(), "ok": True, "name": "steward-x.tar", "size": 2048, "error": ""}
+    bad = {"at": at(4).isoformat(), "ok": False, "name": "", "size": 0, "error": "HTTP 403: the key is wrong."}
+    assert job.verdict(DEFAULTS, False)["tone"] == "off"
+    assert job.verdict(_settings(**{"backup.enabled": True}), False)["tone"] == "warn"
+    assert job.verdict(_settings(**{"backup.enabled": True, "backup.s3_bucket": ""}), False)["tone"] == "bad"
+    good = job.verdict(_settings(**{"backup.enabled": True, "backup.history": [ok],
+                                    "backup.last_success_at": ok["at"]}), False)
+    assert good["tone"] == "ok" and "steward-x.tar (2.0 KiB)" in good["text"] and good["moment"] == at(3)
+    failed = job.verdict(_settings(**{"backup.enabled": True, "backup.history": [bad, ok],
+                                      "backup.last_success_at": ok["at"]}), False)
+    assert failed["tone"] == "bad" and failed["error"] == bad["error"] and failed["good"] == at(3)
+    assert job.verdict(DEFAULTS, True)["text"] == "A backup is being made now"
+
+
+def test_test_button_reads_an_append_only_refusal_as_reached(monkeypatch):
+    target = job.target_from(_settings())
+
+    async def denied(*a, **k):
+        raise S3Error("Listing steward refused: HTTP 403 AccessDenied: the key isn't allowed to do that.")
+
+    async def wrong(*a, **k):
+        raise S3Error("Listing steward refused: HTTP 403 SignatureDoesNotMatch: the secret key is wrong.")
+
+    async def two(*a, **k):
+        return [StoredObject("steward-20261011T030000Z.tar", 1, ""), StoredObject("steward-20261012T030000Z.tar", 1, "")]
+
+    monkeypatch.setattr(job, "list_objects", denied)
+    ok, said = asyncio.run(job.test_target(target))
+    assert ok and "append-only" in said
+    monkeypatch.setattr(job, "list_objects", wrong)
+    ok, said = asyncio.run(job.test_target(target))
+    assert not ok and "secret key is wrong" in said
+    monkeypatch.setattr(job, "list_objects", two)
+    ok, said = asyncio.run(job.test_target(target))
+    assert ok and "2 Steward backups, the newest steward-20261012T030000Z.tar" in said
+
+
+async def _render(app, **ctx):
+    async with app.test_request_context("/settings/backups/"):
+        session["user_role"] = "admin"
+        session["tz"] = "America/New_York"
+        return await render_template("settings/backups.html", **ctx)
+
+
+async def test_the_page_leads_with_the_verdict_and_lists_runs_in_the_readers_zone(app):
+    rows = job.history_rows([
+        {"at": at(4).isoformat(), "ok": False, "by": "schedule", "name": "", "size": 0, "secs": 1.2,
+         "error": "HTTP 403 SignatureDoesNotMatch: the secret key is wrong."},
+        {"at": at(3).isoformat(), "ok": True, "by": "admin", "name": "steward-20261012T030000Z.tar",
+         "size": 2048, "secs": 3.4, "error": ""},
+    ])
+    settings = _settings(**{"backup.enabled": True})
+    v = {"tone": "bad", "text": "The last backup failed", "moment": at(4),
+         "error": rows[0]["error"], "good": at(3), "failed": True}
+    page = await _render(app, settings=settings, flash=None, shown_hour=3, verdict=v,
+                         history=rows, has_secret=True)
+    main = page[page.index("<main"):]
+    assert main.index("The last backup failed") < main.index("Bucket") < main.index("Recent backups")
+    assert "the secret key is wrong" in main and "The newest good backup was sent" in main
+    assert "kept; type to change" in main and 'value="SK"' not in main
+    assert "the schedule" in main and "steward-20261012T030000Z.tar" in main and "2.0 KiB" in main
+    assert "UTC" not in main
+    assert '<a href="/settings/backups/" aria-current="page">' in page

tests/core/test_backup_s3.py

@@ -0,0 +1,112 @@
+"""The backup's S3 client (steward/backup/s3.py): signatures match AWS's
+published Signature Version 4 examples, and the calls read S3's answers."""
+import asyncio
+from datetime import datetime, timezone
+
+import httpx
+import pytest
+
+from steward.backup import s3
+
+# AWS's example credentials and moment ("Signature Calculations for the
+# Authorization Header", Amazon S3 API reference).
+KEY_ID = "AKIAIOSFODNN7EXAMPLE"
+SECRET = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+AT = datetime(2013, 5, 24, tzinfo=timezone.utc)
+
+
+def _signature(headers):
+    return headers["Authorization"].rsplit("Signature=", 1)[1]
+
+
+def test_aws_example_get_object():
+    h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {"Range": "bytes=0-9"},
+                s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
+    assert "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date," in h["Authorization"]
+    assert "Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request" in h["Authorization"]
+    assert _signature(h) == "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
+
+
+def test_aws_example_list_objects():
+    h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", {},
+                s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
+    assert _signature(h) == "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"
+
+
+def test_a_changed_secret_changes_the_signature():
+    a = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {},
+                s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
+    b = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {},
+                s3.EMPTY_SHA256, KEY_ID, SECRET + "x", "us-east-1", AT)
+    assert _signature(a) != _signature(b)
+
+
+TARGET = s3.S3Target("http://archives:8081/s3", "steward", KEY_ID, SECRET)
+
+
+def test_paths_are_path_style():
+    assert TARGET.url("steward-20261012T000000Z.tar") == "http://archives:8081/s3/steward/steward-20261012T000000Z.tar"
+    assert TARGET.url() == "http://archives:8081/s3/steward/"
+
+
+def test_put_sends_the_file_signed(tmp_path):
+    f = tmp_path / "b.tar"
+    f.write_bytes(b"x" * 5000)
+    seen = {}
+
+    def handler(request: httpx.Request):
+        seen["method"], seen["url"] = request.method, str(request.url)
+        seen["body"] = request.read()
+        seen["auth"] = request.headers["authorization"]
+        seen["sha"] = request.headers["x-amz-content-sha256"]
+        return httpx.Response(200)
+
+    asyncio.run(s3.put_file(TARGET, "b.tar", f, transport=httpx.MockTransport(handler)))
+    assert seen["method"] == "PUT"
+    assert seen["url"] == "http://archives:8081/s3/steward/b.tar"
+    assert seen["body"] == b"x" * 5000
+    assert seen["sha"] == s3.file_sha256(f)
+    assert seen["auth"].startswith("AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/")
+
+
+def test_a_refusal_says_why(tmp_path):
+    f = tmp_path / "b.tar"
+    f.write_bytes(b"x")
+    body = b"<Error><Code>SignatureDoesNotMatch</Code><Message>no</Message></Error>"
+    transport = httpx.MockTransport(lambda r: httpx.Response(403, content=body))
+    with pytest.raises(s3.S3Error, match="HTTP 403 SignatureDoesNotMatch: the secret key is wrong"):
+        asyncio.run(s3.put_file(TARGET, "b.tar", f, transport=transport))
+
+
+def test_unreachable_says_so(tmp_path):
+    f = tmp_path / "b.tar"
+    f.write_bytes(b"x")
+
+    def handler(request):
+        raise httpx.ConnectError("refused", request=request)
+
+    with pytest.raises(s3.S3Error, match="Couldn't reach http://archives:8081/s3"):
+        asyncio.run(s3.put_file(TARGET, "b.tar", f, transport=httpx.MockTransport(handler)))
+
+
+def test_list_reads_contents():
+    xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Name>steward</Name><KeyCount>2</KeyCount>
+  <Contents><Key>steward-20261011T000000Z.tar</Key><Size>1200</Size>
+    <LastModified>2026-10-11T00:00:01.000Z</LastModified></Contents>
+  <Contents><Key>steward-20261012T000000Z.tar</Key><Size>1300</Size>
+    <LastModified>2026-10-12T00:00:01.000Z</LastModified></Contents>
+</ListBucketResult>"""
+    seen = {}
+
+    def handler(request):
+        seen["params"] = dict(request.url.params)
+        return httpx.Response(200, content=xml)
+
+    got = asyncio.run(s3.list_objects(TARGET, "steward-", transport=httpx.MockTransport(handler)))
+    assert seen["params"] == {"list-type": "2", "max-keys": "1000", "prefix": "steward-"}
+    assert got == [
+        s3.StoredObject("steward-20261011T000000Z.tar", 1200, "2026-10-11T00:00:01.000Z"),
+        s3.StoredObject("steward-20261012T000000Z.tar", 1300, "2026-10-12T00:00:01.000Z"),
+    ]

tests/integration/test_backup_job.py

@@ -0,0 +1,169 @@
+"""Integration: the backup job records each run against Postgres (#5673).
+pg_dump isn't in this lane (the publish smoke step runs the real one), so the
+dump and the upload are stubbed; what is real is everything Steward records:
+history, the last-success time, metrics, audit, and the sealed secret key.
+Requires STEWARD_DATABASE_URL."""
+from __future__ import annotations
+
+import asyncio
+import os
+import time
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+async def _configure(app, **over):
+    from steward.core.settings import set_setting
+    values = {"backup.enabled": True, "backup.s3_endpoint": "http://archives:8081",
+              "backup.s3_bucket": "steward", "backup.s3_access_key": "AK",
+              "backup.s3_secret_key": "SK", "backup.history": [],
+              "backup.last_success_at": "", "backup.last_attempt_at": "", **over}
+    async with app.db_sessionmaker() as db:
+        async with db.begin():
+            for k, v in values.items():
+                await set_setting(db, k, v)
+
+
+def _stub(monkeypatch, *, fail_upload: str = ""):
+    from steward.backup import job
+    from steward.backup.dump import Made
+    from steward.backup.s3 import S3Error
+    sent = []
+
+    async def make_backup(database_url, secret_key, heads, workdir, now=None):
+        path = workdir / "steward-stub.tar"
+        path.write_bytes(b"x" * 4096)
+        return Made(path, path.name, 4096, {"migration_heads": heads})
+
+    async def put_file(target, key, path, **kw):
+        if fail_upload:
+            raise S3Error(fail_upload)
+        sent.append((target.bucket, key, path.read_bytes()))
+
+    monkeypatch.setattr(job, "make_backup", make_backup)
+    monkeypatch.setattr(job, "put_file", put_file)
+    return sent
+
+
+async def _state(app):
+    from sqlalchemy import select
+    from steward.core.settings import get_all_settings
+    from steward.models.audit import AuditEvent
+    from steward.models.metrics import PluginMetricLatest
+    async with app.db_sessionmaker() as db:
+        settings = await get_all_settings(db)
+        metrics = dict((await db.execute(select(PluginMetricLatest.metric_name, PluginMetricLatest.value).where(
+            PluginMetricLatest.source_module == "steward",
+            PluginMetricLatest.resource_name == "backup"))).all())
+        audits = list((await db.execute(select(AuditEvent.action).where(
+            AuditEvent.action.like("backup.%")).order_by(AuditEvent.timestamp.desc(), AuditEvent.id.desc()).limit(1))).scalars())
+    return settings, metrics, audits
+
+
+def test_a_sent_backup_is_recorded_and_a_failed_one_says_why(app, monkeypatch):
+    from steward.backup import job
+
+    async def _go():
+        await _configure(app)
+        sent = _stub(monkeypatch)
+        entry = await job.run_backup(app, by="admin-x")
+        ok_state = await _state(app)
+
+        _stub(monkeypatch, fail_upload="Upload of steward-stub.tar refused: HTTP 403 AccessDenied.")
+        failed = await job.run_backup(app)
+        bad_state = await _state(app)
+        return sent, entry, ok_state, failed, bad_state
+
+    sent, entry, (settings, metrics, audits), failed, (settings2, _, audits2) = asyncio.run(_go())
+    assert sent == [("steward", "steward-stub.tar", b"x" * 4096)]
+    assert entry["ok"] and entry["by"] == "admin-x" and entry["size"] == 4096
+    assert settings["backup.history"][0]["name"] == "steward-stub.tar"
+    assert settings["backup.last_success_at"] == entry["at"]
+    assert metrics["backup_bytes"] == 4096 and metrics["backup_age_secs"] == 0
+    assert audits == ["backup.made"]
+
+    assert not failed["ok"] and "AccessDenied" in failed["error"]
+    assert [h["ok"] for h in settings2["backup.history"]] == [False, True]
+    assert settings2["backup.last_success_at"] == entry["at"]       # the good one still stands
+    assert settings2["backup.last_attempt_at"] == failed["at"]
+    assert audits2 == ["backup.failed"]
+
+
+def test_the_tick_records_the_age_and_backs_up_when_due(app, monkeypatch):
+    from steward.backup import job
+
+    async def _go():
+        long_ago = (datetime.now(timezone.utc) - timedelta(days=2)).isoformat()
+        await _configure(app, **{"backup.last_success_at": long_ago, "backup.last_attempt_at": long_ago})
+        sent = _stub(monkeypatch)
+        await job.tick(app)
+        settings, metrics, _ = await _state(app)
+        await job.tick(app)                    # not due again until the next slot
+        return sent, settings
+
+    sent, settings = asyncio.run(_go())
+    assert len(sent) == 1
+    assert settings["backup.history"][0]["by"] == "schedule"
+
+
+def test_the_tick_does_nothing_while_backups_are_off(app, monkeypatch):
+    from steward.backup import job
+
+    async def _go():
+        await _configure(app, **{"backup.enabled": False})
+        sent = _stub(monkeypatch)
+        await job.tick(app)
+        return sent
+
+    assert asyncio.run(_go()) == []
+
+
+def test_the_secret_key_is_sealed_and_a_blank_box_keeps_it(app):
+    from sqlalchemy import select
+    from steward.models.settings import AppSetting
+    from steward.models.users import User, UserRole
+
+    async def _go():
+        await _configure(app)
+        uid = str(uuid.uuid4())
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add(User(id=uid, username=f"bk-{uid[:8]}", email=f"bk-{uid[:8]}@test",
+                           password_hash="x", role=UserRole.admin, is_active=True))
+        client = app.test_client()
+        async with client.session_transaction() as sess:
+            sess.update(user_id=uid, user_role="admin", username=f"bk-{uid[:8]}", _csrf="test-token",
+                        login_at=time.time(), tz="UTC")
+        form = {"csrf_token": "test-token", "backup.enabled": "on", "backup.s3_endpoint": "http://archives:8081",
+                "backup.s3_bucket": "steward", "backup.s3_region": "", "backup.s3_access_key": "AK2",
+                "backup.s3_secret_key": "new-secret", "backup.schedule_hour": "4"}
+        r1 = await client.post("/settings/backups/", form=form)
+        async with app.db_sessionmaker() as db:
+            stored = (await db.execute(select(AppSetting.value_json).where(
+                AppSetting.key == "backup.s3_secret_key"))).scalar()
+        r2 = await client.post("/settings/backups/", form=form | {"backup.s3_secret_key": ""})
+        page = await client.get("/settings/backups/")
+        from steward.core.settings import get_all_settings
+        async with app.db_sessionmaker() as db:
+            settings = await get_all_settings(db)
+        return r1.status_code, stored, r2.status_code, page.status_code, await page.get_data(as_text=True), settings
+
+    s1, stored, s2, s3, page, settings = asyncio.run(_go())
+    assert s1 == 302 and s2 == 302 and s3 == 200
+    assert "enc:v1:" in stored and "new-secret" not in stored
+    assert settings["backup.s3_secret_key"] == "new-secret"
+    assert settings["backup.s3_access_key"] == "AK2" and settings["backup.s3_region"] == "us-east-1"
+    assert settings["backup.schedule_hour"] == 4 and settings["backup.schedule_tz"] == "UTC"
+    assert "new-secret" not in page and "kept; type to change" in page