feat(backup): daily backups to an S3 bucket, Settings → Backups, and the guide (#5673, #5675)

#5673 "S3 client and the backup job: settings, schedule, history, metrics": - steward/backup/s3.py: SigV4 over httpx, path-style; put and list only (an append-only key is enough; retention is the bucket's). Signing is held to AWS's published examples. - steward/backup/job.py: a 15-minute tick. The daily backup goes at the hour chosen on the reader's clock; a missed day is made up, and a failure is retried an hour later. History of the last 30 runs, audit (backup.made / backup.failed), and backup_age_secs / backup_bytes under "Steward itself", resource backup, for alert rules. - Settings → Backups: verdict first, then the bucket (secret sealed; a blank box keeps it), the schedule, Test the bucket, Back up now, and the recent runs. #5675 "Docs: backing up to Archives, and restoring": docs/core/backups.md. Also: the audit page now words the Prometheus target actions from #5668, which showed as codes, and a guard fails on any logged action without wording. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 07:23:58 -04:00
parent 357bb6480d
commit 49799bd0b3
16 changed files with 1254 additions and 5 deletions
@@ -0,0 +1,38 @@
+"""Every action Steward writes to the audit log reads as words on the audit
+page (steward/audit/service.py ACTIONS), not as its code."""
+import ast
+import pathlib
+import re
+
+import steward
+from steward.audit.service import ACTIONS
+
+_REPO = pathlib.Path(steward.__file__).parent.parent
+_CODE = re.compile(r"[a-z_]+(\.[a-z_]+)+")
+# The writers: log_audit, and audit_request (imported as _audit in places).
+_WRITERS = {"log_audit", "audit_request", "_audit"}
+
+
+def logged_actions(source: str) -> set[str]:
+    """Literal action codes handed to a writer (positionally or as action=)."""
+    found = set()
+    for call in ast.walk(ast.parse(source)):
+        if not (isinstance(call, ast.Call) and getattr(call.func, "id", getattr(call.func, "attr", "")) in _WRITERS):
+            continue
+        args = list(call.args) + [k.value for k in call.keywords if k.arg == "action"]
+        found |= {a.value for a in args
+                  if isinstance(a, ast.Constant) and isinstance(a.value, str) and _CODE.fullmatch(a.value)}
+    return found
+
+
+def test_every_logged_action_is_worded():
+    files = [p for root in ("steward", "plugins") for p in (_REPO / root).rglob("*.py")
+             if any(w in p.read_text() for w in _WRITERS)]
+    logged = set().union(*(logged_actions(p.read_text()) for p in files))
+    assert "settings.saved" in logged and "backup.made" in logged      # the walk finds them
+    assert logged - set(ACTIONS) == set(), "Word these in ACTIONS in steward/audit/service.py"
+
+
+def test_the_guard_can_fail():
+    code = 'async def f(app):\n    await log_audit(app, None, "", "widget.exploded", entity_type="widget")\n'
+    assert logged_actions(code) - set(ACTIONS) == {"widget.exploded"}
@@ -0,0 +1,123 @@
+"""Steward's backup job (steward/backup/job.py): when a backup is due, what
+the Backups page says, and the Test button's words."""
+import asyncio
+from datetime import datetime, timezone
+from zoneinfo import ZoneInfo
+
+from quart import render_template, session
+
+from steward.backup import job
+from steward.backup.s3 import S3Error, StoredObject
+from steward.core.settings import DEFAULTS
+
+UTC = timezone.utc
+NY = ZoneInfo("America/New_York")
+
+
+def at(h, m=0, day=12):
+    return datetime(2026, 10, day, h, m, tzinfo=UTC)
+
+
+def test_the_slot_is_todays_hour_once_it_has_come_else_yesterdays():
+    assert job.last_slot(at(2), 3, UTC) == at(3, day=11)
+    assert job.last_slot(at(4), 3, UTC) == at(3)
+    # 03:00 in New York in October is 07:00 UTC.
+    assert job.last_slot(at(8), 3, NY) == at(7)
+
+
+def test_due_once_per_slot_and_retried_after_an_hour():
+    assert job.is_due(at(4), 3, UTC, last_success=None, last_attempt=None)
+    assert not job.is_due(at(4), 3, UTC, last_success=at(3, 1), last_attempt=at(3, 1))
+    # Yesterday's success doesn't cover today's slot: a missed day is made up.
+    assert job.is_due(at(9), 3, UTC, last_success=at(3, 1, day=11), last_attempt=at(3, 1, day=11))
+    # A failure at 03:01 waits an hour before the next try.
+    assert not job.is_due(at(3, 30), 3, UTC, last_success=None, last_attempt=at(3, 1))
+    assert job.is_due(at(4, 2), 3, UTC, last_success=None, last_attempt=at(3, 1))
+
+
+def test_history_keeps_the_newest_thirty():
+    history = [{"n": i} for i in range(30)]
+    kept = job.history_with(history, {"n": "new"})
+    assert len(kept) == job.HISTORY_KEEP and kept[0] == {"n": "new"} and kept[-1] == {"n": 28}
+
+
+def _settings(**over):
+    return {**DEFAULTS, "backup.s3_endpoint": "http://archives:8081", "backup.s3_bucket": "steward",
+            "backup.s3_access_key": "AK", "backup.s3_secret_key": "SK", **over}
+
+
+def test_a_target_needs_every_field():
+    t = job.target_from(_settings())
+    assert t.endpoint == "http://archives:8081" and t.bucket == "steward" and t.region == "us-east-1"
+    assert job.target_from(_settings(**{"backup.s3_secret_key": ""})) is None
+
+
+def test_the_hour_is_shown_on_the_readers_clock():
+    assert job.shown_hour(3, NY, UTC, now=at(12)) == 7
+    assert job.shown_hour(3, UTC, UTC, now=at(12)) == 3
+
+
+def test_the_verdict_names_what_matters_first():
+    ok = {"at": at(3).isoformat(), "ok": True, "name": "steward-x.tar", "size": 2048, "error": ""}
+    bad = {"at": at(4).isoformat(), "ok": False, "name": "", "size": 0, "error": "HTTP 403: the key is wrong."}
+    assert job.verdict(DEFAULTS, False)["tone"] == "off"
+    assert job.verdict(_settings(**{"backup.enabled": True}), False)["tone"] == "warn"
+    assert job.verdict(_settings(**{"backup.enabled": True, "backup.s3_bucket": ""}), False)["tone"] == "bad"
+    good = job.verdict(_settings(**{"backup.enabled": True, "backup.history": [ok],
+                                    "backup.last_success_at": ok["at"]}), False)
+    assert good["tone"] == "ok" and "steward-x.tar (2.0 KiB)" in good["text"] and good["moment"] == at(3)
+    failed = job.verdict(_settings(**{"backup.enabled": True, "backup.history": [bad, ok],
+                                      "backup.last_success_at": ok["at"]}), False)
+    assert failed["tone"] == "bad" and failed["error"] == bad["error"] and failed["good"] == at(3)
+    assert job.verdict(DEFAULTS, True)["text"] == "A backup is being made now"
+
+
+def test_test_button_reads_an_append_only_refusal_as_reached(monkeypatch):
+    target = job.target_from(_settings())
+
+    async def denied(*a, **k):
+        raise S3Error("Listing steward refused: HTTP 403 AccessDenied: the key isn't allowed to do that.")
+
+    async def wrong(*a, **k):
+        raise S3Error("Listing steward refused: HTTP 403 SignatureDoesNotMatch: the secret key is wrong.")
+
+    async def two(*a, **k):
+        return [StoredObject("steward-20261011T030000Z.tar", 1, ""), StoredObject("steward-20261012T030000Z.tar", 1, "")]
+
+    monkeypatch.setattr(job, "list_objects", denied)
+    ok, said = asyncio.run(job.test_target(target))
+    assert ok and "append-only" in said
+    monkeypatch.setattr(job, "list_objects", wrong)
+    ok, said = asyncio.run(job.test_target(target))
+    assert not ok and "secret key is wrong" in said
+    monkeypatch.setattr(job, "list_objects", two)
+    ok, said = asyncio.run(job.test_target(target))
+    assert ok and "2 Steward backups, the newest steward-20261012T030000Z.tar" in said
+
+
+async def _render(app, **ctx):
+    async with app.test_request_context("/settings/backups/"):
+        session["user_role"] = "admin"
+        session["tz"] = "America/New_York"
+        return await render_template("settings/backups.html", **ctx)
+
+
+async def test_the_page_leads_with_the_verdict_and_lists_runs_in_the_readers_zone(app):
+    rows = job.history_rows([
+        {"at": at(4).isoformat(), "ok": False, "by": "schedule", "name": "", "size": 0, "secs": 1.2,
+         "error": "HTTP 403 SignatureDoesNotMatch: the secret key is wrong."},
+        {"at": at(3).isoformat(), "ok": True, "by": "admin", "name": "steward-20261012T030000Z.tar",
+         "size": 2048, "secs": 3.4, "error": ""},
+    ])
+    settings = _settings(**{"backup.enabled": True})
+    v = {"tone": "bad", "text": "The last backup failed", "moment": at(4),
+         "error": rows[0]["error"], "good": at(3), "failed": True}
+    page = await _render(app, settings=settings, flash=None, shown_hour=3, verdict=v,
+                         history=rows, has_secret=True)
+    main = page[page.index("<main"):]
+    assert main.index("The last backup failed") < main.index("Bucket") < main.index("Recent backups")
+    assert "the secret key is wrong" in main and "The newest good backup was sent" in main
+    assert "kept; type to change" in main and 'value="SK"' not in main
+    assert "the schedule" in main and "steward-20261012T030000Z.tar" in main and "2.0 KiB" in main
+    assert "UTC" not in main
+    assert '<a href="/settings/backups/" aria-current="page">' in page
@@ -0,0 +1,112 @@
+"""The backup's S3 client (steward/backup/s3.py): signatures match AWS's
+published Signature Version 4 examples, and the calls read S3's answers."""
+import asyncio
+from datetime import datetime, timezone
+
+import httpx
+import pytest
+
+from steward.backup import s3
+
+# AWS's example credentials and moment ("Signature Calculations for the
+# Authorization Header", Amazon S3 API reference).
+KEY_ID = "AKIAIOSFODNN7EXAMPLE"
+SECRET = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+AT = datetime(2013, 5, 24, tzinfo=timezone.utc)
+
+
+def _signature(headers):
+    return headers["Authorization"].rsplit("Signature=", 1)[1]
+
+
+def test_aws_example_get_object():
+    h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {"Range": "bytes=0-9"},
+                s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
+    assert "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date," in h["Authorization"]
+    assert "Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request" in h["Authorization"]
+    assert _signature(h) == "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
+
+
+def test_aws_example_list_objects():
+    h = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", {},
+                s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
+    assert _signature(h) == "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"
+
+
+def test_a_changed_secret_changes_the_signature():
+    a = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {},
+                s3.EMPTY_SHA256, KEY_ID, SECRET, "us-east-1", AT)
+    b = s3.sign("GET", "https://examplebucket.s3.amazonaws.com/test.txt", {},
+                s3.EMPTY_SHA256, KEY_ID, SECRET + "x", "us-east-1", AT)
+    assert _signature(a) != _signature(b)
+
+
+TARGET = s3.S3Target("http://archives:8081/s3", "steward", KEY_ID, SECRET)
+
+
+def test_paths_are_path_style():
+    assert TARGET.url("steward-20261012T000000Z.tar") == "http://archives:8081/s3/steward/steward-20261012T000000Z.tar"
+    assert TARGET.url() == "http://archives:8081/s3/steward/"
+
+
+def test_put_sends_the_file_signed(tmp_path):
+    f = tmp_path / "b.tar"
+    f.write_bytes(b"x" * 5000)
+    seen = {}
+
+    def handler(request: httpx.Request):
+        seen["method"], seen["url"] = request.method, str(request.url)
+        seen["body"] = request.read()
+        seen["auth"] = request.headers["authorization"]
+        seen["sha"] = request.headers["x-amz-content-sha256"]
+        return httpx.Response(200)
+
+    asyncio.run(s3.put_file(TARGET, "b.tar", f, transport=httpx.MockTransport(handler)))
+    assert seen["method"] == "PUT"
+    assert seen["url"] == "http://archives:8081/s3/steward/b.tar"
+    assert seen["body"] == b"x" * 5000
+    assert seen["sha"] == s3.file_sha256(f)
+    assert seen["auth"].startswith("AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/")
+
+
+def test_a_refusal_says_why(tmp_path):
+    f = tmp_path / "b.tar"
+    f.write_bytes(b"x")
+    body = b"<Error><Code>SignatureDoesNotMatch</Code><Message>no</Message></Error>"
+    transport = httpx.MockTransport(lambda r: httpx.Response(403, content=body))
+    with pytest.raises(s3.S3Error, match="HTTP 403 SignatureDoesNotMatch: the secret key is wrong"):
+        asyncio.run(s3.put_file(TARGET, "b.tar", f, transport=transport))
+
+
+def test_unreachable_says_so(tmp_path):
+    f = tmp_path / "b.tar"
+    f.write_bytes(b"x")
+
+    def handler(request):
+        raise httpx.ConnectError("refused", request=request)
+
+    with pytest.raises(s3.S3Error, match="Couldn't reach http://archives:8081/s3"):
+        asyncio.run(s3.put_file(TARGET, "b.tar", f, transport=httpx.MockTransport(handler)))
+
+
+def test_list_reads_contents():
+    xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Name>steward</Name><KeyCount>2</KeyCount>
+  <Contents><Key>steward-20261011T000000Z.tar</Key><Size>1200</Size>
+    <LastModified>2026-10-11T00:00:01.000Z</LastModified></Contents>
+  <Contents><Key>steward-20261012T000000Z.tar</Key><Size>1300</Size>
+    <LastModified>2026-10-12T00:00:01.000Z</LastModified></Contents>
+</ListBucketResult>"""
+    seen = {}
+
+    def handler(request):
+        seen["params"] = dict(request.url.params)
+        return httpx.Response(200, content=xml)
+
+    got = asyncio.run(s3.list_objects(TARGET, "steward-", transport=httpx.MockTransport(handler)))
+    assert seen["params"] == {"list-type": "2", "max-keys": "1000", "prefix": "steward-"}
+    assert got == [
+        s3.StoredObject("steward-20261011T000000Z.tar", 1200, "2026-10-11T00:00:01.000Z"),
+        s3.StoredObject("steward-20261012T000000Z.tar", 1300, "2026-10-12T00:00:01.000Z"),
+    ]