bvandeusenandClaude Opus 5.5
49799bd0b3
#5673 "S3 client and the backup job: settings, schedule, history, metrics": - steward/backup/s3.py: SigV4 over httpx, path-style; put and list only (an append-only key is enough; retention is the bucket's). Signing is held to AWS's published examples. - steward/backup/job.py: a 15-minute tick. The daily backup goes at the hour chosen on the reader's clock; a missed day is made up, and a failure is retried an hour later. History of the last 30 runs, audit (backup.made / backup.failed), and backup_age_secs / backup_bytes under "Steward itself", resource backup, for alert rules. - Settings → Backups: verdict first, then the bucket (secret sealed; a blank box keeps it), the schedule, Test the bucket, Back up now, and the recent runs. #5675 "Docs: backing up to Archives, and restoring": docs/core/backups.md. Also: the audit page now words the Prometheus target actions from #5668, which showed as codes, and a guard fails on any logged action without wording. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
124 lines
5.9 KiB
Python
124 lines
5.9 KiB
Python
"""Steward's backup job (steward/backup/job.py): when a backup is due, what
|
|
the Backups page says, and the Test button's words."""
|
|
import asyncio
|
|
from datetime import datetime, timezone
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from quart import render_template, session
|
|
|
|
from steward.backup import job
|
|
from steward.backup.s3 import S3Error, StoredObject
|
|
from steward.core.settings import DEFAULTS
|
|
|
|
UTC = timezone.utc
|
|
NY = ZoneInfo("America/New_York")
|
|
|
|
|
|
def at(h, m=0, day=12):
|
|
return datetime(2026, 10, day, h, m, tzinfo=UTC)
|
|
|
|
|
|
def test_the_slot_is_todays_hour_once_it_has_come_else_yesterdays():
|
|
assert job.last_slot(at(2), 3, UTC) == at(3, day=11)
|
|
assert job.last_slot(at(4), 3, UTC) == at(3)
|
|
# 03:00 in New York in October is 07:00 UTC.
|
|
assert job.last_slot(at(8), 3, NY) == at(7)
|
|
|
|
|
|
def test_due_once_per_slot_and_retried_after_an_hour():
|
|
assert job.is_due(at(4), 3, UTC, last_success=None, last_attempt=None)
|
|
assert not job.is_due(at(4), 3, UTC, last_success=at(3, 1), last_attempt=at(3, 1))
|
|
# Yesterday's success doesn't cover today's slot: a missed day is made up.
|
|
assert job.is_due(at(9), 3, UTC, last_success=at(3, 1, day=11), last_attempt=at(3, 1, day=11))
|
|
# A failure at 03:01 waits an hour before the next try.
|
|
assert not job.is_due(at(3, 30), 3, UTC, last_success=None, last_attempt=at(3, 1))
|
|
assert job.is_due(at(4, 2), 3, UTC, last_success=None, last_attempt=at(3, 1))
|
|
|
|
|
|
def test_history_keeps_the_newest_thirty():
|
|
history = [{"n": i} for i in range(30)]
|
|
kept = job.history_with(history, {"n": "new"})
|
|
assert len(kept) == job.HISTORY_KEEP and kept[0] == {"n": "new"} and kept[-1] == {"n": 28}
|
|
|
|
|
|
def _settings(**over):
|
|
return {**DEFAULTS, "backup.s3_endpoint": "http://archives:8081", "backup.s3_bucket": "steward",
|
|
"backup.s3_access_key": "AK", "backup.s3_secret_key": "SK", **over}
|
|
|
|
|
|
def test_a_target_needs_every_field():
|
|
t = job.target_from(_settings())
|
|
assert t.endpoint == "http://archives:8081" and t.bucket == "steward" and t.region == "us-east-1"
|
|
assert job.target_from(_settings(**{"backup.s3_secret_key": ""})) is None
|
|
|
|
|
|
def test_the_hour_is_shown_on_the_readers_clock():
|
|
assert job.shown_hour(3, NY, UTC, now=at(12)) == 7
|
|
assert job.shown_hour(3, UTC, UTC, now=at(12)) == 3
|
|
|
|
|
|
def test_the_verdict_names_what_matters_first():
|
|
ok = {"at": at(3).isoformat(), "ok": True, "name": "steward-x.tar", "size": 2048, "error": ""}
|
|
bad = {"at": at(4).isoformat(), "ok": False, "name": "", "size": 0, "error": "HTTP 403: the key is wrong."}
|
|
assert job.verdict(DEFAULTS, False)["tone"] == "off"
|
|
assert job.verdict(_settings(**{"backup.enabled": True}), False)["tone"] == "warn"
|
|
assert job.verdict(_settings(**{"backup.enabled": True, "backup.s3_bucket": ""}), False)["tone"] == "bad"
|
|
good = job.verdict(_settings(**{"backup.enabled": True, "backup.history": [ok],
|
|
"backup.last_success_at": ok["at"]}), False)
|
|
assert good["tone"] == "ok" and "steward-x.tar (2.0 KiB)" in good["text"] and good["moment"] == at(3)
|
|
failed = job.verdict(_settings(**{"backup.enabled": True, "backup.history": [bad, ok],
|
|
"backup.last_success_at": ok["at"]}), False)
|
|
assert failed["tone"] == "bad" and failed["error"] == bad["error"] and failed["good"] == at(3)
|
|
assert job.verdict(DEFAULTS, True)["text"] == "A backup is being made now"
|
|
|
|
|
|
def test_test_button_reads_an_append_only_refusal_as_reached(monkeypatch):
|
|
target = job.target_from(_settings())
|
|
|
|
async def denied(*a, **k):
|
|
raise S3Error("Listing steward refused: HTTP 403 AccessDenied: the key isn't allowed to do that.")
|
|
|
|
async def wrong(*a, **k):
|
|
raise S3Error("Listing steward refused: HTTP 403 SignatureDoesNotMatch: the secret key is wrong.")
|
|
|
|
async def two(*a, **k):
|
|
return [StoredObject("steward-20261011T030000Z.tar", 1, ""), StoredObject("steward-20261012T030000Z.tar", 1, "")]
|
|
|
|
monkeypatch.setattr(job, "list_objects", denied)
|
|
ok, said = asyncio.run(job.test_target(target))
|
|
assert ok and "append-only" in said
|
|
monkeypatch.setattr(job, "list_objects", wrong)
|
|
ok, said = asyncio.run(job.test_target(target))
|
|
assert not ok and "secret key is wrong" in said
|
|
monkeypatch.setattr(job, "list_objects", two)
|
|
ok, said = asyncio.run(job.test_target(target))
|
|
assert ok and "2 Steward backups, the newest steward-20261012T030000Z.tar" in said
|
|
|
|
|
|
async def _render(app, **ctx):
|
|
async with app.test_request_context("/settings/backups/"):
|
|
session["user_role"] = "admin"
|
|
session["tz"] = "America/New_York"
|
|
return await render_template("settings/backups.html", **ctx)
|
|
|
|
|
|
async def test_the_page_leads_with_the_verdict_and_lists_runs_in_the_readers_zone(app):
|
|
rows = job.history_rows([
|
|
{"at": at(4).isoformat(), "ok": False, "by": "schedule", "name": "", "size": 0, "secs": 1.2,
|
|
"error": "HTTP 403 SignatureDoesNotMatch: the secret key is wrong."},
|
|
{"at": at(3).isoformat(), "ok": True, "by": "admin", "name": "steward-20261012T030000Z.tar",
|
|
"size": 2048, "secs": 3.4, "error": ""},
|
|
])
|
|
settings = _settings(**{"backup.enabled": True})
|
|
v = {"tone": "bad", "text": "The last backup failed", "moment": at(4),
|
|
"error": rows[0]["error"], "good": at(3), "failed": True}
|
|
page = await _render(app, settings=settings, flash=None, shown_hour=3, verdict=v,
|
|
history=rows, has_secret=True)
|
|
main = page[page.index("<main"):]
|
|
assert main.index("The last backup failed") < main.index("Bucket") < main.index("Recent backups")
|
|
assert "the secret key is wrong" in main and "The newest good backup was sent" in main
|
|
assert "kept; type to change" in main and 'value="SK"' not in main
|
|
assert "the schedule" in main and "steward-20261012T030000Z.tar" in main and "2.0 KiB" in main
|
|
assert "UTC" not in main
|
|
assert '<a href="/settings/backups/" aria-current="page">' in page
|