bvandeusenandClaude Opus 5.5
e8b75143f1
#5823 "Check a backup: compare what it would restore with what is in place, without restoring". - steward/backup/check.py: pg_restore writes the backup's rows as COPY text with no database involved. The rows in place go out through COPY in the same form. Tables are compared on their primary keys, over the columns both sides have, and grouped by area. - Sealed values are opened with each side's own key and compared as plaintext digests, so a secret sealed twice still matches. They are never shown. A backup whose key can't open its own secrets fails the check. - What Steward records as it runs (last-seen times, a schedule's last run, backup history, scraped UniFi fields) is left out of the comparison: contents.ACTIVITY, STATE_SETTINGS and HAND_SET_ONLY. - Settings → Backups: - a Check button on each sent backup, which fetches it from the bucket (new s3.get_file); - "Check a file" to upload one, with the size limit raised before the CSRF check reads the form; - a results page. - Each backup Steward sends checks itself once written, and its history row says how that went. - New CLI command `steward check-backup FILE`. CI's image smoke step checks a real backup against its source and expects a match. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
182 lines
8.9 KiB
Python
182 lines
8.9 KiB
Python
"""Checking a backup against what is in place (steward/backup/check.py, #5823):
|
|
COPY text reads back exactly, rows are compared on their keys, secrets are
|
|
compared by what they open to and never shown, and what Steward records as
|
|
it runs is left out of the comparison."""
|
|
from datetime import datetime, timezone
|
|
|
|
from quart import render_template
|
|
|
|
from steward.backup import check
|
|
from steward.backup.check import (AreaCheck, CheckReport, RowDiff, TableCheck, _Opener, _Side,
|
|
compare_table, copy_rows, normalise, schema_of, script_tables,
|
|
unescape)
|
|
from steward.core.crypto import _make_fernet
|
|
from tests.test_style_tokens import hex_colours
|
|
|
|
KEY_A, KEY_B = "backup-key", "here-key"
|
|
|
|
|
|
def _seal(key: str, plain: str) -> str:
|
|
return "enc:v1:" + _make_fernet(key).encrypt(plain.encode()).decode()
|
|
|
|
|
|
def _compare(table, backup_cols, backup_rows, here_cols, here_rows, *, types=None, key=("id",),
|
|
keys=(KEY_A, KEY_B)):
|
|
return compare_table(table, _Side(backup_cols, backup_rows), _Side(here_cols, here_rows),
|
|
types=types or {}, key=list(key),
|
|
openers=(_Opener(keys[0]), _Opener(keys[1])), labels={})
|
|
|
|
|
|
def test_copy_text_reads_back_exactly():
|
|
assert unescape(b"\\N") is None
|
|
assert unescape(b"plain") == "plain"
|
|
assert unescape(b"a\\tb\\nc\\\\d") == "a\tb\nc\\d"
|
|
assert unescape(b"\\101\\x42") == "AB"
|
|
assert unescape("caf\u00e9".encode()) == "caf\u00e9"
|
|
assert copy_rows(b"1\tx\\ty\n2\t\\N\n") == [["1", "x\ty"], ["2", None]]
|
|
|
|
|
|
def test_a_restore_script_gives_each_tables_rows():
|
|
script = (b"SET client_encoding = 'UTF8';\n"
|
|
b'COPY public.app_settings (key, value_json, updated_at) FROM stdin;\n'
|
|
b'smtp.port\t587\t2026-10-15 14:00:00+00\n'
|
|
b'\\.\n\n'
|
|
b'COPY public.audit_events (id, "timestamp", action) FROM stdin;\n'
|
|
b'a1\t2026-10-15 14:00:00+00\tbackup.made\n'
|
|
b'\\.\n')
|
|
tables = script_tables(script)
|
|
assert tables["app_settings"] == (["key", "value_json", "updated_at"],
|
|
[["smtp.port", "587", "2026-10-15 14:00:00+00"]])
|
|
assert tables["audit_events"][0] == ["id", "timestamp", "action"]
|
|
|
|
|
|
def test_an_instant_is_the_same_whatever_zone_prints_it():
|
|
a = normalise("2026-10-15 16:00:00+02", "timestamp with time zone")
|
|
b = normalise("2026-10-15 14:00:00+00", "timestamp with time zone")
|
|
assert a == b == datetime(2026, 10, 15, 14, tzinfo=timezone.utc)
|
|
assert normalise('{"b": 1, "a": 2}', "jsonb") == normalise('{"a":2,"b":1}', "jsonb")
|
|
|
|
|
|
def test_rows_are_compared_on_their_key():
|
|
cols = ["id", "name", "target"]
|
|
result = _compare("monitors", cols,
|
|
[["1", "web", "https://a"], ["2", "dns", "1.1.1.1"], ["3", "gone", "x"]],
|
|
cols,
|
|
[["1", "web", "https://a"], ["2", "dns", "9.9.9.9"], ["4", "new", "y"]])
|
|
assert result.same == 1
|
|
assert [(d.label, d.columns) for d in result.differ] == [("dns", ["target"])]
|
|
assert result.only_backup == ["gone"] and result.only_here == ["new"]
|
|
assert result.changes == 3
|
|
|
|
|
|
def test_a_setting_that_differs_shows_both_values():
|
|
cols = ["key", "value_json", "updated_at"]
|
|
result = _compare("app_settings", cols, [["smtp.port", "587", "2026-10-01 00:00:00+00"]],
|
|
cols, [["smtp.port", "25", "2026-10-15 00:00:00+00"]], key=("key",))
|
|
[d] = result.differ
|
|
assert (d.label, d.secret, d.backup_value, d.here_value) == ("smtp.port", False, "587", "25")
|
|
|
|
|
|
def test_a_secret_sealed_twice_matches_and_one_that_changed_is_never_shown():
|
|
cols = ["key", "value_json"]
|
|
same = _compare("app_settings", cols, [["smtp.password", f'"{_seal(KEY_A, "hunter2")}"']],
|
|
cols, [["smtp.password", f'"{_seal(KEY_B, "hunter2")}"']], key=("key",))
|
|
assert same.same == 1 and not same.differ
|
|
|
|
changed = _compare("app_settings", cols, [["smtp.password", f'"{_seal(KEY_A, "old")}"']],
|
|
cols, [["smtp.password", f'"{_seal(KEY_B, "new")}"']], key=("key",))
|
|
[d] = changed.differ
|
|
assert d.secret and d.backup_value is None and d.here_value is None
|
|
|
|
|
|
def test_a_backup_whose_key_cant_open_its_secrets_is_counted():
|
|
cols = ["key", "value_json"]
|
|
openers = (_Opener(KEY_A), _Opener(KEY_B))
|
|
compare_table("app_settings", _Side(cols, [["ldap.bind_password", f'"{_seal("other", "x")}"']]),
|
|
_Side(cols, []), types={}, key=["key"], openers=openers, labels={})
|
|
assert (openers[0].seen, openers[0].failed) == (1, 1)
|
|
|
|
|
|
def test_what_steward_records_as_it_runs_is_not_a_difference():
|
|
cols = ["id", "host_id", "token_hash", "last_seen_at", "agent_version", "updated_at"]
|
|
result = _compare("host_agent_registrations", cols,
|
|
[["r1", "h1", "abc", "2026-10-15 01:00:00+00", "1.19.0", "x"]], cols,
|
|
[["r1", "h1", "abc", "2026-10-15 14:00:00+00", "1.20.0", "y"]])
|
|
assert result.same == 1 and not result.differ
|
|
|
|
settings = ["key", "value_json"]
|
|
state = _compare("app_settings", settings, [["backup.history", "[]"]],
|
|
settings, [["backup.history", '[{"ok": true}]']], key=("key",))
|
|
assert state.same == 0 and state.changes == 0
|
|
|
|
|
|
def test_a_network_client_counts_only_once_someone_named_it():
|
|
cols = ["mac", "name", "note", "ip"]
|
|
result = _compare("unifi_known_clients", cols, [["aa", "printer", None, "10.0.0.5"]], cols,
|
|
[["aa", "printer", None, "10.0.0.9"], ["bb", None, None, "10.0.0.7"]],
|
|
key=("mac",))
|
|
assert result.same == 1 and result.changes == 0
|
|
|
|
|
|
def test_columns_one_side_lacks_are_noted_not_compared():
|
|
result = _compare("monitors", ["id", "name"], [["1", "web"]],
|
|
["id", "name", "retries"], [["1", "web", "3"]])
|
|
assert result.same == 1 and "retries" in result.note
|
|
|
|
|
|
def test_the_schema_is_same_older_or_newer():
|
|
known = {"0001", "0002", "0003"}
|
|
assert schema_of(["0003"], ["0003"], known) == "same"
|
|
assert schema_of(["0002"], ["0003"], known) == "older"
|
|
assert schema_of(["0009"], ["0003"], known) == "newer"
|
|
|
|
|
|
def test_this_steward_knows_its_own_revisions():
|
|
revisions = check.known_revisions()
|
|
assert len(revisions) > 10 and any(r.startswith("host_agent_") for r in revisions)
|
|
|
|
|
|
def _report(**over):
|
|
tables = [TableCheck("app_settings", same=12,
|
|
differ=[RowDiff("smtp.port", ["value_json"], False, "587", "25"),
|
|
RowDiff("smtp.password", ["value_json"], True)]),
|
|
]
|
|
hosts = [TableCheck("hosts", same=3, only_here=["nas"])]
|
|
fields = dict(name="steward-20261015T030000Z.tar",
|
|
manifest={"created_at": "2026-10-15T03:00:00+00:00", "steward_version": "2026.10.15"},
|
|
schema="same", backup_heads=["a"], here_heads=["a"], key_same=True, sealed=4,
|
|
unopened=0, areas=[AreaCheck("Settings", tables), AreaCheck("Hosts and agents", hosts)])
|
|
return CheckReport(**(fields | over))
|
|
|
|
|
|
def test_the_summary_says_whether_it_matches():
|
|
assert _report().summary() == "3 differences from what is in place: 2 in Settings, 1 in Hosts and agents."
|
|
same = _report(areas=[AreaCheck("Settings", [TableCheck("app_settings", same=12)])])
|
|
assert same.summary() == "Matches what is in place: 12 rows across 1 table." and same.ok
|
|
bad = _report(unopened=4)
|
|
assert not bad.ok and "doesn't open 4 of its 4 sealed values" in bad.summary()
|
|
assert _report().as_entry() == {"ok": True, "changes": 3, "rows": 18,
|
|
"summary": _report().summary()}
|
|
|
|
|
|
async def test_the_check_page_shows_what_differs_and_hides_secrets(app):
|
|
async with app.test_request_context("/settings/backups/check/", method="POST"):
|
|
page = await render_template("settings/backup_check.html", report=_report(), error="",
|
|
name="steward-20261015T030000Z.tar")
|
|
page = " ".join(page.split())
|
|
assert 'href="/settings/backups/"' in page # the trail back
|
|
assert "3 differences from what is in place" in page
|
|
assert "<strong>smtp.port</strong> differs: the backup has <code>587</code>, here it is <code>25</code>." in page
|
|
assert "<strong>smtp.password</strong> differs (a sealed value; not shown)." in page
|
|
assert "<strong>nas</strong> is only here: a restore removes it." in page
|
|
assert "The same as this Steward's." in page or "The same as this Steward's." in page
|
|
assert not hex_colours(page), hex_colours(page)
|
|
|
|
|
|
async def test_a_check_that_couldnt_run_says_why(app):
|
|
async with app.test_request_context("/settings/backups/check/", method="POST"):
|
|
page = await render_template("settings/backup_check.html", report=None,
|
|
error="Reading x refused: HTTP 403 AccessDenied.", name="x")
|
|
assert "x couldn't be checked." in page or "x couldn't be checked." in page
|
|
assert "AccessDenied" in page
|