feat(backup): check a backup against what is in place, changing nothing (#5823)

#5823 "Check a backup: compare what it would restore with what is in
place, without restoring".

- steward/backup/check.py: pg_restore writes the backup's rows as COPY
  text with no database involved. The rows in place go out through COPY in
  the same form. Tables are compared on their primary keys, over the columns
  both sides have, and grouped by area.
- Sealed values are opened with each side's own key and compared as
  plaintext digests, so a secret sealed twice still matches. They are never
  shown. A backup whose key can't open its own secrets fails the check.
- What Steward records as it runs (last-seen times, a schedule's last run,
  backup history, scraped UniFi fields) is left out of the comparison:
  contents.ACTIVITY, STATE_SETTINGS and HAND_SET_ONLY.
- Settings → Backups:
  - a Check button on each sent backup, which fetches it from the bucket
    (new s3.get_file);
  - "Check a file" to upload one, with the size limit raised before the
    CSRF check reads the form;
  - a results page.
- Each backup Steward sends checks itself once written, and its history
  row says how that went.
- New CLI command `steward check-backup FILE`. CI's image smoke step checks a
  real backup against its source and expects a match.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_backup_check.py

@@ -0,0 +1,181 @@
+"""Checking a backup against what is in place (steward/backup/check.py, #5823):
+COPY text reads back exactly, rows are compared on their keys, secrets are
+compared by what they open to and never shown, and what Steward records as
+it runs is left out of the comparison."""
+from datetime import datetime, timezone
+
+from quart import render_template
+
+from steward.backup import check
+from steward.backup.check import (AreaCheck, CheckReport, RowDiff, TableCheck, _Opener, _Side,
+                                  compare_table, copy_rows, normalise, schema_of, script_tables,
+                                  unescape)
+from steward.core.crypto import _make_fernet
+from tests.test_style_tokens import hex_colours
+
+KEY_A, KEY_B = "backup-key", "here-key"
+
+
+def _seal(key: str, plain: str) -> str:
+    return "enc:v1:" + _make_fernet(key).encrypt(plain.encode()).decode()
+
+
+def _compare(table, backup_cols, backup_rows, here_cols, here_rows, *, types=None, key=("id",),
+             keys=(KEY_A, KEY_B)):
+    return compare_table(table, _Side(backup_cols, backup_rows), _Side(here_cols, here_rows),
+                         types=types or {}, key=list(key),
+                         openers=(_Opener(keys[0]), _Opener(keys[1])), labels={})
+
+
+def test_copy_text_reads_back_exactly():
+    assert unescape(b"\\N") is None
+    assert unescape(b"plain") == "plain"
+    assert unescape(b"a\\tb\\nc\\\\d") == "a\tb\nc\\d"
+    assert unescape(b"\\101\\x42") == "AB"
+    assert unescape("caf\u00e9".encode()) == "caf\u00e9"
+    assert copy_rows(b"1\tx\\ty\n2\t\\N\n") == [["1", "x\ty"], ["2", None]]
+
+
+def test_a_restore_script_gives_each_tables_rows():
+    script = (b"SET client_encoding = 'UTF8';\n"
+              b'COPY public.app_settings (key, value_json, updated_at) FROM stdin;\n'
+              b'smtp.port\t587\t2026-10-15 14:00:00+00\n'
+              b'\\.\n\n'
+              b'COPY public.audit_events (id, "timestamp", action) FROM stdin;\n'
+              b'a1\t2026-10-15 14:00:00+00\tbackup.made\n'
+              b'\\.\n')
+    tables = script_tables(script)
+    assert tables["app_settings"] == (["key", "value_json", "updated_at"],
+                                      [["smtp.port", "587", "2026-10-15 14:00:00+00"]])
+    assert tables["audit_events"][0] == ["id", "timestamp", "action"]
+
+
+def test_an_instant_is_the_same_whatever_zone_prints_it():
+    a = normalise("2026-10-15 16:00:00+02", "timestamp with time zone")
+    b = normalise("2026-10-15 14:00:00+00", "timestamp with time zone")
+    assert a == b == datetime(2026, 10, 15, 14, tzinfo=timezone.utc)
+    assert normalise('{"b": 1, "a": 2}', "jsonb") == normalise('{"a":2,"b":1}', "jsonb")
+
+
+def test_rows_are_compared_on_their_key():
+    cols = ["id", "name", "target"]
+    result = _compare("monitors", cols,
+                      [["1", "web", "https://a"], ["2", "dns", "1.1.1.1"], ["3", "gone", "x"]],
+                      cols,
+                      [["1", "web", "https://a"], ["2", "dns", "9.9.9.9"], ["4", "new", "y"]])
+    assert result.same == 1
+    assert [(d.label, d.columns) for d in result.differ] == [("dns", ["target"])]
+    assert result.only_backup == ["gone"] and result.only_here == ["new"]
+    assert result.changes == 3
+
+
+def test_a_setting_that_differs_shows_both_values():
+    cols = ["key", "value_json", "updated_at"]
+    result = _compare("app_settings", cols, [["smtp.port", "587", "2026-10-01 00:00:00+00"]],
+                      cols, [["smtp.port", "25", "2026-10-15 00:00:00+00"]], key=("key",))
+    [d] = result.differ
+    assert (d.label, d.secret, d.backup_value, d.here_value) == ("smtp.port", False, "587", "25")
+
+
+def test_a_secret_sealed_twice_matches_and_one_that_changed_is_never_shown():
+    cols = ["key", "value_json"]
+    same = _compare("app_settings", cols, [["smtp.password", f'"{_seal(KEY_A, "hunter2")}"']],
+                    cols, [["smtp.password", f'"{_seal(KEY_B, "hunter2")}"']], key=("key",))
+    assert same.same == 1 and not same.differ
+
+    changed = _compare("app_settings", cols, [["smtp.password", f'"{_seal(KEY_A, "old")}"']],
+                       cols, [["smtp.password", f'"{_seal(KEY_B, "new")}"']], key=("key",))
+    [d] = changed.differ
+    assert d.secret and d.backup_value is None and d.here_value is None
+
+
+def test_a_backup_whose_key_cant_open_its_secrets_is_counted():
+    cols = ["key", "value_json"]
+    openers = (_Opener(KEY_A), _Opener(KEY_B))
+    compare_table("app_settings", _Side(cols, [["ldap.bind_password", f'"{_seal("other", "x")}"']]),
+                  _Side(cols, []), types={}, key=["key"], openers=openers, labels={})
+    assert (openers[0].seen, openers[0].failed) == (1, 1)
+
+
+def test_what_steward_records_as_it_runs_is_not_a_difference():
+    cols = ["id", "host_id", "token_hash", "last_seen_at", "agent_version", "updated_at"]
+    result = _compare("host_agent_registrations", cols,
+                      [["r1", "h1", "abc", "2026-10-15 01:00:00+00", "1.19.0", "x"]], cols,
+                      [["r1", "h1", "abc", "2026-10-15 14:00:00+00", "1.20.0", "y"]])
+    assert result.same == 1 and not result.differ
+
+    settings = ["key", "value_json"]
+    state = _compare("app_settings", settings, [["backup.history", "[]"]],
+                     settings, [["backup.history", '[{"ok": true}]']], key=("key",))
+    assert state.same == 0 and state.changes == 0
+
+
+def test_a_network_client_counts_only_once_someone_named_it():
+    cols = ["mac", "name", "note", "ip"]
+    result = _compare("unifi_known_clients", cols, [["aa", "printer", None, "10.0.0.5"]], cols,
+                      [["aa", "printer", None, "10.0.0.9"], ["bb", None, None, "10.0.0.7"]],
+                      key=("mac",))
+    assert result.same == 1 and result.changes == 0
+
+
+def test_columns_one_side_lacks_are_noted_not_compared():
+    result = _compare("monitors", ["id", "name"], [["1", "web"]],
+                      ["id", "name", "retries"], [["1", "web", "3"]])
+    assert result.same == 1 and "retries" in result.note
+
+
+def test_the_schema_is_same_older_or_newer():
+    known = {"0001", "0002", "0003"}
+    assert schema_of(["0003"], ["0003"], known) == "same"
+    assert schema_of(["0002"], ["0003"], known) == "older"
+    assert schema_of(["0009"], ["0003"], known) == "newer"
+
+
+def test_this_steward_knows_its_own_revisions():
+    revisions = check.known_revisions()
+    assert len(revisions) > 10 and any(r.startswith("host_agent_") for r in revisions)
+
+
+def _report(**over):
+    tables = [TableCheck("app_settings", same=12,
+                         differ=[RowDiff("smtp.port", ["value_json"], False, "587", "25"),
+                                 RowDiff("smtp.password", ["value_json"], True)]),
+              ]
+    hosts = [TableCheck("hosts", same=3, only_here=["nas"])]
+    fields = dict(name="steward-20261015T030000Z.tar",
+                  manifest={"created_at": "2026-10-15T03:00:00+00:00", "steward_version": "2026.10.15"},
+                  schema="same", backup_heads=["a"], here_heads=["a"], key_same=True, sealed=4,
+                  unopened=0, areas=[AreaCheck("Settings", tables), AreaCheck("Hosts and agents", hosts)])
+    return CheckReport(**(fields | over))
+
+
+def test_the_summary_says_whether_it_matches():
+    assert _report().summary() == "3 differences from what is in place: 2 in Settings, 1 in Hosts and agents."
+    same = _report(areas=[AreaCheck("Settings", [TableCheck("app_settings", same=12)])])
+    assert same.summary() == "Matches what is in place: 12 rows across 1 table." and same.ok
+    bad = _report(unopened=4)
+    assert not bad.ok and "doesn't open 4 of its 4 sealed values" in bad.summary()
+    assert _report().as_entry() == {"ok": True, "changes": 3, "rows": 18,
+                                    "summary": _report().summary()}
+
+
+async def test_the_check_page_shows_what_differs_and_hides_secrets(app):
+    async with app.test_request_context("/settings/backups/check/", method="POST"):
+        page = await render_template("settings/backup_check.html", report=_report(), error="",
+                                     name="steward-20261015T030000Z.tar")
+    page = " ".join(page.split())
+    assert 'href="/settings/backups/"' in page                     # the trail back
+    assert "3 differences from what is in place" in page
+    assert "<strong>smtp.port</strong> differs: the backup has <code>587</code>, here it is <code>25</code>." in page
+    assert "<strong>smtp.password</strong> differs (a sealed value; not shown)." in page
+    assert "<strong>nas</strong> is only here: a restore removes it." in page
+    assert "The same as this Steward&#39;s." in page or "The same as this Steward's." in page
+    assert not hex_colours(page), hex_colours(page)
+
+
+async def test_a_check_that_couldnt_run_says_why(app):
+    async with app.test_request_context("/settings/backups/check/", method="POST"):
+        page = await render_template("settings/backup_check.html", report=None,
+                                     error="Reading x refused: HTTP 403 AccessDenied.", name="x")
+    assert "x couldn&#39;t be checked." in page or "x couldn't be checked." in page
+    assert "AccessDenied" in page

tests/core/test_backup_s3.py

@@ -110,3 +110,27 @@ def test_list_reads_contents():
         s3.StoredObject("steward-20261011T000000Z.tar", 1200, "2026-10-11T00:00:01.000Z"),
         s3.StoredObject("steward-20261012T000000Z.tar", 1300, "2026-10-12T00:00:01.000Z"),
     ]
+
+
+def test_get_streams_the_object_to_a_file(tmp_path):
+    seen = {}
+
+    def handler(request):
+        seen["method"], seen["url"] = request.method, str(request.url)
+        seen["auth"] = request.headers["authorization"]
+        return httpx.Response(200, content=b"y" * 3000)
+
+    out = tmp_path / "b.tar"
+    size = asyncio.run(s3.get_file(TARGET, "b.tar", out, transport=httpx.MockTransport(handler)))
+    assert (seen["method"], seen["url"]) == ("GET", "http://archives:8081/s3/steward/b.tar")
+    assert seen["auth"].startswith("AWS4-HMAC-SHA256 ")
+    assert size == 3000 and out.read_bytes() == b"y" * 3000
+
+
+def test_a_refused_get_leaves_no_file(tmp_path):
+    body = b"<Error><Code>AccessDenied</Code><Message>no</Message></Error>"
+    out = tmp_path / "b.tar"
+    with pytest.raises(s3.S3Error, match="Reading b.tar refused: HTTP 403 AccessDenied"):
+        asyncio.run(s3.get_file(TARGET, "b.tar", out,
+                                transport=httpx.MockTransport(lambda r: httpx.Response(403, content=body))))
+    assert not out.exists()

tests/integration/test_backup_check.py

@@ -0,0 +1,86 @@
+"""Integration: checking a backup against Postgres (#5823). pg_restore isn't in
+this lane (the publish smoke step checks a real backup with the image), so
+the backup's rows are a snapshot of this database taken through the check's
+own reader; what is real is everything read in place, the comparison and the
+report. Requires STEWARD_DATABASE_URL."""
+from __future__ import annotations
+
+import asyncio
+import os
+import uuid
+from datetime import datetime, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def test_a_check_finds_a_changed_setting_a_deleted_monitor_and_a_new_host(app, monkeypatch, tmp_path):
+    from sqlalchemy import delete
+    from steward.backup import check, contents, dump, job
+    from steward.core.settings import set_setting
+    from steward.models.hosts import Host
+    from steward.models.monitors import Monitor
+
+    tag = uuid.uuid4().hex[:8]
+    setting, monitor_name, host_name = f"test.check_{tag}", f"check-mon-{tag}", f"check-host-{tag}"
+    url = app.config["DATABASE_URL"]
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await set_setting(s, setting, "before")
+                mon = Monitor(name=monitor_name, type="tcp", target="10.9.9.9", config_json="{}")
+                s.add(mon)
+        # The "backup": this database as it stands, read the way a check reads.
+        snapshot, *_ = await check._here(url, sorted(contents.KEPT))
+        theirs = {t: (side.columns, side.rows) for t, side in snapshot.items()}
+
+        async def backup_rows(_dump):
+            return theirs
+        monkeypatch.setattr(check, "_backup_rows", backup_rows)
+
+        path = tmp_path / "steward-20261015T030000Z.tar"
+        (tmp_path / "steward.dump").write_bytes(b"PGDMP")
+        heads = await job.migration_heads(url)
+        dump.pack(path, tmp_path / "steward.dump", app.config["SECRET_KEY"],
+                  dump.manifest(datetime.now(timezone.utc), heads))
+
+        same = await job.check_file(app, path)
+
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await set_setting(s, setting, "after")
+                await s.execute(delete(Monitor).where(Monitor.name == monitor_name))
+                s.add(Host(name=host_name, address="10.9.9.10"))
+        changed = await job.check_file(app, path)
+        return same, changed
+
+    same, changed = asyncio.run(_go())
+    assert same.ok and same.changes == 0, same.summary()
+    assert same.schema == "same" and same.key_same and same.rows > 0
+
+    by_table = {t.table: t for a in changed.areas for t in a.tables}
+    [diff] = [d for d in by_table["app_settings"].differ if d.label == setting]
+    assert (diff.backup_value, diff.here_value) == ('"before"', '"after"')
+    assert monitor_name in by_table["monitors"].only_backup
+    assert host_name in by_table["hosts"].only_here
+    assert changed.changes == 3, changed.summary()
+
+
+def test_a_file_that_isnt_a_backup_fails_the_check(app, tmp_path):
+    from steward.backup import job
+    from steward.backup.dump import BackupError
+
+    bad = tmp_path / "steward-20261015T030000Z.tar"
+    bad.write_bytes(b"not a tar")
+    with pytest.raises(BackupError, match="isn't a Steward backup"):
+        asyncio.run(job.check_file(app, bad))

tests/integration/test_backup_job.py

@@ -89,6 +89,8 @@ def test_a_sent_backup_is_recorded_and_a_failed_one_says_why(app, monkeypatch):
     sent, entry, (settings, metrics, audits), failed, (settings2, _, audits2) = asyncio.run(_go())
     assert sent == [("steward", "steward-stub.tar", b"x" * 4096)]
     assert entry["ok"] and entry["by"] == "admin-x" and entry["size"] == 4096
+    # The stub isn't a real backup, so its self-check (#5823) says so; it was sent anyway.
+    assert not entry["check"]["ok"] and "isn't a Steward backup" in entry["check"]["summary"]
     assert settings["backup.history"][0]["name"] == "steward-stub.tar"
     assert settings["backup.last_success_at"] == entry["at"]
     assert metrics["backup_bytes"] == 4096 and metrics["backup_age_secs"] == 0