diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 4166a1e..3a70543 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -235,6 +235,11 @@ jobs: sql -d steward -c "INSERT INTO plugin_metrics (id, source_module, resource_name, metric_name, value, recorded_at) VALUES ('smoke-metric', 'smoke', 'smoke', 'smoke', 1, now())" sql -d steward -c "INSERT INTO audit_events (id, timestamp, username, action) VALUES ('smoke-audit', now(), 'smoke', 'smoke.backup')" docker exec -u app "$SMOKE-app" steward backup --to /tmp/backups + # Checked against the database it was made from, it matches (#5823): + # a real pg_restore read, and what a running Steward writes by itself + # (last-seen times, backup history) left out of the comparison. + docker exec -u app "$SMOKE-app" sh -c 'steward check-backup /tmp/backups/steward-*.tar' | tee /tmp/smoke-check.txt + grep -q "Matches what is in place" /tmp/smoke-check.txt sql -d steward -c "CREATE DATABASE restored" docker exec -u app \ -e "STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@$SMOKE-pg:5432/restored" \ diff --git a/docs/core/backups.md b/docs/core/backups.md index a94174a..3a34c05 100644 --- a/docs/core/backups.md +++ b/docs/core/backups.md @@ -79,6 +79,42 @@ While backups are on, Steward records two metrics under the source A rule such as `backup_age_secs > 93600` (26 hours) fires when a daily backup has been missed. +## Checking a backup + +**Settings → Backups → Check** compares a backup with what is in place, without +changing anything. It shows what a restore would bring back, change or remove. +Each row under Recent backups has its own Check button, and **Check a file** +takes a file downloaded from the bucket. From a shell: + +```sh +docker exec -u app steward check-backup /tmp/steward-20261015T030000Z.tar +``` + +The check reports: +- for each area (Settings, Users and API tokens, Hosts and agents, Monitors, + Alert rules, Ansible, Dashboards, Plugins), how many entries are the same, + differ, are only in the backup or are only here; +- each setting that differs, with both values. A password or key shows only + "differs": sealed values are opened with their own key and compared, never + shown; +- whether the backup's secret key is the one in use, and whether it opens the + backup's own sealed values. A backup whose key can't open them fails the + check, because its secrets would not come back; +- whether the backup was made by this Steward's schema, an older one or a + newer one. + +What Steward records as it runs is left out of the comparison: when an agent +or monitor was last seen, a schedule's last run, the backup history. These +change by the minute. The list is `ACTIVITY` and `STATE_SETTINGS` in +`steward/backup/contents.py`. + +Each backup Steward sends is checked once it is written, and its row under +Recent backups says so. A backup is sent even when its check can't run. + +Checking a backup on the bucket reads it back from there. An append-only key +may not be allowed to; the page then says AccessDenied, and the file can be +downloaded and checked with **Check a file** instead. + ## Making a backup by hand ```sh diff --git a/steward/auth/csrf.py b/steward/auth/csrf.py index 93a1dcc..561ea1d 100644 --- a/steward/auth/csrf.py +++ b/steward/auth/csrf.py @@ -47,10 +47,20 @@ def rotate() -> None: session.pop(SESSION_KEY, None) +# Forms that send a file past the app's default body limit (16 MiB). This +# check reads the form first, so the limit is raised here, before it does: +# a backup file (settings only is small; an older one carries the audit log). +UPLOAD_LIMITS: dict[str, int] = { + "/settings/backups/check/file/": 512 * 1024 * 1024, +} + + async def check_request() -> Response | None: """before_request hook: refuse an unsafe request without the session's token.""" if request.method not in UNSAFE_METHODS or request.path in EXEMPT_PATHS: return None + if request.path in UPLOAD_LIMITS: + request.max_content_length = UPLOAD_LIMITS[request.path] expected = session.get(SESSION_KEY) sent = request.headers.get(HEADER, "") if not sent: diff --git a/steward/backup/check.py b/steward/backup/check.py new file mode 100644 index 0000000..a74b807 --- /dev/null +++ b/steward/backup/check.py @@ -0,0 +1,481 @@ +"""Checking a backup: what it would restore, compared with what is in place, +without restoring anything (#5823). + +The backup's rows come out of pg_restore as COPY text, with no database +involved; the rows in place go out through COPY in the same form. They are +compared table by table on their primary keys, over the columns both sides +have. Nothing is written. + +What is compared is what a restore would replace: every table whose rows +the backup carries and this Steward keeps (contents.py). Columns that are +Steward's record of what it last saw (contents.ACTIVITY) and settings it +writes as it runs (contents.STATE_SETTINGS) are left out of the comparison: +they change by the minute and say nothing about whether the backup is good. + +A sealed value (crypto's ``enc:v1:`` tokens) is opened with its own side's +key and compared as plaintext, so a secret sealed twice still matches. It +is never shown: the report says only that it differs. A backup whose key +doesn't open its own sealed values is reported, since its secrets would not +come back. +""" +from __future__ import annotations + +import asyncio +import hashlib +import io +import json +import re +import tempfile +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + +from .contents import (ACTIVITY, ACTIVITY_EVERYWHERE, AREAS, EXCLUDED, HAND_SET_ONLY, + STATE_SETTINGS) +from .dump import BackupError, read_backup +from .restore import _async_url + +CHECK_DEADLINE_SECONDS = 5 * 60 +# Names shown for a row, the first a table has; else its primary key. +LABEL_COLUMNS = ("name", "username", "key", "title", "widget_key", "label", "mac") +# A setting's two values are shown when they fit; longer ones are cut. +SHOWN_CHARS = 120 +_TOKEN = re.compile(r"enc:v1:[A-Za-z0-9_\-=]+") +_COPY = re.compile(r'^COPY ([^ ]+) \((.*)\) FROM stdin;$') + + +@dataclass +class RowDiff: + label: str + columns: list[str] + secret: bool = False + backup_value: str | None = None # shown only for a setting that isn't secret + here_value: str | None = None + + +@dataclass +class TableCheck: + table: str + same: int = 0 + differ: list[RowDiff] = field(default_factory=list) + only_backup: list[str] = field(default_factory=list) + only_here: list[str] = field(default_factory=list) + note: str = "" # a table one side lacks, columns one side lacks + + @property + def changes(self) -> int: + return len(self.differ) + len(self.only_backup) + len(self.only_here) + + +@dataclass +class AreaCheck: + name: str + tables: list[TableCheck] + + @property + def same(self) -> int: + return sum(t.same for t in self.tables) + + @property + def changes(self) -> int: + return sum(t.changes for t in self.tables) + + +@dataclass +class CheckReport: + name: str + manifest: dict + schema: str # "same", "older" or "newer" + backup_heads: list[str] + here_heads: list[str] + key_same: bool + sealed: int # sealed values in the backup + unopened: int # of those, how many its own key couldn't open + areas: list[AreaCheck] + checked_at: str = "" + + @property + def made_at(self) -> datetime | None: + try: + return datetime.fromisoformat(self.manifest.get("created_at") or "") + except ValueError: + return None + + @property + def rows(self) -> int: + return sum(a.same + a.changes for a in self.areas) + + @property + def changes(self) -> int: + return sum(a.changes for a in self.areas) + + @property + def ok(self) -> bool: + """The backup reads back whole: its key opens its secrets. Whether it + matches what is in place is `changes`; a difference isn't a fault.""" + return self.unopened == 0 + + def summary(self) -> str: + if self.unopened: + return (f"The backup's key doesn't open {self.unopened} of its {self.sealed} sealed " + f"value{'s' if self.sealed != 1 else ''}: those secrets wouldn't come back.") + if not self.changes: + tables = sum(len(a.tables) for a in self.areas) + return f"Matches what is in place: {self.rows} rows across {tables} table{'s' if tables != 1 else ''}." + parts = [f"{a.changes} in {a.name}" for a in self.areas if a.changes] + return (f"{self.changes} difference{'s' if self.changes != 1 else ''} from what is in place: " + + ", ".join(parts) + ".") + + def as_entry(self) -> dict: + """What a backup's history row keeps of its own check.""" + return {"ok": self.ok, "changes": self.changes, "rows": self.rows, "summary": self.summary()} + + +# ── COPY text ─────────────────────────────────────────────────────────────── + +_ESCAPES = {ord("b"): b"\b", ord("f"): b"\f", ord("n"): b"\n", ord("r"): b"\r", + ord("t"): b"\t", ord("v"): b"\v"} + + +def unescape(field: bytes) -> str | None: + """One field of COPY's text format: \\N is NULL, backslash escapes are + undone (named, octal \\ooo and hex \\xhh).""" + if field == b"\\N": + return None + if b"\\" not in field: + return field.decode("utf-8", errors="replace") + out = bytearray() + i, n = 0, len(field) + while i < n: + c = field[i] + if c != 0x5C or i + 1 == n: + out.append(c) + i += 1 + continue + nxt = field[i + 1] + if nxt in _ESCAPES: + out += _ESCAPES[nxt] + i += 2 + elif 0x30 <= nxt <= 0x37: + j = i + 1 + while j < n and j < i + 4 and 0x30 <= field[j] <= 0x37: + j += 1 + out.append(int(field[i + 1:j], 8) & 0xFF) + i = j + elif nxt == ord("x") and i + 2 < n and chr(field[i + 2]) in "0123456789abcdefABCDEF": + j = i + 2 + while j < n and j < i + 4 and chr(field[j]) in "0123456789abcdefABCDEF": + j += 1 + out.append(int(field[i + 2:j], 16)) + i = j + else: + out.append(nxt) + i += 2 + return bytes(out).decode("utf-8", errors="replace") + + +def copy_rows(data: bytes) -> list[list[str | None]]: + """COPY text rows (no header, no terminator) as lists of fields.""" + return [[unescape(f) for f in line.split(b"\t")] + for line in data.split(b"\n") if line and line != b"\\."] + + +def _ident(name: str) -> str: + name = name.strip() + return name[1:-1].replace('""', '"') if name.startswith('"') and name.endswith('"') else name + + +def script_tables(script: bytes) -> dict[str, tuple[list[str], list[list[str | None]]]]: + """Each table's (columns, rows) from a pg_restore --data-only script.""" + out: dict[str, tuple[list[str], list[list[str | None]]]] = {} + lines = iter(script.split(b"\n")) + for line in lines: + m = _COPY.match(line.decode("utf-8", errors="replace")) + if not m: + continue + table = _ident(m.group(1).split(".")[-1]) + columns = [_ident(c) for c in m.group(2).split(",")] + body = [] + for row in lines: + if row == b"\\.": + break + body.append(row) + out[table] = (columns, copy_rows(b"\n".join(body))) + return out + + +# ── Comparing ─────────────────────────────────────────────────────────────── + +def _fernet(key: str): + from steward.core.crypto import _make_fernet + return _make_fernet(key) if key else None + + +class _Opener: + """Opens one side's sealed values, counting what it couldn't.""" + + def __init__(self, key: str): + self.fernet = _fernet(key) + self.seen = 0 + self.failed = 0 + + def open(self, value: str) -> tuple[str, bool]: + """`value` with each sealed token replaced by a digest of its + plaintext (never the plaintext itself), and whether it held one.""" + if "enc:v1:" not in value: + return value, False + + def plain(m: re.Match) -> str: + self.seen += 1 + try: + opened = self.fernet.decrypt(m.group(0)[len("enc:v1:"):].encode("ascii")) + except Exception: + self.failed += 1 + return "\x00unopened:" + m.group(0) + return "\x00sealed:" + hashlib.sha256(opened).hexdigest() + return _TOKEN.sub(plain, value), True + + +def _offset(v: str) -> str: + return re.sub(r"([+-]\d\d)$", r"\1:00", v) + + +def normalise(value: str | None, data_type: str) -> object: + """A value as compared: an instant for a timestamp with zone (the two + sides may print it in different zones), parsed JSON, a float.""" + if value is None: + return None + try: + if data_type == "timestamp with time zone": + return datetime.fromisoformat(_offset(value)).astimezone(timezone.utc) + if data_type in ("json", "jsonb"): + return json.dumps(json.loads(value), sort_keys=True) + if data_type in ("double precision", "real", "numeric"): + return float(value) + except ValueError: + pass + return value + + +def _shown(value: str | None) -> str | None: + if value is None: + return None + return value if len(value) <= SHOWN_CHARS else value[:SHOWN_CHARS - 1] + "…" + + +@dataclass +class _Side: + columns: list[str] + rows: list[list[str | None]] + + +def compare_table(table: str, backup: _Side, here: _Side, *, types: dict[str, str], + key: list[str], openers: tuple[_Opener, _Opener], + labels: dict[str, dict[str, str]]) -> TableCheck: + """One table's rows, backup against here.""" + result = TableCheck(table) + skip = ACTIVITY.get(table, frozenset()) | ACTIVITY_EVERYWHERE + common = [c for c in backup.columns if c in here.columns and c not in skip] + lacking = sorted((set(backup.columns) ^ set(here.columns)) - skip) + if lacking: + result.note = "Columns only one side has: " + ", ".join(lacking) + "." + key = [c for c in key if c in common] or common + hand_set = HAND_SET_ONLY.get(table) + + def index(side: _Side, opener: _Opener) -> dict[tuple, dict[str, object]]: + at = {c: side.columns.index(c) for c in common} + out = {} + for raw in side.rows: + row: dict[str, object] = {} + sealed: set[str] = set() + for c in common: + v = raw[at[c]] + if isinstance(v, str): + v, was = opener.open(v) + if was: + sealed.add(c) + row[c] = normalise(v, types.get(c, "")) + if table == "app_settings" and row.get("key") in STATE_SETTINGS: + continue + if hand_set and not any(raw[side.columns.index(c)] for c in hand_set if c in side.columns): + continue + row["\x00sealed"] = sealed + row["\x00raw"] = {c: raw[at[c]] for c in common} + out[tuple(str(row[c]) for c in key)] = row + return out + + theirs, ours = index(backup, openers[0]), index(here, openers[1]) + + def label(row: dict) -> str: + raw = row["\x00raw"] + if table == "host_agent_registrations" and raw.get("host_id"): + return labels.get("hosts", {}).get(raw["host_id"], raw["host_id"]) + if table == "ansible_target_groups": + return (f"{labels.get('ansible_targets', {}).get(raw.get('target_id'), raw.get('target_id'))} in " + f"{labels.get('ansible_groups', {}).get(raw.get('group_id'), raw.get('group_id'))}") + for c in LABEL_COLUMNS: + if raw.get(c): + return str(raw[c]) + return " / ".join(str(raw.get(c)) for c in key) + + for k in sorted(theirs.keys() | ours.keys()): + b, h = theirs.get(k), ours.get(k) + if h is None: + result.only_backup.append(label(b)) + elif b is None: + result.only_here.append(label(h)) + else: + cols = [c for c in common if b[c] != h[c]] + if not cols: + result.same += 1 + continue + secret = bool((b["\x00sealed"] | h["\x00sealed"]) & set(cols)) + diff = RowDiff(label(b), cols, secret) + if table == "app_settings" and not secret: + diff.backup_value = _shown(b["\x00raw"].get("value_json")) + diff.here_value = _shown(h["\x00raw"].get("value_json")) + result.differ.append(diff) + return result + + +def _labels(tables: dict[str, _Side]) -> dict[str, dict[str, str]]: + """id → name for the tables other rows are named through.""" + out: dict[str, dict[str, str]] = {} + for t in ("hosts", "ansible_targets", "ansible_groups"): + side = tables.get(t) + if side and "id" in side.columns and "name" in side.columns: + i, n = side.columns.index("id"), side.columns.index("name") + out[t] = {r[i]: r[n] for r in side.rows} + return out + + +# ── Reading both sides ────────────────────────────────────────────────────── + +async def _backup_rows(dump: Path) -> dict[str, tuple[list[str], list[list[str | None]]]]: + """Every table's rows in the dump, as pg_restore would load them. Run + without a database: pg_restore writes the script and nothing connects.""" + try: + proc = await asyncio.create_subprocess_exec( + "pg_restore", "--data-only", "--no-owner", "--no-privileges", "--file=-", str(dump), + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) + except FileNotFoundError: + raise BackupError("pg_restore isn't installed here.") from None + try: + out, err = await asyncio.wait_for(proc.communicate(), CHECK_DEADLINE_SECONDS) + except TimeoutError: + proc.kill() + await proc.wait() + raise BackupError(f"Reading the backup didn't finish within " + f"{CHECK_DEADLINE_SECONDS // 60} minutes.") from None + if proc.returncode != 0: + lines = [ln for ln in err.decode(errors="replace").splitlines() if ln.strip()] + raise BackupError("The backup can't be read: " + (lines[-1] if lines else + f"pg_restore exited {proc.returncode}.")) + return script_tables(out) + + +async def _here(database_url: str, tables: list[str]): + """The rows in place for `tables`, with each column's type, each table's + primary key and the migration heads.""" + engine = create_async_engine(_async_url(database_url)) + try: + async with engine.connect() as conn: + types: dict[str, dict[str, str]] = {} + order: dict[str, list[str]] = {} + for t, c, dt in (await conn.execute(text( + "SELECT table_name, column_name, data_type FROM information_schema.columns " + "WHERE table_schema = 'public' AND table_name = ANY(:t) " + "ORDER BY table_name, ordinal_position"), {"t": tables})).all(): + types.setdefault(t, {})[c] = dt + order.setdefault(t, []).append(c) + keys: dict[str, list[str]] = {} + for t, c in (await conn.execute(text( + "SELECT cl.relname, a.attname FROM pg_index i " + "JOIN pg_class cl ON cl.oid = i.indrelid " + "JOIN pg_namespace ns ON ns.oid = cl.relnamespace " + "JOIN pg_attribute a ON a.attrelid = cl.oid AND a.attnum = ANY(i.indkey) " + "WHERE i.indisprimary AND ns.nspname = 'public' AND cl.relname = ANY(:t) " + "ORDER BY cl.relname, a.attnum"), {"t": tables})).all(): + keys.setdefault(t, []).append(c) + heads = sorted((await conn.execute(text("SELECT version_num FROM alembic_version"))).scalars()) + raw = (await conn.get_raw_connection()).driver_connection + rows: dict[str, _Side] = {} + for t in tables: + if t not in order: + continue + buf = io.BytesIO() + await raw.copy_from_table(t, columns=order[t], output=buf, format="text") + rows[t] = _Side(order[t], copy_rows(buf.getvalue())) + return rows, types, keys, heads + finally: + await engine.dispose() + + +def known_revisions(plugin_dirs: list[Path] | None = None) -> set[str]: + """Every migration revision this Steward has, core and plugins: a backup + head not among them was made by a newer Steward.""" + import steward + roots = [Path(steward.__file__).parent / "migrations" / "versions"] + for d in plugin_dirs or [Path(steward.__file__).parent.parent / "plugins"]: + if Path(d).is_dir(): + roots += [p / "migrations" / "versions" for p in Path(d).iterdir()] + found = set() + for root in roots: + if root.is_dir(): + for f in root.glob("*.py"): + m = re.search(r'^revision\s*(?::\s*str\s*)?=\s*["\']([^"\']+)["\']', + f.read_text(errors="replace"), re.M) + if m: + found.add(m.group(1)) + return found + + +def schema_of(backup_heads: list[str], here_heads: list[str], known: set[str]) -> str: + if sorted(backup_heads) == sorted(here_heads): + return "same" + return "newer" if any(h not in known for h in backup_heads) else "older" + + +async def check_backup(path: Path, database_url: str, here_key: str, *, + plugin_dirs: list[Path] | None = None, name: str = "") -> CheckReport: + """Compare the backup at `path` with the database at `database_url`. + Raises BackupError when the file isn't a backup or can't be read.""" + with tempfile.TemporaryDirectory(prefix="steward-check-") as tmp: + info, dump, backup_key = await asyncio.to_thread(read_backup, path, Path(tmp)) + theirs = await _backup_rows(dump) + tables = sorted(t for t in theirs if t not in EXCLUDED) + here, types, keys, heads = await _here(database_url, tables) + openers = (_Opener(backup_key), _Opener(here_key)) + backup_sides = {t: _Side(*theirs[t]) for t in tables} + labels = _labels(backup_sides) + for t, names in _labels(here).items(): # a name in place wins + labels[t] = {**labels.get(t, {}), **names} + + checks: dict[str, TableCheck] = {} + for t in tables: + if t not in here: + checks[t] = TableCheck(t, only_backup=[f"{len(theirs[t][1])} rows"], + note="This Steward has no such table.") + continue + checks[t] = compare_table(t, backup_sides[t], here[t], types=types.get(t, {}), + key=keys.get(t, []), openers=openers, labels=labels) + areas = [] + placed: set[str] = set() + for area, members in AREAS: + found = [checks[t] for t in members if t in checks] + placed |= set(members) + if found: + areas.append(AreaCheck(area, found)) + rest = [checks[t] for t in tables if t not in placed] + if rest: + areas.append(AreaCheck("Other", rest)) + backup_heads = sorted(info.get("migration_heads") or []) + return CheckReport( + name=name or path.name, manifest=info, + schema=schema_of(backup_heads, heads, known_revisions(plugin_dirs)), + backup_heads=backup_heads, here_heads=heads, key_same=backup_key == here_key, + sealed=openers[0].seen, unopened=openers[0].failed, areas=areas, + checked_at=datetime.now(timezone.utc).isoformat()) diff --git a/steward/backup/contents.py b/steward/backup/contents.py index 087a0f1..2e1f776 100644 --- a/steward/backup/contents.py +++ b/steward/backup/contents.py @@ -59,3 +59,55 @@ EXCLUDED: frozenset[str] = frozenset({ "unifi_events", "unifi_fw_rules", "unifi_networks", "unifi_port_forwards", "unifi_speedtest_results", "unifi_wan_stats", "unifi_wlan_stats", }) + +# ── What a check compares (#5823) ──────────────────────────────────────────── +# Checking a backup compares its kept rows with the ones in place. Some columns +# of kept tables are Steward's own record of what it last saw, not something +# anyone set up: they change by the minute, so comparing them would call every +# backup out of date. They are left out of the comparison (a restore still +# carries them). `updated_at` is bookkeeping wherever it appears. +ACTIVITY: dict[str, frozenset[str]] = { + "api_tokens": frozenset({"last_used_at"}), + # What the agent reports about its host; the enrolment (host_id, + # token_hash, token_created_at) is what is compared. + "host_agent_registrations": frozenset({ + "agent_version", "kernel", "distro", "arch", "host_ip", "last_seen_at", + "clock_offset_s", "clock_checked_at", "updates_manager", "updates_pending", + "updates_security", "updates_checked_at", "updates_error", "reboot_required", + "reboot_required_since", "reboot_packages", "maint_helper", "drive_health", + "drive_health_at"}), + "host_maintenance_plans": frozenset({"last_slot_at"}), + "monitors": frozenset({"last_checked_at"}), + "ansible_schedules": frozenset({"last_run_at", "last_run_id", "last_error"}), + "snmp_devices": frozenset({"last_polled_at", "last_ok_at", "last_error", "missing"}), + "prometheus_targets": frozenset({"last_scraped_at", "last_ok_at", "last_error", "last_kept", + "last_dropped", "last_capped"}), + # Scraped from the controller; the name and note are what someone set. + "unifi_known_clients": frozenset({"hostname", "ip", "mac_vendor", "is_blocked", "first_seen", + "last_seen", "scraped_at"}), +} +ACTIVITY_EVERYWHERE: frozenset[str] = frozenset({"updated_at"}) + +# Settings Steward writes as it runs: a record of its own work, not setup. +STATE_SETTINGS: frozenset[str] = frozenset({ + "backup.history", "backup.last_attempt_at", "backup.last_success_at", + "reports.last_sent_at", "cleanup.last_run", +}) + +# A network client only counts once someone has named it or noted it; the +# rest are the controller's, and come and go as devices do. +HAND_SET_ONLY: dict[str, tuple[str, ...]] = {"unifi_known_clients": ("name", "note")} + +# How a check groups what it found, in the order it shows them. A table +# not named here is shown under "Other". +AREAS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("Settings", ("app_settings",)), + ("Users and API tokens", ("users", "api_tokens")), + ("Hosts and agents", ("hosts", "host_agent_registrations", "host_maintenance_plans")), + ("Monitors", ("monitors",)), + ("Alert rules and maintenance windows", ("alert_rules", "maintenance_windows")), + ("Ansible", ("ansible_targets", "ansible_groups", "ansible_target_groups", "ansible_schedules")), + ("Dashboards", ("dashboards", "dashboard_widgets", "dashboard_share_tokens")), + ("Plugins", ("snmp_devices", "prometheus_targets", "prometheus_widget_templates", + "unifi_known_clients")), +) diff --git a/steward/backup/job.py b/steward/backup/job.py index 4ee93b8..0830541 100644 --- a/steward/backup/job.py +++ b/steward/backup/job.py @@ -22,7 +22,7 @@ from sqlalchemy import text from sqlalchemy.ext.asyncio import create_async_engine from .dump import BackupError, make_backup -from .s3 import S3Error, S3Target, list_objects, put_file +from .s3 import S3Error, S3Target, get_file, list_objects, put_file logger = logging.getLogger(__name__) @@ -133,7 +133,8 @@ async def run_backup(app, by: str = "schedule", user_id: str | None = None) -> d with tempfile.TemporaryDirectory(prefix="steward-backup-") as tmp: made = await make_backup(db_url, app.config["SECRET_KEY"], heads, Path(tmp), now=at) await put_file(target, made.name, made.path) - entry.update(ok=True, name=made.name, size=made.size) + entry.update(ok=True, name=made.name, size=made.size) + entry["check"] = await _self_check(app, made.path, made.name) except (BackupError, S3Error) as exc: entry["error"] = str(exc) except Exception as exc: # recorded and logged: a run never takes the scheduler down @@ -148,6 +149,49 @@ async def run_backup(app, by: str = "schedule", user_id: str | None = None) -> d return entry +async def _self_check(app, path: Path, name: str) -> dict: + """A backup just sent, read back and compared with what it was made from + (#5823): proof the file restores, kept on its history row. A check that + can't run says why; the backup was sent either way.""" + try: + report = await check_file(app, path, name) + except Exception as exc: + logger.warning("Backup %s couldn't be checked: %s", name, exc) + return {"ok": False, "changes": 0, "rows": 0, "summary": f"Couldn't check it: {exc}"} + return report.as_entry() + + +async def check_file(app, path: Path, name: str = ""): + """Compare a backup file with this Steward's database (check.py).""" + from .check import check_backup + return await check_backup(path, app.config["DATABASE_URL"], app.config["SECRET_KEY"], + plugin_dirs=[Path(d) for d in app.config["PLUGIN_DIRS"]], + name=name or path.name) + + +def is_backup_name(name: str) -> bool: + """A file name Steward's backups have, and nothing else: no path, no + other object in the bucket.""" + import re + return bool(re.fullmatch(r"steward-\d{8}T\d{6}Z\.tar", name or "")) + + +async def check_stored(app, name: str): + """Fetch one of this Steward's backups from the bucket and check it. + Raises BackupError or S3Error with what went wrong.""" + from steward.core.settings import get_all_settings + if not is_backup_name(name): + raise BackupError(f"{name!r} isn't the name of a Steward backup.") + async with app.db_sessionmaker() as db: + target = target_from(await get_all_settings(db)) + if target is None: + raise BackupError("The bucket isn't set up: fill in the endpoint, bucket and keys.") + with tempfile.TemporaryDirectory(prefix="steward-check-") as tmp: + path = Path(tmp) / name + await get_file(target, name, path) + return await check_file(app, path, name) + + def start_backup(app, by: str, user_id: str | None) -> bool: """Start a backup in the background for a person; False if one is running.""" if _running.locked(): diff --git a/steward/backup/s3.py b/steward/backup/s3.py index c6cb151..733e6e9 100644 --- a/steward/backup/s3.py +++ b/steward/backup/s3.py @@ -3,10 +3,11 @@ path-style addressing (endpoint/bucket/key), the three calls a backup needs. Fabled Archives speaks S3 with path-style buckets and append-only keys by default (System 180 ruling: backups go there with an append-only key), so -Steward only ever puts and lists; it never deletes. Retention is the -target's business. +Steward puts, lists and reads back (to check or restore a backup); it never +deletes. Retention is the target's business. A key that may not read says +so as AccessDenied, and the backup can be uploaded by hand instead. -Written by hand rather than with boto3: three calls don't justify a large +Written by hand rather than with boto3: four calls don't justify a large dependency, and signing is a pure function tests can hold to AWS's own published examples. """ @@ -189,3 +190,32 @@ async def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *, out.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0), fields.get("LastModified", ""))) return out + + +async def get_file(target: S3Target, key: str, path: Path, *, + transport: httpx.AsyncBaseTransport | None = None, + now: datetime | None = None) -> int: + """Download one object to `path`, streamed; returns its size. A partial + download is removed.""" + url = target.url(key) + headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key, + target.region, now or datetime.now(timezone.utc)) + size = 0 + try: + async with httpx.AsyncClient(verify=target.verify_tls, timeout=UPLOAD_TIMEOUT_SECONDS, + transport=transport) as client: + async with client.stream("GET", url, headers=headers) as response: + if response.status_code != 200: + await response.aread() + raise S3Error(f"Reading {key} refused: {_says(response)}") + with open(path, "wb") as f: + async for chunk in response.aiter_bytes(CHUNK): + f.write(chunk) + size += len(chunk) + except httpx.HTTPError as exc: + path.unlink(missing_ok=True) + raise _reach_error(target, exc) from None + except S3Error: + path.unlink(missing_ok=True) + raise + return size diff --git a/steward/cli.py b/steward/cli.py index 9f261d6..bfc098a 100644 --- a/steward/cli.py +++ b/steward/cli.py @@ -81,6 +81,41 @@ def backup_command(ctx: click.Context, out_dir: Path) -> None: click.echo(f"Wrote {made.path} ({made.size} bytes). It holds the secret key: keep it safe.") +@main.command("check-backup") +@click.argument("backup_file", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.pass_context +def check_backup_command(ctx: click.Context, backup_file: Path) -> None: + """Compare a backup file with this Steward's database, changing nothing: + what a restore would bring back, area by area. Exits 1 when the backup + can't be read or its key doesn't open its secrets.""" + from .backup.check import check_backup + from .backup.dump import BackupError + from .config import load_bootstrap + + boot = load_bootstrap(Path(ctx.obj["config"])) + try: + report = asyncio.run(check_backup(backup_file, boot["database_url"], boot["secret_key"], + plugin_dirs=[Path(d) for d in boot["plugin_dirs"]])) + except BackupError as exc: + raise click.ClickException(str(exc)) from None + click.echo(report.summary()) + click.echo(f"Made {report.manifest.get('created_at', '?')} by Steward " + f"{report.manifest.get('steward_version', '?')}; schema: {report.schema}; " + f"secret key: {'the one in use' if report.key_same else 'a different one'}.") + for area in report.areas: + for t in area.tables: + for d in t.differ: + click.echo(f" {area.name}: {d.label} differs ({', '.join(d.columns)})") + for label in t.only_backup: + click.echo(f" {area.name}: {label} is only in the backup") + for label in t.only_here: + click.echo(f" {area.name}: {label} is only here") + if t.note: + click.echo(f" {area.name} ({t.table}): {t.note}") + if not report.ok: + raise SystemExit(1) + + @main.command("restore") @click.argument("backup_file", type=click.Path(exists=True, dir_okay=False, path_type=Path)) @click.option("--wipe", is_flag=True, default=False, diff --git a/steward/settings/routes.py b/steward/settings/routes.py index dcec521..c029f4d 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -776,6 +776,54 @@ async def backup_now(): return await _backups_page({"ok": started, "message": message}) +async def _check_page(report=None, error: str = "", name: str = ""): + return await render_template("settings/backup_check.html", report=report, error=error, + name=name) + + +@settings_bp.post("/backups/check/") +@require_role(UserRole.admin) +async def check_stored_backup(): + """Check one of the backups on the bucket against what is in place + (#5823). Read-only: nothing here changes.""" + from steward.backup import job + from steward.backup.dump import BackupError + from steward.backup.s3 import S3Error + name = (await request.form).get("name", "") + try: + report = await job.check_stored(current_app._get_current_object(), name) + except (BackupError, S3Error) as exc: + error = str(exc) + if "AccessDenied" in error: + error += (" The bucket's key may not read backups back; download the file from the " + "bucket and check it with Check a file instead.") + return await _check_page(error=error, name=name) + return await _check_page(report=report, name=name) + + +@settings_bp.post("/backups/check/file/") +@require_role(UserRole.admin) +async def check_uploaded_backup(): + """Check a backup file sent from the browser. Its size limit is raised + before the form is read (auth/csrf.py UPLOAD_LIMITS).""" + import tempfile + + from steward.backup import job + from steward.backup.dump import BackupError + upload = (await request.files).get("file") + if upload is None or not upload.filename: + return await _check_page(error="Choose a backup file to check.") + name = Path(upload.filename).name + with tempfile.TemporaryDirectory(prefix="steward-check-") as tmp: + path = Path(tmp) / "upload.tar" + await upload.save(path) + try: + report = await job.check_file(current_app._get_current_object(), path, name) + except BackupError as exc: + return await _check_page(error=str(exc), name=name) + return await _check_page(report=report, name=name) + + def _build_plugin_cfg_from_form(plugin: dict, form) -> dict: """Extract config values for one plugin from a form submission.""" name = plugin["_dir"] diff --git a/steward/static/steward.css b/steward/static/steward.css index 000b1b6..083520b 100644 --- a/steward/static/steward.css +++ b/steward/static/steward.css @@ -398,6 +398,8 @@ a.stat-tile:hover { border-color: var(--border-mid); background: var(--bg-elevat /* A label/value list:
…
…
. */ .kv { display: grid; grid-template-columns: max-content 1fr; gap: 0.35rem 1.25rem; font-size: 0.88rem; } .kv dt { color: var(--text-muted); } +/* What a backup check found, one line per row (settings/backup_check.html). */ +.check-list { margin: 0 0 0 1.1rem; padding: 0; font-size: 0.88rem; line-height: 1.6; } /* An Ansible run just started from the run form (ansible/run_started.html). */ .run-started-head { display: flex; align-items: center; gap: 0.75rem; margin-bottom: 0.75rem; font-size: 0.9rem; } .run-started-link { margin-left: auto; font-size: 0.82rem; } diff --git a/steward/templates/settings/backup_check.html b/steward/templates/settings/backup_check.html new file mode 100644 index 0000000..f4ef323 --- /dev/null +++ b/steward/templates/settings/backup_check.html @@ -0,0 +1,74 @@ +{# settings/backup_check.html — a backup compared with what is in place (#5823): + the verdict, what the backup is (when, which Steward, its key), then each + area's count and the rows that differ. Nothing was changed to show it. #} +{% extends "base.html" %} +{% from "_macros.html" import crumbs %} +{% from "settings/_tabs.html" import settings_tabs %} +{% block title %}Settings — Backups — Check — Steward{% endblock %} +{% block section_tabs %}{{ settings_tabs("backups") }}{% endblock %} +{% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("Backups", "/settings/backups/"), ("Check " ~ (name or "a backup"), "")]) }}{% endblock %} +{% block content %} +{% if error %} +

{{ name or "The backup" }} couldn't be checked. {{ error }}

+{% else %} +{% set tone = "tone-crit" if not report.ok else ("tone-ok" if not report.changes else "tone-warn") %} +

{{ report.summary() }}

+ +
+
+
File
{{ report.name }}
+
Made
{% if report.made_at %}{{ report.made_at|when }}{% else %}—{% endif %} + by Steward {{ report.manifest.steward_version or "—" }}
+
Schema
+ {%- if report.schema == "same" %}The same as this Steward's. + {%- elif report.schema == "older" %}Older than this Steward's. A restore would replace everything, and Steward would bring it up to date on start. + {%- else %}Newer than this Steward's. This Steward can't restore it; update Steward first.{% endif %}
+
Secret key
+ {%- if report.key_same %}The one in use here.{% else %}A different one. A restore would put it in place, and passwords and keys stored since would need entering again.{% endif %} + {%- if report.sealed %} It opens {{ report.sealed - report.unopened }} of the backup's {{ report.sealed }} sealed value{{ "s" if report.sealed != 1 }}.{% endif %}
+
+
+ +
+ + + + {% for a in report.areas %} + {% set differ = a.tables|sum(attribute="differ", start=[])|length %} + {% set only_b = a.tables|sum(attribute="only_backup", start=[])|length %} + {% set only_h = a.tables|sum(attribute="only_here", start=[])|length %} + + + + + + + + {% endfor %} + +
AreaSameDifferOnly in the backupOnly here
{{ a.name }}{{ a.same }}{% if differ %}{{ differ }}{% else %}—{% endif %}{% if only_b %}{{ only_b }}{% else %}—{% endif %}{% if only_h %}{{ only_h }}{% else %}—{% endif %}
+
+ +{% for a in report.areas if a.changes or a.tables|selectattr("note")|list %} +

{{ a.name }}

+ +{% endfor %} + +

Left out of the comparison: what Steward records as it runs + (when an agent or monitor was last seen, a schedule's last run, the backup history), since it changes by the + minute. A restore still brings those back as they were in the backup.

+{% endif %} +

Back to Backups

+{% endblock %} diff --git a/steward/templates/settings/backups.html b/steward/templates/settings/backups.html index 13f1176..bdb9fe1 100644 --- a/steward/templates/settings/backups.html +++ b/steward/templates/settings/backups.html @@ -1,6 +1,7 @@ {# settings/backups.html — Steward's own backups (#5673): a verdict, the bucket they go to (Fabled Archives over S3) and the daily hour, then the recent - runs. Restoring is `steward restore`; see docs/core/backups.md. #} + runs, each with its own check and a Check button (#5823), and a file to + check. Restoring is `steward restore`; see docs/core/backups.md. #} {% extends "base.html" %} {% from "_macros.html" import crumbs %} {% from "settings/_tabs.html" import settings_tabs %} @@ -115,17 +116,25 @@ {% if history %}
- + {% for r in history %} - + {% endfor %} @@ -134,6 +143,18 @@ {% else %}

None yet.

{% endif %} -

To bring Steward back from one, see - steward restore in the backups guide (docs/core/backups.md).

+

Check compares a backup with what is in place: what + a restore would bring back, change or remove, area by area. It changes nothing. To bring Steward back + from one, see steward restore in the backups guide (docs/core/backups.md).

+ +
+

Check a file

+
{{ csrf_field() }} + + + +

A steward-<time>.tar downloaded from + the bucket, or made with steward backup.

+
{% endblock %} diff --git a/tests/core/test_backup_check.py b/tests/core/test_backup_check.py new file mode 100644 index 0000000..000552b --- /dev/null +++ b/tests/core/test_backup_check.py @@ -0,0 +1,181 @@ +"""Checking a backup against what is in place (steward/backup/check.py, #5823): +COPY text reads back exactly, rows are compared on their keys, secrets are +compared by what they open to and never shown, and what Steward records as +it runs is left out of the comparison.""" +from datetime import datetime, timezone + +from quart import render_template + +from steward.backup import check +from steward.backup.check import (AreaCheck, CheckReport, RowDiff, TableCheck, _Opener, _Side, + compare_table, copy_rows, normalise, schema_of, script_tables, + unescape) +from steward.core.crypto import _make_fernet +from tests.test_style_tokens import hex_colours + +KEY_A, KEY_B = "backup-key", "here-key" + + +def _seal(key: str, plain: str) -> str: + return "enc:v1:" + _make_fernet(key).encrypt(plain.encode()).decode() + + +def _compare(table, backup_cols, backup_rows, here_cols, here_rows, *, types=None, key=("id",), + keys=(KEY_A, KEY_B)): + return compare_table(table, _Side(backup_cols, backup_rows), _Side(here_cols, here_rows), + types=types or {}, key=list(key), + openers=(_Opener(keys[0]), _Opener(keys[1])), labels={}) + + +def test_copy_text_reads_back_exactly(): + assert unescape(b"\\N") is None + assert unescape(b"plain") == "plain" + assert unescape(b"a\\tb\\nc\\\\d") == "a\tb\nc\\d" + assert unescape(b"\\101\\x42") == "AB" + assert unescape("caf\u00e9".encode()) == "caf\u00e9" + assert copy_rows(b"1\tx\\ty\n2\t\\N\n") == [["1", "x\ty"], ["2", None]] + + +def test_a_restore_script_gives_each_tables_rows(): + script = (b"SET client_encoding = 'UTF8';\n" + b'COPY public.app_settings (key, value_json, updated_at) FROM stdin;\n' + b'smtp.port\t587\t2026-10-15 14:00:00+00\n' + b'\\.\n\n' + b'COPY public.audit_events (id, "timestamp", action) FROM stdin;\n' + b'a1\t2026-10-15 14:00:00+00\tbackup.made\n' + b'\\.\n') + tables = script_tables(script) + assert tables["app_settings"] == (["key", "value_json", "updated_at"], + [["smtp.port", "587", "2026-10-15 14:00:00+00"]]) + assert tables["audit_events"][0] == ["id", "timestamp", "action"] + + +def test_an_instant_is_the_same_whatever_zone_prints_it(): + a = normalise("2026-10-15 16:00:00+02", "timestamp with time zone") + b = normalise("2026-10-15 14:00:00+00", "timestamp with time zone") + assert a == b == datetime(2026, 10, 15, 14, tzinfo=timezone.utc) + assert normalise('{"b": 1, "a": 2}', "jsonb") == normalise('{"a":2,"b":1}', "jsonb") + + +def test_rows_are_compared_on_their_key(): + cols = ["id", "name", "target"] + result = _compare("monitors", cols, + [["1", "web", "https://a"], ["2", "dns", "1.1.1.1"], ["3", "gone", "x"]], + cols, + [["1", "web", "https://a"], ["2", "dns", "9.9.9.9"], ["4", "new", "y"]]) + assert result.same == 1 + assert [(d.label, d.columns) for d in result.differ] == [("dns", ["target"])] + assert result.only_backup == ["gone"] and result.only_here == ["new"] + assert result.changes == 3 + + +def test_a_setting_that_differs_shows_both_values(): + cols = ["key", "value_json", "updated_at"] + result = _compare("app_settings", cols, [["smtp.port", "587", "2026-10-01 00:00:00+00"]], + cols, [["smtp.port", "25", "2026-10-15 00:00:00+00"]], key=("key",)) + [d] = result.differ + assert (d.label, d.secret, d.backup_value, d.here_value) == ("smtp.port", False, "587", "25") + + +def test_a_secret_sealed_twice_matches_and_one_that_changed_is_never_shown(): + cols = ["key", "value_json"] + same = _compare("app_settings", cols, [["smtp.password", f'"{_seal(KEY_A, "hunter2")}"']], + cols, [["smtp.password", f'"{_seal(KEY_B, "hunter2")}"']], key=("key",)) + assert same.same == 1 and not same.differ + + changed = _compare("app_settings", cols, [["smtp.password", f'"{_seal(KEY_A, "old")}"']], + cols, [["smtp.password", f'"{_seal(KEY_B, "new")}"']], key=("key",)) + [d] = changed.differ + assert d.secret and d.backup_value is None and d.here_value is None + + +def test_a_backup_whose_key_cant_open_its_secrets_is_counted(): + cols = ["key", "value_json"] + openers = (_Opener(KEY_A), _Opener(KEY_B)) + compare_table("app_settings", _Side(cols, [["ldap.bind_password", f'"{_seal("other", "x")}"']]), + _Side(cols, []), types={}, key=["key"], openers=openers, labels={}) + assert (openers[0].seen, openers[0].failed) == (1, 1) + + +def test_what_steward_records_as_it_runs_is_not_a_difference(): + cols = ["id", "host_id", "token_hash", "last_seen_at", "agent_version", "updated_at"] + result = _compare("host_agent_registrations", cols, + [["r1", "h1", "abc", "2026-10-15 01:00:00+00", "1.19.0", "x"]], cols, + [["r1", "h1", "abc", "2026-10-15 14:00:00+00", "1.20.0", "y"]]) + assert result.same == 1 and not result.differ + + settings = ["key", "value_json"] + state = _compare("app_settings", settings, [["backup.history", "[]"]], + settings, [["backup.history", '[{"ok": true}]']], key=("key",)) + assert state.same == 0 and state.changes == 0 + + +def test_a_network_client_counts_only_once_someone_named_it(): + cols = ["mac", "name", "note", "ip"] + result = _compare("unifi_known_clients", cols, [["aa", "printer", None, "10.0.0.5"]], cols, + [["aa", "printer", None, "10.0.0.9"], ["bb", None, None, "10.0.0.7"]], + key=("mac",)) + assert result.same == 1 and result.changes == 0 + + +def test_columns_one_side_lacks_are_noted_not_compared(): + result = _compare("monitors", ["id", "name"], [["1", "web"]], + ["id", "name", "retries"], [["1", "web", "3"]]) + assert result.same == 1 and "retries" in result.note + + +def test_the_schema_is_same_older_or_newer(): + known = {"0001", "0002", "0003"} + assert schema_of(["0003"], ["0003"], known) == "same" + assert schema_of(["0002"], ["0003"], known) == "older" + assert schema_of(["0009"], ["0003"], known) == "newer" + + +def test_this_steward_knows_its_own_revisions(): + revisions = check.known_revisions() + assert len(revisions) > 10 and any(r.startswith("host_agent_") for r in revisions) + + +def _report(**over): + tables = [TableCheck("app_settings", same=12, + differ=[RowDiff("smtp.port", ["value_json"], False, "587", "25"), + RowDiff("smtp.password", ["value_json"], True)]), + ] + hosts = [TableCheck("hosts", same=3, only_here=["nas"])] + fields = dict(name="steward-20261015T030000Z.tar", + manifest={"created_at": "2026-10-15T03:00:00+00:00", "steward_version": "2026.10.15"}, + schema="same", backup_heads=["a"], here_heads=["a"], key_same=True, sealed=4, + unopened=0, areas=[AreaCheck("Settings", tables), AreaCheck("Hosts and agents", hosts)]) + return CheckReport(**(fields | over)) + + +def test_the_summary_says_whether_it_matches(): + assert _report().summary() == "3 differences from what is in place: 2 in Settings, 1 in Hosts and agents." + same = _report(areas=[AreaCheck("Settings", [TableCheck("app_settings", same=12)])]) + assert same.summary() == "Matches what is in place: 12 rows across 1 table." and same.ok + bad = _report(unopened=4) + assert not bad.ok and "doesn't open 4 of its 4 sealed values" in bad.summary() + assert _report().as_entry() == {"ok": True, "changes": 3, "rows": 18, + "summary": _report().summary()} + + +async def test_the_check_page_shows_what_differs_and_hides_secrets(app): + async with app.test_request_context("/settings/backups/check/", method="POST"): + page = await render_template("settings/backup_check.html", report=_report(), error="", + name="steward-20261015T030000Z.tar") + page = " ".join(page.split()) + assert 'href="/settings/backups/"' in page # the trail back + assert "3 differences from what is in place" in page + assert "smtp.port differs: the backup has 587, here it is 25." in page + assert "smtp.password differs (a sealed value; not shown)." in page + assert "nas is only here: a restore removes it." in page + assert "The same as this Steward's." in page or "The same as this Steward's." in page + assert not hex_colours(page), hex_colours(page) + + +async def test_a_check_that_couldnt_run_says_why(app): + async with app.test_request_context("/settings/backups/check/", method="POST"): + page = await render_template("settings/backup_check.html", report=None, + error="Reading x refused: HTTP 403 AccessDenied.", name="x") + assert "x couldn't be checked." in page or "x couldn't be checked." in page + assert "AccessDenied" in page diff --git a/tests/core/test_backup_s3.py b/tests/core/test_backup_s3.py index 4638723..aea6805 100644 --- a/tests/core/test_backup_s3.py +++ b/tests/core/test_backup_s3.py @@ -110,3 +110,27 @@ def test_list_reads_contents(): s3.StoredObject("steward-20261011T000000Z.tar", 1200, "2026-10-11T00:00:01.000Z"), s3.StoredObject("steward-20261012T000000Z.tar", 1300, "2026-10-12T00:00:01.000Z"), ] + + +def test_get_streams_the_object_to_a_file(tmp_path): + seen = {} + + def handler(request): + seen["method"], seen["url"] = request.method, str(request.url) + seen["auth"] = request.headers["authorization"] + return httpx.Response(200, content=b"y" * 3000) + + out = tmp_path / "b.tar" + size = asyncio.run(s3.get_file(TARGET, "b.tar", out, transport=httpx.MockTransport(handler))) + assert (seen["method"], seen["url"]) == ("GET", "http://archives:8081/s3/steward/b.tar") + assert seen["auth"].startswith("AWS4-HMAC-SHA256 ") + assert size == 3000 and out.read_bytes() == b"y" * 3000 + + +def test_a_refused_get_leaves_no_file(tmp_path): + body = b"AccessDeniedno" + out = tmp_path / "b.tar" + with pytest.raises(s3.S3Error, match="Reading b.tar refused: HTTP 403 AccessDenied"): + asyncio.run(s3.get_file(TARGET, "b.tar", out, + transport=httpx.MockTransport(lambda r: httpx.Response(403, content=body)))) + assert not out.exists() diff --git a/tests/integration/test_backup_check.py b/tests/integration/test_backup_check.py new file mode 100644 index 0000000..cabeb36 --- /dev/null +++ b/tests/integration/test_backup_check.py @@ -0,0 +1,86 @@ +"""Integration: checking a backup against Postgres (#5823). pg_restore isn't in +this lane (the publish smoke step checks a real backup with the image), so +the backup's rows are a snapshot of this database taken through the check's +own reader; what is real is everything read in place, the comparison and the +report. Requires STEWARD_DATABASE_URL.""" +from __future__ import annotations + +import asyncio +import os +import uuid +from datetime import datetime, timezone + +import pytest + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +def test_a_check_finds_a_changed_setting_a_deleted_monitor_and_a_new_host(app, monkeypatch, tmp_path): + from sqlalchemy import delete + from steward.backup import check, contents, dump, job + from steward.core.settings import set_setting + from steward.models.hosts import Host + from steward.models.monitors import Monitor + + tag = uuid.uuid4().hex[:8] + setting, monitor_name, host_name = f"test.check_{tag}", f"check-mon-{tag}", f"check-host-{tag}" + url = app.config["DATABASE_URL"] + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + await set_setting(s, setting, "before") + mon = Monitor(name=monitor_name, type="tcp", target="10.9.9.9", config_json="{}") + s.add(mon) + # The "backup": this database as it stands, read the way a check reads. + snapshot, *_ = await check._here(url, sorted(contents.KEPT)) + theirs = {t: (side.columns, side.rows) for t, side in snapshot.items()} + + async def backup_rows(_dump): + return theirs + monkeypatch.setattr(check, "_backup_rows", backup_rows) + + path = tmp_path / "steward-20261015T030000Z.tar" + (tmp_path / "steward.dump").write_bytes(b"PGDMP") + heads = await job.migration_heads(url) + dump.pack(path, tmp_path / "steward.dump", app.config["SECRET_KEY"], + dump.manifest(datetime.now(timezone.utc), heads)) + + same = await job.check_file(app, path) + + async with app.db_sessionmaker() as s: + async with s.begin(): + await set_setting(s, setting, "after") + await s.execute(delete(Monitor).where(Monitor.name == monitor_name)) + s.add(Host(name=host_name, address="10.9.9.10")) + changed = await job.check_file(app, path) + return same, changed + + same, changed = asyncio.run(_go()) + assert same.ok and same.changes == 0, same.summary() + assert same.schema == "same" and same.key_same and same.rows > 0 + + by_table = {t.table: t for a in changed.areas for t in a.tables} + [diff] = [d for d in by_table["app_settings"].differ if d.label == setting] + assert (diff.backup_value, diff.here_value) == ('"before"', '"after"') + assert monitor_name in by_table["monitors"].only_backup + assert host_name in by_table["hosts"].only_here + assert changed.changes == 3, changed.summary() + + +def test_a_file_that_isnt_a_backup_fails_the_check(app, tmp_path): + from steward.backup import job + from steward.backup.dump import BackupError + + bad = tmp_path / "steward-20261015T030000Z.tar" + bad.write_bytes(b"not a tar") + with pytest.raises(BackupError, match="isn't a Steward backup"): + asyncio.run(job.check_file(app, bad)) diff --git a/tests/integration/test_backup_job.py b/tests/integration/test_backup_job.py index cc40a64..1e7dd16 100644 --- a/tests/integration/test_backup_job.py +++ b/tests/integration/test_backup_job.py @@ -89,6 +89,8 @@ def test_a_sent_backup_is_recorded_and_a_failed_one_says_why(app, monkeypatch): sent, entry, (settings, metrics, audits), failed, (settings2, _, audits2) = asyncio.run(_go()) assert sent == [("steward", "steward-stub.tar", b"x" * 4096)] assert entry["ok"] and entry["by"] == "admin-x" and entry["size"] == 4096 + # The stub isn't a real backup, so its self-check (#5823) says so; it was sent anyway. + assert not entry["check"]["ok"] and "isn't a Steward backup" in entry["check"]["summary"] assert settings["backup.history"][0]["name"] == "steward-stub.tar" assert settings["backup.last_success_at"] == entry["at"] assert metrics["backup_bytes"] == 4096 and metrics["backup_age_secs"] == 0
WhenOutcomeFileSizeTookStarted by
WhenOutcomeFileSizeTookStarted by
{{ r.at|when if r.at else "—" }}{% if r.ok %}Sent{% else %}Failed + {% if r.ok %}Sent + {%- if r.check %}
+ {%- if r.check.ok and not r.check.changes %}Checked: reads back whole{% else %}{{ r.check.summary }}{% endif %}
{% endif %} + {%- else %}Failed
{{ r.error }}
{% endif %}		 {{ r.name or "—" }} {{ r.size|bytes if r.ok else "—" }} {{ r.secs|duration }} {{ "the schedule" if r.by == "schedule" else r.by }}{% if r.ok and r.name %} +
{{ csrf_field() }} + + +
{% endif %}