From b875160c40884f8472fad751f04badb2b51ed1a1 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 12 Oct 2026 07:14:04 -0400 Subject: [PATCH] feat(backup): `steward backup` and `steward restore`; CI restores a dump and boots on it (#5672, #5674) The backup file (#5672 "Backup file: pg_dump of config + audit + alert history, the secret key and a manifest"): steward-.tar holding a pg_dump --format=custom with metrics', check results' and Docker/Traefik/ UniFi rows left out, the secret key, and a manifest. Every table is named KEPT or EXCLUDED in steward/backup/contents.py, and a guard test fails on a new one. The image gains postgresql-client-17 (approved, System 180). The restore (#5674 "Restore: `steward restore` and a CI test that a dump brings Steward back"): pg_restore in its three sections. Between data and constraints, a kept row pointing at a left-out row gets what its foreign key's ON DELETE says (alert_events.ansible_run_id is cleared). It refuses a non-empty database unless --wipe, and puts the backup's key in /data/secret.key. Migrations run on Steward's next start. `steward` stays the server command: it is now a click group that serves when given no subcommand. The publish smoke step backs up the booted image's database, restores it into a second one, checks a marker setting came back and plugin_metrics is empty, and starts Steward on it. Co-Authored-By: Claude Opus 5.5 --- .forgejo/workflows/ci.yml | 33 +++- Dockerfile | 3 + steward/backup/__init__.py | 1 + steward/backup/contents.py | 44 +++++ steward/backup/dump.py | 147 ++++++++++++++++ steward/backup/job.py | 16 ++ steward/backup/restore.py | 158 ++++++++++++++++++ steward/cli.py | 67 +++++++- tests/core/test_backup_contents.py | 112 +++++++++++++ .../integration/test_backup_restore_fixes.py | 62 +++++++ 10 files changed, 639 insertions(+), 4 deletions(-) create mode 100644 steward/backup/__init__.py create mode 100644 steward/backup/contents.py create mode 100644 steward/backup/dump.py create mode 100644 steward/backup/job.py create mode 100644 steward/backup/restore.py create mode 100644 tests/core/test_backup_contents.py create mode 100644 tests/integration/test_backup_restore_fixes.py diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index f5700b6..c9b8dd7 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -221,6 +221,36 @@ jobs: exit 1 fi + # A backup made by the image restores into an empty database, and + # Steward starts on it (milestone 507). It carries configuration + # (the marker setting) and leaves metrics' rows out. + sql() { docker exec "$SMOKE-pg" psql -U steward -v ON_ERROR_STOP=1 -tA "$@"; } + sql -d steward -c "INSERT INTO app_settings (key, value_json, updated_at) VALUES ('smoke.backup', '\"kept\"', now())" + sql -d steward -c "INSERT INTO plugin_metrics (id, source_module, resource_name, metric_name, value, recorded_at) VALUES ('smoke-metric', 'smoke', 'smoke', 'smoke', 1, now())" + docker exec -u app "$SMOKE-app" steward backup --to /tmp/backups + sql -d steward -c "CREATE DATABASE restored" + docker exec -u app \ + -e "STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@$SMOKE-pg:5432/restored" \ + "$SMOKE-app" sh -c 'steward restore /tmp/backups/steward-*.tar' + test "$(sql -d restored -c "SELECT value_json FROM app_settings WHERE key = 'smoke.backup'")" = '"kept"' + test "$(sql -d restored -c "SELECT count(*) FROM plugin_metrics")" = "0" + test "$(sql -d restored -c "SELECT count(*) FROM alembic_version")" -ge 1 + docker run -d --name "$SMOKE-restored" --network "$SMOKE-net" \ + -e "STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@$SMOKE-pg:5432/restored" \ + -e STEWARD_SECRET_KEY=smoke-test-placeholder \ + "$IMAGE" + up="" + for _ in $(seq 1 60); do + if docker exec "$SMOKE-restored" python -c 'import urllib.request + urllib.request.urlopen("http://127.0.0.1:5000/health", timeout=5)' >/dev/null 2>&1; then up=1; break; fi + sleep 2 + done + if [ -z "$up" ]; then + echo "FATAL: Steward never started on the restored database" + docker logs "$SMOKE-restored" 2>&1 | tail -80 + exit 1 + fi + # A stop is a clean shutdown: SIGTERM through tini to Hypercorn, which # finishes within its graceful timeout and exits 0. docker stop -t 30 "$SMOKE-app" @@ -246,7 +276,8 @@ jobs: - name: Smoke clean-up if: always() run: | - docker rm -f "steward-smoke-${{ github.run_id }}-app" "steward-smoke-${{ github.run_id }}-pg" >/dev/null 2>&1 || true + docker rm -f "steward-smoke-${{ github.run_id }}-app" "steward-smoke-${{ github.run_id }}-restored" \ + "steward-smoke-${{ github.run_id }}-pg" >/dev/null 2>&1 || true docker network rm "steward-smoke-${{ github.run_id }}-net" >/dev/null 2>&1 || true - name: Prune dangling layers if: always() diff --git a/Dockerfile b/Dockerfile index dd6cdef..280e1cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \ # tzdata — the zone database, so times show in each reader's own zone # (milestone 490); Debian's slim base no longer carries it tzdata \ + # postgresql-client-17 — pg_dump and pg_restore for Steward's own + # backups (milestone 507); 17 reads and writes a Postgres 16 server + postgresql-client-17 \ && rm -rf /var/lib/apt/lists/* # Upgrade pip to suppress the version notice diff --git a/steward/backup/__init__.py b/steward/backup/__init__.py new file mode 100644 index 0000000..95260db --- /dev/null +++ b/steward/backup/__init__.py @@ -0,0 +1 @@ +"""Steward backing itself up, and coming back from a backup (milestone 507).""" diff --git a/steward/backup/contents.py b/steward/backup/contents.py new file mode 100644 index 0000000..bcf1862 --- /dev/null +++ b/steward/backup/contents.py @@ -0,0 +1,44 @@ +"""What a backup carries. Every table Steward defines is named here as kept +or left out, and tests/core/test_backup_contents.py fails on one that is +neither, so a new table is a decision someone makes in this file. + +The operator's ruling (System 180 "Backup & Restore", 2026-10-12): a backup +carries configuration, the audit log and alert history. Metrics, check +results and the Docker, Traefik and UniFi history are left out: their tables +come back with their structure and no rows, and fill again as Steward runs. + +A table that is in neither list (an external plugin's) is backed up whole: +a table nobody thought about is kept, never lost. +""" +from __future__ import annotations + +# Configuration, and the history the operator chose to keep. +KEPT: frozenset[str] = frozenset({ + "app_settings", "users", "user_invites", "api_tokens", + "hosts", "host_agent_registrations", + "monitors", + "alert_rules", "alert_states", "maintenance_windows", + "ansible_targets", "ansible_groups", "ansible_schedules", + "dashboards", "dashboard_widgets", "dashboard_share_tokens", + "snmp_devices", "prometheus_targets", + # Names given to network clients by hand (UniFi's alias column). + "unifi_known_clients", + # History kept by the ruling. + "audit_events", "alert_events", +}) + +# Rows that are measurements, logs or in-flight work: left out. +EXCLUDED: frozenset[str] = frozenset({ + "plugin_metrics", "plugin_metric_latest", "plugin_metrics_hourly", + "monitor_results", + "email_outbox", "password_resets", + # Run history; its logs live on disk, not in the database. + "ansible_runs", + "docker_containers", "docker_disk_usage", "docker_events", "docker_host_status", + "docker_images", "docker_logs", "docker_metrics", "docker_metrics_hourly", + "docker_swarm_nodes", "docker_swarm_services", + "traefik_access_summaries", "traefik_certs", "traefik_global_stats", "traefik_metrics", + "unifi_alarms", "unifi_client_snapshots", "unifi_devices", "unifi_dpi_snapshots", + "unifi_events", "unifi_fw_rules", "unifi_networks", "unifi_port_forwards", + "unifi_speedtest_results", "unifi_wan_stats", "unifi_wlan_stats", +}) diff --git a/steward/backup/dump.py b/steward/backup/dump.py new file mode 100644 index 0000000..3ec3500 --- /dev/null +++ b/steward/backup/dump.py @@ -0,0 +1,147 @@ +"""Making one backup file: steward-.tar holding + + steward.dump pg_dump --format=custom; excluded tables' rows left out + secret.key the key Steward's stored secrets are sealed with + manifest.json what this is: Steward version, migration heads, time, + the tables whose rows were left out + +The secret key is in the file by the operator's choice (System 180), so a +restore is complete on its own: treat every copy as sensitive. +""" +from __future__ import annotations + +import asyncio +import io +import json +import os +import tarfile +import tempfile +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +from sqlalchemy.engine import make_url + +from .contents import EXCLUDED + +FORMAT = 1 +DUMP_NAME = "steward.dump" +KEY_NAME = "secret.key" +MANIFEST_NAME = "manifest.json" +# A dump past this is given up: Steward's configuration is megabytes. +DUMP_DEADLINE_SECONDS = 15 * 60 + + +class BackupError(Exception): + """Why a backup couldn't be made, said to the person.""" + + +@dataclass +class Made: + path: Path + name: str + size: int + manifest: dict + + +def file_name(at: datetime) -> str: + return f"steward-{at.astimezone(timezone.utc):%Y%m%dT%H%M%SZ}.tar" + + +def pg_env(database_url: str) -> dict[str, str]: + """libpq's environment for the app's database URL, so the password is + never on a command line.""" + url = make_url(database_url) + env = {k: v for k, v in os.environ.items() if not k.startswith("PG")} + pairs = {"PGHOST": url.host, "PGPORT": url.port, "PGUSER": url.username, + "PGPASSWORD": url.password, "PGDATABASE": url.database} + env.update({k: str(v) for k, v in pairs.items() if v not in (None, "")}) + ssl = url.query.get("ssl") or url.query.get("sslmode") + if isinstance(ssl, str) and ssl: + env["PGSSLMODE"] = {"true": "require", "false": "disable"}.get(ssl.lower(), ssl) + return env + + +def dump_command(path: Path) -> list[str]: + args = ["pg_dump", "--format=custom", "--no-owner", "--no-privileges", f"--file={path}"] + args += [f"--exclude-table-data={t}" for t in sorted(EXCLUDED)] + return args + + +async def run_dump(database_url: str, path: Path) -> None: + try: + proc = await asyncio.create_subprocess_exec( + *dump_command(path), env=pg_env(database_url), + stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE) + except FileNotFoundError: + raise BackupError("pg_dump isn't installed in this image.") from None + try: + _, err = await asyncio.wait_for(proc.communicate(), DUMP_DEADLINE_SECONDS) + except TimeoutError: + proc.kill() + await proc.wait() + raise BackupError(f"pg_dump didn't finish within {DUMP_DEADLINE_SECONDS // 60} minutes.") from None + if proc.returncode != 0: + lines = [ln for ln in err.decode(errors="replace").splitlines() if ln.strip()] + raise BackupError(lines[-1] if lines else f"pg_dump exited {proc.returncode}.") + + +def manifest(at: datetime, heads: list[str]) -> dict: + from steward import __version__ + return {"format": FORMAT, "steward_version": __version__, "created_at": at.isoformat(), + "migration_heads": sorted(heads), "rows_left_out": sorted(EXCLUDED), + "contains_secret_key": True} + + +def pack(out: Path, dump: Path, secret_key: str, info: dict) -> None: + """The tar, written next to `out` and renamed into place.""" + part = out.with_name(out.name + ".part") + with tarfile.open(part, "w") as tar: + tar.add(dump, arcname=DUMP_NAME) + for name, data in ((KEY_NAME, secret_key.encode()), + (MANIFEST_NAME, json.dumps(info, indent=2).encode())): + member = tarfile.TarInfo(name) + member.size = len(data) + member.mode = 0o600 + member.mtime = int(datetime.now(timezone.utc).timestamp()) + tar.addfile(member, io.BytesIO(data)) + part.chmod(0o600) + part.replace(out) + + +async def make_backup(database_url: str, secret_key: str, heads: list[str], workdir: Path, + now: datetime | None = None) -> Made: + """Dump the database and pack the backup file into `workdir`.""" + at = now or datetime.now(timezone.utc) + if not secret_key: + raise BackupError("Steward has no secret key to put in the backup.") + out = workdir / file_name(at) + with tempfile.TemporaryDirectory(dir=workdir) as tmp: + dump = Path(tmp) / DUMP_NAME + await run_dump(database_url, dump) + info = manifest(at, heads) + await asyncio.to_thread(pack, out, dump, secret_key, info) + return Made(out, out.name, out.stat().st_size, info) + + +def read_backup(path: Path, into: Path) -> tuple[dict, Path, str]: + """Unpack a backup file into `into`: (manifest, dump path, secret key). + Refuses anything that isn't one.""" + try: + tar = tarfile.open(path, "r") + except (tarfile.TarError, OSError) as exc: + raise BackupError(f"{path} isn't a Steward backup: {exc}.") from None + with tar: + names = set(tar.getnames()) + if not {DUMP_NAME, KEY_NAME, MANIFEST_NAME} <= names: + raise BackupError(f"{path} isn't a Steward backup: it lacks " + f"{', '.join(sorted({DUMP_NAME, KEY_NAME, MANIFEST_NAME} - names))}.") + info = json.loads(tar.extractfile(MANIFEST_NAME).read()) + if info.get("format") != FORMAT: + raise BackupError(f"{path} is backup format {info.get('format')}; this Steward reads {FORMAT}.") + key = tar.extractfile(KEY_NAME).read().decode().strip() + dump = into / DUMP_NAME + with tar.extractfile(DUMP_NAME) as src, open(dump, "wb") as dst: + while chunk := src.read(1 << 20): + dst.write(chunk) + return info, dump, key diff --git a/steward/backup/job.py b/steward/backup/job.py new file mode 100644 index 0000000..6cfaa9a --- /dev/null +++ b/steward/backup/job.py @@ -0,0 +1,16 @@ +"""Steward's own backups as a scheduled job: make the file, send it, record +how it went (milestone 507).""" +from __future__ import annotations + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + + +async def migration_heads(database_url: str) -> list[str]: + """The migration revisions the database is at, for the manifest.""" + engine = create_async_engine(database_url) + try: + async with engine.connect() as conn: + return list((await conn.execute(text("SELECT version_num FROM alembic_version"))).scalars()) + finally: + await engine.dispose() diff --git a/steward/backup/restore.py b/steward/backup/restore.py new file mode 100644 index 0000000..26697b1 --- /dev/null +++ b/steward/backup/restore.py @@ -0,0 +1,158 @@ +"""Bringing Steward back from a backup file into an empty database. + +pg_restore runs in its three sections so the rows can be put right between +the data and the constraints: a kept table may point at a row of a table +whose rows were left out (an alert event naming the Ansible run it +started). Each such reference gets what its foreign key's ON DELETE says +would happen had that row been deleted: SET NULL clears it, anything else +removes the row. Then the constraints and indexes go on. + +Migrations are not run here: the next start of Steward runs them, as it +always does, so a backup made by an older Steward comes up to date. +""" +from __future__ import annotations + +import asyncio +import logging +import tempfile +from dataclasses import dataclass +from pathlib import Path + +from sqlalchemy import MetaData, text +from sqlalchemy.ext.asyncio import create_async_engine + +from .contents import EXCLUDED +from .dump import BackupError, pg_env, read_backup + +logger = logging.getLogger(__name__) + +RESTORE_DEADLINE_SECONDS = 30 * 60 + + +@dataclass +class Restored: + manifest: dict + secret_key: str + fixed: int # rows put right between data and constraints + + +def all_metadata() -> MetaData: + """Steward's tables, every bundled plugin's included, without an app: + the restore runs before anything else would load them.""" + import importlib + import pkgutil + + import sys + + import plugins + import steward.models # noqa: F401 — registers core tables + from steward.models.base import Base + for info in pkgutil.iter_modules(plugins.__path__): + # A plugin the app already loaded is under the loader's name; its + # models imported a second time would map every table twice. + loaded = f"_steward_plugin_{info.name}" + package = loaded if loaded in sys.modules else f"plugins.{info.name}" + try: + importlib.import_module(f"{package}.models") + except ImportError: # no models, or an optional dependency missing + continue + return Base.metadata + + +def dangling_fixes(metadata: MetaData) -> list[str]: + """SQL putting right the rows of kept tables that point at a table whose + rows were left out, by each foreign key's own ON DELETE.""" + out: list[str] = [] + for table in sorted(metadata.tables.values(), key=lambda t: t.name): + if table.name in EXCLUDED: + continue + for fk in sorted(table.foreign_keys, key=lambda f: f.parent.name): + parent = fk.column.table.name + if parent not in EXCLUDED: + continue + child, col, pcol = table.name, fk.parent.name, fk.column.name + gone = (f'"{col}" IS NOT NULL AND NOT EXISTS ' + f'(SELECT 1 FROM "{parent}" p WHERE p."{pcol}" = "{child}"."{col}")') + if (fk.ondelete or "").upper() == "SET NULL": + out.append(f'UPDATE "{child}" SET "{col}" = NULL WHERE {gone}') + else: + out.append(f'DELETE FROM "{child}" WHERE {gone}') + return out + + +async def _pg_restore(database_url: str, dump: Path, section: str) -> None: + args = ["pg_restore", "--no-owner", "--no-privileges", "--exit-on-error", + f"--section={section}", "--dbname", make_dbname(database_url), str(dump)] + try: + proc = await asyncio.create_subprocess_exec( + *args, env=pg_env(database_url), + stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE) + except FileNotFoundError: + raise BackupError("pg_restore isn't installed here.") from None + try: + _, err = await asyncio.wait_for(proc.communicate(), RESTORE_DEADLINE_SECONDS) + except TimeoutError: + proc.kill() + await proc.wait() + raise BackupError(f"pg_restore ({section}) didn't finish within " + f"{RESTORE_DEADLINE_SECONDS // 60} minutes.") from None + if proc.returncode != 0: + lines = [ln for ln in err.decode(errors="replace").splitlines() if ln.strip()] + raise BackupError(f"pg_restore ({section}): " + (lines[-1] if lines else f"exited {proc.returncode}.")) + + +def make_dbname(database_url: str) -> str: + """The database name for pg_restore's --dbname; host, user and password + come from the environment pg_env builds.""" + from sqlalchemy.engine import make_url + return make_url(database_url).database or "" + + +def _async_url(database_url: str) -> str: + from sqlalchemy.engine import make_url + url = make_url(database_url) + return str(url.set(drivername="postgresql+asyncpg")) if url.drivername == "postgresql" else database_url + + +async def _tables(database_url: str) -> list[str]: + engine = create_async_engine(_async_url(database_url)) + try: + async with engine.connect() as conn: + return list((await conn.execute(text( + "SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename"))).scalars()) + finally: + await engine.dispose() + + +async def _execute(database_url: str, statements: list[str]) -> int: + engine = create_async_engine(_async_url(database_url)) + changed = 0 + try: + async with engine.begin() as conn: + for sql in statements: + result = await conn.execute(text(sql)) + changed += max(result.rowcount or 0, 0) + finally: + await engine.dispose() + return changed + + +async def restore(path: Path, database_url: str, *, wipe: bool = False, + metadata: MetaData | None = None) -> Restored: + """Restore a backup file into the database at `database_url`. It must be + empty unless `wipe`, which drops everything in its public schema first.""" + existing = await _tables(database_url) + if existing and not wipe: + raise BackupError(f"The database already has {len(existing)} tables " + f"({', '.join(existing[:3])}{'…' if len(existing) > 3 else ''}). " + "Restore into an empty database, or pass --wipe to replace what is there.") + if existing: + logger.warning("Restore: dropping %d tables in the public schema first", len(existing)) + await _execute(database_url, ["DROP SCHEMA public CASCADE", "CREATE SCHEMA public"]) + with tempfile.TemporaryDirectory() as tmp: + info, dump, key = await asyncio.to_thread(read_backup, path, Path(tmp)) + await _pg_restore(database_url, dump, "pre-data") + await _pg_restore(database_url, dump, "data") + fixed = await _execute(database_url, dangling_fixes(metadata or all_metadata())) + await _pg_restore(database_url, dump, "post-data") + return Restored(info, key, fixed) diff --git a/steward/cli.py b/steward/cli.py index c65ad6a..cd4fce7 100644 --- a/steward/cli.py +++ b/steward/cli.py @@ -37,20 +37,81 @@ async def _serve(app, config) -> None: await serve(app, config, shutdown_trigger=stop.wait) -@click.command() +@click.group(invoke_without_command=True) @click.option("--config", default="config.yaml", help="Path to config.yaml") @click.option("--host", default="0.0.0.0") @click.option("--port", default=5000, type=int) @click.option("--debug", is_flag=True, default=False, help="Quart's development server with the reloader, not Hypercorn.") -def main(config: str, host: str, port: int, debug: bool) -> None: - """Start the Steward server.""" +@click.pass_context +def main(ctx: click.Context, config: str, host: str, port: int, debug: bool) -> None: + """Start the Steward server, or run one of the commands below.""" # Before anything logs; create_app applies the log.level setting once # settings load. configure_logging() + ctx.obj = {"config": config} + if ctx.invoked_subcommand is not None: + return from .app import create_app app = create_app(config_path=Path(config)) if debug: app.run(host=host, port=port, debug=True) return asyncio.run(_serve(app, hypercorn_config(host, port))) + + +@main.command("backup") +@click.option("--to", "out_dir", default=".", type=click.Path(file_okay=False, path_type=Path), + help="The folder to write the backup file into.") +@click.pass_context +def backup_command(ctx: click.Context, out_dir: Path) -> None: + """Write a backup file of this Steward (configuration, audit log, alert + history and the secret key) into a folder, without sending it anywhere.""" + from .backup.dump import BackupError, make_backup + from .backup.job import migration_heads + from .config import load_bootstrap + + boot = load_bootstrap(Path(ctx.obj["config"])) + out_dir.mkdir(parents=True, exist_ok=True) + try: + heads = asyncio.run(migration_heads(boot["database_url"])) + made = asyncio.run(make_backup(boot["database_url"], boot["secret_key"], heads, out_dir)) + except BackupError as exc: + raise click.ClickException(str(exc)) from None + click.echo(f"Wrote {made.path} ({made.size} bytes). It holds the secret key: keep it safe.") + + +@main.command("restore") +@click.argument("backup_file", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--wipe", is_flag=True, default=False, + help="Drop everything in the database first. Without it the database must be empty.") +@click.pass_context +def restore_command(ctx: click.Context, backup_file: Path, wipe: bool) -> None: + """Bring Steward back from a backup file into its (empty) database, and + put the backup's secret key in place. Start Steward afterwards: it runs + any migrations the backup is behind on.""" + from .backup.dump import BackupError + from .backup.restore import restore + from .config import _SECRET_KEY_FILE, _env, load_bootstrap + + boot = load_bootstrap(Path(ctx.obj["config"])) + try: + done = asyncio.run(restore(backup_file, boot["database_url"], wipe=wipe)) + except BackupError as exc: + raise click.ClickException(str(exc)) from None + made = done.manifest.get("created_at", "?") + click.echo(f"Restored the backup made {made} (Steward {done.manifest.get('steward_version', '?')}).") + if done.fixed: + click.echo(f"Cleared {done.fixed} references to rows a backup leaves out (e.g. Ansible runs).") + env_key = _env("SECRET_KEY") + if env_key: + if env_key != done.secret_key: + click.echo("STEWARD_SECRET_KEY is set and differs from the backup's key: set it to the " + "backup's key, or the restored passwords and keys can't be read.", err=True) + raise SystemExit(2) + else: + _SECRET_KEY_FILE.parent.mkdir(parents=True, exist_ok=True) + _SECRET_KEY_FILE.write_text(done.secret_key) + _SECRET_KEY_FILE.chmod(0o600) + click.echo(f"Wrote the backup's secret key to {_SECRET_KEY_FILE}.") + click.echo("Start Steward now; it brings the database up to date on start.") diff --git a/tests/core/test_backup_contents.py b/tests/core/test_backup_contents.py new file mode 100644 index 0000000..093e4ee --- /dev/null +++ b/tests/core/test_backup_contents.py @@ -0,0 +1,112 @@ +"""What a backup carries (steward/backup/contents.py) and the file it makes +(steward/backup/dump.py): every table is a decision, and the tar reads back.""" +import tarfile +from datetime import datetime, timezone + +import pytest +from sqlalchemy import Column, ForeignKey, Integer, MetaData, String, Table + +from steward.backup import contents, dump +from steward.backup.restore import all_metadata, dangling_fixes + + +def undecided(tables: set[str]) -> set[str]: + return tables - contents.KEPT - contents.EXCLUDED + + +def test_every_table_is_kept_or_left_out(): + tables = set(all_metadata().tables) + assert "prometheus_targets" in tables and "plugin_metrics" in tables # plugins were loaded + assert undecided(tables) == set(), ( + "Name these in steward/backup/contents.py as KEPT or EXCLUDED") + + +def test_the_guard_can_fail(): + assert undecided({"app_settings", "plugin_metrics", "a_new_table"}) == {"a_new_table"} + + +def test_no_table_is_both_and_none_is_named_for_nothing(): + assert contents.KEPT & contents.EXCLUDED == set() + stale = (contents.KEPT | contents.EXCLUDED) - set(all_metadata().tables) + assert stale == set(), "contents.py names tables that no longer exist" + + +def test_an_alert_event_keeps_its_row_when_its_ansible_run_is_left_out(): + fixes = dangling_fixes(all_metadata()) + assert ('UPDATE "alert_events" SET "ansible_run_id" = NULL WHERE "ansible_run_id" IS NOT NULL ' + 'AND NOT EXISTS (SELECT 1 FROM "ansible_runs" p WHERE p."id" = "alert_events"."ansible_run_id")' + ) in fixes + + +def test_a_cascading_reference_removes_the_row(): + md = MetaData() + Table("plugin_metrics", md, Column("id", Integer, primary_key=True)) + Table("notes", md, Column("id", Integer, primary_key=True), + Column("metric_id", Integer, ForeignKey("plugin_metrics.id", ondelete="CASCADE")), + Column("owner", String, ForeignKey("users.name"))) + Table("users", md, Column("name", String, primary_key=True)) + assert dangling_fixes(md) == [ + 'DELETE FROM "notes" WHERE "metric_id" IS NOT NULL AND NOT EXISTS ' + '(SELECT 1 FROM "plugin_metrics" p WHERE p."id" = "notes"."metric_id")' + ] + + +def test_dump_leaves_out_each_excluded_tables_rows(tmp_path): + args = dump.dump_command(tmp_path / "x.dump") + assert "--format=custom" in args + assert "--exclude-table-data=plugin_metrics" in args + assert "--exclude-table-data=audit_events" not in args + + +def test_password_goes_in_the_environment_not_the_command_line(): + env = dump.pg_env("postgresql+asyncpg://steward:s3cret@db:5433/steward?ssl=true") + assert env["PGHOST"] == "db" and env["PGPORT"] == "5433" + assert env["PGUSER"] == "steward" and env["PGPASSWORD"] == "s3cret" + assert env["PGDATABASE"] == "steward" and env["PGSSLMODE"] == "require" + + +def test_file_name_is_utc(): + at = datetime(2026, 10, 12, 21, 5, 9, tzinfo=timezone.utc) + assert dump.file_name(at) == "steward-20261012T210509Z.tar" + + +def test_a_packed_backup_reads_back(tmp_path): + at = datetime(2026, 10, 12, 21, 5, 9, tzinfo=timezone.utc) + src = tmp_path / "in.dump" + src.write_bytes(b"PGDMP" + b"x" * 3000) + info = dump.manifest(at, ["0004_app_settings", "prometheus_001_targets"]) + out = tmp_path / dump.file_name(at) + dump.pack(out, src, "the-key", info) + + assert out.stat().st_mode & 0o777 == 0o600 + assert not out.with_name(out.name + ".part").exists() + into = tmp_path / "out" + into.mkdir() + got, dumped, key = dump.read_backup(out, into) + assert key == "the-key" + assert dumped.read_bytes() == src.read_bytes() + assert got["format"] == dump.FORMAT and got["contains_secret_key"] is True + assert got["created_at"] == "2026-10-12T21:05:09+00:00" + assert "plugin_metrics" in got["rows_left_out"] + + +def test_read_backup_refuses_what_isnt_one(tmp_path): + plain = tmp_path / "plain.tar" + with tarfile.open(plain, "w") as tar: + tar.add(__file__, arcname="notes.txt") + with pytest.raises(dump.BackupError, match="lacks manifest.json, secret.key, steward.dump"): + dump.read_backup(plain, tmp_path) + + (tmp_path / "junk.tar").write_bytes(b"not a tar") + with pytest.raises(dump.BackupError, match="isn't a Steward backup"): + dump.read_backup(tmp_path / "junk.tar", tmp_path) + + +def test_read_backup_refuses_another_format(tmp_path): + src = tmp_path / "in.dump" + src.write_bytes(b"PGDMP") + info = dump.manifest(datetime.now(timezone.utc), []) | {"format": 99} + out = tmp_path / "b.tar" + dump.pack(out, src, "k", info) + with pytest.raises(dump.BackupError, match="format 99"): + dump.read_backup(out, tmp_path) diff --git a/tests/integration/test_backup_restore_fixes.py b/tests/integration/test_backup_restore_fixes.py new file mode 100644 index 0000000..32f422e --- /dev/null +++ b/tests/integration/test_backup_restore_fixes.py @@ -0,0 +1,62 @@ +"""Integration: between pg_restore's data and its constraints, an alert event +naming an Ansible run the backup left out keeps its row and loses the link +(steward/backup/restore.py, #5674). Requires STEWARD_DATABASE_URL.""" +from __future__ import annotations + +import asyncio +import os +import uuid + +import pytest + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +def test_a_dangling_ansible_run_link_is_cleared_and_the_event_kept(app): + from sqlalchemy import select, text + + from steward.backup.restore import _execute, all_metadata, dangling_fixes + from steward.models.alerts import AlertEvent, AlertOperator, AlertRule + from steward.models.users import User, UserRole + + uid, rule_id = str(uuid.uuid4()), str(uuid.uuid4()) + gone_run, dangling, plain = str(uuid.uuid4()), str(uuid.uuid4()), str(uuid.uuid4()) + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add(User(id=uid, username=f"u{uid[:8]}", email=f"{uid[:8]}@e.test", + password_hash="x", role=UserRole.admin, is_active=True)) + async with s.begin(): + s.add(AlertRule(id=rule_id, name="backup fixes", source_module="host_agent", + resource_name="srv1", metric_name="cpu_pct", + operator=AlertOperator.gt, threshold=90.0, + consecutive_failures_required=1, enabled=True, created_by=uid)) + # The state pg_restore's data section leaves: the run's row was + # never restored. Replica mode lets the row in without its parent. + async with s.begin(): + await s.execute(text("SET LOCAL session_replication_role = replica")) + s.add(AlertEvent(id=dangling, rule_id=rule_id, from_state="pending", + to_state="firing", ansible_run_id=gone_run)) + s.add(AlertEvent(id=plain, rule_id=rule_id, from_state="firing", + to_state="inactive")) + + fixed = await _execute(os.environ["STEWARD_DATABASE_URL"], dangling_fixes(all_metadata())) + + async with app.db_sessionmaker() as s: + rows = dict((await s.execute( + select(AlertEvent.id, AlertEvent.ansible_run_id) + .where(AlertEvent.rule_id == rule_id))).all()) + return fixed, rows + + fixed, rows = asyncio.run(_go()) + assert fixed >= 1 + assert rows == {dangling: None, plain: None}