feat(backup): steward backup and steward restore; CI restores a dump and boots on it (#5672, #5674)

The backup file (#5672 "Backup file: pg_dump of config + audit + alert
history, the secret key and a manifest"): steward-<UTC>.tar holding a
pg_dump --format=custom with metrics', check results' and Docker/Traefik/
UniFi rows left out, the secret key, and a manifest. Every table is named
KEPT or EXCLUDED in steward/backup/contents.py, and a guard test fails on a
new one. The image gains postgresql-client-17 (approved, System 180).

The restore (#5674 "Restore: `steward restore` and a CI test that a dump
brings Steward back"): pg_restore in its three sections. Between data and
constraints, a kept row pointing at a left-out row gets what its foreign
key's ON DELETE says (alert_events.ansible_run_id is cleared). It refuses
a non-empty database unless --wipe, and puts the backup's key in
/data/secret.key. Migrations run on Steward's next start.

`steward` stays the server command: it is now a click group that serves
when given no subcommand.

The publish smoke step backs up the booted image's database, restores it
into a second one, checks a marker setting came back and plugin_metrics
is empty, and starts Steward on it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/backup/__init__.py

@@ -0,0 +1 @@
+"""Steward backing itself up, and coming back from a backup (milestone 507)."""

steward/backup/contents.py

@@ -0,0 +1,44 @@
+"""What a backup carries. Every table Steward defines is named here as kept
+or left out, and tests/core/test_backup_contents.py fails on one that is
+neither, so a new table is a decision someone makes in this file.
+
+The operator's ruling (System 180 "Backup & Restore", 2026-10-12): a backup
+carries configuration, the audit log and alert history. Metrics, check
+results and the Docker, Traefik and UniFi history are left out: their tables
+come back with their structure and no rows, and fill again as Steward runs.
+
+A table that is in neither list (an external plugin's) is backed up whole:
+a table nobody thought about is kept, never lost.
+"""
+from __future__ import annotations
+
+# Configuration, and the history the operator chose to keep.
+KEPT: frozenset[str] = frozenset({
+    "app_settings", "users", "user_invites", "api_tokens",
+    "hosts", "host_agent_registrations",
+    "monitors",
+    "alert_rules", "alert_states", "maintenance_windows",
+    "ansible_targets", "ansible_groups", "ansible_schedules",
+    "dashboards", "dashboard_widgets", "dashboard_share_tokens",
+    "snmp_devices", "prometheus_targets",
+    # Names given to network clients by hand (UniFi's alias column).
+    "unifi_known_clients",
+    # History kept by the ruling.
+    "audit_events", "alert_events",
+})
+
+# Rows that are measurements, logs or in-flight work: left out.
+EXCLUDED: frozenset[str] = frozenset({
+    "plugin_metrics", "plugin_metric_latest", "plugin_metrics_hourly",
+    "monitor_results",
+    "email_outbox", "password_resets",
+    # Run history; its logs live on disk, not in the database.
+    "ansible_runs",
+    "docker_containers", "docker_disk_usage", "docker_events", "docker_host_status",
+    "docker_images", "docker_logs", "docker_metrics", "docker_metrics_hourly",
+    "docker_swarm_nodes", "docker_swarm_services",
+    "traefik_access_summaries", "traefik_certs", "traefik_global_stats", "traefik_metrics",
+    "unifi_alarms", "unifi_client_snapshots", "unifi_devices", "unifi_dpi_snapshots",
+    "unifi_events", "unifi_fw_rules", "unifi_networks", "unifi_port_forwards",
+    "unifi_speedtest_results", "unifi_wan_stats", "unifi_wlan_stats",
+})

steward/backup/dump.py

@@ -0,0 +1,147 @@
+"""Making one backup file: steward-<UTC time>.tar holding
+
+    steward.dump    pg_dump --format=custom; excluded tables' rows left out
+    secret.key      the key Steward's stored secrets are sealed with
+    manifest.json   what this is: Steward version, migration heads, time,
+                    the tables whose rows were left out
+
+The secret key is in the file by the operator's choice (System 180), so a
+restore is complete on its own: treat every copy as sensitive.
+"""
+from __future__ import annotations
+
+import asyncio
+import io
+import json
+import os
+import tarfile
+import tempfile
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+
+from sqlalchemy.engine import make_url
+
+from .contents import EXCLUDED
+
+FORMAT = 1
+DUMP_NAME = "steward.dump"
+KEY_NAME = "secret.key"
+MANIFEST_NAME = "manifest.json"
+# A dump past this is given up: Steward's configuration is megabytes.
+DUMP_DEADLINE_SECONDS = 15 * 60
+
+
+class BackupError(Exception):
+    """Why a backup couldn't be made, said to the person."""
+
+
+@dataclass
+class Made:
+    path: Path
+    name: str
+    size: int
+    manifest: dict
+
+
+def file_name(at: datetime) -> str:
+    return f"steward-{at.astimezone(timezone.utc):%Y%m%dT%H%M%SZ}.tar"
+
+
+def pg_env(database_url: str) -> dict[str, str]:
+    """libpq's environment for the app's database URL, so the password is
+    never on a command line."""
+    url = make_url(database_url)
+    env = {k: v for k, v in os.environ.items() if not k.startswith("PG")}
+    pairs = {"PGHOST": url.host, "PGPORT": url.port, "PGUSER": url.username,
+             "PGPASSWORD": url.password, "PGDATABASE": url.database}
+    env.update({k: str(v) for k, v in pairs.items() if v not in (None, "")})
+    ssl = url.query.get("ssl") or url.query.get("sslmode")
+    if isinstance(ssl, str) and ssl:
+        env["PGSSLMODE"] = {"true": "require", "false": "disable"}.get(ssl.lower(), ssl)
+    return env
+
+
+def dump_command(path: Path) -> list[str]:
+    args = ["pg_dump", "--format=custom", "--no-owner", "--no-privileges", f"--file={path}"]
+    args += [f"--exclude-table-data={t}" for t in sorted(EXCLUDED)]
+    return args
+
+
+async def run_dump(database_url: str, path: Path) -> None:
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            *dump_command(path), env=pg_env(database_url),
+            stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE)
+    except FileNotFoundError:
+        raise BackupError("pg_dump isn't installed in this image.") from None
+    try:
+        _, err = await asyncio.wait_for(proc.communicate(), DUMP_DEADLINE_SECONDS)
+    except TimeoutError:
+        proc.kill()
+        await proc.wait()
+        raise BackupError(f"pg_dump didn't finish within {DUMP_DEADLINE_SECONDS // 60} minutes.") from None
+    if proc.returncode != 0:
+        lines = [ln for ln in err.decode(errors="replace").splitlines() if ln.strip()]
+        raise BackupError(lines[-1] if lines else f"pg_dump exited {proc.returncode}.")
+
+
+def manifest(at: datetime, heads: list[str]) -> dict:
+    from steward import __version__
+    return {"format": FORMAT, "steward_version": __version__, "created_at": at.isoformat(),
+            "migration_heads": sorted(heads), "rows_left_out": sorted(EXCLUDED),
+            "contains_secret_key": True}
+
+
+def pack(out: Path, dump: Path, secret_key: str, info: dict) -> None:
+    """The tar, written next to `out` and renamed into place."""
+    part = out.with_name(out.name + ".part")
+    with tarfile.open(part, "w") as tar:
+        tar.add(dump, arcname=DUMP_NAME)
+        for name, data in ((KEY_NAME, secret_key.encode()),
+                           (MANIFEST_NAME, json.dumps(info, indent=2).encode())):
+            member = tarfile.TarInfo(name)
+            member.size = len(data)
+            member.mode = 0o600
+            member.mtime = int(datetime.now(timezone.utc).timestamp())
+            tar.addfile(member, io.BytesIO(data))
+    part.chmod(0o600)
+    part.replace(out)
+
+
+async def make_backup(database_url: str, secret_key: str, heads: list[str], workdir: Path,
+                      now: datetime | None = None) -> Made:
+    """Dump the database and pack the backup file into `workdir`."""
+    at = now or datetime.now(timezone.utc)
+    if not secret_key:
+        raise BackupError("Steward has no secret key to put in the backup.")
+    out = workdir / file_name(at)
+    with tempfile.TemporaryDirectory(dir=workdir) as tmp:
+        dump = Path(tmp) / DUMP_NAME
+        await run_dump(database_url, dump)
+        info = manifest(at, heads)
+        await asyncio.to_thread(pack, out, dump, secret_key, info)
+    return Made(out, out.name, out.stat().st_size, info)
+
+
+def read_backup(path: Path, into: Path) -> tuple[dict, Path, str]:
+    """Unpack a backup file into `into`: (manifest, dump path, secret key).
+    Refuses anything that isn't one."""
+    try:
+        tar = tarfile.open(path, "r")
+    except (tarfile.TarError, OSError) as exc:
+        raise BackupError(f"{path} isn't a Steward backup: {exc}.") from None
+    with tar:
+        names = set(tar.getnames())
+        if not {DUMP_NAME, KEY_NAME, MANIFEST_NAME} <= names:
+            raise BackupError(f"{path} isn't a Steward backup: it lacks "
+                              f"{', '.join(sorted({DUMP_NAME, KEY_NAME, MANIFEST_NAME} - names))}.")
+        info = json.loads(tar.extractfile(MANIFEST_NAME).read())
+        if info.get("format") != FORMAT:
+            raise BackupError(f"{path} is backup format {info.get('format')}; this Steward reads {FORMAT}.")
+        key = tar.extractfile(KEY_NAME).read().decode().strip()
+        dump = into / DUMP_NAME
+        with tar.extractfile(DUMP_NAME) as src, open(dump, "wb") as dst:
+            while chunk := src.read(1 << 20):
+                dst.write(chunk)
+    return info, dump, key

steward/backup/job.py

@@ -0,0 +1,16 @@
+"""Steward's own backups as a scheduled job: make the file, send it, record
+how it went (milestone 507)."""
+from __future__ import annotations
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import create_async_engine
+
+
+async def migration_heads(database_url: str) -> list[str]:
+    """The migration revisions the database is at, for the manifest."""
+    engine = create_async_engine(database_url)
+    try:
+        async with engine.connect() as conn:
+            return list((await conn.execute(text("SELECT version_num FROM alembic_version"))).scalars())
+    finally:
+        await engine.dispose()

steward/backup/restore.py

@@ -0,0 +1,158 @@
+"""Bringing Steward back from a backup file into an empty database.
+
+pg_restore runs in its three sections so the rows can be put right between
+the data and the constraints: a kept table may point at a row of a table
+whose rows were left out (an alert event naming the Ansible run it
+started). Each such reference gets what its foreign key's ON DELETE says
+would happen had that row been deleted: SET NULL clears it, anything else
+removes the row. Then the constraints and indexes go on.
+
+Migrations are not run here: the next start of Steward runs them, as it
+always does, so a backup made by an older Steward comes up to date.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+
+from sqlalchemy import MetaData, text
+from sqlalchemy.ext.asyncio import create_async_engine
+
+from .contents import EXCLUDED
+from .dump import BackupError, pg_env, read_backup
+
+logger = logging.getLogger(__name__)
+
+RESTORE_DEADLINE_SECONDS = 30 * 60
+
+
+@dataclass
+class Restored:
+    manifest: dict
+    secret_key: str
+    fixed: int                 # rows put right between data and constraints
+
+
+def all_metadata() -> MetaData:
+    """Steward's tables, every bundled plugin's included, without an app:
+    the restore runs before anything else would load them."""
+    import importlib
+    import pkgutil
+
+    import sys
+
+    import plugins
+    import steward.models  # noqa: F401 — registers core tables
+    from steward.models.base import Base
+    for info in pkgutil.iter_modules(plugins.__path__):
+        # A plugin the app already loaded is under the loader's name; its
+        # models imported a second time would map every table twice.
+        loaded = f"_steward_plugin_{info.name}"
+        package = loaded if loaded in sys.modules else f"plugins.{info.name}"
+        try:
+            importlib.import_module(f"{package}.models")
+        except ImportError:       # no models, or an optional dependency missing
+            continue
+    return Base.metadata
+
+
+def dangling_fixes(metadata: MetaData) -> list[str]:
+    """SQL putting right the rows of kept tables that point at a table whose
+    rows were left out, by each foreign key's own ON DELETE."""
+    out: list[str] = []
+    for table in sorted(metadata.tables.values(), key=lambda t: t.name):
+        if table.name in EXCLUDED:
+            continue
+        for fk in sorted(table.foreign_keys, key=lambda f: f.parent.name):
+            parent = fk.column.table.name
+            if parent not in EXCLUDED:
+                continue
+            child, col, pcol = table.name, fk.parent.name, fk.column.name
+            gone = (f'"{col}" IS NOT NULL AND NOT EXISTS '
+                    f'(SELECT 1 FROM "{parent}" p WHERE p."{pcol}" = "{child}"."{col}")')
+            if (fk.ondelete or "").upper() == "SET NULL":
+                out.append(f'UPDATE "{child}" SET "{col}" = NULL WHERE {gone}')
+            else:
+                out.append(f'DELETE FROM "{child}" WHERE {gone}')
+    return out
+
+
+async def _pg_restore(database_url: str, dump: Path, section: str) -> None:
+    args = ["pg_restore", "--no-owner", "--no-privileges", "--exit-on-error",
+            f"--section={section}", "--dbname", make_dbname(database_url), str(dump)]
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            *args, env=pg_env(database_url),
+            stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE)
+    except FileNotFoundError:
+        raise BackupError("pg_restore isn't installed here.") from None
+    try:
+        _, err = await asyncio.wait_for(proc.communicate(), RESTORE_DEADLINE_SECONDS)
+    except TimeoutError:
+        proc.kill()
+        await proc.wait()
+        raise BackupError(f"pg_restore ({section}) didn't finish within "
+                          f"{RESTORE_DEADLINE_SECONDS // 60} minutes.") from None
+    if proc.returncode != 0:
+        lines = [ln for ln in err.decode(errors="replace").splitlines() if ln.strip()]
+        raise BackupError(f"pg_restore ({section}): " + (lines[-1] if lines else f"exited {proc.returncode}."))
+
+
+def make_dbname(database_url: str) -> str:
+    """The database name for pg_restore's --dbname; host, user and password
+    come from the environment pg_env builds."""
+    from sqlalchemy.engine import make_url
+    return make_url(database_url).database or ""
+
+
+def _async_url(database_url: str) -> str:
+    from sqlalchemy.engine import make_url
+    url = make_url(database_url)
+    return str(url.set(drivername="postgresql+asyncpg")) if url.drivername == "postgresql" else database_url
+
+
+async def _tables(database_url: str) -> list[str]:
+    engine = create_async_engine(_async_url(database_url))
+    try:
+        async with engine.connect() as conn:
+            return list((await conn.execute(text(
+                "SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename"))).scalars())
+    finally:
+        await engine.dispose()
+
+
+async def _execute(database_url: str, statements: list[str]) -> int:
+    engine = create_async_engine(_async_url(database_url))
+    changed = 0
+    try:
+        async with engine.begin() as conn:
+            for sql in statements:
+                result = await conn.execute(text(sql))
+                changed += max(result.rowcount or 0, 0)
+    finally:
+        await engine.dispose()
+    return changed
+
+
+async def restore(path: Path, database_url: str, *, wipe: bool = False,
+                  metadata: MetaData | None = None) -> Restored:
+    """Restore a backup file into the database at `database_url`. It must be
+    empty unless `wipe`, which drops everything in its public schema first."""
+    existing = await _tables(database_url)
+    if existing and not wipe:
+        raise BackupError(f"The database already has {len(existing)} tables "
+                          f"({', '.join(existing[:3])}{'…' if len(existing) > 3 else ''}). "
+                          "Restore into an empty database, or pass --wipe to replace what is there.")
+    if existing:
+        logger.warning("Restore: dropping %d tables in the public schema first", len(existing))
+        await _execute(database_url, ["DROP SCHEMA public CASCADE", "CREATE SCHEMA public"])
+    with tempfile.TemporaryDirectory() as tmp:
+        info, dump, key = await asyncio.to_thread(read_backup, path, Path(tmp))
+        await _pg_restore(database_url, dump, "pre-data")
+        await _pg_restore(database_url, dump, "data")
+        fixed = await _execute(database_url, dangling_fixes(metadata or all_metadata()))
+        await _pg_restore(database_url, dump, "post-data")
+    return Restored(info, key, fixed)

steward/cli.py

@@ -37,20 +37,81 @@ async def _serve(app, config) -> None:
     await serve(app, config, shutdown_trigger=stop.wait)
 
 
-@click.command()
+@click.group(invoke_without_command=True)
 @click.option("--config", default="config.yaml", help="Path to config.yaml")
 @click.option("--host", default="0.0.0.0")
 @click.option("--port", default=5000, type=int)
 @click.option("--debug", is_flag=True, default=False,
               help="Quart's development server with the reloader, not Hypercorn.")
-def main(config: str, host: str, port: int, debug: bool) -> None:
-    """Start the Steward server."""
+@click.pass_context
+def main(ctx: click.Context, config: str, host: str, port: int, debug: bool) -> None:
+    """Start the Steward server, or run one of the commands below."""
     # Before anything logs; create_app applies the log.level setting once
     # settings load.
     configure_logging()
+    ctx.obj = {"config": config}
+    if ctx.invoked_subcommand is not None:
+        return
     from .app import create_app
     app = create_app(config_path=Path(config))
     if debug:
         app.run(host=host, port=port, debug=True)
         return
     asyncio.run(_serve(app, hypercorn_config(host, port)))
+
+
+@main.command("backup")
+@click.option("--to", "out_dir", default=".", type=click.Path(file_okay=False, path_type=Path),
+              help="The folder to write the backup file into.")
+@click.pass_context
+def backup_command(ctx: click.Context, out_dir: Path) -> None:
+    """Write a backup file of this Steward (configuration, audit log, alert
+    history and the secret key) into a folder, without sending it anywhere."""
+    from .backup.dump import BackupError, make_backup
+    from .backup.job import migration_heads
+    from .config import load_bootstrap
+
+    boot = load_bootstrap(Path(ctx.obj["config"]))
+    out_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        heads = asyncio.run(migration_heads(boot["database_url"]))
+        made = asyncio.run(make_backup(boot["database_url"], boot["secret_key"], heads, out_dir))
+    except BackupError as exc:
+        raise click.ClickException(str(exc)) from None
+    click.echo(f"Wrote {made.path} ({made.size} bytes). It holds the secret key: keep it safe.")
+
+
+@main.command("restore")
+@click.argument("backup_file", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.option("--wipe", is_flag=True, default=False,
+              help="Drop everything in the database first. Without it the database must be empty.")
+@click.pass_context
+def restore_command(ctx: click.Context, backup_file: Path, wipe: bool) -> None:
+    """Bring Steward back from a backup file into its (empty) database, and
+    put the backup's secret key in place. Start Steward afterwards: it runs
+    any migrations the backup is behind on."""
+    from .backup.dump import BackupError
+    from .backup.restore import restore
+    from .config import _SECRET_KEY_FILE, _env, load_bootstrap
+
+    boot = load_bootstrap(Path(ctx.obj["config"]))
+    try:
+        done = asyncio.run(restore(backup_file, boot["database_url"], wipe=wipe))
+    except BackupError as exc:
+        raise click.ClickException(str(exc)) from None
+    made = done.manifest.get("created_at", "?")
+    click.echo(f"Restored the backup made {made} (Steward {done.manifest.get('steward_version', '?')}).")
+    if done.fixed:
+        click.echo(f"Cleared {done.fixed} references to rows a backup leaves out (e.g. Ansible runs).")
+    env_key = _env("SECRET_KEY")
+    if env_key:
+        if env_key != done.secret_key:
+            click.echo("STEWARD_SECRET_KEY is set and differs from the backup's key: set it to the "
+                       "backup's key, or the restored passwords and keys can't be read.", err=True)
+            raise SystemExit(2)
+    else:
+        _SECRET_KEY_FILE.parent.mkdir(parents=True, exist_ok=True)
+        _SECRET_KEY_FILE.write_text(done.secret_key)
+        _SECRET_KEY_FILE.chmod(0o600)
+        click.echo(f"Wrote the backup's secret key to {_SECRET_KEY_FILE}.")
+    click.echo("Start Steward now; it brings the database up to date on start.")