feat(backup): steward backup and steward restore; CI restores a dump and boots on it (#5672, #5674)

The backup file (#5672 "Backup file: pg_dump of config + audit + alert
history, the secret key and a manifest"): steward-<UTC>.tar holding a
pg_dump --format=custom with metrics', check results' and Docker/Traefik/
UniFi rows left out, the secret key, and a manifest. Every table is named
KEPT or EXCLUDED in steward/backup/contents.py, and a guard test fails on a
new one. The image gains postgresql-client-17 (approved, System 180).

The restore (#5674 "Restore: `steward restore` and a CI test that a dump
brings Steward back"): pg_restore in its three sections. Between data and
constraints, a kept row pointing at a left-out row gets what its foreign
key's ON DELETE says (alert_events.ansible_run_id is cleared). It refuses
a non-empty database unless --wipe, and puts the backup's key in
/data/secret.key. Migrations run on Steward's next start.

`steward` stays the server command: it is now a click group that serves
when given no subcommand.

The publish smoke step backs up the booted image's database, restores it
into a second one, checks a marker setting came back and plugin_metrics
is empty, and starts Steward on it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_backup_contents.py

@@ -0,0 +1,112 @@
+"""What a backup carries (steward/backup/contents.py) and the file it makes
+(steward/backup/dump.py): every table is a decision, and the tar reads back."""
+import tarfile
+from datetime import datetime, timezone
+
+import pytest
+from sqlalchemy import Column, ForeignKey, Integer, MetaData, String, Table
+
+from steward.backup import contents, dump
+from steward.backup.restore import all_metadata, dangling_fixes
+
+
+def undecided(tables: set[str]) -> set[str]:
+    return tables - contents.KEPT - contents.EXCLUDED
+
+
+def test_every_table_is_kept_or_left_out():
+    tables = set(all_metadata().tables)
+    assert "prometheus_targets" in tables and "plugin_metrics" in tables   # plugins were loaded
+    assert undecided(tables) == set(), (
+        "Name these in steward/backup/contents.py as KEPT or EXCLUDED")
+
+
+def test_the_guard_can_fail():
+    assert undecided({"app_settings", "plugin_metrics", "a_new_table"}) == {"a_new_table"}
+
+
+def test_no_table_is_both_and_none_is_named_for_nothing():
+    assert contents.KEPT & contents.EXCLUDED == set()
+    stale = (contents.KEPT | contents.EXCLUDED) - set(all_metadata().tables)
+    assert stale == set(), "contents.py names tables that no longer exist"
+
+
+def test_an_alert_event_keeps_its_row_when_its_ansible_run_is_left_out():
+    fixes = dangling_fixes(all_metadata())
+    assert ('UPDATE "alert_events" SET "ansible_run_id" = NULL WHERE "ansible_run_id" IS NOT NULL '
+            'AND NOT EXISTS (SELECT 1 FROM "ansible_runs" p WHERE p."id" = "alert_events"."ansible_run_id")'
+            ) in fixes
+
+
+def test_a_cascading_reference_removes_the_row():
+    md = MetaData()
+    Table("plugin_metrics", md, Column("id", Integer, primary_key=True))
+    Table("notes", md, Column("id", Integer, primary_key=True),
+          Column("metric_id", Integer, ForeignKey("plugin_metrics.id", ondelete="CASCADE")),
+          Column("owner", String, ForeignKey("users.name")))
+    Table("users", md, Column("name", String, primary_key=True))
+    assert dangling_fixes(md) == [
+        'DELETE FROM "notes" WHERE "metric_id" IS NOT NULL AND NOT EXISTS '
+        '(SELECT 1 FROM "plugin_metrics" p WHERE p."id" = "notes"."metric_id")'
+    ]
+
+
+def test_dump_leaves_out_each_excluded_tables_rows(tmp_path):
+    args = dump.dump_command(tmp_path / "x.dump")
+    assert "--format=custom" in args
+    assert "--exclude-table-data=plugin_metrics" in args
+    assert "--exclude-table-data=audit_events" not in args
+
+
+def test_password_goes_in_the_environment_not_the_command_line():
+    env = dump.pg_env("postgresql+asyncpg://steward:s3cret@db:5433/steward?ssl=true")
+    assert env["PGHOST"] == "db" and env["PGPORT"] == "5433"
+    assert env["PGUSER"] == "steward" and env["PGPASSWORD"] == "s3cret"
+    assert env["PGDATABASE"] == "steward" and env["PGSSLMODE"] == "require"
+
+
+def test_file_name_is_utc():
+    at = datetime(2026, 10, 12, 21, 5, 9, tzinfo=timezone.utc)
+    assert dump.file_name(at) == "steward-20261012T210509Z.tar"
+
+
+def test_a_packed_backup_reads_back(tmp_path):
+    at = datetime(2026, 10, 12, 21, 5, 9, tzinfo=timezone.utc)
+    src = tmp_path / "in.dump"
+    src.write_bytes(b"PGDMP" + b"x" * 3000)
+    info = dump.manifest(at, ["0004_app_settings", "prometheus_001_targets"])
+    out = tmp_path / dump.file_name(at)
+    dump.pack(out, src, "the-key", info)
+
+    assert out.stat().st_mode & 0o777 == 0o600
+    assert not out.with_name(out.name + ".part").exists()
+    into = tmp_path / "out"
+    into.mkdir()
+    got, dumped, key = dump.read_backup(out, into)
+    assert key == "the-key"
+    assert dumped.read_bytes() == src.read_bytes()
+    assert got["format"] == dump.FORMAT and got["contains_secret_key"] is True
+    assert got["created_at"] == "2026-10-12T21:05:09+00:00"
+    assert "plugin_metrics" in got["rows_left_out"]
+
+
+def test_read_backup_refuses_what_isnt_one(tmp_path):
+    plain = tmp_path / "plain.tar"
+    with tarfile.open(plain, "w") as tar:
+        tar.add(__file__, arcname="notes.txt")
+    with pytest.raises(dump.BackupError, match="lacks manifest.json, secret.key, steward.dump"):
+        dump.read_backup(plain, tmp_path)
+
+    (tmp_path / "junk.tar").write_bytes(b"not a tar")
+    with pytest.raises(dump.BackupError, match="isn't a Steward backup"):
+        dump.read_backup(tmp_path / "junk.tar", tmp_path)
+
+
+def test_read_backup_refuses_another_format(tmp_path):
+    src = tmp_path / "in.dump"
+    src.write_bytes(b"PGDMP")
+    info = dump.manifest(datetime.now(timezone.utc), []) | {"format": 99}
+    out = tmp_path / "b.tar"
+    dump.pack(out, src, "k", info)
+    with pytest.raises(dump.BackupError, match="format 99"):
+        dump.read_backup(out, tmp_path)

tests/integration/test_backup_restore_fixes.py

@@ -0,0 +1,62 @@
+"""Integration: between pg_restore's data and its constraints, an alert event
+naming an Ansible run the backup left out keeps its row and loses the link
+(steward/backup/restore.py, #5674). Requires STEWARD_DATABASE_URL."""
+from __future__ import annotations
+
+import asyncio
+import os
+import uuid
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def test_a_dangling_ansible_run_link_is_cleared_and_the_event_kept(app):
+    from sqlalchemy import select, text
+
+    from steward.backup.restore import _execute, all_metadata, dangling_fixes
+    from steward.models.alerts import AlertEvent, AlertOperator, AlertRule
+    from steward.models.users import User, UserRole
+
+    uid, rule_id = str(uuid.uuid4()), str(uuid.uuid4())
+    gone_run, dangling, plain = str(uuid.uuid4()), str(uuid.uuid4()), str(uuid.uuid4())
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add(User(id=uid, username=f"u{uid[:8]}", email=f"{uid[:8]}@e.test",
+                           password_hash="x", role=UserRole.admin, is_active=True))
+            async with s.begin():
+                s.add(AlertRule(id=rule_id, name="backup fixes", source_module="host_agent",
+                                resource_name="srv1", metric_name="cpu_pct",
+                                operator=AlertOperator.gt, threshold=90.0,
+                                consecutive_failures_required=1, enabled=True, created_by=uid))
+            # The state pg_restore's data section leaves: the run's row was
+            # never restored. Replica mode lets the row in without its parent.
+            async with s.begin():
+                await s.execute(text("SET LOCAL session_replication_role = replica"))
+                s.add(AlertEvent(id=dangling, rule_id=rule_id, from_state="pending",
+                                 to_state="firing", ansible_run_id=gone_run))
+                s.add(AlertEvent(id=plain, rule_id=rule_id, from_state="firing",
+                                 to_state="inactive"))
+
+        fixed = await _execute(os.environ["STEWARD_DATABASE_URL"], dangling_fixes(all_metadata()))
+
+        async with app.db_sessionmaker() as s:
+            rows = dict((await s.execute(
+                select(AlertEvent.id, AlertEvent.ansible_run_id)
+                .where(AlertEvent.rule_id == rule_id))).all())
+        return fixed, rows
+
+    fixed, rows = asyncio.run(_go())
+    assert fixed >= 1
+    assert rows == {dangling: None, plain: None}