feat(backup): steward backup and steward restore; CI restores a dump and boots on it (#5672, #5674)

The backup file (#5672 "Backup file: pg_dump of config + audit + alert
history, the secret key and a manifest"): steward-<UTC>.tar holding a
pg_dump --format=custom with metrics', check results' and Docker/Traefik/
UniFi rows left out, the secret key, and a manifest. Every table is named
KEPT or EXCLUDED in steward/backup/contents.py, and a guard test fails on a
new one. The image gains postgresql-client-17 (approved, System 180).

The restore (#5674 "Restore: `steward restore` and a CI test that a dump
brings Steward back"): pg_restore in its three sections. Between data and
constraints, a kept row pointing at a left-out row gets what its foreign
key's ON DELETE says (alert_events.ansible_run_id is cleared). It refuses
a non-empty database unless --wipe, and puts the backup's key in
/data/secret.key. Migrations run on Steward's next start.

`steward` stays the server command: it is now a click group that serves
when given no subcommand.

The publish smoke step backs up the booted image's database, restores it
into a second one, checks a marker setting came back and plugin_metrics
is empty, and starts Steward on it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/cli.py

@@ -37,20 +37,81 @@ async def _serve(app, config) -> None:
     await serve(app, config, shutdown_trigger=stop.wait)
 
 
-@click.command()
+@click.group(invoke_without_command=True)
 @click.option("--config", default="config.yaml", help="Path to config.yaml")
 @click.option("--host", default="0.0.0.0")
 @click.option("--port", default=5000, type=int)
 @click.option("--debug", is_flag=True, default=False,
               help="Quart's development server with the reloader, not Hypercorn.")
-def main(config: str, host: str, port: int, debug: bool) -> None:
-    """Start the Steward server."""
+@click.pass_context
+def main(ctx: click.Context, config: str, host: str, port: int, debug: bool) -> None:
+    """Start the Steward server, or run one of the commands below."""
     # Before anything logs; create_app applies the log.level setting once
     # settings load.
     configure_logging()
+    ctx.obj = {"config": config}
+    if ctx.invoked_subcommand is not None:
+        return
     from .app import create_app
     app = create_app(config_path=Path(config))
     if debug:
         app.run(host=host, port=port, debug=True)
         return
     asyncio.run(_serve(app, hypercorn_config(host, port)))
+
+
+@main.command("backup")
+@click.option("--to", "out_dir", default=".", type=click.Path(file_okay=False, path_type=Path),
+              help="The folder to write the backup file into.")
+@click.pass_context
+def backup_command(ctx: click.Context, out_dir: Path) -> None:
+    """Write a backup file of this Steward (configuration, audit log, alert
+    history and the secret key) into a folder, without sending it anywhere."""
+    from .backup.dump import BackupError, make_backup
+    from .backup.job import migration_heads
+    from .config import load_bootstrap
+
+    boot = load_bootstrap(Path(ctx.obj["config"]))
+    out_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        heads = asyncio.run(migration_heads(boot["database_url"]))
+        made = asyncio.run(make_backup(boot["database_url"], boot["secret_key"], heads, out_dir))
+    except BackupError as exc:
+        raise click.ClickException(str(exc)) from None
+    click.echo(f"Wrote {made.path} ({made.size} bytes). It holds the secret key: keep it safe.")
+
+
+@main.command("restore")
+@click.argument("backup_file", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.option("--wipe", is_flag=True, default=False,
+              help="Drop everything in the database first. Without it the database must be empty.")
+@click.pass_context
+def restore_command(ctx: click.Context, backup_file: Path, wipe: bool) -> None:
+    """Bring Steward back from a backup file into its (empty) database, and
+    put the backup's secret key in place. Start Steward afterwards: it runs
+    any migrations the backup is behind on."""
+    from .backup.dump import BackupError
+    from .backup.restore import restore
+    from .config import _SECRET_KEY_FILE, _env, load_bootstrap
+
+    boot = load_bootstrap(Path(ctx.obj["config"]))
+    try:
+        done = asyncio.run(restore(backup_file, boot["database_url"], wipe=wipe))
+    except BackupError as exc:
+        raise click.ClickException(str(exc)) from None
+    made = done.manifest.get("created_at", "?")
+    click.echo(f"Restored the backup made {made} (Steward {done.manifest.get('steward_version', '?')}).")
+    if done.fixed:
+        click.echo(f"Cleared {done.fixed} references to rows a backup leaves out (e.g. Ansible runs).")
+    env_key = _env("SECRET_KEY")
+    if env_key:
+        if env_key != done.secret_key:
+            click.echo("STEWARD_SECRET_KEY is set and differs from the backup's key: set it to the "
+                       "backup's key, or the restored passwords and keys can't be read.", err=True)
+            raise SystemExit(2)
+    else:
+        _SECRET_KEY_FILE.parent.mkdir(parents=True, exist_ok=True)
+        _SECRET_KEY_FILE.write_text(done.secret_key)
+        _SECRET_KEY_FILE.chmod(0o600)
+        click.echo(f"Wrote the backup's secret key to {_SECRET_KEY_FILE}.")
+    click.echo("Start Steward now; it brings the database up to date on start.")