feat(backup): restore a same-version backup in place, history kept, all or nothing (#5815)

#5815 "Restore engine: in place for a same-version backup".

restore_in_place (steward/backup/restore.py) runs in one transaction:
1. It keeps aside what Steward records as it runs: ACTIVITY columns of rows
   that come back, and STATE_SETTINGS.
2. It drops every foreign key touching a replaced table.
3. It truncates the replaced tables and COPYs the backup's rows in
   unparsed, through asyncpg.
4. It puts the kept state back and moves sequences past their rows.
5. It puts right the rows whose parent didn't come back, by each key's own
   ON DELETE. A RESTRICT key that may be empty is cleared, so a run whose
   user is gone keeps its record.
6. It re-adds the keys, which checks every row.

A failure anywhere rolls the whole restore back. A backup made at another
schema is refused with NotSameSchema ("older": a full replace is the way;
"newer": update Steward).

The COPY-text reading moves into steward/backup/pgdata.py, shared by the
check and the restore. The integration tests stand in for pg_restore with
this database's own COPY output:
- settings come back while the audit log and the history of what remains
  stay;
- history of what didn't come back goes;
- a failure part-way changes nothing;
- a newer schema is refused.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_backup_check.py

@@ -8,8 +8,9 @@ from quart import render_template
 
 from steward.backup import check
 from steward.backup.check import (AreaCheck, CheckReport, RowDiff, TableCheck, _Opener, _Side,
-                                  compare_table, copy_rows, normalise, schema_of, script_tables,
-                                  unescape)
+                                  compare_table, normalise, schema_of)
+from steward.backup.pgdata import (copy_rows, script_blocks, script_from, script_tables,
+                                   unescape)
 from steward.core.crypto import _make_fernet
 from tests.test_style_tokens import hex_colours
 
@@ -48,6 +49,9 @@ def test_a_restore_script_gives_each_tables_rows():
     assert tables["app_settings"] == (["key", "value_json", "updated_at"],
                                       [["smtp.port", "587", "2026-10-15 14:00:00+00"]])
     assert tables["audit_events"][0] == ["id", "timestamp", "action"]
+    # Handed on unparsed, a block is the COPY text as the dump holds it.
+    assert script_blocks(script)["app_settings"][1] == b"smtp.port\t587\t2026-10-15 14:00:00+00\n"
+    assert script_tables(script_from(script_blocks(script))) == tables
 
 
 def test_an_instant_is_the_same_whatever_zone_prints_it():

tests/core/test_backup_restore_in_place.py

@@ -0,0 +1,32 @@
+"""How an in-place restore puts right a row whose parent didn't come back
+(steward/backup/restore.py, #5815): by the row's own foreign key."""
+from steward.backup.restore import NotSameSchema, _ForeignKey, fk_fix
+
+
+def _fk(on_delete, nullable=True, columns=("rule_id",)):
+    return _ForeignKey("fk", "alert_events", "alert_rules", list(columns), ["id"] * len(columns),
+                       on_delete, nullable, "FOREIGN KEY (rule_id) REFERENCES alert_rules(id)")
+
+
+def test_a_cascading_key_removes_the_row():
+    assert fk_fix(_fk("c")) == (
+        'DELETE FROM public."alert_events" x WHERE x."rule_id" IS NOT NULL AND NOT EXISTS '
+        '(SELECT 1 FROM public."alert_rules" p WHERE p."id" = x."rule_id")')
+
+
+def test_set_null_and_a_restricting_key_that_may_be_empty_clear_the_link():
+    for action in ("n", "a", "r"):
+        assert fk_fix(_fk(action)).startswith('UPDATE public."alert_events" x SET "rule_id" = NULL')
+
+
+def test_a_restricting_key_that_may_not_be_empty_removes_the_row():
+    assert fk_fix(_fk("r", nullable=False)).startswith('DELETE FROM public."alert_events"')
+
+
+def test_a_key_over_several_columns_is_left_to_the_constraint():
+    assert fk_fix(_fk("c", columns=("a", "b"))) is None
+
+
+def test_another_schema_says_what_to_do():
+    assert "update Steward first" in str(NotSameSchema("newer", ["x"], ["y"]))
+    assert "replaces everything" in str(NotSameSchema("older", ["x"], ["y"]))

tests/integration/test_backup_restore_in_place.py

@@ -0,0 +1,177 @@
+"""Integration: a backup put back in place on a running Steward (#5815). It
+replaces what the backup carries and keeps the rest: settings, hosts and
+rules come back as they were; the audit log and the history of what is
+still there stay; history of what didn't come back goes with it. A failure
+part-way changes nothing. pg_restore isn't in this lane, so the backup's
+rows are this database's own, read out as COPY text before the changes.
+Requires STEWARD_DATABASE_URL."""
+from __future__ import annotations
+
+import asyncio
+import io
+import os
+import uuid
+from datetime import datetime, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+async def _snapshot(app) -> bytes:
+    """The kept tables as they stand, as a --data-only script."""
+    from sqlalchemy import text
+    from steward.backup import contents
+    from steward.backup.pgdata import script_from
+    blocks = {}
+    async with app._db_engine.connect() as conn:
+        order: dict[str, list[str]] = {}
+        for t, c in (await conn.execute(text(
+                "SELECT table_name, column_name FROM information_schema.columns "
+                "WHERE table_schema = 'public' AND table_name = ANY(:t) "
+                "ORDER BY table_name, ordinal_position"), {"t": sorted(contents.KEPT)})).all():
+            order.setdefault(t, []).append(c)
+        raw = (await conn.get_raw_connection()).driver_connection
+        for t, cols in order.items():
+            buf = io.BytesIO()
+            await raw.copy_from_table(t, columns=cols, output=buf, format="text")
+            blocks[t] = (cols, buf.getvalue())
+    return script_from(blocks)
+
+
+async def _backup_file(app, tmp_path, heads=None):
+    from steward.backup import dump, job
+    (tmp_path / "steward.dump").write_bytes(b"PGDMP")
+    path = tmp_path / "steward-20261015T030000Z.tar"
+    heads = heads if heads is not None else await job.migration_heads(app.config["DATABASE_URL"])
+    dump.pack(path, tmp_path / "steward.dump", app.config["SECRET_KEY"],
+              dump.manifest(datetime.now(timezone.utc), heads))
+    return path
+
+
+def test_a_backup_goes_back_in_place_and_history_stays(app, monkeypatch, tmp_path):
+    from sqlalchemy import func, select
+    from steward.backup import pgdata
+    from steward.backup.restore import restore_in_place
+    from steward.core.settings import get_all_settings, set_setting
+    from steward.models.alerts import AlertEvent, AlertOperator, AlertRule
+    from steward.models.audit import AuditEvent
+    from steward.models.hosts import Host
+    from steward.models.users import User, UserRole
+
+    tag = uuid.uuid4().hex[:8]
+    setting = f"test.in_place_{tag}"
+
+    def rule(name, uid):
+        return AlertRule(name=name, source_module="host_agent", resource_name=f"r-{tag}",
+                         metric_name="cpu_pct", operator=AlertOperator.gt, threshold=90.0,
+                         consecutive_failures_required=1, enabled=True, created_by=uid)
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                user = User(username=f"ip-{tag}", email=f"ip-{tag}@test", password_hash="x",
+                            role=UserRole.admin)
+                s.add(user)
+                await s.flush()
+                await set_setting(s, setting, "before")
+                kept_host = Host(name=f"kept-{tag}", address="10.7.0.1")
+                kept_rule = rule(f"kept {tag}", user.id)
+                s.add_all([kept_host, kept_rule])
+                await s.flush()
+                s.add(AlertEvent(rule_id=kept_rule.id, from_state="inactive", to_state="firing"))
+        ids = {"user": user.id, "kept_host": kept_host.id, "kept_rule": kept_rule.id}
+        script = await _snapshot(app)
+
+        async def restore_script(_dump):
+            return script
+        monkeypatch.setattr(pgdata, "restore_script", restore_script)
+        path = await _backup_file(app, tmp_path)
+
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await set_setting(s, setting, "after")
+                await set_setting(s, "backup.last_attempt_at", f"kept-{tag}")   # Steward's own record
+                new_host = Host(name=f"new-{tag}", address="10.7.0.2")
+                new_rule = rule(f"new {tag}", ids["user"])
+                s.add_all([new_host, new_rule])
+                await s.flush()
+                s.add(AlertEvent(rule_id=new_rule.id, from_state="inactive", to_state="firing"))
+                s.add(AuditEvent(username="test", action=f"test.{tag}"))
+        ids |= {"new_host": new_host.id, "new_rule": new_rule.id}
+
+        done = await restore_in_place(path, app.config["DATABASE_URL"])
+
+        async with app.db_sessionmaker() as s:
+            settings = await get_all_settings(s)
+            hosts = set((await s.execute(select(Host.id).where(
+                Host.id.in_([ids["kept_host"], ids["new_host"]])))).scalars())
+            events = dict((await s.execute(select(AlertEvent.rule_id, func.count()).where(
+                AlertEvent.rule_id.in_([ids["kept_rule"], ids["new_rule"]]))
+                .group_by(AlertEvent.rule_id))).all())
+            audits = (await s.execute(select(func.count()).select_from(AuditEvent).where(
+                AuditEvent.action == f"test.{tag}"))).scalar_one()
+        return ids, done, settings, hosts, events, audits
+
+    ids, done, settings, hosts, events, audits = asyncio.run(_go())
+    assert settings[setting] == "before"                       # settings came back
+    assert settings["backup.last_attempt_at"] == f"kept-{tag}"  # Steward's own record stayed
+    assert hosts == {ids["kept_host"]}                          # the new host is gone
+    assert events == {ids["kept_rule"]: 1}                      # its rule's history went with it
+    assert audits == 1                                          # the audit log stayed
+    assert "app_settings" in done.tables and "audit_events" not in done.tables
+    assert done.fixed >= 1
+
+
+def test_a_restore_that_fails_part_way_changes_nothing(app, monkeypatch, tmp_path):
+    from steward.backup import pgdata
+    from steward.backup.dump import BackupError
+    from steward.backup.restore import restore_in_place
+    from steward.core.settings import get_all_settings, set_setting
+
+    tag = uuid.uuid4().hex[:8]
+    setting = f"test.undone_{tag}"
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await set_setting(s, setting, "here")
+        script = await _snapshot(app)
+        # A monitor row with too few fields: COPY refuses it after the
+        # settings have already been replaced in the same transaction.
+        script += b"COPY public.monitors (id, name) FROM stdin;\nonly-one-field\n\\.\n"
+
+        async def restore_script(_dump):
+            return script
+        monkeypatch.setattr(pgdata, "restore_script", restore_script)
+        path = await _backup_file(app, tmp_path)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await set_setting(s, setting, "changed")
+        with pytest.raises(BackupError, match="undone, nothing changed"):
+            await restore_in_place(path, app.config["DATABASE_URL"])
+        async with app.db_sessionmaker() as s:
+            return (await get_all_settings(s))[setting]
+
+    assert asyncio.run(_go()) == "changed"
+
+
+def test_a_backup_from_another_schema_is_refused_in_place(app, tmp_path):
+    from steward.backup.restore import NotSameSchema, restore_in_place
+
+    async def _go():
+        path = await _backup_file(app, tmp_path, heads=["9999_from_the_future"])
+        with pytest.raises(NotSameSchema) as caught:
+            await restore_in_place(path, app.config["DATABASE_URL"])
+        return caught.value
+
+    refused = asyncio.run(_go())
+    assert refused.schema == "newer" and "newer Steward" in str(refused)