feat(backup): restore from Settings → Backups, with restoring mode (#5816, #5817)

#5816 "Restoring mode": while a backup goes back in, the scheduler holds,
ingest and HTMX answer 503 with Retry-After (agents keep their samples),
and every page says what is happening. A safety backup of Steward as it
stands is made first, to /data/backups and the bucket. The backup's key
is put in place, the restore audited, and Steward exits with code 3 so
Swarm, compose or systemd starts it fresh. A failed in-place restore
changes nothing and says why on Backups; a full replace that fails after
the wipe restarts into first-run setup.

#5817 "Settings → Backups: Restore": a Restore button on each sent backup
and on the file card. The chosen backup is checked (the same comparison
as Check, now a shared partial) and held server-side for an hour under a
token; it starts only once the admin types "restore", and an older
backup also needs "Replace everything" ticked. A newer backup, or one
whose key can't open its secrets, gets no button.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_restoring.py

@@ -0,0 +1,145 @@
+"""Restoring mode (steward/backup/restoring.py, #5816): while a backup goes back
+in, pages say so and agents are told to retry; the scheduler holds; a restore
+saves a safety backup first, puts the backup's key in place, audits itself
+and asks for a restart; a failed one says why and changes nothing."""
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+
+from steward.backup import restoring
+from steward.backup.dump import BackupError
+
+
+@pytest.fixture(autouse=True)
+def fresh_state(monkeypatch):
+    monkeypatch.setattr(restoring, "STATE", restoring.Restoring())
+
+
+async def test_while_restoring_pages_say_so_and_agents_are_told_to_retry(client):
+    restoring.STATE.active, restoring.STATE.name = True, "steward-20261015T030000Z.tar"
+    restoring.STATE.by, restoring.STATE.step = "admin", "Restoring"
+    page = await client.get("/", headers={"Accept": "text/html"})
+    assert page.status_code == 503 and page.headers["Retry-After"] == "30"
+    body = await page.get_data(as_text=True)
+    assert "Steward is restoring a backup" in body and "steward-20261015T030000Z.tar" in body
+    assert 'http-equiv="refresh"' in body
+
+    ingest = await client.post("/plugins/host_agent/ingest", json={})
+    assert ingest.status_code == 503 and ingest.headers["Retry-After"] == "30"
+    assert "restoring a backup" in await ingest.get_data(as_text=True)
+
+
+async def test_when_not_restoring_the_gate_stays_out_of_the_way(app):
+    async with app.test_request_context("/", headers={"Accept": "text/html"}):
+        assert await restoring.gate() is None
+
+
+async def test_the_scheduler_holds_while_paused():
+    from steward.core.scheduler import ScheduledTask, start_scheduler
+    ran = []
+    paused = {"now": True}
+
+    async def tick():
+        ran.append(1)
+
+    task = asyncio.create_task(start_scheduler(
+        [ScheduledTask("t", tick, interval_seconds=0, run_on_startup=False)],
+        paused=lambda: paused["now"]))
+    await asyncio.sleep(1.5)
+    held = len(ran)
+    paused["now"] = False
+    await asyncio.sleep(1.5)
+    task.cancel()
+    assert held == 0 and ran
+
+
+def _app():
+    return SimpleNamespace(config={"DATABASE_URL": "postgresql+asyncpg://x/y", "SECRET_KEY": "k-here",
+                                   "PLUGIN_DIRS": ["plugins"]})
+
+
+def _stub(monkeypatch, tmp_path, *, restore_error=None, key="k-backup"):
+    from steward import config
+    from steward.backup import restore
+    from steward.core import audit
+    seen = {"audits": [], "restarts": 0}
+
+    async def safety(app):
+        seen["safety"] = True
+        return "steward-20261015T140000Z.tar"
+
+    async def in_place(path, url, plugin_dirs=None):
+        if restore_error:
+            raise BackupError(restore_error)
+        return restore.RestoredInPlace({}, key, ["app_settings", "hosts"], 12, 1)
+
+    async def log_audit(app, user_id, username, action, **kw):
+        seen["audits"].append((action, kw.get("detail")))
+
+    monkeypatch.setattr(restoring, "_backup_key", lambda path: key)
+    monkeypatch.setattr(restoring, "_safety_backup", safety)
+    monkeypatch.setattr(restore, "restore_in_place", in_place)
+    monkeypatch.setattr(audit, "log_audit", log_audit)
+    monkeypatch.setattr(restoring, "_restart_soon", lambda: seen.__setitem__("restarts", seen["restarts"] + 1))
+    monkeypatch.setattr(config, "_SECRET_KEY_FILE", tmp_path / "secret.key")
+    monkeypatch.delenv("STEWARD_SECRET_KEY", raising=False)
+    return seen
+
+
+async def _restore(tmp_path):
+    assert restoring.start_restore(_app(), tmp_path / "b.tar", "steward-20261015T030000Z.tar",
+                                   username="admin")
+    await asyncio.gather(*list(restoring._tasks))
+    await asyncio.gather(*list(restoring._tasks))      # the failure audit, when there is one
+
+
+async def test_a_restore_saves_a_safety_backup_puts_the_key_in_place_and_restarts(monkeypatch, tmp_path):
+    seen = _stub(monkeypatch, tmp_path)
+    await _restore(tmp_path)
+    assert seen["safety"] and seen["restarts"] == 1
+    assert (tmp_path / "secret.key").read_text() == "k-backup"
+    [(action, detail)] = seen["audits"]
+    assert action == "backup.restored"
+    assert detail == {"mode": "in place", "tables": 2, "rows": 12, "put_right": 1,
+                      "safety_backup": "steward-20261015T140000Z.tar"}
+    assert restoring.STATE.step.startswith("Restored")
+
+
+async def test_a_failed_restore_says_why_and_steward_carries_on(monkeypatch, tmp_path):
+    seen = _stub(monkeypatch, tmp_path, restore_error="The restore was undone, nothing changed: x")
+    await _restore(tmp_path)
+    assert not restoring.STATE.active and "undone" in restoring.STATE.error
+    assert seen["restarts"] == 0 and not (tmp_path / "secret.key").exists()
+    assert [a for a, _ in seen["audits"]] == ["backup.restore_failed"]
+
+
+async def test_a_pinned_key_that_differs_stops_it_before_anything_is_saved(monkeypatch, tmp_path):
+    seen = _stub(monkeypatch, tmp_path)
+    monkeypatch.setenv("STEWARD_SECRET_KEY", "k-pinned")
+    await _restore(tmp_path)
+    assert "STEWARD_SECRET_KEY" in restoring.STATE.error and "safety" not in seen
+
+
+async def test_one_restore_at_a_time(monkeypatch, tmp_path):
+    _stub(monkeypatch, tmp_path)
+    restoring.STATE.active = True
+    assert not restoring.start_restore(_app(), tmp_path / "b.tar", "x", username="admin")
+
+
+def test_a_chosen_backup_is_held_by_token_until_it_is_too_old(monkeypatch, tmp_path):
+    import os
+    import time
+    monkeypatch.setattr(restoring, "SAFETY_DIR", tmp_path)
+    held = restoring.stage()
+    held.path.write_bytes(b"x")
+    held.name, held.schema, held.ok = "steward-20261015T030000Z.tar", "same", True
+    held.save()
+    again = restoring.staged(held.token)
+    assert (again.name, again.schema, again.ok, again.path) == (held.name, "same", True, held.path)
+    assert restoring.staged("../../etc") is None and restoring.staged("0" * 32) is None
+
+    old = time.time() - restoring.STAGED_SECONDS - 1
+    os.utime(held.dir, (old, old))
+    restoring.stage()
+    assert restoring.staged(held.token) is None

tests/integration/test_backup_restore_routes.py

@@ -0,0 +1,159 @@
+"""Integration: Restore on Settings → Backups (#5817). A chosen backup is
+checked and held; nothing starts until the person types the word, an older
+backup also needs Replace everything ticked, a newer one has no button, and
+only an admin gets this far. The restore itself is restoring.py's (stubbed
+here; tests/core/test_restoring.py covers it)."""
+from __future__ import annotations
+
+import asyncio
+import io
+import os
+import time
+import uuid
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+CSRF = {"X-CSRF-Token": "test-token"}
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+async def _client(app, role):
+    from steward.models.users import User, UserRole
+    uid, tag = str(uuid.uuid4()), uuid.uuid4().hex[:8]
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            s.add(User(id=uid, username=f"rs-{tag}", email=f"rs-{tag}@test",
+                       password_hash="x", role=UserRole(role), is_active=True))
+    client = app.test_client()
+    async with client.session_transaction() as sess:
+        sess.update(user_id=uid, user_role=role, username=f"rs-{tag}",
+                    _csrf="test-token", login_at=time.time())
+    return client
+
+
+def _report(schema="same", unopened=0):
+    from steward.backup.check import AreaCheck, CheckReport, TableCheck
+    return CheckReport(name="steward-20261015T030000Z.tar",
+                       manifest={"created_at": "2026-10-15T03:00:00+00:00"}, schema=schema,
+                       backup_heads=["a"], here_heads=["a"], key_same=True, sealed=1,
+                       unopened=unopened,
+                       areas=[AreaCheck("Settings", [TableCheck("app_settings", same=4)])])
+
+
+@pytest.fixture
+def stubs(monkeypatch, tmp_path):
+    from steward.backup import job, restoring
+    seen = {"started": [], "schema": "same", "unopened": 0}
+
+    async def check_file(app, path, name=""):
+        seen["checked"] = path.read_bytes()
+        return _report(seen["schema"], seen["unopened"])
+
+    def start_restore(app, path, name, *, username, replace_all=False, owned_dir=None):
+        seen["started"].append({"path": path, "name": name, "replace_all": replace_all,
+                                "owned_dir": owned_dir})
+        return True
+
+    monkeypatch.setattr(restoring, "SAFETY_DIR", tmp_path)
+    monkeypatch.setattr(job, "check_file", check_file)
+    monkeypatch.setattr(restoring, "start_restore", start_restore)
+    return seen
+
+
+async def _upload(client, name="steward-20261015T030000Z.tar"):
+    from quart.datastructures import FileStorage
+    return await client.post("/settings/backups/restore/file/", headers=CSRF, files={
+        "file": FileStorage(io.BytesIO(b"a backup"), filename=name)})
+
+
+def _token(page: str) -> str:
+    import re
+    return re.search(r'name="token" value="([0-9a-f]{32})"', page).group(1)
+
+
+def test_nothing_starts_until_the_word_is_typed(app, stubs):
+    async def _go():
+        admin = await _client(app, "admin")
+        page = await (await _upload(admin)).get_data(as_text=True)
+        assert stubs["checked"] == b"a backup"
+        assert "Matches what is in place" in page and "Type <code>restore</code> to confirm" in page
+        token = _token(page)
+
+        wrong = await admin.post("/settings/backups/restore/start/", headers=CSRF,
+                                 form={"token": token, "confirm": "yes"})
+        assert wrong.status_code == 200 and not stubs["started"]
+        assert "Type restore to confirm." in await wrong.get_data(as_text=True)
+
+        go = await admin.post("/settings/backups/restore/start/", headers=CSRF,
+                              form={"token": token, "confirm": " Restore "})
+        assert go.status_code == 302 and go.headers["Location"].endswith("/settings/backups/")
+        [started] = stubs["started"]
+        assert started["name"] == "steward-20261015T030000Z.tar" and not started["replace_all"]
+        assert started["path"].read_bytes() == b"a backup" and started["owned_dir"].name == token
+
+        gone = await admin.post("/settings/backups/restore/start/", headers=CSRF,
+                                form={"token": "0" * 32, "confirm": "restore"})
+        assert "no longer held" in await gone.get_data(as_text=True)
+    asyncio.run(_go())
+
+
+def test_an_older_backup_also_needs_replace_everything(app, stubs):
+    stubs["schema"] = "older"
+
+    async def _go():
+        admin = await _client(app, "admin")
+        page = await (await _upload(admin)).get_data(as_text=True)
+        assert 'name="replace_all"' in page and "replaces everything" in page
+        token = _token(page)
+        await admin.post("/settings/backups/restore/start/", headers=CSRF,
+                         form={"token": token, "confirm": "restore"})
+        assert not stubs["started"]
+        await admin.post("/settings/backups/restore/start/", headers=CSRF,
+                         form={"token": token, "confirm": "restore", "replace_all": "on"})
+        assert [s["replace_all"] for s in stubs["started"]] == [True]
+    asyncio.run(_go())
+
+
+def test_a_backup_that_cant_go_in_has_no_button(app, stubs):
+    async def _go():
+        admin = await _client(app, "admin")
+        stubs["schema"] = "newer"
+        newer = await (await _upload(admin)).get_data(as_text=True)
+        stubs["schema"], stubs["unopened"] = "same", 1
+        unopened = await (await _upload(admin)).get_data(as_text=True)
+        return newer, unopened
+    newer, unopened = asyncio.run(_go())
+    assert "Update Steward, then restore it." in newer and 'name="token"' not in newer
+    assert "wouldn&#39;t come back" in unopened and 'name="token"' not in unopened
+
+
+def test_only_an_admin_restores(app, stubs):
+    async def _go():
+        operator = await _client(app, "operator")
+        return (await _upload(operator)).status_code, (await operator.post(
+            "/settings/backups/restore/", headers=CSRF, form={"name": "steward-20261015T030000Z.tar"})
+        ).status_code
+    assert asyncio.run(_go()) == (403, 403)
+    assert not stubs["started"]
+
+
+def test_the_backups_page_offers_restore_and_says_when_one_failed(app, stubs, monkeypatch):
+    from steward.backup import restoring
+    monkeypatch.setattr(restoring, "STATE", restoring.Restoring(
+        name="steward-20261014T030000Z.tar", error="The restore was undone, nothing changed: x"))
+
+    async def _go():
+        admin = await _client(app, "admin")
+        return await (await admin.get("/settings/backups/")).get_data(as_text=True)
+    page = asyncio.run(_go())
+    assert 'formaction="/settings/backups/restore/file/"' in page
+    assert "The restore of steward-20261014T030000Z.tar" in page and "nothing changed: x" in page