"""Restoring mode (steward/backup/restoring.py, #5816): while a backup goes back in, pages say so and agents are told to retry; the scheduler holds; a restore saves a safety backup first, puts the backup's key in place, audits itself and asks for a restart; a failed one says why and changes nothing.""" import asyncio from types import SimpleNamespace import pytest from steward.backup import restoring from steward.backup.dump import BackupError @pytest.fixture(autouse=True) def fresh_state(monkeypatch): monkeypatch.setattr(restoring, "STATE", restoring.Restoring()) async def test_while_restoring_pages_say_so_and_agents_are_told_to_retry(client): restoring.STATE.active, restoring.STATE.name = True, "steward-20261015T030000Z.tar" restoring.STATE.by, restoring.STATE.step = "admin", "Restoring" page = await client.get("/", headers={"Accept": "text/html"}) assert page.status_code == 503 and page.headers["Retry-After"] == "30" body = await page.get_data(as_text=True) assert "Steward is restoring a backup" in body and "steward-20261015T030000Z.tar" in body assert 'http-equiv="refresh"' in body ingest = await client.post("/plugins/host_agent/ingest", json={}) assert ingest.status_code == 503 and ingest.headers["Retry-After"] == "30" assert "restoring a backup" in await ingest.get_data(as_text=True) async def test_when_not_restoring_the_gate_stays_out_of_the_way(app): async with app.test_request_context("/", headers={"Accept": "text/html"}): assert await restoring.gate() is None async def test_the_scheduler_holds_while_paused(): from steward.core.scheduler import ScheduledTask, start_scheduler ran = [] paused = {"now": True} async def tick(): ran.append(1) task = asyncio.create_task(start_scheduler( [ScheduledTask("t", tick, interval_seconds=0, run_on_startup=False)], paused=lambda: paused["now"])) await asyncio.sleep(1.5) held = len(ran) paused["now"] = False await asyncio.sleep(1.5) task.cancel() assert held == 0 and ran def _app(): return SimpleNamespace(config={"DATABASE_URL": "postgresql+asyncpg://x/y", "SECRET_KEY": "k-here", "PLUGIN_DIRS": ["plugins"]}) def _stub(monkeypatch, tmp_path, *, restore_error=None, key="k-backup"): from steward import config from steward.backup import restore from steward.core import audit seen = {"audits": [], "restarts": 0} async def safety(app, username): seen["safety"] = username return "steward-20261015T140000Z.tar" async def in_place(path, url, plugin_dirs=None): if restore_error: raise BackupError(restore_error) return restore.RestoredInPlace({}, key, ["app_settings", "hosts"], 12, 1) async def log_audit(app, user_id, username, action, **kw): seen["audits"].append((action, kw.get("detail"))) monkeypatch.setattr(restoring, "_backup_key", lambda path: key) monkeypatch.setattr(restoring, "_safety_backup", safety) monkeypatch.setattr(restore, "restore_in_place", in_place) monkeypatch.setattr(audit, "log_audit", log_audit) monkeypatch.setattr(restoring, "_restart_soon", lambda: seen.__setitem__("restarts", seen["restarts"] + 1)) monkeypatch.setattr(config, "_SECRET_KEY_FILE", tmp_path / "secret.key") monkeypatch.delenv("STEWARD_SECRET_KEY", raising=False) return seen async def _restore(tmp_path): assert restoring.start_restore(_app(), tmp_path / "b.tar", "steward-20261015T030000Z.tar", username="admin") await asyncio.gather(*list(restoring._tasks)) await asyncio.gather(*list(restoring._tasks)) # the failure audit, when there is one async def test_a_restore_saves_a_safety_backup_puts_the_key_in_place_and_restarts(monkeypatch, tmp_path): seen = _stub(monkeypatch, tmp_path) await _restore(tmp_path) assert seen["safety"] == "admin" and seen["restarts"] == 1 # its audit row names who restored assert (tmp_path / "secret.key").read_text() == "k-backup" [(action, detail)] = seen["audits"] assert action == "backup.restored" assert detail == {"mode": "in place", "tables": 2, "rows": 12, "put_right": 1, "safety_backup": "steward-20261015T140000Z.tar"} assert restoring.STATE.step.startswith("Restored") async def test_a_failed_restore_says_why_and_steward_carries_on(monkeypatch, tmp_path): seen = _stub(monkeypatch, tmp_path, restore_error="The restore was undone, nothing changed: x") await _restore(tmp_path) assert not restoring.STATE.active and "undone" in restoring.STATE.error assert seen["restarts"] == 0 and not (tmp_path / "secret.key").exists() assert [a for a, _ in seen["audits"]] == ["backup.restore_failed"] async def test_a_pinned_key_that_differs_stops_it_before_anything_is_saved(monkeypatch, tmp_path): seen = _stub(monkeypatch, tmp_path) monkeypatch.setenv("STEWARD_SECRET_KEY", "k-pinned") await _restore(tmp_path) assert "STEWARD_SECRET_KEY" in restoring.STATE.error and "safety" not in seen async def test_one_restore_at_a_time(monkeypatch, tmp_path): _stub(monkeypatch, tmp_path) restoring.STATE.active = True assert not restoring.start_restore(_app(), tmp_path / "b.tar", "x", username="admin") def test_a_chosen_backup_is_held_by_token_until_it_is_too_old(monkeypatch, tmp_path): import os import time monkeypatch.setattr(restoring, "SAFETY_DIR", tmp_path) held = restoring.stage() held.path.write_bytes(b"x") held.name, held.schema, held.ok = "steward-20261015T030000Z.tar", "same", True held.save() again = restoring.staged(held.token) assert (again.name, again.schema, again.ok, again.path) == (held.name, "same", True, held.path) assert restoring.staged("../../etc") is None and restoring.staged("0" * 32) is None old = time.time() - restoring.STAGED_SECONDS - 1 os.utime(held.dir, (old, old)) restoring.stage() assert restoring.staged(held.token) is None