FabledSteward/tests/core/test_restoring.py

"""Restoring mode (steward/backup/restoring.py, #5816): while a backup goes back
in, pages say so and agents are told to retry; the scheduler holds; a restore
saves a safety backup first, puts the backup's key in place, audits itself
and asks for a restart; a failed one says why and changes nothing."""
import asyncio
from types import SimpleNamespace

import pytest

from steward.backup import restoring
from steward.backup.dump import BackupError


@pytest.fixture(autouse=True)
def fresh_state(monkeypatch):
    monkeypatch.setattr(restoring, "STATE", restoring.Restoring())


async def test_while_restoring_pages_say_so_and_agents_are_told_to_retry(client):
    restoring.STATE.active, restoring.STATE.name = True, "steward-20261015T030000Z.tar"
    restoring.STATE.by, restoring.STATE.step = "admin", "Restoring"
    page = await client.get("/", headers={"Accept": "text/html"})
    assert page.status_code == 503 and page.headers["Retry-After"] == "30"
    body = await page.get_data(as_text=True)
    assert "Steward is restoring a backup" in body and "steward-20261015T030000Z.tar" in body
    assert 'http-equiv="refresh"' in body

    ingest = await client.post("/plugins/host_agent/ingest", json={})
    assert ingest.status_code == 503 and ingest.headers["Retry-After"] == "30"
    assert "restoring a backup" in await ingest.get_data(as_text=True)


async def test_when_not_restoring_the_gate_stays_out_of_the_way(app):
    async with app.test_request_context("/", headers={"Accept": "text/html"}):
        assert await restoring.gate() is None


async def test_the_scheduler_holds_while_paused():
    from steward.core.scheduler import ScheduledTask, start_scheduler
    ran = []
    paused = {"now": True}

    async def tick():
        ran.append(1)

    task = asyncio.create_task(start_scheduler(
        [ScheduledTask("t", tick, interval_seconds=0, run_on_startup=False)],
        paused=lambda: paused["now"]))
    await asyncio.sleep(1.5)
    held = len(ran)
    paused["now"] = False
    await asyncio.sleep(1.5)
    task.cancel()
    assert held == 0 and ran


def _app():
    return SimpleNamespace(config={"DATABASE_URL": "postgresql+asyncpg://x/y", "SECRET_KEY": "k-here",
                                   "PLUGIN_DIRS": ["plugins"]})


def _stub(monkeypatch, tmp_path, *, restore_error=None, key="k-backup"):
    from steward import config
    from steward.backup import restore
    from steward.core import audit
    seen = {"audits": [], "restarts": 0}

    async def safety(app):
        seen["safety"] = True
        return "steward-20261015T140000Z.tar"

    async def in_place(path, url, plugin_dirs=None):
        if restore_error:
            raise BackupError(restore_error)
        return restore.RestoredInPlace({}, key, ["app_settings", "hosts"], 12, 1)

    async def log_audit(app, user_id, username, action, **kw):
        seen["audits"].append((action, kw.get("detail")))

    monkeypatch.setattr(restoring, "_backup_key", lambda path: key)
    monkeypatch.setattr(restoring, "_safety_backup", safety)
    monkeypatch.setattr(restore, "restore_in_place", in_place)
    monkeypatch.setattr(audit, "log_audit", log_audit)
    monkeypatch.setattr(restoring, "_restart_soon", lambda: seen.__setitem__("restarts", seen["restarts"] + 1))
    monkeypatch.setattr(config, "_SECRET_KEY_FILE", tmp_path / "secret.key")
    monkeypatch.delenv("STEWARD_SECRET_KEY", raising=False)
    return seen


async def _restore(tmp_path):
    assert restoring.start_restore(_app(), tmp_path / "b.tar", "steward-20261015T030000Z.tar",
                                   username="admin")
    await asyncio.gather(*list(restoring._tasks))
    await asyncio.gather(*list(restoring._tasks))      # the failure audit, when there is one


async def test_a_restore_saves_a_safety_backup_puts_the_key_in_place_and_restarts(monkeypatch, tmp_path):
    seen = _stub(monkeypatch, tmp_path)
    await _restore(tmp_path)
    assert seen["safety"] and seen["restarts"] == 1
    assert (tmp_path / "secret.key").read_text() == "k-backup"
    [(action, detail)] = seen["audits"]
    assert action == "backup.restored"
    assert detail == {"mode": "in place", "tables": 2, "rows": 12, "put_right": 1,
                      "safety_backup": "steward-20261015T140000Z.tar"}
    assert restoring.STATE.step.startswith("Restored")


async def test_a_failed_restore_says_why_and_steward_carries_on(monkeypatch, tmp_path):
    seen = _stub(monkeypatch, tmp_path, restore_error="The restore was undone, nothing changed: x")
    await _restore(tmp_path)
    assert not restoring.STATE.active and "undone" in restoring.STATE.error
    assert seen["restarts"] == 0 and not (tmp_path / "secret.key").exists()
    assert [a for a, _ in seen["audits"]] == ["backup.restore_failed"]


async def test_a_pinned_key_that_differs_stops_it_before_anything_is_saved(monkeypatch, tmp_path):
    seen = _stub(monkeypatch, tmp_path)
    monkeypatch.setenv("STEWARD_SECRET_KEY", "k-pinned")
    await _restore(tmp_path)
    assert "STEWARD_SECRET_KEY" in restoring.STATE.error and "safety" not in seen


async def test_one_restore_at_a_time(monkeypatch, tmp_path):
    _stub(monkeypatch, tmp_path)
    restoring.STATE.active = True
    assert not restoring.start_restore(_app(), tmp_path / "b.tar", "x", username="admin")


def test_a_chosen_backup_is_held_by_token_until_it_is_too_old(monkeypatch, tmp_path):
    import os
    import time
    monkeypatch.setattr(restoring, "SAFETY_DIR", tmp_path)
    held = restoring.stage()
    held.path.write_bytes(b"x")
    held.name, held.schema, held.ok = "steward-20261015T030000Z.tar", "same", True
    held.save()
    again = restoring.staged(held.token)
    assert (again.name, again.schema, again.ok, again.path) == (held.name, "same", True, held.path)
    assert restoring.staged("../../etc") is None and restoring.staged("0" * 32) is None

    old = time.time() - restoring.STAGED_SECONDS - 1
    os.utime(held.dir, (old, old))
    restoring.stage()
    assert restoring.staged(held.token) is None