# tests/plugins/host_agent/test_maintenance.py
"""#5767: a host's maintenance job, step by step, from the agent's reports.

The transitions are model-free, so a job here is a SimpleNamespace with the
row's attributes; the database side is tests/integration/test_host_maintenance_job.py.
"""
import json
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace

import pytest

from plugins.host_agent import agent as a
from plugins.host_agent import maintenance as m

T0 = datetime(2026, 10, 14, 23, 0, tzinfo=timezone.utc)


def _job(**kw):
    base = dict(id="job-1", host_id="h1", state="queued", state_at=T0, created_at=T0,
                reboot_policy="if_required", step_id=None, step_seen=False, node_id=None,
                drain_host_id=None, packages=None, reboot_needed=None, back_at=None,
                detail=None, finished_at=None, drained_at=None, docker_update=None, checked=None,
                came_back_at=None)
    base.update(kw)
    return SimpleNamespace(**base)


def _result(job, state, **kw):
    return {"kind": "maintenance", "id": job.step_id, "state": state, **kw}


def test_an_update_with_no_reboot_needed_is_done_after_the_update():
    job = _job()
    m.start(job, T0)
    assert m.handoff(job) == {"id": "job-1-update", "action": "update"}
    assert m.apply_result(job, _result(job, "running"), T0)
    assert m.handoff(job) is None                       # taken: not handed again
    m.apply_result(job, _result(job, "done", ok=True, packages=3, reboot_required=False), T0)
    assert (job.state, job.packages, job.finished_at) == ("done", 3, T0)
    assert m.summary(job) == "Done: 3 packages updated, no reboot needed"


def test_a_swarm_node_is_drained_rebooted_and_made_active_again():
    job = _job(node_id="n" * 25, drain_host_id="mgr")
    m.start(job, T0)
    m.apply_result(job, _result(job, "done", ok=True, packages=12, reboot_required=True), T0)
    assert job.state == "draining" and m.executor(job) == "mgr"
    assert m.handoff(job) == {"id": "job-1-drain", "action": "drain", "node": "n" * 25}
    m.apply_result(job, _result(job, "done", ok=True), T0)
    # The node's tasks get 5 minutes to move before the reboot (#6098).
    assert job.state == "settling" and job.drained_at == T0 and m.handoff(job) is None
    assert not m.settle(job, T0 + timedelta(minutes=4))
    assert m.settle(job, T0 + timedelta(minutes=5))
    assert job.state == "rebooting" and m.executor(job) == "h1"
    assert m.handoff(job) == {"id": "job-1-reboot", "action": "reboot"}
    t1 = T0 + timedelta(minutes=9)                      # the reboot went out at T0+5
    assert m.apply_boot(job, T0 + timedelta(minutes=7), t1)
    assert job.state == "undraining" and job.back_at == t1 and m.executor(job) == "mgr"
    m.apply_result(job, _result(job, "done", ok=True), t1)
    assert job.state == "done"
    assert m.summary(job) == "Done: 12 packages updated, rebooted"


def test_a_job_that_may_never_reboot_stops_after_the_update():
    job = _job(reboot_policy="never")
    m.start(job, T0)
    m.apply_result(job, _result(job, "done", ok=True, packages=1, reboot_required=True), T0)
    assert job.state == "done"
    assert "reboot needed, not allowed" in m.summary(job)


def test_only_a_boot_after_the_reboot_was_handed_out_completes_it():
    job = _job(state="rebooting", step_id="job-1-reboot", state_at=T0)
    assert not m.apply_boot(job, T0 - timedelta(days=3), T0)    # the agent restarted, not the host
    assert not m.apply_boot(job, None, T0)
    assert job.state == "rebooting"
    assert m.apply_boot(job, T0 - timedelta(seconds=30), T0)     # within the clock slack
    assert job.state == "done"


def test_a_failed_step_fails_the_job_and_says_which():
    job = _job(state="undraining", step_id="job-1-undrain", node_id="n" * 25, drain_host_id="mgr")
    m.apply_result(job, _result(job, "failed", ok=False, detail="This node is not a swarm manager."), T0)
    assert job.state == "failed"
    assert job.detail.startswith("undrain failed: This node is not a swarm manager.")
    assert "still drained" in job.detail


def test_a_late_or_repeated_result_does_nothing():
    job = _job()
    m.start(job, T0)
    stale = {"kind": "maintenance", "id": "job-1-drain", "state": "done", "ok": True}
    assert not m.apply_result(job, stale, T0)
    assert m.apply_result(job, _result(job, "running"), T0)
    assert not m.apply_result(job, _result(job, "running"), T0)
    m.apply_result(job, _result(job, "done", ok=True, reboot_required=False), T0)
    assert not m.apply_result(job, {"kind": "maintenance", "id": "job-1-update", "state": "done"}, T0)


@pytest.mark.parametrize("state,minutes,expected", [
    ("queued", 59, False), ("queued", 61, True),
    ("updating", 91, True), ("rebooting", 29, False), ("rebooting", 31, True),
])
def test_a_job_is_given_up_past_its_state_deadline(state, minutes, expected):
    job = _job(state=state, step_id=f"job-1-{m.STEP_ACTION.get(state, 'x')}")
    assert m.check_deadline(job, T0 + timedelta(minutes=minutes)) is expected
    if expected:
        assert job.state == "timed_out" and "within" in job.detail


def test_the_window_lasts_the_rest_of_the_job_and_ends_with_it():
    job = _job(state="rebooting", step_id="job-1-reboot")
    assert m.window_end(job, T0) == T0 + timedelta(minutes=30 + 10 + 15)
    job.state = "done"
    assert m.window_end(job, T0) == T0


def test_a_manager_drains_itself_and_a_worker_is_drained_by_a_manager():
    nodes = {"docker02": "a" * 25, "docker03": "b" * 25}
    managers = {"h2": "a" * 25}
    assert m.pick_drainer(nodes, managers, "Docker02", "h2", {"h2", "h3"}) == ("a" * 25, "h2")
    assert m.pick_drainer(nodes, managers, "Docker03", "h3", {"h2", "h3"}) == ("b" * 25, "h2")
    assert m.pick_drainer(nodes, managers, "Util01", "h4", {"h2"}) == (None, None)
    with pytest.raises(m.CannotStart, match="no swarm manager"):
        m.pick_drainer(nodes, managers, "Docker03", "h3", {"h3"})


def test_a_node_the_operator_keeps_drained_is_left_drained():
    # A manager kept free of work (availability drain) is neither drained
    # before nor made active after: the job only updates and reboots it.
    nodes = {"util01": "c" * 25, "docker02": "a" * 25}
    managers = {"h1": "c" * 25, "h2": "a" * 25}
    assert m.pick_drainer(nodes, managers, "Util01", "h1", {"h1", "h2"}, "drain") == ("c" * 25, None)
    assert m.pick_drainer(nodes, managers, "Util01", "h1", {"h1", "h2"}, "pause") == ("c" * 25, None)
    assert m.pick_drainer(nodes, managers, "Util01", "h1", {"h1", "h2"}, "active") == ("c" * 25, "h1")


# ── a check first: Docker upgraded, drain before; containers seen again (#6098)

NODE = "n" * 25
CONTAINERS = [{"name": "ak-outpost", "restart": "unless-stopped"}, {"name": "scratch", "restart": "no"}]


def _checked(job, **kw):
    m.apply_result(job, _result(job, "done", ok=True, pending=7, **kw), T0)


def test_a_new_enough_agent_checks_first():
    assert m.can_check("1.24.0") and m.can_check("1.30.2") and m.can_check("2.0.0")
    assert not m.can_check("1.23.9") and not m.can_check(None) and not m.can_check("dev")
    job = _job()
    m.start(job, T0, check=True)
    assert job.state == "checking" and m.handoff(job) == {"id": "job-1-check", "action": "check"}


def test_an_update_that_upgrades_docker_drains_the_node_before_it():
    job = _job(node_id=NODE, drain_host_id="mgr")
    m.start(job, T0, check=True)
    _checked(job, docker_update=True, docker_packages=["docker-ce"], containers=[])
    assert job.state == "draining" and job.docker_update is True
    assert job.checked == {"pending": 7, "docker_packages": ["docker-ce"], "containers": []}
    m.apply_result(job, _result(job, "done", ok=True), T0)
    assert job.state == "settling"
    assert m.settle(job, T0 + timedelta(minutes=5)) and job.state == "updating"
    m.apply_result(job, _result(job, "done", ok=True, packages=9, reboot_required=True), T0)
    assert job.state == "rebooting"                             # drained already: no second drain
    assert m.apply_boot(job, T0 + timedelta(minutes=8), T0 + timedelta(minutes=9))
    assert job.state == "undraining"
    m.apply_result(job, _result(job, "done", ok=True), T0)
    assert job.state == "done" and m.summary(job) == "Done: Docker upgraded, 9 packages updated, rebooted"


def test_a_drained_node_with_no_reboot_is_made_active_after_the_update():
    job = _job(node_id=NODE, drain_host_id="mgr", reboot_policy="never")
    m.start(job, T0, check=True)
    _checked(job, docker_update=True)
    m.apply_result(job, _result(job, "done", ok=True), T0)
    m.settle(job, T0 + timedelta(minutes=5))
    m.apply_result(job, _result(job, "done", ok=True, reboot_required=True), T0)
    assert job.state == "undraining"


def test_an_update_without_docker_doesnt_drain_unless_it_reboots():
    job = _job(node_id=NODE, drain_host_id="mgr")
    m.start(job, T0, check=True)
    _checked(job, docker_update=False)
    assert job.state == "updating"
    m.apply_result(job, _result(job, "done", ok=True, reboot_required=False), T0)
    assert job.state == "done" and job.drained_at is None    # never drained, so no undrain


def test_docker_on_a_node_the_operator_keeps_drained_isnt_drained_again():
    job = _job(node_id=NODE, drain_host_id=None)
    m.start(job, T0, check=True)
    _checked(job, docker_update=True)
    assert job.state == "updating"


def test_the_containers_outside_the_swarm_are_waited_for_after_the_reboot():
    job = _job()
    m.start(job, T0, check=True)
    _checked(job, docker_update=False, containers=CONTAINERS)
    m.apply_result(job, _result(job, "done", ok=True, reboot_required=True), T0)
    t1 = T0 + timedelta(minutes=6)
    assert m.apply_boot(job, T0 + timedelta(minutes=5), t1)
    assert job.state == "recovering" and m.handoff(job) is None
    assert not m.recovered(job, {"ak-outpost"}, t1 + timedelta(minutes=2))      # still waiting
    assert m.recovered(job, {"ak-outpost", "scratch", "other"}, t1 + timedelta(minutes=3))
    assert job.state == "done"


def test_a_container_that_doesnt_run_again_fails_the_job_by_name():
    job = _job(state="recovering", state_at=T0, checked={"containers": CONTAINERS})
    assert not m.recovered(job, {"ak-outpost"}, T0 + timedelta(minutes=9))
    assert m.recovered(job, {"ak-outpost"}, T0 + timedelta(minutes=10))
    assert job.state == "failed"
    assert job.detail == "not running again 10 min after the job: scratch (no restart policy)"


def test_a_job_with_nothing_outside_the_swarm_ends_at_the_boot():
    job = _job(state="rebooting", step_id="job-1-reboot", checked={"containers": []})
    assert m.apply_boot(job, T0, T0) and job.state == "done"


def test_what_a_check_said_is_kept_bounded():
    job = _job(state="checking", step_id="job-1-check")
    many = [{"name": f"c{i}" * 40, "restart": "always"} for i in range(60)] + ["junk", {"name": 3}]
    _checked(job, docker_update="yes", docker_packages=["docker-ce", 4] * 9, containers=many)
    assert job.docker_update is False                           # only a real True counts
    assert len(job.checked["containers"]) == m.CONTAINERS_KEPT
    assert len(job.checked["containers"][0]["name"]) == 64
    assert job.checked["docker_packages"] == ["docker-ce"] * 9


@pytest.mark.parametrize("state, minutes", [("checking", 31), ("settling", 16), ("recovering", 16)])
def test_the_new_states_are_given_up_too(state, minutes):
    job = _job(state=state, step_id="job-1-check" if state == "checking" else None)
    assert m.check_deadline(job, T0 + timedelta(minutes=minutes)) and job.state == "timed_out"


# ── the agent's side ─────────────────────────────────────────────────────────

def test_the_agent_leaves_each_handed_step_once(tmp_path):
    requested: set = set()
    steps = [{"id": "job-1-update", "action": "update"}, {"id": "job-1-x", "action": "shell"}, "junk"]
    assert a.take_maintenance(steps, requested, requests_dir=str(tmp_path)) == ["job-1-update"]
    assert json.loads((tmp_path / "job-1-update.json").read_text()) == {
        "id": "job-1-update", "action": "update"}
    (tmp_path / "job-1-update.json").unlink()          # the helper took it
    assert a.take_maintenance(steps, requested, requests_dir=str(tmp_path)) == []
    assert not (tmp_path / "job-1-update.json").exists()
    assert a.take_maintenance(None, requested, requests_dir=str(tmp_path)) == []


def test_the_agent_reads_stewards_answer_when_asked(monkeypatch):
    class Resp:
        status = 200

        def read(self, n):
            return b'{"ok": true, "maintenance": [{"id": "job-1-reboot", "action": "reboot"}]}'

        def __enter__(self):
            return self

        def __exit__(self, *exc):
            return False

    monkeypatch.setattr(a.urllib.request, "urlopen", lambda req, timeout: Resp())
    answer: dict = {}
    assert a.post_payload("https://x", "tok", {}, answer) == (True, 200)
    assert answer["maintenance"] == [{"id": "job-1-reboot", "action": "reboot"}]


def test_a_late_return_stops_flagging_its_host_once_it_came_back():
    """#6175: one answer for the figure, the overview and the pages."""
    late = _job(state="failed", finished_at=T0,
                detail=f"{m.NOT_BACK} 30 min: services below their replicas: portal_app")
    assert m.still_failed(late) and m.summary(late) == "Failed"
    late.came_back_at = T0 + timedelta(minutes=45)
    assert not m.still_failed(late)
    assert m.summary(late) == "Back in service late, 2026-10-14 23:45Z"
    # An update that failed, or a job that timed out, flags its host whatever came back.
    for job in (_job(state="failed", detail="dpkg was interrupted", came_back_at=T0),
                _job(state="timed_out", detail="no answer within 60 min", came_back_at=T0)):
        assert m.still_failed(job)
    assert not m.still_failed(_job(state="done"))