# tests/plugins/host_agent/test_maintenance.py """#5767: a host's maintenance job, step by step, from the agent's reports. The transitions are model-free, so a job here is a SimpleNamespace with the row's attributes; the database side is tests/integration/test_host_maintenance_job.py. """ import json from datetime import datetime, timedelta, timezone from types import SimpleNamespace import pytest from plugins.host_agent import agent as a from plugins.host_agent import maintenance as m T0 = datetime(2026, 10, 14, 23, 0, tzinfo=timezone.utc) def _job(**kw): base = dict(id="job-1", host_id="h1", state="queued", state_at=T0, created_at=T0, reboot_policy="if_required", step_id=None, step_seen=False, node_id=None, drain_host_id=None, packages=None, reboot_needed=None, back_at=None, detail=None, finished_at=None) base.update(kw) return SimpleNamespace(**base) def _result(job, state, **kw): return {"kind": "maintenance", "id": job.step_id, "state": state, **kw} def test_an_update_with_no_reboot_needed_is_done_after_the_update(): job = _job() m.start(job, T0) assert m.handoff(job) == {"id": "job-1-update", "action": "update"} assert m.apply_result(job, _result(job, "running"), T0) assert m.handoff(job) is None # taken: not handed again m.apply_result(job, _result(job, "done", ok=True, packages=3, reboot_required=False), T0) assert (job.state, job.packages, job.finished_at) == ("done", 3, T0) assert m.summary(job) == "Done: 3 packages updated, no reboot needed" def test_a_swarm_node_is_drained_rebooted_and_made_active_again(): job = _job(node_id="n" * 25, drain_host_id="mgr") m.start(job, T0) m.apply_result(job, _result(job, "done", ok=True, packages=12, reboot_required=True), T0) assert job.state == "draining" and m.executor(job) == "mgr" assert m.handoff(job) == {"id": "job-1-drain", "action": "drain", "node": "n" * 25} m.apply_result(job, _result(job, "done", ok=True), T0) assert job.state == "rebooting" and m.executor(job) == "h1" assert m.handoff(job) == {"id": "job-1-reboot", "action": "reboot"} t1 = T0 + timedelta(minutes=4) assert m.apply_boot(job, T0 + timedelta(minutes=2), t1) assert job.state == "undraining" and job.back_at == t1 and m.executor(job) == "mgr" m.apply_result(job, _result(job, "done", ok=True), t1) assert job.state == "done" assert m.summary(job) == "Done: 12 packages updated, rebooted" def test_a_job_that_may_never_reboot_stops_after_the_update(): job = _job(reboot_policy="never") m.start(job, T0) m.apply_result(job, _result(job, "done", ok=True, packages=1, reboot_required=True), T0) assert job.state == "done" assert "reboot needed, not allowed" in m.summary(job) def test_only_a_boot_after_the_reboot_was_handed_out_completes_it(): job = _job(state="rebooting", step_id="job-1-reboot", state_at=T0) assert not m.apply_boot(job, T0 - timedelta(days=3), T0) # the agent restarted, not the host assert not m.apply_boot(job, None, T0) assert job.state == "rebooting" assert m.apply_boot(job, T0 - timedelta(seconds=30), T0) # within the clock slack assert job.state == "done" def test_a_failed_step_fails_the_job_and_says_which(): job = _job(state="undraining", step_id="job-1-undrain", node_id="n" * 25, drain_host_id="mgr") m.apply_result(job, _result(job, "failed", ok=False, detail="This node is not a swarm manager."), T0) assert job.state == "failed" assert job.detail.startswith("undrain failed: This node is not a swarm manager.") assert "still drained" in job.detail def test_a_late_or_repeated_result_does_nothing(): job = _job() m.start(job, T0) stale = {"kind": "maintenance", "id": "job-1-drain", "state": "done", "ok": True} assert not m.apply_result(job, stale, T0) assert m.apply_result(job, _result(job, "running"), T0) assert not m.apply_result(job, _result(job, "running"), T0) m.apply_result(job, _result(job, "done", ok=True, reboot_required=False), T0) assert not m.apply_result(job, {"kind": "maintenance", "id": "job-1-update", "state": "done"}, T0) @pytest.mark.parametrize("state,minutes,expected", [ ("queued", 59, False), ("queued", 61, True), ("updating", 91, True), ("rebooting", 29, False), ("rebooting", 31, True), ]) def test_a_job_is_given_up_past_its_state_deadline(state, minutes, expected): job = _job(state=state, step_id=f"job-1-{m.STEP_ACTION.get(state, 'x')}") assert m.check_deadline(job, T0 + timedelta(minutes=minutes)) is expected if expected: assert job.state == "timed_out" and "within" in job.detail def test_the_window_lasts_the_rest_of_the_job_and_ends_with_it(): job = _job(state="rebooting", step_id="job-1-reboot") assert m.window_end(job, T0) == T0 + timedelta(minutes=30 + 10) job.state = "done" assert m.window_end(job, T0) == T0 def test_a_manager_drains_itself_and_a_worker_is_drained_by_a_manager(): nodes = {"docker02": "a" * 25, "docker03": "b" * 25} managers = {"h2": "a" * 25} assert m.pick_drainer(nodes, managers, "Docker02", "h2", {"h2", "h3"}) == ("a" * 25, "h2") assert m.pick_drainer(nodes, managers, "Docker03", "h3", {"h2", "h3"}) == ("b" * 25, "h2") assert m.pick_drainer(nodes, managers, "Util01", "h4", {"h2"}) == (None, None) with pytest.raises(m.CannotStart, match="no swarm manager"): m.pick_drainer(nodes, managers, "Docker03", "h3", {"h3"}) def test_a_node_the_operator_keeps_drained_is_left_drained(): # A manager kept free of work (availability drain) is neither drained # before nor made active after: the job only updates and reboots it. nodes = {"util01": "c" * 25, "docker02": "a" * 25} managers = {"h1": "c" * 25, "h2": "a" * 25} assert m.pick_drainer(nodes, managers, "Util01", "h1", {"h1", "h2"}, "drain") == ("c" * 25, None) assert m.pick_drainer(nodes, managers, "Util01", "h1", {"h1", "h2"}, "pause") == ("c" * 25, None) assert m.pick_drainer(nodes, managers, "Util01", "h1", {"h1", "h2"}, "active") == ("c" * 25, "h1") # ── the agent's side ───────────────────────────────────────────────────────── def test_the_agent_leaves_each_handed_step_once(tmp_path): requested: set = set() steps = [{"id": "job-1-update", "action": "update"}, {"id": "job-1-x", "action": "shell"}, "junk"] assert a.take_maintenance(steps, requested, requests_dir=str(tmp_path)) == ["job-1-update"] assert json.loads((tmp_path / "job-1-update.json").read_text()) == { "id": "job-1-update", "action": "update"} (tmp_path / "job-1-update.json").unlink() # the helper took it assert a.take_maintenance(steps, requested, requests_dir=str(tmp_path)) == [] assert not (tmp_path / "job-1-update.json").exists() assert a.take_maintenance(None, requested, requests_dir=str(tmp_path)) == [] def test_the_agent_reads_stewards_answer_when_asked(monkeypatch): class Resp: status = 200 def read(self, n): return b'{"ok": true, "maintenance": [{"id": "job-1-reboot", "action": "reboot"}]}' def __enter__(self): return self def __exit__(self, *exc): return False monkeypatch.setattr(a.urllib.request, "urlopen", lambda req, timeout: Resp()) answer: dict = {} assert a.post_payload("https://x", "tok", {}, answer) == (True, 200) assert answer["maintenance"] == [{"id": "job-1-reboot", "action": "reboot"}]