bvandeusenandClaude Opus 5.5
d102fc3333
On 2026-10-25 the 09:00 job on VDNT-Docker02 drained the node, which ran Traefik and Steward. The agent's result reaches Steward through Traefik, and Traefik took 11.5 minutes to start on Docker03 (its image had to be pulled). The job gave up at the 10-minute drain deadline and finished timed_out where it stood, so the node stayed drained for hours. - A job that fails or times out while its node may be drained (the drain was handed out, or went through and wasn't undone) goes to undraining first. The outcome is kept in the new column ends_as, and the job finishes as that once the node is active again. If the undrain goes wrong too, detail says both. - The drain deadline is 30 minutes instead of 10, so a result held up while the proxy moves still lands. - Migration host_agent_012_ends_as. get_maintenance_job returns ends_as. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
355 lines
17 KiB
Python
355 lines
17 KiB
Python
# tests/plugins/host_agent/test_maintenance.py
|
|
"""#5767: a host's maintenance job, step by step, from the agent's reports.
|
|
|
|
The transitions are model-free, so a job here is a SimpleNamespace with the
|
|
row's attributes; the database side is tests/integration/test_host_maintenance_job.py.
|
|
"""
|
|
import json
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from plugins.host_agent import agent as a
|
|
from plugins.host_agent import maintenance as m
|
|
|
|
T0 = datetime(2026, 10, 14, 23, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
def _job(**kw):
|
|
base = dict(id="job-1", host_id="h1", state="queued", state_at=T0, created_at=T0,
|
|
reboot_policy="if_required", step_id=None, step_seen=False, node_id=None,
|
|
drain_host_id=None, packages=None, reboot_needed=None, back_at=None,
|
|
detail=None, finished_at=None, drained_at=None, docker_update=None, checked=None,
|
|
came_back_at=None, ends_as=None)
|
|
base.update(kw)
|
|
return SimpleNamespace(**base)
|
|
|
|
|
|
def _result(job, state, **kw):
|
|
return {"kind": "maintenance", "id": job.step_id, "state": state, **kw}
|
|
|
|
|
|
def test_an_update_with_no_reboot_needed_is_done_after_the_update():
|
|
job = _job()
|
|
m.start(job, T0)
|
|
assert m.handoff(job) == {"id": "job-1-update", "action": "update"}
|
|
assert m.apply_result(job, _result(job, "running"), T0)
|
|
assert m.handoff(job) is None # taken: not handed again
|
|
m.apply_result(job, _result(job, "done", ok=True, packages=3, reboot_required=False), T0)
|
|
assert (job.state, job.packages, job.finished_at) == ("done", 3, T0)
|
|
assert m.summary(job) == "Done: 3 packages updated, no reboot needed"
|
|
|
|
|
|
def test_a_swarm_node_is_drained_rebooted_and_made_active_again():
|
|
job = _job(node_id="n" * 25, drain_host_id="mgr")
|
|
m.start(job, T0)
|
|
m.apply_result(job, _result(job, "done", ok=True, packages=12, reboot_required=True), T0)
|
|
assert job.state == "draining" and m.executor(job) == "mgr"
|
|
assert m.handoff(job) == {"id": "job-1-drain", "action": "drain", "node": "n" * 25}
|
|
m.apply_result(job, _result(job, "done", ok=True), T0)
|
|
# The node's tasks get 5 minutes to move before the reboot (#6098).
|
|
assert job.state == "settling" and job.drained_at == T0 and m.handoff(job) is None
|
|
assert not m.settle(job, T0 + timedelta(minutes=4))
|
|
assert m.settle(job, T0 + timedelta(minutes=5))
|
|
assert job.state == "rebooting" and m.executor(job) == "h1"
|
|
assert m.handoff(job) == {"id": "job-1-reboot", "action": "reboot"}
|
|
t1 = T0 + timedelta(minutes=9) # the reboot went out at T0+5
|
|
assert m.apply_boot(job, T0 + timedelta(minutes=7), t1)
|
|
assert job.state == "undraining" and job.back_at == t1 and m.executor(job) == "mgr"
|
|
m.apply_result(job, _result(job, "done", ok=True), t1)
|
|
assert job.state == "done"
|
|
assert m.summary(job) == "Done: 12 packages updated, rebooted"
|
|
|
|
|
|
def test_a_job_that_may_never_reboot_stops_after_the_update():
|
|
job = _job(reboot_policy="never")
|
|
m.start(job, T0)
|
|
m.apply_result(job, _result(job, "done", ok=True, packages=1, reboot_required=True), T0)
|
|
assert job.state == "done"
|
|
assert "reboot needed, not allowed" in m.summary(job)
|
|
|
|
|
|
def test_only_a_boot_after_the_reboot_was_handed_out_completes_it():
|
|
job = _job(state="rebooting", step_id="job-1-reboot", state_at=T0)
|
|
assert not m.apply_boot(job, T0 - timedelta(days=3), T0) # the agent restarted, not the host
|
|
assert not m.apply_boot(job, None, T0)
|
|
assert job.state == "rebooting"
|
|
assert m.apply_boot(job, T0 - timedelta(seconds=30), T0) # within the clock slack
|
|
assert job.state == "done"
|
|
|
|
|
|
def test_a_failed_step_fails_the_job_and_says_which():
|
|
job = _job(state="undraining", step_id="job-1-undrain", node_id="n" * 25, drain_host_id="mgr")
|
|
m.apply_result(job, _result(job, "failed", ok=False, detail="This node is not a swarm manager."), T0)
|
|
assert job.state == "failed"
|
|
assert job.detail.startswith("undrain failed: This node is not a swarm manager.")
|
|
assert "still drained" in job.detail
|
|
|
|
|
|
def test_a_late_or_repeated_result_does_nothing():
|
|
job = _job()
|
|
m.start(job, T0)
|
|
stale = {"kind": "maintenance", "id": "job-1-drain", "state": "done", "ok": True}
|
|
assert not m.apply_result(job, stale, T0)
|
|
assert m.apply_result(job, _result(job, "running"), T0)
|
|
assert not m.apply_result(job, _result(job, "running"), T0)
|
|
m.apply_result(job, _result(job, "done", ok=True, reboot_required=False), T0)
|
|
assert not m.apply_result(job, {"kind": "maintenance", "id": "job-1-update", "state": "done"}, T0)
|
|
|
|
|
|
@pytest.mark.parametrize("state,minutes,expected", [
|
|
("queued", 59, False), ("queued", 61, True),
|
|
("updating", 91, True), ("rebooting", 29, False), ("rebooting", 31, True),
|
|
])
|
|
def test_a_job_is_given_up_past_its_state_deadline(state, minutes, expected):
|
|
job = _job(state=state, step_id=f"job-1-{m.STEP_ACTION.get(state, 'x')}")
|
|
assert m.check_deadline(job, T0 + timedelta(minutes=minutes)) is expected
|
|
if expected:
|
|
assert job.state == "timed_out" and "within" in job.detail
|
|
|
|
|
|
def test_the_window_lasts_the_rest_of_the_job_and_ends_with_it():
|
|
job = _job(state="rebooting", step_id="job-1-reboot")
|
|
assert m.window_end(job, T0) == T0 + timedelta(minutes=30 + 10 + 15)
|
|
job.state = "done"
|
|
assert m.window_end(job, T0) == T0
|
|
|
|
|
|
def test_a_manager_drains_itself_and_a_worker_is_drained_by_a_manager():
|
|
nodes = {"docker02": "a" * 25, "docker03": "b" * 25}
|
|
managers = {"h2": "a" * 25}
|
|
assert m.pick_drainer(nodes, managers, "Docker02", "h2", {"h2", "h3"}) == ("a" * 25, "h2")
|
|
assert m.pick_drainer(nodes, managers, "Docker03", "h3", {"h2", "h3"}) == ("b" * 25, "h2")
|
|
assert m.pick_drainer(nodes, managers, "Util01", "h4", {"h2"}) == (None, None)
|
|
with pytest.raises(m.CannotStart, match="no swarm manager"):
|
|
m.pick_drainer(nodes, managers, "Docker03", "h3", {"h3"})
|
|
|
|
|
|
def test_a_node_the_operator_keeps_drained_is_left_drained():
|
|
# A manager kept free of work (availability drain) is neither drained
|
|
# before nor made active after: the job only updates and reboots it.
|
|
nodes = {"util01": "c" * 25, "docker02": "a" * 25}
|
|
managers = {"h1": "c" * 25, "h2": "a" * 25}
|
|
assert m.pick_drainer(nodes, managers, "Util01", "h1", {"h1", "h2"}, "drain") == ("c" * 25, None)
|
|
assert m.pick_drainer(nodes, managers, "Util01", "h1", {"h1", "h2"}, "pause") == ("c" * 25, None)
|
|
assert m.pick_drainer(nodes, managers, "Util01", "h1", {"h1", "h2"}, "active") == ("c" * 25, "h1")
|
|
|
|
|
|
# ── a check first: Docker upgraded, drain before; containers seen again (#6098)
|
|
|
|
NODE = "n" * 25
|
|
CONTAINERS = [{"name": "ak-outpost", "restart": "unless-stopped"}, {"name": "scratch", "restart": "no"}]
|
|
|
|
|
|
def _checked(job, **kw):
|
|
m.apply_result(job, _result(job, "done", ok=True, pending=7, **kw), T0)
|
|
|
|
|
|
def test_a_new_enough_agent_checks_first():
|
|
assert m.can_check("1.24.0") and m.can_check("1.30.2") and m.can_check("2.0.0")
|
|
assert not m.can_check("1.23.9") and not m.can_check(None) and not m.can_check("dev")
|
|
job = _job()
|
|
m.start(job, T0, check=True)
|
|
assert job.state == "checking" and m.handoff(job) == {"id": "job-1-check", "action": "check"}
|
|
|
|
|
|
def test_an_update_that_upgrades_docker_drains_the_node_before_it():
|
|
job = _job(node_id=NODE, drain_host_id="mgr")
|
|
m.start(job, T0, check=True)
|
|
_checked(job, docker_update=True, docker_packages=["docker-ce"], containers=[])
|
|
assert job.state == "draining" and job.docker_update is True
|
|
assert job.checked == {"pending": 7, "docker_packages": ["docker-ce"], "containers": []}
|
|
m.apply_result(job, _result(job, "done", ok=True), T0)
|
|
assert job.state == "settling"
|
|
assert m.settle(job, T0 + timedelta(minutes=5)) and job.state == "updating"
|
|
m.apply_result(job, _result(job, "done", ok=True, packages=9, reboot_required=True), T0)
|
|
assert job.state == "rebooting" # drained already: no second drain
|
|
assert m.apply_boot(job, T0 + timedelta(minutes=8), T0 + timedelta(minutes=9))
|
|
assert job.state == "undraining"
|
|
m.apply_result(job, _result(job, "done", ok=True), T0)
|
|
assert job.state == "done" and m.summary(job) == "Done: Docker upgraded, 9 packages updated, rebooted"
|
|
|
|
|
|
def test_a_drained_node_with_no_reboot_is_made_active_after_the_update():
|
|
job = _job(node_id=NODE, drain_host_id="mgr", reboot_policy="never")
|
|
m.start(job, T0, check=True)
|
|
_checked(job, docker_update=True)
|
|
m.apply_result(job, _result(job, "done", ok=True), T0)
|
|
m.settle(job, T0 + timedelta(minutes=5))
|
|
m.apply_result(job, _result(job, "done", ok=True, reboot_required=True), T0)
|
|
assert job.state == "undraining"
|
|
|
|
|
|
def test_an_update_without_docker_doesnt_drain_unless_it_reboots():
|
|
job = _job(node_id=NODE, drain_host_id="mgr")
|
|
m.start(job, T0, check=True)
|
|
_checked(job, docker_update=False)
|
|
assert job.state == "updating"
|
|
m.apply_result(job, _result(job, "done", ok=True, reboot_required=False), T0)
|
|
assert job.state == "done" and job.drained_at is None # never drained, so no undrain
|
|
|
|
|
|
def test_docker_on_a_node_the_operator_keeps_drained_isnt_drained_again():
|
|
job = _job(node_id=NODE, drain_host_id=None)
|
|
m.start(job, T0, check=True)
|
|
_checked(job, docker_update=True)
|
|
assert job.state == "updating"
|
|
|
|
|
|
def test_the_containers_outside_the_swarm_are_waited_for_after_the_reboot():
|
|
job = _job()
|
|
m.start(job, T0, check=True)
|
|
_checked(job, docker_update=False, containers=CONTAINERS)
|
|
m.apply_result(job, _result(job, "done", ok=True, reboot_required=True), T0)
|
|
t1 = T0 + timedelta(minutes=6)
|
|
assert m.apply_boot(job, T0 + timedelta(minutes=5), t1)
|
|
assert job.state == "recovering" and m.handoff(job) is None
|
|
assert not m.recovered(job, {"ak-outpost"}, t1 + timedelta(minutes=2)) # still waiting
|
|
assert m.recovered(job, {"ak-outpost", "scratch", "other"}, t1 + timedelta(minutes=3))
|
|
assert job.state == "done"
|
|
|
|
|
|
def test_a_container_that_doesnt_run_again_fails_the_job_by_name():
|
|
job = _job(state="recovering", state_at=T0, checked={"containers": CONTAINERS})
|
|
assert not m.recovered(job, {"ak-outpost"}, T0 + timedelta(minutes=9))
|
|
assert m.recovered(job, {"ak-outpost"}, T0 + timedelta(minutes=10))
|
|
assert job.state == "failed"
|
|
assert job.detail == "not running again 10 min after the job: scratch (no restart policy)"
|
|
|
|
|
|
def test_a_job_with_nothing_outside_the_swarm_ends_at_the_boot():
|
|
job = _job(state="rebooting", step_id="job-1-reboot", checked={"containers": []})
|
|
assert m.apply_boot(job, T0, T0) and job.state == "done"
|
|
|
|
|
|
def test_what_a_check_said_is_kept_bounded():
|
|
job = _job(state="checking", step_id="job-1-check")
|
|
many = [{"name": f"c{i}" * 40, "restart": "always"} for i in range(60)] + ["junk", {"name": 3}]
|
|
_checked(job, docker_update="yes", docker_packages=["docker-ce", 4] * 9, containers=many)
|
|
assert job.docker_update is False # only a real True counts
|
|
assert len(job.checked["containers"]) == m.CONTAINERS_KEPT
|
|
assert len(job.checked["containers"][0]["name"]) == 64
|
|
assert job.checked["docker_packages"] == ["docker-ce"] * 9
|
|
|
|
|
|
@pytest.mark.parametrize("state, minutes", [("checking", 31), ("settling", 16), ("recovering", 16)])
|
|
def test_the_new_states_are_given_up_too(state, minutes):
|
|
job = _job(state=state, step_id="job-1-check" if state == "checking" else None)
|
|
assert m.check_deadline(job, T0 + timedelta(minutes=minutes)) and job.state == "timed_out"
|
|
|
|
|
|
def _drained_job(**kw):
|
|
return _job(node_id="n" * 25, drain_host_id="mgr", docker_update=True, **kw)
|
|
|
|
|
|
def test_a_drain_whose_result_is_late_still_gets_its_time():
|
|
# Draining the node that runs the proxy moves it, and the drain's result
|
|
# waits for it to run again elsewhere.
|
|
job = _drained_job(state="draining", step_id="job-1-drain")
|
|
assert not m.check_deadline(job, T0 + timedelta(minutes=29))
|
|
m.apply_result(job, _result(job, "done", ok=True), T0 + timedelta(minutes=12))
|
|
assert job.state == "settling"
|
|
|
|
|
|
def test_a_drain_given_up_makes_the_node_active_again_before_the_job_ends():
|
|
job = _drained_job(state="draining", step_id="job-1-drain")
|
|
assert m.check_deadline(job, T0 + timedelta(minutes=31))
|
|
assert job.state == "undraining" and job.finished_at is None
|
|
assert m.executor(job) == "mgr"
|
|
assert m.handoff(job) == {"id": "job-1-undrain", "action": "undrain", "node": "n" * 25}
|
|
late = {"kind": "maintenance", "id": "job-1-drain", "state": "done", "ok": True}
|
|
assert not m.apply_result(job, late, T0 + timedelta(minutes=32)) # the drain's late result
|
|
t1 = T0 + timedelta(minutes=33)
|
|
m.apply_result(job, _result(job, "done", ok=True), t1)
|
|
assert (job.state, job.finished_at) == ("timed_out", t1)
|
|
assert job.detail == "the node wasn't drained within 30 min"
|
|
assert m.still_failed(job)
|
|
|
|
|
|
@pytest.mark.parametrize("state, step, minutes", [
|
|
("settling", None, 16), ("updating", "job-1-update", 91), ("rebooting", "job-1-reboot", 31),
|
|
])
|
|
def test_a_job_given_up_after_its_drain_makes_the_node_active_again(state, step, minutes):
|
|
job = _drained_job(state=state, step_id=step, drained_at=T0)
|
|
assert m.check_deadline(job, T0 + timedelta(minutes=minutes))
|
|
assert job.state == "undraining" and job.ends_as == "timed_out"
|
|
|
|
|
|
def test_a_failed_step_after_the_drain_makes_the_node_active_again_and_fails():
|
|
job = _drained_job(state="updating", step_id="job-1-update", drained_at=T0)
|
|
m.apply_result(job, _result(job, "failed", ok=False, detail="dpkg lock"), T0)
|
|
assert job.state == "undraining"
|
|
m.apply_result(job, _result(job, "done", ok=True), T0)
|
|
assert job.state == "failed" and job.detail == "update failed: dpkg lock"
|
|
|
|
|
|
def test_an_undrain_that_goes_wrong_after_giving_up_says_both():
|
|
job = _drained_job(state="draining", step_id="job-1-drain")
|
|
m.check_deadline(job, T0 + timedelta(minutes=31))
|
|
assert m.check_deadline(job, T0 + timedelta(minutes=42))
|
|
assert job.state == "timed_out"
|
|
assert job.detail == ("the node wasn't drained within 30 min; "
|
|
"then the node wasn't made active again within 10 min")
|
|
job = _drained_job(state="updating", step_id="job-1-update", drained_at=T0)
|
|
m.apply_result(job, _result(job, "failed", ok=False, detail="dpkg lock"), T0)
|
|
m.apply_result(job, _result(job, "failed", ok=False, detail="not a manager"), T0)
|
|
assert job.state == "failed"
|
|
assert job.detail == ("update failed: dpkg lock; then undrain failed: not a manager "
|
|
"(the node is still drained)")
|
|
|
|
|
|
def test_a_job_that_never_drained_ends_where_it_is_given_up():
|
|
job = _job(state="updating", step_id="job-1-update", node_id="n" * 25, drain_host_id="mgr")
|
|
assert m.check_deadline(job, T0 + timedelta(minutes=91)) and job.state == "timed_out"
|
|
kept = _job(state="draining", step_id="job-1-drain", node_id="n" * 25) # kept drained: no drainer
|
|
assert m.check_deadline(kept, T0 + timedelta(minutes=31)) and kept.state == "timed_out"
|
|
|
|
|
|
# ── the agent's side ─────────────────────────────────────────────────────────
|
|
|
|
def test_the_agent_leaves_each_handed_step_once(tmp_path):
|
|
requested: set = set()
|
|
steps = [{"id": "job-1-update", "action": "update"}, {"id": "job-1-x", "action": "shell"}, "junk"]
|
|
assert a.take_maintenance(steps, requested, requests_dir=str(tmp_path)) == ["job-1-update"]
|
|
assert json.loads((tmp_path / "job-1-update.json").read_text()) == {
|
|
"id": "job-1-update", "action": "update"}
|
|
(tmp_path / "job-1-update.json").unlink() # the helper took it
|
|
assert a.take_maintenance(steps, requested, requests_dir=str(tmp_path)) == []
|
|
assert not (tmp_path / "job-1-update.json").exists()
|
|
assert a.take_maintenance(None, requested, requests_dir=str(tmp_path)) == []
|
|
|
|
|
|
def test_the_agent_reads_stewards_answer_when_asked(monkeypatch):
|
|
class Resp:
|
|
status = 200
|
|
|
|
def read(self, n):
|
|
return b'{"ok": true, "maintenance": [{"id": "job-1-reboot", "action": "reboot"}]}'
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
monkeypatch.setattr(a.urllib.request, "urlopen", lambda req, timeout: Resp())
|
|
answer: dict = {}
|
|
assert a.post_payload("https://x", "tok", {}, answer) == (True, 200)
|
|
assert answer["maintenance"] == [{"id": "job-1-reboot", "action": "reboot"}]
|
|
|
|
|
|
def test_a_late_return_stops_flagging_its_host_once_it_came_back():
|
|
"""#6175: one answer for the figure, the overview and the pages."""
|
|
late = _job(state="failed", finished_at=T0,
|
|
detail=f"{m.NOT_BACK} 30 min: services below their replicas: portal_app")
|
|
assert m.still_failed(late) and m.summary(late) == "Failed"
|
|
late.came_back_at = T0 + timedelta(minutes=45)
|
|
assert not m.still_failed(late)
|
|
assert m.summary(late) == "Back in service late, 2026-10-14 23:45Z"
|
|
# An update that failed, or a job that timed out, flags its host whatever came back.
|
|
for job in (_job(state="failed", detail="dpkg was interrupted", came_back_at=T0),
|
|
_job(state="timed_out", detail="no answer within 60 min", came_back_at=T0)):
|
|
assert m.still_failed(job)
|
|
assert not m.still_failed(_job(state="done"))
|