fix(maintenance): check first, drain before a Docker upgrade, wait 5 min after a drain, see containers back (#6098)

On 2026-10-18 an update on VDNT-Docker02 restarted the Docker daemon six
seconds in, before the drain that came only with the reboot, and every
container on the node stopped. Docker's own packages restart the daemon from
their install scripts; needrestart's list-only mode doesn't stop that.

- The helper gains a `check` action (agent 1.24.0): a dry run of the update
  for each package manager, whether Docker is among it, and the running
  containers outside the swarm with their restart policies.
- A swarm node is drained before an update that upgrades Docker, else only
  before a reboot, and at most once a job.
- After a drain the job settles 5 minutes before the update or reboot.
- Once the work is over, the containers that ran outside the swarm must be
  seen running again within 10 minutes, or the job fails naming them.
- An older agent gets no check and its job runs as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/plugins/host_agent/test_maint_helper.py

@@ -255,3 +255,82 @@ def test_a_pacman_host_needs_a_reboot_once_the_running_kernel_is_replaced(maint,
     assert maint.reboot_required() is False
     (tmp_path / os.uname().release).rmdir()
     assert maint.reboot_required() is True
+
+
+# ── check: what an update would change (#6098) ──────────────────────────────
+
+APT_SIM = """Reading package lists...
+Calculating upgrade...
+The following packages will be upgraded:
+  containerd.io docker-ce docker-ce-cli openssl
+Inst docker-ce [5:27.0.3-1~debian.12~bookworm] (5:27.1.1-1~debian.12~bookworm Docker CE:bookworm [amd64])
+Inst docker-ce-cli [5:27.0.3-1] (5:27.1.1-1 Docker CE:bookworm [amd64])
+Inst openssl [3.0.13-1] (3.0.14-1 Debian-Security:12/stable-security [amd64])
+Conf docker-ce (5:27.1.1-1~debian.12~bookworm Docker CE:bookworm [amd64])
+"""
+
+
+def test_a_check_reads_what_each_manager_would_upgrade(maint):
+    assert maint.pending_names("apt", APT_SIM) == ["docker-ce", "docker-ce-cli", "openssl"]
+    dnf = ("\nmoby-engine.x86_64        26.1.3-1.fc40      updates\n"
+           "kernel-core.x86_64        6.10.3-200.fc40    updates\n"
+           "Obsoleting Packages\n")
+    assert maint.pending_names("dnf", dnf) == ["moby-engine", "kernel-core"]
+    table = ("S | Repository | Name   | Current Version | Available Version | Arch\n"
+             "--+------------+--------+-----------------+-------------------+-------\n"
+             "v | repo-oss   | docker | 24.0.7          | 25.0.3            | x86_64\n"
+             "v | repo-oss   | vim    | 9.0             | 9.1               | x86_64\n")
+    assert maint.pending_names("zypper", table) == ["docker", "vim"]
+    dup = ("The following 2 packages are going to be upgraded:\n"
+           "  containerd runc\n\nThe following package is going to be installed:\n  foo\n"
+           "2 packages to upgrade, 1 new.\n")
+    assert maint.pending_names("zypper", dup) == ["containerd", "runc", "foo"]
+    assert maint.pending_names("pacman", "docker 1:27.0-1 -> 1:27.1-1\nlinux 6.10-1 -> 6.10.3-1\n") == [
+        "docker", "linux"]
+
+
+@pytest.mark.parametrize("name, restarts_docker", [
+    ("docker-ce", True), ("docker.io", True), ("containerd.io", True), ("moby-engine", True),
+    ("docker", True), ("containerd", True),
+    ("docker-ce-cli", False), ("docker-compose-plugin", False), ("docker-buildx-plugin", False),
+    ("runc", False), ("python3-docker", False)])
+def test_only_the_packages_that_restart_the_daemon_count_as_docker(maint, name, restarts_docker):
+    assert bool(maint.DOCKER_PACKAGES.match(name)) is restarts_docker
+
+
+def test_the_check_names_the_containers_outside_the_swarm(maint, monkeypatch):
+    monkeypatch.setattr(maint.shutil, "which", lambda b: "/usr/bin/docker")
+    inspected = ("/ak-outpost|unless-stopped|\n"
+                 "/traefik_router.abc.def|no|t0k3n\n"           # a swarm task: the drain moves it
+                 "/scratch||\n")
+
+    def fake_run(cmd, deadline=0, env=None, ok_codes=(0,), keep=0):
+        return True, ("id1 id2 id3\n" if cmd[1] == "ps" else inspected)
+
+    monkeypatch.setattr(maint, "run", fake_run)
+    assert maint.standalone_containers() == [{"name": "ak-outpost", "restart": "unless-stopped"},
+                                             {"name": "scratch", "restart": "no"}]
+
+
+def test_a_check_says_whether_docker_is_upgraded_and_changes_nothing(maint, monkeypatch):
+    _manager(monkeypatch, maint, ("apt-get",))
+    ran = []
+
+    def fake_run(cmd, deadline=0, env=None, ok_codes=(0,), keep=0):
+        ran.append(cmd)
+        return True, APT_SIM if "-s" in cmd else "Hit:1 http://deb.debian.org bookworm InRelease\n"
+
+    monkeypatch.setattr(maint, "run", fake_run)
+    monkeypatch.setattr(maint, "standalone_containers", lambda: [{"name": "ak-outpost", "restart": "always"}])
+    got = maint.do_check()
+    assert got == {"ok": True, "pending": 3, "docker_update": True, "docker_packages": ["docker-ce"],
+                   "containers": [{"name": "ak-outpost", "restart": "always"}]}
+    assert ran == [["apt-get", "update"], ["apt-get", "-s", "dist-upgrade"]]   # a simulation only
+    assert "check" in maint.ACTIONS
+
+
+def test_a_check_that_cant_list_says_which_step(maint, monkeypatch):
+    _manager(monkeypatch, maint, ("dnf",))
+    monkeypatch.setattr(maint, "run", lambda cmd, **kw: (False, "Curl error (6): Couldn't resolve host"))
+    got = maint.do_check()
+    assert not got["ok"] and got["detail"].startswith("dnf check-update failed: Curl error")

tests/plugins/host_agent/test_maintenance.py

@@ -20,7 +20,7 @@ def _job(**kw):
     base = dict(id="job-1", host_id="h1", state="queued", state_at=T0, created_at=T0,
                 reboot_policy="if_required", step_id=None, step_seen=False, node_id=None,
                 drain_host_id=None, packages=None, reboot_needed=None, back_at=None,
-                detail=None, finished_at=None)
+                detail=None, finished_at=None, drained_at=None, docker_update=None, checked=None)
     base.update(kw)
     return SimpleNamespace(**base)
 
@@ -47,6 +47,10 @@ def test_a_swarm_node_is_drained_rebooted_and_made_active_again():
     assert job.state == "draining" and m.executor(job) == "mgr"
     assert m.handoff(job) == {"id": "job-1-drain", "action": "drain", "node": "n" * 25}
     m.apply_result(job, _result(job, "done", ok=True), T0)
+    # The node's tasks get 5 minutes to move before the reboot (#6098).
+    assert job.state == "settling" and job.drained_at == T0 and m.handoff(job) is None
+    assert not m.settle(job, T0 + timedelta(minutes=4))
+    assert m.settle(job, T0 + timedelta(minutes=5))
     assert job.state == "rebooting" and m.executor(job) == "h1"
     assert m.handoff(job) == {"id": "job-1-reboot", "action": "reboot"}
     t1 = T0 + timedelta(minutes=4)
@@ -106,7 +110,7 @@ def test_a_job_is_given_up_past_its_state_deadline(state, minutes, expected):
 
 def test_the_window_lasts_the_rest_of_the_job_and_ends_with_it():
     job = _job(state="rebooting", step_id="job-1-reboot")
-    assert m.window_end(job, T0) == T0 + timedelta(minutes=30 + 10)
+    assert m.window_end(job, T0) == T0 + timedelta(minutes=30 + 10 + 15)
     job.state = "done"
     assert m.window_end(job, T0) == T0
 
@@ -131,6 +135,109 @@ def test_a_node_the_operator_keeps_drained_is_left_drained():
     assert m.pick_drainer(nodes, managers, "Util01", "h1", {"h1", "h2"}, "active") == ("c" * 25, "h1")
 
 
+# ── a check first: Docker upgraded, drain before; containers seen again (#6098)
+
+NODE = "n" * 25
+CONTAINERS = [{"name": "ak-outpost", "restart": "unless-stopped"}, {"name": "scratch", "restart": "no"}]
+
+
+def _checked(job, **kw):
+    m.apply_result(job, _result(job, "done", ok=True, pending=7, **kw), T0)
+
+
+def test_a_new_enough_agent_checks_first():
+    assert m.can_check("1.24.0") and m.can_check("1.30.2") and m.can_check("2.0.0")
+    assert not m.can_check("1.23.9") and not m.can_check(None) and not m.can_check("dev")
+    job = _job()
+    m.start(job, T0, check=True)
+    assert job.state == "checking" and m.handoff(job) == {"id": "job-1-check", "action": "check"}
+
+
+def test_an_update_that_upgrades_docker_drains_the_node_before_it():
+    job = _job(node_id=NODE, drain_host_id="mgr")
+    m.start(job, T0, check=True)
+    _checked(job, docker_update=True, docker_packages=["docker-ce"], containers=[])
+    assert job.state == "draining" and job.docker_update is True
+    assert job.checked == {"pending": 7, "docker_packages": ["docker-ce"], "containers": []}
+    m.apply_result(job, _result(job, "done", ok=True), T0)
+    assert job.state == "settling"
+    assert m.settle(job, T0 + timedelta(minutes=5)) and job.state == "updating"
+    m.apply_result(job, _result(job, "done", ok=True, packages=9, reboot_required=True), T0)
+    assert job.state == "rebooting"                             # drained already: no second drain
+    assert m.apply_boot(job, T0 + timedelta(minutes=8), T0 + timedelta(minutes=9))
+    assert job.state == "undraining"
+    m.apply_result(job, _result(job, "done", ok=True), T0)
+    assert job.state == "done" and m.summary(job) == "Done: Docker upgraded, 9 packages updated, rebooted"
+
+
+def test_a_drained_node_with_no_reboot_is_made_active_after_the_update():
+    job = _job(node_id=NODE, drain_host_id="mgr", reboot_policy="never")
+    m.start(job, T0, check=True)
+    _checked(job, docker_update=True)
+    m.apply_result(job, _result(job, "done", ok=True), T0)
+    m.settle(job, T0 + timedelta(minutes=5))
+    m.apply_result(job, _result(job, "done", ok=True, reboot_required=True), T0)
+    assert job.state == "undraining"
+
+
+def test_an_update_without_docker_doesnt_drain_unless_it_reboots():
+    job = _job(node_id=NODE, drain_host_id="mgr")
+    m.start(job, T0, check=True)
+    _checked(job, docker_update=False)
+    assert job.state == "updating"
+    m.apply_result(job, _result(job, "done", ok=True, reboot_required=False), T0)
+    assert job.state == "done" and job.drained_at is None    # never drained, so no undrain
+
+
+def test_docker_on_a_node_the_operator_keeps_drained_isnt_drained_again():
+    job = _job(node_id=NODE, drain_host_id=None)
+    m.start(job, T0, check=True)
+    _checked(job, docker_update=True)
+    assert job.state == "updating"
+
+
+def test_the_containers_outside_the_swarm_are_waited_for_after_the_reboot():
+    job = _job()
+    m.start(job, T0, check=True)
+    _checked(job, docker_update=False, containers=CONTAINERS)
+    m.apply_result(job, _result(job, "done", ok=True, reboot_required=True), T0)
+    t1 = T0 + timedelta(minutes=6)
+    assert m.apply_boot(job, T0 + timedelta(minutes=5), t1)
+    assert job.state == "recovering" and m.handoff(job) is None
+    assert not m.recovered(job, {"ak-outpost"}, t1 + timedelta(minutes=2))      # still waiting
+    assert m.recovered(job, {"ak-outpost", "scratch", "other"}, t1 + timedelta(minutes=3))
+    assert job.state == "done"
+
+
+def test_a_container_that_doesnt_run_again_fails_the_job_by_name():
+    job = _job(state="recovering", state_at=T0, checked={"containers": CONTAINERS})
+    assert not m.recovered(job, {"ak-outpost"}, T0 + timedelta(minutes=9))
+    assert m.recovered(job, {"ak-outpost"}, T0 + timedelta(minutes=10))
+    assert job.state == "failed"
+    assert job.detail == "not running again 10 min after the job: scratch (no restart policy)"
+
+
+def test_a_job_with_nothing_outside_the_swarm_ends_at_the_boot():
+    job = _job(state="rebooting", step_id="job-1-reboot", checked={"containers": []})
+    assert m.apply_boot(job, T0, T0) and job.state == "done"
+
+
+def test_what_a_check_said_is_kept_bounded():
+    job = _job(state="checking", step_id="job-1-check")
+    many = [{"name": f"c{i}" * 40, "restart": "always"} for i in range(60)] + ["junk", {"name": 3}]
+    _checked(job, docker_update="yes", docker_packages=["docker-ce", 4] * 9, containers=many)
+    assert job.docker_update is False                           # only a real True counts
+    assert len(job.checked["containers"]) == m.CONTAINERS_KEPT
+    assert len(job.checked["containers"][0]["name"]) == 64
+    assert job.checked["docker_packages"] == ["docker-ce"] * 9
+
+
+@pytest.mark.parametrize("state, minutes", [("checking", 31), ("settling", 16), ("recovering", 16)])
+def test_the_new_states_are_given_up_too(state, minutes):
+    job = _job(state=state, step_id="job-1-check" if state == "checking" else None)
+    assert m.check_deadline(job, T0 + timedelta(minutes=minutes)) and job.state == "timed_out"
+
+
 # ── the agent's side ─────────────────────────────────────────────────────────
 
 def test_the_agent_leaves_each_handed_step_once(tmp_path):

tests/plugins/host_agent/test_mcp_tools.py

@@ -31,7 +31,9 @@ def test_a_job_row_says_what_it_did_and_when_it_gives_up():
     job = NS(id="j1", host_id="h1", state="rebooting", state_at=NOW - timedelta(minutes=5),
              reboot_policy="if_required", packages=12, reboot_needed=True, back_at=None,
              node_id="n1", drain_host_id="h2", detail=None, created_by=None, run_id="r1",
-             created_at=NOW - timedelta(minutes=20), finished_at=None)
+             created_at=NOW - timedelta(minutes=20), finished_at=None,
+             drained_at=NOW - timedelta(minutes=11), docker_update=False,
+             checked={"pending": 12, "docker_packages": [], "containers": []})
     row = _job_row(job, {"h1": "web-1", "h2": "mgr-1"}, NOW)
     assert (row["host"], row["drained_by"], row["label"], row["active"]) == (
         "web-1", "mgr-1", "Rebooting", True)