fix(maintenance): check first, drain before a Docker upgrade, wait 5 min after a drain, see containers back (#6098)

On 2026-10-18 an update on VDNT-Docker02 restarted the Docker daemon six
seconds in, before the drain that came only with the reboot, and every
container on the node stopped. Docker's own packages restart the daemon from
their install scripts; needrestart's list-only mode doesn't stop that.

- The helper gains a `check` action (agent 1.24.0): a dry run of the update
  for each package manager, whether Docker is among it, and the running
  containers outside the swarm with their restart policies.
- A swarm node is drained before an update that upgrades Docker, else only
  before a reboot, and at most once a job.
- After a drain the job settles 5 minutes before the update or reboot.
- Once the work is over, the containers that ran outside the swarm must be
  seen running again within 10 minutes, or the job fails naming them.
- An older agent gets no check and its job runs as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/maintenance.py

@@ -4,8 +4,18 @@
 Steward plans and the host's agent acts: a job is one host's update and, only
 when the update says one is needed, its reboot, with the host's swarm node
 drained first and made active again after. Each action is one step the agent
-hands to its root helper (maint.py, #5766). Everything lives on the job's row,
-so Steward restarting, or the host Steward runs on rebooting, loses nothing:
+hands to its root helper (maint.py, #5766).
+
+A job first asks the helper what the update would change (#6098). Docker's
+own packages restart the daemon from their install scripts, stopping every
+container on the host, so a swarm node is drained before an update that
+upgrades Docker; otherwise only before a reboot. After a drain the node's
+tasks get DRAIN_SETTLE to move before the update or reboot goes ahead. The
+containers running outside the swarm when the job began must be running again
+when it ends, within RECOVER_LIMIT, or the job fails naming them.
+
+Everything lives on the job's row, so Steward restarting, or the host Steward
+runs on rebooting, loses nothing:
 the next post from the agent carries the step's result, and the `started`
 event a host sends after booting is what completes a reboot.
 
@@ -13,27 +23,47 @@ The transitions here are model-free (a job is anything with the attributes),
 so they're unit-tested without a database; routes.ingest and the scheduler's
 tick apply them.
 
-States: queued → updating → [draining →] rebooting → [undraining →] done,
-or failed / timed_out. A job with no reboot needed goes updating → done.
+States: queued → [checking →] [draining → settling →] updating →
+[draining → settling →] [rebooting →] [undraining →] [recovering →] done, or
+failed / timed_out. A node is drained at most once a job: before the update
+when Docker is upgraded, else before the reboot. settling and recovering are
+Steward's own waits, moved on by the minute tick. An agent older than
+CHECK_AGENT gets no check: its job starts at updating.
 """
 from __future__ import annotations
 
 from datetime import datetime, timedelta, timezone
 
-ACTIVE = ("queued", "updating", "draining", "rebooting", "undraining")
+ACTIVE = ("queued", "checking", "draining", "settling", "updating", "rebooting",
+          "undraining", "recovering")
 FINISHED = ("done", "failed", "timed_out")
 # The helper action each handed state asks for.
-STEP_ACTION = {"updating": "update", "draining": "drain", "rebooting": "reboot",
-               "undraining": "undrain"}
+STEP_ACTION = {"checking": "check", "updating": "update", "draining": "drain",
+               "rebooting": "reboot", "undraining": "undrain"}
 # How long a state may last before the job is given up. Queued counts from the
 # job's creation (the host may be silent); the rest from their hand-off.
 DEADLINES = {
     "queued": timedelta(hours=1),
-    "updating": timedelta(minutes=90),     # the helper itself stops at 60
+    "checking": timedelta(minutes=30),     # two listings, the helper stops each at 10
     "draining": timedelta(minutes=10),
+    "settling": timedelta(minutes=15),     # the tick moves it on at DRAIN_SETTLE
+    "updating": timedelta(minutes=90),     # the helper itself stops at 60
     "rebooting": timedelta(minutes=30),
     "undraining": timedelta(minutes=10),
+    "recovering": timedelta(minutes=15),   # the tick fails it at RECOVER_LIMIT
 }
+# After a drain, how long the node's swarm tasks get to move before the
+# update or the reboot goes ahead (the operator, #6098: "drain wait 5 minutes
+# and then reboot").
+DRAIN_SETTLE = timedelta(minutes=5)
+# How long the containers that ran outside the swarm before the job get to
+# run again after it.
+RECOVER_LIMIT = timedelta(minutes=10)
+# The first agent whose helper answers a check (#6098).
+CHECK_AGENT = (1, 24, 0)
+# A restart policy under which Docker starts a container again after a reboot.
+RESTARTS = ("always", "unless-stopped")
+CONTAINERS_KEPT = 40
 # A host's clock and Steward's differ by seconds; a boot this far before the
 # reboot was handed out is still taken as that reboot.
 BOOT_SLACK = timedelta(seconds=60)
@@ -60,10 +90,30 @@ def _finish(job, state: str, now: datetime, detail: str | None = None) -> None:
     _enter(job, state, now)
 
 
-def start(job, now: datetime) -> None:
-    """The host's agent is about to be handed the job: update first."""
+def can_check(agent_version) -> bool:
+    """Whether an agent this version hands the helper a check."""
+    try:
+        return tuple(int(x) for x in str(agent_version).split(".")[:3]) >= CHECK_AGENT
+    except ValueError:
+        return False
+
+
+def start(job, now: datetime, check: bool = False) -> None:
+    """The host's agent is about to be handed the job: what the update would
+    change first, when its helper can say, else the update."""
     if job.state == "queued":
-        _enter(job, "updating", now)
+        _enter(job, "checking" if check else "updating", now)
+
+
+def drains(job) -> bool:
+    """Whether the job drains the host's swarm node: not when the host isn't
+    in a swarm, nor when the operator keeps its node drained."""
+    return bool(job.node_id and job.drain_host_id)
+
+
+def outside_swarm(job) -> list:
+    """The containers the check found running outside the swarm."""
+    return (job.checked or {}).get("containers") or []
 
 
 def executor(job) -> str:
@@ -97,15 +147,46 @@ def wants_reboot(job) -> bool:
     return bool(job.reboot_needed) and job.reboot_policy == "if_required"
 
 
+def _checked(event: dict) -> dict:
+    """What a check said, kept on the job: bounded, and only the fields used."""
+    containers = []
+    for c in event.get("containers") or []:
+        if isinstance(c, dict) and isinstance(c.get("name"), str) and c["name"]:
+            containers.append({"name": c["name"][:64], "restart": str(c.get("restart") or "no")[:16]})
+    packages = [str(p)[:64] for p in event.get("docker_packages") or [] if isinstance(p, str)]
+    pending = event.get("pending")
+    return {"pending": pending if isinstance(pending, int) else None,
+            "docker_packages": packages[:10], "containers": containers[:CONTAINERS_KEPT]}
+
+
+def _after_check(job, event: dict, now: datetime) -> None:
+    job.docker_update = event.get("docker_update") is True
+    job.checked = _checked(event)
+    _enter(job, "draining" if job.docker_update and drains(job) else "updating", now)
+
+
 def _after_update(job, now: datetime) -> None:
     if not wants_reboot(job):
-        _enter(job, "done", now)
-    elif job.node_id and job.drain_host_id:
+        _after_work(job, now)
+    elif drains(job) and job.drained_at is None:
         _enter(job, "draining", now)
     else:
         _enter(job, "rebooting", now)
 
 
+def _after_work(job, now: datetime) -> None:
+    """The update (and any reboot) is over: make the node active again if the
+    job drained it, then see the containers back."""
+    if job.drained_at is not None and drains(job):
+        _enter(job, "undraining", now)
+    else:
+        _recover(job, now)
+
+
+def _recover(job, now: datetime) -> None:
+    _enter(job, "recovering" if outside_swarm(job) else "done", now)
+
+
 def apply_result(job, event: dict, now: datetime) -> bool:
     """Apply a `maintenance` event for this job's current step; True when the
     job changed. A result for a step the job has moved past is ignored, so a
@@ -127,15 +208,18 @@ def apply_result(job, event: dict, now: datetime) -> bool:
             detail += " (the node is still drained)"
         _finish(job, "failed", now, detail)
         return True
-    if job.state == "updating":
+    if job.state == "checking":
+        _after_check(job, event, now)
+    elif job.state == "updating":
         if isinstance(event.get("packages"), int):
             job.packages = event["packages"]
         job.reboot_needed = event.get("reboot_required") is True
         _after_update(job, now)
     elif job.state == "draining":
-        _enter(job, "rebooting", now)
+        job.drained_at = now
+        _enter(job, "settling", now)
     elif job.state == "undraining":
-        _enter(job, "done", now)
+        _recover(job, now)
     else:                               # rebooting: the boot completes it, not this
         return False
     return True
@@ -149,10 +233,36 @@ def apply_boot(job, boot_at: datetime | None, now: datetime) -> bool:
     if boot_at < job.state_at - BOOT_SLACK:
         return False                    # the agent restarted; the host didn't
     job.back_at = now
-    if job.node_id and job.drain_host_id:
-        _enter(job, "undraining", now)
-    else:
+    _after_work(job, now)
+    return True
+
+
+def settle(job, now: datetime) -> bool:
+    """A drained node has had DRAIN_SETTLE for its tasks to move: on to the
+    update, or to the reboot when the update is done (reboot_needed is set)."""
+    if job.state != "settling" or job.state_at is None or now - job.state_at < DRAIN_SETTLE:
+        return False
+    _enter(job, "updating" if job.reboot_needed is None else "rebooting", now)
+    return True
+
+
+def recovered(job, running: set, now: datetime) -> bool:
+    """Done once every container that ran outside the swarm before the job
+    runs again (`running`: the host's containers seen running since the job
+    came to this state); failed, naming the rest, past RECOVER_LIMIT."""
+    if job.state != "recovering" or job.state_at is None:
+        return False
+    missing = [c for c in outside_swarm(job) if c["name"] not in running]
+    if not missing:
         _enter(job, "done", now)
+        return True
+    if now - job.state_at < RECOVER_LIMIT:
+        return False
+    names = [c["name"] + ("" if c["restart"] in RESTARTS else " (no restart policy)")
+             for c in missing[:6]]
+    more = f" and {len(missing) - 6} more" if len(missing) > 6 else ""
+    _finish(job, "failed", now, f"not running again {int(RECOVER_LIMIT.total_seconds() // 60)} min "
+                                f"after the job: {', '.join(names)}{more}")
     return True
 
 
@@ -163,10 +273,13 @@ def check_deadline(job, now: datetime) -> bool:
     if limit is None or since is None or now - since <= limit:
         return False
     waiting = {"queued": "the host's agent didn't check in",
-               "updating": "the update didn't finish",
+               "checking": "the check of what the update would change didn't finish",
                "draining": "the node wasn't drained",
+               "settling": "the wait after the drain didn't end",
+               "updating": "the update didn't finish",
                "rebooting": "the host didn't come back",
-               "undraining": "the node wasn't made active again"}[job.state]
+               "undraining": "the node wasn't made active again",
+               "recovering": "its containers weren't seen again"}[job.state]
     _finish(job, "timed_out", now, f"{waiting} within {int(limit.total_seconds() // 60)} min")
     return True
 
@@ -269,9 +382,11 @@ async def apply_events(session, host_id: str, events, now: datetime) -> int:
     return len(changed)
 
 
-async def handoffs(session, host_id: str, host_name: str, now: datetime) -> list[dict]:
+async def handoffs(session, host_id: str, host_name: str, now: datetime,
+                   agent_version: str | None = None) -> list[dict]:
     """The steps this host's agent should carry out now. A queued job for the
-    host starts here, when its agent is known to be listening."""
+    host starts here, when its agent is known to be listening, with a check
+    when the agent is new enough to hand one over."""
     from sqlalchemy import or_, select
     from .models import HostMaintenanceJob
     jobs = (await session.execute(select(HostMaintenanceJob).where(
@@ -282,7 +397,7 @@ async def handoffs(session, host_id: str, host_name: str, now: datetime) -> list
     out = []
     for job in jobs:
         if job.state == "queued" and job.host_id == host_id:
-            start(job, now)
+            start(job, now, check=can_check(agent_version))
             await _window(session, job, host_name, now)
         if executor(job) == host_id:
             step = handoff(job)
@@ -291,15 +406,36 @@ async def handoffs(session, host_id: str, host_name: str, now: datetime) -> list
     return out
 
 
+async def _running(session, job) -> set:
+    """The job's host's containers seen running since the job came to its
+    state; without the Docker plugin, the ones it waits for, as nothing can
+    say otherwise."""
+    from sqlalchemy import select
+    models = docker_models()
+    if models is None:
+        return {c["name"] for c in outside_swarm(job)}
+    DockerContainer = models.DockerContainer
+    return set((await session.execute(select(DockerContainer.name).where(
+        DockerContainer.host_id == job.host_id, DockerContainer.status == "running",
+        DockerContainer.scraped_at >= job.state_at))).scalars())
+
+
 async def tick(session, now: datetime) -> int:
-    """Give up on jobs past their deadline; returns how many."""
+    """Move on the jobs waiting on Steward (the wait after a drain, the
+    containers seen again) and give up on jobs past their deadline; returns
+    how many were given up."""
     from sqlalchemy import select
     from .models import HostMaintenanceJob
     gave_up = 0
     for job in (await session.execute(select(HostMaintenanceJob).where(
             HostMaintenanceJob.state.in_(ACTIVE)))).scalars():
+        moved = settle(job, now)
+        if job.state == "recovering":
+            moved = recovered(job, await _running(session, job), now) or moved
         if check_deadline(job, now):
             gave_up += 1
+            moved = True
+        if moved:
             await _window(session, job, await _host_name(session, job.host_id), now)
     return gave_up
 
@@ -344,14 +480,19 @@ async def plan_job(session, host, reg, reboot_policy: str = "if_required") -> tu
         HostMaintenanceJob.state.in_(ACTIVE)))).first()
     if busy is not None:
         raise CannotStart("a maintenance job is already running on it")
-    if reboot_policy == "never":
-        return None, None
     nodes, managers, availability = await _swarm(session)
     helpers = set((await session.execute(select(HostAgentRegistration.host_id).where(
         HostAgentRegistration.maint_helper.is_(True)))).scalars())
     node_id = nodes.get(host.name.lower())
-    return pick_drainer(nodes, managers, host.name, host.id, helpers,
-                        availability.get(node_id, "active"))
+    try:
+        return pick_drainer(nodes, managers, host.name, host.id, helpers,
+                            availability.get(node_id, "active"))
+    except CannotStart:
+        # A job that never reboots needs a drain only if Docker is upgraded;
+        # with no manager to drain, it runs undrained rather than not at all.
+        if reboot_policy == "never":
+            return None, None
+        raise
 
 
 async def create_job(session, host, reg, user_id: str | None,
@@ -383,10 +524,13 @@ async def latest_job(session, host_id: str, now: datetime, days: int = 366):
 
 LABELS = {
     "queued": "Waiting for the agent",
-    "updating": "Updating",
+    "checking": "Checking what the update will change",
     "draining": "Draining its swarm node",
+    "settling": "Giving its swarm tasks 5 min to move",
+    "updating": "Updating",
     "rebooting": "Rebooting",
     "undraining": "Making its swarm node active again",
+    "recovering": "Waiting for its containers to run again",
     "done": "Done",
     "failed": "Failed",
     "timed_out": "Timed out",
@@ -398,6 +542,8 @@ def summary(job) -> str:
     if job.state not in FINISHED:
         return LABELS[job.state]
     parts = []
+    if job.docker_update:
+        parts.append("Docker upgraded")
     if job.packages is not None:
         parts.append(f"{job.packages} package{'s' if job.packages != 1 else ''} updated")
     if job.back_at is not None: