fix(maintenance): check first, drain before a Docker upgrade, wait 5 min after a drain, see containers back (#6098)

On 2026-10-18 an update on VDNT-Docker02 restarted the Docker daemon six
seconds in, before the drain that came only with the reboot, and every
container on the node stopped. Docker's own packages restart the daemon from
their install scripts; needrestart's list-only mode doesn't stop that.

- The helper gains a `check` action (agent 1.24.0): a dry run of the update
  for each package manager, whether Docker is among it, and the running
  containers outside the swarm with their restart policies.
- A swarm node is drained before an update that upgrades Docker, else only
  before a reboot, and at most once a job.
- After a drain the job settles 5 minutes before the update or reboot.
- Once the work is over, the containers that ran outside the swarm must be
  seen running again within 10 minutes, or the job fails naming them.
- An older agent gets no check and its job runs as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/agent.py

@@ -22,7 +22,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.23.0"
+AGENT_VERSION = "1.24.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -1663,13 +1663,13 @@ def collect_reboot(path: str = REBOOT_REQUIRED, modules: str = KERNEL_MODULES):
 # The agent can't update or reboot its host: it runs unprivileged under
 # NoNewPrivileges. It leaves a request in MAINT_REQUESTS, the one directory its
 # unit lets it write; steward-maint.path starts the root helper (maint.py),
-# which does only update / reboot / drain / undrain and writes each outcome to
-# MAINT_RESULTS. The agent reads those back and reports each new state once.
+# which does only check / update / reboot / drain / undrain and writes each
+# outcome to MAINT_RESULTS. The agent reads those back and reports each new state once.
 
 MAINT_REQUESTS = "/var/lib/steward-agent/requests"
 MAINT_RESULTS = "/var/lib/steward-maint/results"
 MAINT_PATH_UNIT = "/etc/systemd/system/steward-maint.path"
-MAINT_ACTIONS = ("update", "reboot", "drain", "undrain", "drives")
+MAINT_ACTIONS = ("check", "update", "reboot", "drain", "undrain", "drives")
 MAINT_RESULT_MAX_BYTES = 8192
 # The helper keeps results a week; only this recent a one is reported, so a
 # job a reboot interrupted still reaches Steward from the agent's next start
@@ -1677,7 +1677,9 @@ MAINT_RESULT_MAX_BYTES = 8192
 MAINT_REPORT_WINDOW_SECONDS = 86400
 _MAINT_ID = re.compile(r"^[A-Za-z0-9-]{1,64}$")
 _MAINT_FIELDS = ("action", "state", "ok", "packages", "reboot_required", "node",
-                 "detail", "started_at", "finished_at")
+                 "detail", "started_at", "finished_at",
+                 # a check's answer (#6098)
+                 "pending", "docker_update", "docker_packages", "containers")
 
 
 def maint_helper_installed(unit: str = MAINT_PATH_UNIT,

plugins/host_agent/files/maint.py

@@ -12,7 +12,9 @@ make its host update, reboot or drain, and nothing else.
 
 One action only reads: `drives` (#5789) asks smartctl for each disk's health,
 which needs root, and writes a summary to DRIVES, never a result: it is not a
-maintenance job.
+maintenance job. `check` (#6098) changes nothing but the package lists: it
+says what an update would upgrade, whether Docker is among it, and which
+containers run outside the swarm.
 
 Stdlib only; served by Steward at /plugins/host_agent/files/maint.py.
 """
@@ -30,12 +32,14 @@ from datetime import datetime, timezone
 REQUESTS = "/var/lib/steward-agent/requests"
 RESULTS = "/var/lib/steward-maint/results"
 DRIVES = "/var/lib/steward-maint/drives.json"
-ACTIONS = ("update", "reboot", "drain", "undrain", "drives")
+ACTIONS = ("check", "update", "reboot", "drain", "undrain", "drives")
 REQUEST_MAX_BYTES = 4096
 RESULTS_KEPT_SECONDS = 7 * 86400
 UPDATE_DEADLINE_SECONDS = 3000
 COMMAND_DEADLINE_SECONDS = 600
 OUTPUT_KEPT_CHARS = 2000
+# A listing of what an update would change is read whole, not just its tail.
+LISTING_KEPT_CHARS = 1_000_000
 _ID = re.compile(r"^[A-Za-z0-9-]{1,64}$")
 _NODE = re.compile(r"^[a-z0-9]{25}$")
 
@@ -77,9 +81,10 @@ def valid(req) -> bool:
 
 
 def run(cmd: list, deadline: int = COMMAND_DEADLINE_SECONDS, env: dict | None = None,
-        ok_codes: tuple = (0,)):
+        ok_codes: tuple = (0,), keep: int = OUTPUT_KEPT_CHARS):
     """(ok, output tail) of a command, bounded by `deadline`. `ok_codes` are
-    the exit codes that mean it worked (zypper says more than 0)."""
+    the exit codes that mean it worked (zypper says more than 0); `keep` is
+    how much of the output's end is kept."""
     try:
         done = subprocess.run(cmd, capture_output=True, text=True, timeout=deadline,
                               env=dict(os.environ, LANG="C", LC_ALL="C", **(env or {})))
@@ -88,7 +93,7 @@ def run(cmd: list, deadline: int = COMMAND_DEADLINE_SECONDS, env: dict | None =
     except OSError as exc:
         return False, str(exc)
     out = (done.stdout or "") + (done.stderr or "")
-    return done.returncode in ok_codes, out[-OUTPUT_KEPT_CHARS:]
+    return done.returncode in ok_codes, out[-keep:]
 
 
 REBOOT_MARKER = "/var/run/reboot-required"                 # Debian and Ubuntu
@@ -152,9 +157,10 @@ def _os_release_id(path: str = "/etc/os-release") -> str:
 def update_steps(manager: str) -> list:
     """The commands an update runs on a host with `manager`, in order: each
     (what it is, command, deadline, env, ok exit codes). The last one is the upgrade
-    whose output is counted. Existing config files are kept, and service
-    restarts are left to a reboot: needrestart only lists, so the Docker
-    daemon isn't restarted under its containers mid-upgrade."""
+    whose output is counted. Existing config files are kept, and needrestart
+    only lists, leaving service restarts to a reboot. A package's own install
+    script can still restart its service: Docker's does, which is why the job
+    checks first and drains a swarm node before such an update (#6098)."""
     long = UPDATE_DEADLINE_SECONDS
     if manager == "apt":
         env = {"DEBIAN_FRONTEND": "noninteractive", "NEEDRESTART_MODE": "l"}
@@ -210,6 +216,116 @@ def do_update() -> dict:
             "detail": out[-500:]}
 
 
+# ── what an update would change, and what runs outside the swarm (#6098) ─────
+# Docker's own packages restart the daemon from their install scripts, whatever
+# needrestart is told, and every container on the host stops with it. So the
+# job asks first: a swarm node is drained before an update that upgrades
+# Docker, not after it, and the containers running outside the swarm are
+# noted, to see each of them running again once the job is over.
+
+DOCKER_PACKAGES = re.compile(
+    r"^(docker|docker-ce|docker\.io|docker-engine|moby-engine|containerd|containerd\.io)$")
+CONTAINERS_MAX = 40
+CONTAINER_NAME_MAX = 64
+
+
+def pending_steps(manager: str) -> list:
+    """The commands that list what an update would upgrade, changing nothing
+    but the package lists: each (what it is, command, ok exit codes). The
+    last one's output names the packages."""
+    if manager == "apt":
+        return [("apt-get update", ["apt-get", "update"], (0,)),
+                ("apt-get -s dist-upgrade", ["apt-get", "-s", "dist-upgrade"], (0,))]
+    if manager == "dnf":
+        # check-update exits 100 when there is something to update.
+        return [("dnf check-update", ["dnf", "-q", "check-update", "--refresh"], (0, 100))]
+    if manager == "yum":
+        return [("yum check-update", ["yum", "-q", "check-update"], (0, 100))]
+    if manager == "zypper":
+        if _os_release_id() == "opensuse-tumbleweed":
+            listing = ("zypper dup --dry-run", ["zypper", "--non-interactive", "dup", "--dry-run",
+                                                "--auto-agree-with-licenses"])
+        else:
+            listing = ("zypper list-updates", ["zypper", "--non-interactive", "list-updates"])
+        return [("zypper refresh", ["zypper", "--non-interactive", "refresh"], ZYPPER_OK),
+                (*listing, ZYPPER_OK)]
+    if manager == "pacman":
+        # -Sy refreshes the lists only; the update's -Syu follows it. -Qu exits
+        # 1 when there is nothing to upgrade.
+        return [("pacman -Sy", ["pacman", "-Sy"], (0,)), ("pacman -Qu", ["pacman", "-Qu"], (0, 1))]
+    return []
+
+
+def pending_names(manager: str, output: str) -> list:
+    """The packages a listing from pending_steps says would be upgraded."""
+    if manager == "apt":
+        return re.findall(r"^Inst (\S+)", output, re.MULTILINE)
+    if manager in ("dnf", "yum"):
+        # "docker-ce.x86_64   3:27.1.1-1.el9   docker-ce-stable"
+        return [parts[0].rsplit(".", 1)[0] for parts in (line.split() for line in output.splitlines())
+                if len(parts) == 3 and "." in parts[0]]
+    if manager == "zypper":
+        names, column, listing = [], None, False
+        for line in output.splitlines():
+            cells = [c.strip() for c in line.split("|")]
+            if len(cells) > 2 and column is None and "Name" in cells:
+                column = cells.index("Name")                   # list-updates' table
+            elif column is not None and len(cells) > column and cells[0] == "v":
+                names.append(cells[column])
+            elif re.search(r"going to be (upgraded|installed)", line):
+                listing = True                                 # dup --dry-run's lists
+            elif listing and line.startswith(" ") and line.strip():
+                names.extend(line.split())
+            else:
+                listing = False
+        return names
+    if manager == "pacman":
+        return [line.split()[0] for line in output.splitlines() if line.strip()]
+    return []
+
+
+def standalone_containers() -> list:
+    """The running containers that aren't swarm tasks, each with its restart
+    policy, by name; at most CONTAINERS_MAX, so the result stays small."""
+    if not shutil.which("docker"):
+        return []
+    ok, out = run(["docker", "ps", "-q", "--no-trunc"], deadline=30, keep=LISTING_KEPT_CHARS)
+    ids = out.split() if ok else []
+    if not ids:
+        return []
+    fmt = '{{.Name}}|{{.HostConfig.RestartPolicy.Name}}|{{index .Config.Labels "com.docker.swarm.task.id"}}'
+    ok, out = run(["docker", "inspect", "--format", fmt, *ids], deadline=60, keep=LISTING_KEPT_CHARS)
+    if not ok:
+        return []
+    found = []
+    for line in out.splitlines():
+        parts = line.split("|")
+        if len(parts) != 3 or parts[2].strip():
+            continue                                    # a swarm task: the drain moves it
+        found.append({"name": parts[0].lstrip("/")[:CONTAINER_NAME_MAX],
+                      "restart": parts[1].strip() or "no"})
+    return sorted(found, key=lambda c: c["name"])[:CONTAINERS_MAX]
+
+
+def do_check() -> dict:
+    """What an update would upgrade, whether Docker is among it, and the
+    containers running outside the swarm. Only the package lists change."""
+    manager = host_manager()
+    if manager is None:
+        return {"ok": False, "detail": "no package manager this helper knows "
+                                       "(apt, dnf, yum, zypper, pacman) on this host"}
+    env = {"DEBIAN_FRONTEND": "noninteractive"} if manager == "apt" else None
+    out = ""
+    for label, cmd, ok_codes in pending_steps(manager):
+        ok, out = run(cmd, env=env, ok_codes=ok_codes, keep=LISTING_KEPT_CHARS)
+        if not ok:
+            return {"ok": False, "detail": f"{label} failed: " + out[-500:]}
+    names = pending_names(manager, out)
+    docker = sorted({n for n in names if DOCKER_PACKAGES.match(n)})
+    return {"ok": True, "pending": len(names), "docker_update": bool(docker),
+            "docker_packages": docker[:10], "containers": standalone_containers()}
+
+
 def own_node() -> str | None:
     ok, out = run(["docker", "info", "--format", "{{.Swarm.NodeID}}"], deadline=30)
     node = out.strip() if ok else ""
@@ -387,7 +503,9 @@ def handle(req: dict) -> dict:
         return {**base, "state": "done"}
     write_result({**base, "state": "running"})
     action = req["action"]
-    if action == "update":
+    if action == "check":
+        outcome = do_check()
+    elif action == "update":
         outcome = do_update()
     elif action in ("drain", "undrain"):
         outcome = do_node("drain" if action == "drain" else "active", req.get("node"))

plugins/host_agent/maintenance.py

@@ -4,8 +4,18 @@
 Steward plans and the host's agent acts: a job is one host's update and, only
 when the update says one is needed, its reboot, with the host's swarm node
 drained first and made active again after. Each action is one step the agent
-hands to its root helper (maint.py, #5766). Everything lives on the job's row,
-so Steward restarting, or the host Steward runs on rebooting, loses nothing:
+hands to its root helper (maint.py, #5766).
+
+A job first asks the helper what the update would change (#6098). Docker's
+own packages restart the daemon from their install scripts, stopping every
+container on the host, so a swarm node is drained before an update that
+upgrades Docker; otherwise only before a reboot. After a drain the node's
+tasks get DRAIN_SETTLE to move before the update or reboot goes ahead. The
+containers running outside the swarm when the job began must be running again
+when it ends, within RECOVER_LIMIT, or the job fails naming them.
+
+Everything lives on the job's row, so Steward restarting, or the host Steward
+runs on rebooting, loses nothing:
 the next post from the agent carries the step's result, and the `started`
 event a host sends after booting is what completes a reboot.
 
@@ -13,27 +23,47 @@ The transitions here are model-free (a job is anything with the attributes),
 so they're unit-tested without a database; routes.ingest and the scheduler's
 tick apply them.
 
-States: queued → updating → [draining →] rebooting → [undraining →] done,
-or failed / timed_out. A job with no reboot needed goes updating → done.
+States: queued → [checking →] [draining → settling →] updating →
+[draining → settling →] [rebooting →] [undraining →] [recovering →] done, or
+failed / timed_out. A node is drained at most once a job: before the update
+when Docker is upgraded, else before the reboot. settling and recovering are
+Steward's own waits, moved on by the minute tick. An agent older than
+CHECK_AGENT gets no check: its job starts at updating.
 """
 from __future__ import annotations
 
 from datetime import datetime, timedelta, timezone
 
-ACTIVE = ("queued", "updating", "draining", "rebooting", "undraining")
+ACTIVE = ("queued", "checking", "draining", "settling", "updating", "rebooting",
+          "undraining", "recovering")
 FINISHED = ("done", "failed", "timed_out")
 # The helper action each handed state asks for.
-STEP_ACTION = {"updating": "update", "draining": "drain", "rebooting": "reboot",
-               "undraining": "undrain"}
+STEP_ACTION = {"checking": "check", "updating": "update", "draining": "drain",
+               "rebooting": "reboot", "undraining": "undrain"}
 # How long a state may last before the job is given up. Queued counts from the
 # job's creation (the host may be silent); the rest from their hand-off.
 DEADLINES = {
     "queued": timedelta(hours=1),
-    "updating": timedelta(minutes=90),     # the helper itself stops at 60
+    "checking": timedelta(minutes=30),     # two listings, the helper stops each at 10
     "draining": timedelta(minutes=10),
+    "settling": timedelta(minutes=15),     # the tick moves it on at DRAIN_SETTLE
+    "updating": timedelta(minutes=90),     # the helper itself stops at 60
     "rebooting": timedelta(minutes=30),
     "undraining": timedelta(minutes=10),
+    "recovering": timedelta(minutes=15),   # the tick fails it at RECOVER_LIMIT
 }
+# After a drain, how long the node's swarm tasks get to move before the
+# update or the reboot goes ahead (the operator, #6098: "drain wait 5 minutes
+# and then reboot").
+DRAIN_SETTLE = timedelta(minutes=5)
+# How long the containers that ran outside the swarm before the job get to
+# run again after it.
+RECOVER_LIMIT = timedelta(minutes=10)
+# The first agent whose helper answers a check (#6098).
+CHECK_AGENT = (1, 24, 0)
+# A restart policy under which Docker starts a container again after a reboot.
+RESTARTS = ("always", "unless-stopped")
+CONTAINERS_KEPT = 40
 # A host's clock and Steward's differ by seconds; a boot this far before the
 # reboot was handed out is still taken as that reboot.
 BOOT_SLACK = timedelta(seconds=60)
@@ -60,10 +90,30 @@ def _finish(job, state: str, now: datetime, detail: str | None = None) -> None:
     _enter(job, state, now)
 
 
-def start(job, now: datetime) -> None:
-    """The host's agent is about to be handed the job: update first."""
+def can_check(agent_version) -> bool:
+    """Whether an agent this version hands the helper a check."""
+    try:
+        return tuple(int(x) for x in str(agent_version).split(".")[:3]) >= CHECK_AGENT
+    except ValueError:
+        return False
+
+
+def start(job, now: datetime, check: bool = False) -> None:
+    """The host's agent is about to be handed the job: what the update would
+    change first, when its helper can say, else the update."""
     if job.state == "queued":
-        _enter(job, "updating", now)
+        _enter(job, "checking" if check else "updating", now)
+
+
+def drains(job) -> bool:
+    """Whether the job drains the host's swarm node: not when the host isn't
+    in a swarm, nor when the operator keeps its node drained."""
+    return bool(job.node_id and job.drain_host_id)
+
+
+def outside_swarm(job) -> list:
+    """The containers the check found running outside the swarm."""
+    return (job.checked or {}).get("containers") or []
 
 
 def executor(job) -> str:
@@ -97,15 +147,46 @@ def wants_reboot(job) -> bool:
     return bool(job.reboot_needed) and job.reboot_policy == "if_required"
 
 
+def _checked(event: dict) -> dict:
+    """What a check said, kept on the job: bounded, and only the fields used."""
+    containers = []
+    for c in event.get("containers") or []:
+        if isinstance(c, dict) and isinstance(c.get("name"), str) and c["name"]:
+            containers.append({"name": c["name"][:64], "restart": str(c.get("restart") or "no")[:16]})
+    packages = [str(p)[:64] for p in event.get("docker_packages") or [] if isinstance(p, str)]
+    pending = event.get("pending")
+    return {"pending": pending if isinstance(pending, int) else None,
+            "docker_packages": packages[:10], "containers": containers[:CONTAINERS_KEPT]}
+
+
+def _after_check(job, event: dict, now: datetime) -> None:
+    job.docker_update = event.get("docker_update") is True
+    job.checked = _checked(event)
+    _enter(job, "draining" if job.docker_update and drains(job) else "updating", now)
+
+
 def _after_update(job, now: datetime) -> None:
     if not wants_reboot(job):
-        _enter(job, "done", now)
-    elif job.node_id and job.drain_host_id:
+        _after_work(job, now)
+    elif drains(job) and job.drained_at is None:
         _enter(job, "draining", now)
     else:
         _enter(job, "rebooting", now)
 
 
+def _after_work(job, now: datetime) -> None:
+    """The update (and any reboot) is over: make the node active again if the
+    job drained it, then see the containers back."""
+    if job.drained_at is not None and drains(job):
+        _enter(job, "undraining", now)
+    else:
+        _recover(job, now)
+
+
+def _recover(job, now: datetime) -> None:
+    _enter(job, "recovering" if outside_swarm(job) else "done", now)
+
+
 def apply_result(job, event: dict, now: datetime) -> bool:
     """Apply a `maintenance` event for this job's current step; True when the
     job changed. A result for a step the job has moved past is ignored, so a
@@ -127,15 +208,18 @@ def apply_result(job, event: dict, now: datetime) -> bool:
             detail += " (the node is still drained)"
         _finish(job, "failed", now, detail)
         return True
-    if job.state == "updating":
+    if job.state == "checking":
+        _after_check(job, event, now)
+    elif job.state == "updating":
         if isinstance(event.get("packages"), int):
             job.packages = event["packages"]
         job.reboot_needed = event.get("reboot_required") is True
         _after_update(job, now)
     elif job.state == "draining":
-        _enter(job, "rebooting", now)
+        job.drained_at = now
+        _enter(job, "settling", now)
     elif job.state == "undraining":
-        _enter(job, "done", now)
+        _recover(job, now)
     else:                               # rebooting: the boot completes it, not this
         return False
     return True
@@ -149,10 +233,36 @@ def apply_boot(job, boot_at: datetime | None, now: datetime) -> bool:
     if boot_at < job.state_at - BOOT_SLACK:
         return False                    # the agent restarted; the host didn't
     job.back_at = now
-    if job.node_id and job.drain_host_id:
-        _enter(job, "undraining", now)
-    else:
+    _after_work(job, now)
+    return True
+
+
+def settle(job, now: datetime) -> bool:
+    """A drained node has had DRAIN_SETTLE for its tasks to move: on to the
+    update, or to the reboot when the update is done (reboot_needed is set)."""
+    if job.state != "settling" or job.state_at is None or now - job.state_at < DRAIN_SETTLE:
+        return False
+    _enter(job, "updating" if job.reboot_needed is None else "rebooting", now)
+    return True
+
+
+def recovered(job, running: set, now: datetime) -> bool:
+    """Done once every container that ran outside the swarm before the job
+    runs again (`running`: the host's containers seen running since the job
+    came to this state); failed, naming the rest, past RECOVER_LIMIT."""
+    if job.state != "recovering" or job.state_at is None:
+        return False
+    missing = [c for c in outside_swarm(job) if c["name"] not in running]
+    if not missing:
         _enter(job, "done", now)
+        return True
+    if now - job.state_at < RECOVER_LIMIT:
+        return False
+    names = [c["name"] + ("" if c["restart"] in RESTARTS else " (no restart policy)")
+             for c in missing[:6]]
+    more = f" and {len(missing) - 6} more" if len(missing) > 6 else ""
+    _finish(job, "failed", now, f"not running again {int(RECOVER_LIMIT.total_seconds() // 60)} min "
+                                f"after the job: {', '.join(names)}{more}")
     return True
 
 
@@ -163,10 +273,13 @@ def check_deadline(job, now: datetime) -> bool:
     if limit is None or since is None or now - since <= limit:
         return False
     waiting = {"queued": "the host's agent didn't check in",
-               "updating": "the update didn't finish",
+               "checking": "the check of what the update would change didn't finish",
                "draining": "the node wasn't drained",
+               "settling": "the wait after the drain didn't end",
+               "updating": "the update didn't finish",
                "rebooting": "the host didn't come back",
-               "undraining": "the node wasn't made active again"}[job.state]
+               "undraining": "the node wasn't made active again",
+               "recovering": "its containers weren't seen again"}[job.state]
     _finish(job, "timed_out", now, f"{waiting} within {int(limit.total_seconds() // 60)} min")
     return True
 
@@ -269,9 +382,11 @@ async def apply_events(session, host_id: str, events, now: datetime) -> int:
     return len(changed)
 
 
-async def handoffs(session, host_id: str, host_name: str, now: datetime) -> list[dict]:
+async def handoffs(session, host_id: str, host_name: str, now: datetime,
+                   agent_version: str | None = None) -> list[dict]:
     """The steps this host's agent should carry out now. A queued job for the
-    host starts here, when its agent is known to be listening."""
+    host starts here, when its agent is known to be listening, with a check
+    when the agent is new enough to hand one over."""
     from sqlalchemy import or_, select
     from .models import HostMaintenanceJob
     jobs = (await session.execute(select(HostMaintenanceJob).where(
@@ -282,7 +397,7 @@ async def handoffs(session, host_id: str, host_name: str, now: datetime) -> list
     out = []
     for job in jobs:
         if job.state == "queued" and job.host_id == host_id:
-            start(job, now)
+            start(job, now, check=can_check(agent_version))
             await _window(session, job, host_name, now)
         if executor(job) == host_id:
             step = handoff(job)
@@ -291,15 +406,36 @@ async def handoffs(session, host_id: str, host_name: str, now: datetime) -> list
     return out
 
 
+async def _running(session, job) -> set:
+    """The job's host's containers seen running since the job came to its
+    state; without the Docker plugin, the ones it waits for, as nothing can
+    say otherwise."""
+    from sqlalchemy import select
+    models = docker_models()
+    if models is None:
+        return {c["name"] for c in outside_swarm(job)}
+    DockerContainer = models.DockerContainer
+    return set((await session.execute(select(DockerContainer.name).where(
+        DockerContainer.host_id == job.host_id, DockerContainer.status == "running",
+        DockerContainer.scraped_at >= job.state_at))).scalars())
+
+
 async def tick(session, now: datetime) -> int:
-    """Give up on jobs past their deadline; returns how many."""
+    """Move on the jobs waiting on Steward (the wait after a drain, the
+    containers seen again) and give up on jobs past their deadline; returns
+    how many were given up."""
     from sqlalchemy import select
     from .models import HostMaintenanceJob
     gave_up = 0
     for job in (await session.execute(select(HostMaintenanceJob).where(
             HostMaintenanceJob.state.in_(ACTIVE)))).scalars():
+        moved = settle(job, now)
+        if job.state == "recovering":
+            moved = recovered(job, await _running(session, job), now) or moved
         if check_deadline(job, now):
             gave_up += 1
+            moved = True
+        if moved:
             await _window(session, job, await _host_name(session, job.host_id), now)
     return gave_up
 
@@ -344,14 +480,19 @@ async def plan_job(session, host, reg, reboot_policy: str = "if_required") -> tu
         HostMaintenanceJob.state.in_(ACTIVE)))).first()
     if busy is not None:
         raise CannotStart("a maintenance job is already running on it")
-    if reboot_policy == "never":
-        return None, None
     nodes, managers, availability = await _swarm(session)
     helpers = set((await session.execute(select(HostAgentRegistration.host_id).where(
         HostAgentRegistration.maint_helper.is_(True)))).scalars())
     node_id = nodes.get(host.name.lower())
-    return pick_drainer(nodes, managers, host.name, host.id, helpers,
-                        availability.get(node_id, "active"))
+    try:
+        return pick_drainer(nodes, managers, host.name, host.id, helpers,
+                            availability.get(node_id, "active"))
+    except CannotStart:
+        # A job that never reboots needs a drain only if Docker is upgraded;
+        # with no manager to drain, it runs undrained rather than not at all.
+        if reboot_policy == "never":
+            return None, None
+        raise
 
 
 async def create_job(session, host, reg, user_id: str | None,
@@ -383,10 +524,13 @@ async def latest_job(session, host_id: str, now: datetime, days: int = 366):
 
 LABELS = {
     "queued": "Waiting for the agent",
-    "updating": "Updating",
+    "checking": "Checking what the update will change",
     "draining": "Draining its swarm node",
+    "settling": "Giving its swarm tasks 5 min to move",
+    "updating": "Updating",
     "rebooting": "Rebooting",
     "undraining": "Making its swarm node active again",
+    "recovering": "Waiting for its containers to run again",
     "done": "Done",
     "failed": "Failed",
     "timed_out": "Timed out",
@@ -398,6 +542,8 @@ def summary(job) -> str:
     if job.state not in FINISHED:
         return LABELS[job.state]
     parts = []
+    if job.docker_update:
+        parts.append("Docker upgraded")
     if job.packages is not None:
         parts.append(f"{job.packages} package{'s' if job.packages != 1 else ''} updated")
     if job.back_at is not None:

plugins/host_agent/mcp_tools.py

@@ -118,6 +118,9 @@ def _job_row(job, names: dict, now) -> dict:
         "rebooted_at": iso(job.back_at),
         "swarm_node": job.node_id,
         "drained_by": names.get(job.drain_host_id, job.drain_host_id),
+        "drained_at": iso(job.drained_at),
+        "docker_update": job.docker_update,
+        "checked": job.checked,
         "detail": job.detail,
         "state_since": iso(job.state_at), "state_age_s": age_s(job.state_at, now),
         "gives_up_at": iso(started + deadline) if deadline and started else None,
@@ -192,11 +195,19 @@ async def get_maintenance_job(host: str = "", job: str = "", limit: int = 10) ->
     limit: how many of the host's jobs (default 10, max 50).
 
     A job is one host's update and, only when the update needs it, a reboot,
-    with its swarm node drained first and made active again after. `state`
-    moves queued → updating → [draining →] rebooting → [undraining →] done,
-    or failed / timed_out; `detail` says why a job failed. `summary` is the
-    host page's line ("Done: 12 packages updated, rebooted"). `gives_up_at`
-    is when an active job's current state times out.
+    with its swarm node drained first and made active again after. It first
+    checks what the update would change: `docker_update` is whether it
+    upgrades Docker (whose packages restart the daemon), and `checked` holds
+    {pending, docker_packages, containers}, the containers then running
+    outside the swarm. A swarm node is drained before an update that upgrades
+    Docker, else before a reboot; `drained_at` is when, and its tasks get 5
+    min to move (settling). `state` moves queued → [checking →] [draining →
+    settling →] updating → [draining → settling →] [rebooting →]
+    [undraining →] [recovering →] done, or failed / timed_out. recovering
+    waits up to 10 min for those containers to run again; `detail` says why
+    a job failed, naming any that didn't. `summary` is the host page's line
+    ("Done: 12 packages updated, rebooted"). `gives_up_at` is when an active
+    job's current state times out.
     """
     if bool(host.strip()) == bool(job.strip()):
         raise ValueError("pass either `host` (a name or id) or `job` (a job id).")

plugins/host_agent/migrations/versions/host_agent_010_maint_check.py

@@ -0,0 +1,32 @@
+# plugins/host_agent/migrations/versions/host_agent_010_maint_check.py
+"""host_agent: a maintenance job's check of what its update would change, and
+when it drained its node (#6098)
+
+Revision ID: host_agent_010_maint_check
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_010_maint_check"
+down_revision: Union[str, None] = "host_agent_009_drive_health"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("host_maintenance_jobs", sa.Column("docker_update", sa.Boolean(), nullable=True))
+    op.add_column("host_maintenance_jobs", sa.Column("checked", sa.JSON(), nullable=True))
+    op.add_column("host_maintenance_jobs",
+                  sa.Column("drained_at", sa.DateTime(timezone=True), nullable=True))
+    # A job already past its drain when this lands still makes its node
+    # active again: before this, every job that drained went on to undrain.
+    op.execute("UPDATE host_maintenance_jobs SET drained_at = state_at "
+               "WHERE state IN ('rebooting', 'undraining') "
+               "AND node_id IS NOT NULL AND drain_host_id IS NOT NULL")
+
+
+def downgrade() -> None:
+    op.drop_column("host_maintenance_jobs", "drained_at")
+    op.drop_column("host_maintenance_jobs", "checked")
+    op.drop_column("host_maintenance_jobs", "docker_update")

plugins/host_agent/models.py

@@ -114,6 +114,13 @@ class HostMaintenanceJob(Base):
     # The cadence run this job belongs to (#5768); None for Update now.
     run_id = Column(String(36), ForeignKey("host_maintenance_runs.id", ondelete="SET NULL"),
                     nullable=True, index=True)
+    # What the check said the update would change (#6098): whether it
+    # upgrades Docker, and {pending, docker_packages, containers} with the
+    # containers then running outside the swarm. drained_at is when the job's
+    # drain finished, before the update or before the reboot.
+    docker_update = Column(Boolean, nullable=True)
+    checked = Column(JSON, nullable=True)
+    drained_at = Column(DateTime(timezone=True), nullable=True)
 
 
 class HostMaintenancePlan(Base):

plugins/host_agent/routes.py

@@ -532,7 +532,8 @@ async def ingest():
                         await maintenance.apply_events(session, host.id, events, received_at)
                     if reg.maint_helper:
                         maint_steps = await maintenance.handoffs(
-                            session, host.id, host.name, received_at)
+                            session, host.id, host.name, received_at,
+                            payload.get("agent_version") or reg.agent_version)
             except Exception:
                 maint_steps = []
                 current_app.logger.exception(

plugins/host_agent/scheduler.py

@@ -147,8 +147,9 @@ def make_sensor_task(app) -> ScheduledTask:
 def make_maintenance_task(app) -> ScheduledTask:
     """Gives up on maintenance jobs past their deadline (#5767), starts due
     maintenance plans and moves their runs on one host at a time (#5768), and
-    records each host's maintenance_failed for an alert rule. Every move a job
-    itself makes comes from its agent's posts."""
+    records each host's maintenance_failed for an alert rule. A job's steps
+    move on from its agent's posts; the tick moves on only Steward's own waits
+    (the 5 minutes after a drain, the containers seen running again, #6098)."""
     async def _tick():
         from . import cadence, maintenance
         now = datetime.now(timezone.utc)

plugins/host_agent/templates/maintenance.html

@@ -20,9 +20,11 @@
 {{ page_header("Maintenance") }}
 <p style="color:var(--text-muted);font-size:0.84rem;margin:-0.4rem 0 1.25rem;">
   Each plan updates an inventory group's hosts once a week, one host at a time. A host reboots
-  only when its update needs it, unless the plan says otherwise; a swarm node is drained first
-  and made active again after. The next host starts once the last is back in service. A failure
-  stops the run, and the host's <em>Maintenance failed</em> figure can drive an alert rule.
+  only when its update needs it, unless the plan says otherwise. A swarm node is drained before
+  a reboot, or before the update when it upgrades Docker, given 5 minutes for its tasks to move,
+  and made active again after. Containers running outside the swarm must be running again
+  within 10 minutes. The next host starts once the last is back in service. A failure stops the
+  run, and the host's <em>Maintenance failed</em> figure can drive an alert rule.
 </p>
 
 {% if live %}

plugins/host_agent/templates/panel.html

@@ -93,7 +93,7 @@
     {% else %}
     <form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/maintain"
           style="display:flex;gap:0.6rem;align-items:center;flex-wrap:wrap;margin:0.6rem 0 0;"
-          data-confirm="Update {{ host.name }} now? It reboots only if the update needs it, after its swarm node is drained.">{{ csrf_field() }}
+          data-confirm="Update {{ host.name }} now? It reboots only if the update needs it. A swarm node is drained first when it reboots or Docker is upgraded.">{{ csrf_field() }}
       <button type="submit" class="btn btn-sm">Update now</button>
       <label style="font-size:0.8rem;color:var(--text-muted);display:flex;gap:0.3rem;align-items:center;">
         <input type="checkbox" name="reboot_policy" value="never"> never reboot