fix(maintenance): check first, drain before a Docker upgrade, wait 5 min after a drain, see containers back (#6098)

On 2026-10-18 an update on VDNT-Docker02 restarted the Docker daemon six
seconds in, before the drain that came only with the reboot, and every
container on the node stopped. Docker's own packages restart the daemon from
their install scripts; needrestart's list-only mode doesn't stop that.

- The helper gains a `check` action (agent 1.24.0): a dry run of the update
  for each package manager, whether Docker is among it, and the running
  containers outside the swarm with their restart policies.
- A swarm node is drained before an update that upgrades Docker, else only
  before a reboot, and at most once a job.
- After a drain the job settles 5 minutes before the update or reboot.
- Once the work is over, the containers that ran outside the swarm must be
  seen running again within 10 minutes, or the job fails naming them.
- An older agent gets no check and its job runs as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/files/maint.py

@@ -12,7 +12,9 @@ make its host update, reboot or drain, and nothing else.
 
 One action only reads: `drives` (#5789) asks smartctl for each disk's health,
 which needs root, and writes a summary to DRIVES, never a result: it is not a
-maintenance job.
+maintenance job. `check` (#6098) changes nothing but the package lists: it
+says what an update would upgrade, whether Docker is among it, and which
+containers run outside the swarm.
 
 Stdlib only; served by Steward at /plugins/host_agent/files/maint.py.
 """
@@ -30,12 +32,14 @@ from datetime import datetime, timezone
 REQUESTS = "/var/lib/steward-agent/requests"
 RESULTS = "/var/lib/steward-maint/results"
 DRIVES = "/var/lib/steward-maint/drives.json"
-ACTIONS = ("update", "reboot", "drain", "undrain", "drives")
+ACTIONS = ("check", "update", "reboot", "drain", "undrain", "drives")
 REQUEST_MAX_BYTES = 4096
 RESULTS_KEPT_SECONDS = 7 * 86400
 UPDATE_DEADLINE_SECONDS = 3000
 COMMAND_DEADLINE_SECONDS = 600
 OUTPUT_KEPT_CHARS = 2000
+# A listing of what an update would change is read whole, not just its tail.
+LISTING_KEPT_CHARS = 1_000_000
 _ID = re.compile(r"^[A-Za-z0-9-]{1,64}$")
 _NODE = re.compile(r"^[a-z0-9]{25}$")
 
@@ -77,9 +81,10 @@ def valid(req) -> bool:
 
 
 def run(cmd: list, deadline: int = COMMAND_DEADLINE_SECONDS, env: dict | None = None,
-        ok_codes: tuple = (0,)):
+        ok_codes: tuple = (0,), keep: int = OUTPUT_KEPT_CHARS):
     """(ok, output tail) of a command, bounded by `deadline`. `ok_codes` are
-    the exit codes that mean it worked (zypper says more than 0)."""
+    the exit codes that mean it worked (zypper says more than 0); `keep` is
+    how much of the output's end is kept."""
     try:
         done = subprocess.run(cmd, capture_output=True, text=True, timeout=deadline,
                               env=dict(os.environ, LANG="C", LC_ALL="C", **(env or {})))
@@ -88,7 +93,7 @@ def run(cmd: list, deadline: int = COMMAND_DEADLINE_SECONDS, env: dict | None =
     except OSError as exc:
         return False, str(exc)
     out = (done.stdout or "") + (done.stderr or "")
-    return done.returncode in ok_codes, out[-OUTPUT_KEPT_CHARS:]
+    return done.returncode in ok_codes, out[-keep:]
 
 
 REBOOT_MARKER = "/var/run/reboot-required"                 # Debian and Ubuntu
@@ -152,9 +157,10 @@ def _os_release_id(path: str = "/etc/os-release") -> str:
 def update_steps(manager: str) -> list:
     """The commands an update runs on a host with `manager`, in order: each
     (what it is, command, deadline, env, ok exit codes). The last one is the upgrade
-    whose output is counted. Existing config files are kept, and service
-    restarts are left to a reboot: needrestart only lists, so the Docker
-    daemon isn't restarted under its containers mid-upgrade."""
+    whose output is counted. Existing config files are kept, and needrestart
+    only lists, leaving service restarts to a reboot. A package's own install
+    script can still restart its service: Docker's does, which is why the job
+    checks first and drains a swarm node before such an update (#6098)."""
     long = UPDATE_DEADLINE_SECONDS
     if manager == "apt":
         env = {"DEBIAN_FRONTEND": "noninteractive", "NEEDRESTART_MODE": "l"}
@@ -210,6 +216,116 @@ def do_update() -> dict:
             "detail": out[-500:]}
 
 
+# ── what an update would change, and what runs outside the swarm (#6098) ─────
+# Docker's own packages restart the daemon from their install scripts, whatever
+# needrestart is told, and every container on the host stops with it. So the
+# job asks first: a swarm node is drained before an update that upgrades
+# Docker, not after it, and the containers running outside the swarm are
+# noted, to see each of them running again once the job is over.
+
+DOCKER_PACKAGES = re.compile(
+    r"^(docker|docker-ce|docker\.io|docker-engine|moby-engine|containerd|containerd\.io)$")
+CONTAINERS_MAX = 40
+CONTAINER_NAME_MAX = 64
+
+
+def pending_steps(manager: str) -> list:
+    """The commands that list what an update would upgrade, changing nothing
+    but the package lists: each (what it is, command, ok exit codes). The
+    last one's output names the packages."""
+    if manager == "apt":
+        return [("apt-get update", ["apt-get", "update"], (0,)),
+                ("apt-get -s dist-upgrade", ["apt-get", "-s", "dist-upgrade"], (0,))]
+    if manager == "dnf":
+        # check-update exits 100 when there is something to update.
+        return [("dnf check-update", ["dnf", "-q", "check-update", "--refresh"], (0, 100))]
+    if manager == "yum":
+        return [("yum check-update", ["yum", "-q", "check-update"], (0, 100))]
+    if manager == "zypper":
+        if _os_release_id() == "opensuse-tumbleweed":
+            listing = ("zypper dup --dry-run", ["zypper", "--non-interactive", "dup", "--dry-run",
+                                                "--auto-agree-with-licenses"])
+        else:
+            listing = ("zypper list-updates", ["zypper", "--non-interactive", "list-updates"])
+        return [("zypper refresh", ["zypper", "--non-interactive", "refresh"], ZYPPER_OK),
+                (*listing, ZYPPER_OK)]
+    if manager == "pacman":
+        # -Sy refreshes the lists only; the update's -Syu follows it. -Qu exits
+        # 1 when there is nothing to upgrade.
+        return [("pacman -Sy", ["pacman", "-Sy"], (0,)), ("pacman -Qu", ["pacman", "-Qu"], (0, 1))]
+    return []
+
+
+def pending_names(manager: str, output: str) -> list:
+    """The packages a listing from pending_steps says would be upgraded."""
+    if manager == "apt":
+        return re.findall(r"^Inst (\S+)", output, re.MULTILINE)
+    if manager in ("dnf", "yum"):
+        # "docker-ce.x86_64   3:27.1.1-1.el9   docker-ce-stable"
+        return [parts[0].rsplit(".", 1)[0] for parts in (line.split() for line in output.splitlines())
+                if len(parts) == 3 and "." in parts[0]]
+    if manager == "zypper":
+        names, column, listing = [], None, False
+        for line in output.splitlines():
+            cells = [c.strip() for c in line.split("|")]
+            if len(cells) > 2 and column is None and "Name" in cells:
+                column = cells.index("Name")                   # list-updates' table
+            elif column is not None and len(cells) > column and cells[0] == "v":
+                names.append(cells[column])
+            elif re.search(r"going to be (upgraded|installed)", line):
+                listing = True                                 # dup --dry-run's lists
+            elif listing and line.startswith(" ") and line.strip():
+                names.extend(line.split())
+            else:
+                listing = False
+        return names
+    if manager == "pacman":
+        return [line.split()[0] for line in output.splitlines() if line.strip()]
+    return []
+
+
+def standalone_containers() -> list:
+    """The running containers that aren't swarm tasks, each with its restart
+    policy, by name; at most CONTAINERS_MAX, so the result stays small."""
+    if not shutil.which("docker"):
+        return []
+    ok, out = run(["docker", "ps", "-q", "--no-trunc"], deadline=30, keep=LISTING_KEPT_CHARS)
+    ids = out.split() if ok else []
+    if not ids:
+        return []
+    fmt = '{{.Name}}|{{.HostConfig.RestartPolicy.Name}}|{{index .Config.Labels "com.docker.swarm.task.id"}}'
+    ok, out = run(["docker", "inspect", "--format", fmt, *ids], deadline=60, keep=LISTING_KEPT_CHARS)
+    if not ok:
+        return []
+    found = []
+    for line in out.splitlines():
+        parts = line.split("|")
+        if len(parts) != 3 or parts[2].strip():
+            continue                                    # a swarm task: the drain moves it
+        found.append({"name": parts[0].lstrip("/")[:CONTAINER_NAME_MAX],
+                      "restart": parts[1].strip() or "no"})
+    return sorted(found, key=lambda c: c["name"])[:CONTAINERS_MAX]
+
+
+def do_check() -> dict:
+    """What an update would upgrade, whether Docker is among it, and the
+    containers running outside the swarm. Only the package lists change."""
+    manager = host_manager()
+    if manager is None:
+        return {"ok": False, "detail": "no package manager this helper knows "
+                                       "(apt, dnf, yum, zypper, pacman) on this host"}
+    env = {"DEBIAN_FRONTEND": "noninteractive"} if manager == "apt" else None
+    out = ""
+    for label, cmd, ok_codes in pending_steps(manager):
+        ok, out = run(cmd, env=env, ok_codes=ok_codes, keep=LISTING_KEPT_CHARS)
+        if not ok:
+            return {"ok": False, "detail": f"{label} failed: " + out[-500:]}
+    names = pending_names(manager, out)
+    docker = sorted({n for n in names if DOCKER_PACKAGES.match(n)})
+    return {"ok": True, "pending": len(names), "docker_update": bool(docker),
+            "docker_packages": docker[:10], "containers": standalone_containers()}
+
+
 def own_node() -> str | None:
     ok, out = run(["docker", "info", "--format", "{{.Swarm.NodeID}}"], deadline=30)
     node = out.strip() if ok else ""
@@ -387,7 +503,9 @@ def handle(req: dict) -> dict:
         return {**base, "state": "done"}
     write_result({**base, "state": "running"})
     action = req["action"]
-    if action == "update":
+    if action == "check":
+        outcome = do_check()
+    elif action == "update":
         outcome = do_update()
     elif action in ("drain", "undrain"):
         outcome = do_node("drain" if action == "drain" else "active", req.get("node"))