fix(maintenance): check first, drain before a Docker upgrade, wait 5 min after a drain, see containers back (#6098)

On 2026-10-18 an update on VDNT-Docker02 restarted the Docker daemon six
seconds in, before the drain that came only with the reboot, and every
container on the node stopped. Docker's own packages restart the daemon from
their install scripts; needrestart's list-only mode doesn't stop that.

- The helper gains a `check` action (agent 1.24.0): a dry run of the update
  for each package manager, whether Docker is among it, and the running
  containers outside the swarm with their restart policies.
- A swarm node is drained before an update that upgrades Docker, else only
  before a reboot, and at most once a job.
- After a drain the job settles 5 minutes before the update or reboot.
- Once the work is over, the containers that ran outside the swarm must be
  seen running again within 10 minutes, or the job fails naming them.
- An older agent gets no check and its job runs as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_host_maintenance_job.py

@@ -152,3 +152,71 @@ def test_a_swarm_node_s_job_also_holds_the_swarm_wide_alerts():
     rows, now = asyncio.run(_go())
     assert [(r[0], r[1]) for r in rows] == [("docker", "swarm"), ("host_agent", host_name)]
     assert all(r[2] == now + timedelta(minutes=2) for r in rows)     # ended with the job, not before
+
+
+def test_the_tick_ends_the_wait_after_a_drain_and_waits_for_the_containers():
+    """#6098: settling and recovering are Steward's own waits. The minute tick
+    moves a drained node on after 5 minutes, and finishes a job only once the
+    containers that ran outside the swarm before it are seen running again."""
+    import json
+    from sqlalchemy import text
+    from steward.core.plugin_manager import loaded_plugin_submodule
+    app = _app()
+    m = loaded_plugin_submodule("host_agent", "maintenance")
+
+    async def _go():
+        host_id, _token, job_id = await _seed(app)
+        now = datetime.now(timezone.utc)
+
+        async def _tick(at):
+            async with app.db_sessionmaker() as s:
+                async with s.begin():
+                    await m.tick(s, at)
+            return (await _job(app, job_id))[0]
+
+        async def _set(sql, **kw):
+            async with app.db_sessionmaker() as s:
+                async with s.begin():
+                    await s.execute(text(sql), {"id": job_id, "h": host_id, **kw})
+
+        await _set("UPDATE host_maintenance_jobs SET state = 'settling', state_at = :at, "
+                   "drained_at = :at WHERE id = :id", at=now)
+        assert await _tick(now + timedelta(minutes=4)) == "settling"
+        assert await _tick(now + timedelta(minutes=5)) == "updating"   # not updated yet: update next
+
+        checked = {"pending": 3, "docker_packages": [],
+                   "containers": [{"name": "ak-outpost", "restart": "unless-stopped"},
+                                  {"name": "scratch", "restart": "no"}]}
+        await _set("UPDATE host_maintenance_jobs SET state = 'recovering', state_at = :at, "
+                   "checked = CAST(:c AS JSON) WHERE id = :id", at=now, c=json.dumps(checked))
+        container = ("INSERT INTO docker_containers (host_id, name, container_id, image, status, "
+                     "restart_count, ports_json, scraped_at, oom_killed) "
+                     "VALUES (:h, :n, '', 'img:1', :st, 0, '[]', :at, false)")
+        await _set(container, n="ak-outpost", st="running", at=now + timedelta(seconds=30))
+        await _set(container, n="scratch", st="running", at=now - timedelta(minutes=20))  # before: stale
+        assert await _tick(now + timedelta(minutes=1)) == "recovering"
+        await _set("UPDATE docker_containers SET scraped_at = :at WHERE host_id = :h AND name = 'scratch'",
+                   at=now + timedelta(minutes=2))
+        assert await _tick(now + timedelta(minutes=2)) == "done"
+
+    asyncio.run(_go())
+
+
+def test_a_new_agent_is_handed_a_check_first():
+    app = _app()
+
+    async def _go():
+        from sqlalchemy import text
+        host_id, token, job_id = await _seed(app)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await s.execute(text("UPDATE host_agent_registrations SET agent_version = '1.24.0' "
+                                     "WHERE host_id = :h"), {"h": host_id})
+        assert await _post(app, token) == [{"id": f"{job_id}-check", "action": "check"}]
+        checked = {"kind": "maintenance", "id": f"{job_id}-check", "action": "check", "state": "done",
+                   "ok": True, "pending": 2, "docker_update": True, "docker_packages": ["docker-ce"],
+                   "containers": []}
+        # Not a swarm node: nothing to drain, so on to the update.
+        assert await _post(app, token, [checked]) == [{"id": f"{job_id}-update", "action": "update"}]
+
+    asyncio.run(_go())