fix(host_agent): a node the operator keeps drained stays drained through its maintenance (#5777)

#5777 "Maintenance would make a node the operator keeps drained active after
its reboot". A swarm node already set to drain or pause, such as a manager
kept free of work, is no longer drained before its reboot or made active
after it. Its job only updates and reboots it, and back-in-service needs it
ready but not active.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/maintenance.py

@@ -283,14 +283,18 @@ class CannotStart(Exception):
 
 
 def pick_drainer(node_hostname_to_id: dict, managers: dict, host_name: str,
-                 host_id: str, helpers: set) -> tuple:
+                 host_id: str, helpers: set, availability: str = "active") -> tuple:
     """(node_id, drain_host_id) for a host in a swarm, (None, None) for one
     that isn't. `managers` maps a manager's host id to its swarm node id;
     `helpers` is the hosts whose agent has the maintenance helper. The host
-    drains itself when it is a manager, else another manager does."""
+    drains itself when it is a manager, else another manager does. A node the
+    operator already set to drain or pause (a manager kept free of work) is
+    left as it is: no drain before, and no making it active after."""
     node_id = node_hostname_to_id.get(host_name.lower())
     if node_id is None:
         return None, None
+    if availability != "active":
+        return node_id, None
     if host_id in managers and host_id in helpers:
         return node_id, host_id
     for mgr in sorted(managers):
@@ -316,10 +320,12 @@ async def plan_job(session, host, reg, reboot_policy: str = "if_required") -> tu
         raise CannotStart("a maintenance job is already running on it")
     if reboot_policy == "never":
         return None, None
-    nodes, managers = await _swarm(session)
+    nodes, managers, availability = await _swarm(session)
     helpers = set((await session.execute(select(HostAgentRegistration.host_id).where(
         HostAgentRegistration.maint_helper.is_(True)))).scalars())
-    return pick_drainer(nodes, managers, host.name, host.id, helpers)
+    node_id = nodes.get(host.name.lower())
+    return pick_drainer(nodes, managers, host.name, host.id, helpers,
+                        availability.get(node_id, "active"))
 
 
 async def create_job(session, host, reg, user_id: str | None,
@@ -394,12 +400,13 @@ def docker_models():
 
 
 async def _swarm(session) -> tuple:
-    """({node hostname: node id}, {manager host id: its node id}) from what the
-    Docker plugin last stored; empty when it isn't installed. A manager is a
-    host whose agent reported the swarm's nodes."""
+    """({node hostname: node id}, {manager host id: its node id},
+    {node id: availability}) from what the Docker plugin last stored; empty
+    when it isn't installed. A manager is a host whose agent reported the
+    swarm's nodes."""
     models = docker_models()
     if models is None:
-        return {}, {}
+        return {}, {}, {}
     DockerSwarmNode = models.DockerSwarmNode
     from sqlalchemy import select
     from steward.models.hosts import Host
@@ -409,4 +416,6 @@ async def _swarm(session) -> tuple:
     names = dict((await session.execute(select(Host.id, Host.name).where(
         Host.id.in_(reporters)))).all()) if reporters else {}
     managers = {hid: nodes.get(name.lower()) for hid, name in names.items()}
-    return nodes, managers
+    availability = {n.node_id: (n.availability or "active")
+                    for n in sorted(rows, key=lambda n: n.updated_at)}
+    return nodes, managers, availability