feat(swarm): services short, nodes down and managers lost are alertable (#5788)

On 2026-10-14 nine swarm services sat at 0/1 and nothing fired. From each
manager's swarm report the docker plugin now records three figures on
resource "swarm": swarm_services_short, swarm_nodes_down and
swarm_managers_unreachable. Drained nodes are left out of the last two,
since a drain is deliberate. The figures are recorded at most once every
30 s whichever manager reported, and are offered to alert rules.

Agent 1.19.0 sends each manager's ManagerStatus.Reachability. Older agents
fall back to the node's status.

A maintenance job on a swarm node also keeps a window over docker/swarm,
so its drain and reboot hold those alerts until the job ends.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/agent.py

@@ -22,7 +22,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.18.0"
+AGENT_VERSION = "1.19.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -1170,7 +1170,10 @@ def _swarm_services(services: list, tasks: list) -> list:
 
 
 def _swarm_nodes(nodes: list) -> list:
-    """Normalise node role / availability / status (+ manager leader flag)."""
+    """Normalise node role / availability / status (+ manager leader flag and,
+    from 1.19.0, whether the swarm can reach a manager: "reachable",
+    "unreachable" or "unknown"; "" for a worker), so Steward can count the
+    managers the swarm has lost (#5788)."""
     out = []
     for n in (nodes or []):
         spec = n.get("Spec") or {}
@@ -1184,6 +1187,7 @@ def _swarm_nodes(nodes: list) -> list:
             "availability": (spec.get("Availability") or "active").lower(),
             "status": (status.get("State") or "unknown").lower(),
             "leader": bool(mgr.get("Leader", False)),
+            "reachability": (mgr.get("Reachability") or "").lower(),
         })
     return out
 

plugins/host_agent/maintenance.py

@@ -188,21 +188,47 @@ def utcnow() -> datetime:
 
 async def _window(session, job, host_name: str, now: datetime) -> None:
     """Keep the job's maintenance window over the host until the job ends: one
-    scoped to this host's agent metrics, so an alert the job causes waits."""
+    scoped to this host's agent metrics, so an alert the job causes waits.
+    A swarm node's job keeps a second over the swarm-wide figures (#5788):
+    draining and rebooting a node takes it down and can leave a service
+    pinned to it short, which is the job, not news."""
     from steward.models.alerts import MaintenanceWindow
     if job.window_id:
         w = await session.get(MaintenanceWindow, job.window_id)
         if w is not None:
             w.end_at = max(w.start_at, window_end(job, now))
-        return
-    if job.created_by is None or job.state in FINISHED:
+    elif job.created_by is None or job.state in FINISHED:
         return                          # a window needs an owner; none for a finished job
-    w = MaintenanceWindow(name=f"Maintenance: {host_name}"[:128], start_at=now,
-                          end_at=window_end(job, now), scope_source="host_agent",
-                          scope_resource=host_name, created_by=job.created_by)
-    session.add(w)
-    await session.flush()
-    job.window_id = w.id
+    else:
+        w = MaintenanceWindow(name=f"Maintenance: {host_name}"[:128], start_at=now,
+                              end_at=window_end(job, now), scope_source="host_agent",
+                              scope_resource=host_name, created_by=job.created_by)
+        session.add(w)
+        await session.flush()
+        job.window_id = w.id
+    if job.node_id:
+        await _swarm_window(session, job, host_name, now)
+
+
+SWARM_SCOPE = ("docker", "swarm")       # the swarm-wide figures' source and resource
+
+
+async def _swarm_window(session, job, host_name: str, now: datetime) -> None:
+    """The swarm half of a swarm node's job window, found again by its name
+    and the job's start, so it needs no column of its own."""
+    from sqlalchemy import select
+    from steward.models.alerts import MaintenanceWindow
+    name = f"Maintenance: {host_name} (swarm)"[:128]
+    w = (await session.execute(select(MaintenanceWindow).where(
+        MaintenanceWindow.name == name, MaintenanceWindow.scope_source == SWARM_SCOPE[0],
+        MaintenanceWindow.scope_resource == SWARM_SCOPE[1],
+        MaintenanceWindow.start_at >= job.created_at))).scalars().first()
+    if w is not None:
+        w.end_at = max(w.start_at, window_end(job, now))
+    elif job.created_by is not None and job.state not in FINISHED:
+        session.add(MaintenanceWindow(name=name, start_at=now, end_at=window_end(job, now),
+                                      scope_source=SWARM_SCOPE[0], scope_resource=SWARM_SCOPE[1],
+                                      created_by=job.created_by))
 
 
 async def _host_name(session, host_id: str) -> str: