feat(swarm): services short, nodes down and managers lost are alertable (#5788)

On 2026-10-14 nine swarm services sat at 0/1 and nothing fired. From each
manager's swarm report the docker plugin now records three figures on
resource "swarm": swarm_services_short, swarm_nodes_down and
swarm_managers_unreachable. Drained nodes are left out of the last two,
since a drain is deliberate. The figures are recorded at most once every
30 s whichever manager reported, and are offered to alert rules.

Agent 1.19.0 sends each manager's ManagerStatus.Reachability. Older agents
fall back to the node's status.

A maintenance job on a swarm node also keeps a window over docker/swarm,
so its drain and reboot hold those alerts until the job ends.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/__init__.py

@@ -83,6 +83,10 @@ def get_alert_metrics() -> dict[str, list[str]]:
     """What ingest.py records per container (resource "<host>/<container>"):
     restart_count is cumulative (alert on crash-looping); is_healthy is 1.0 /
     0.0 from the container's HEALTHCHECK, only for containers that define one.
-    Per host (resource "<host>"), what Docker keeps on its disk (#5304)."""
+    Per host (resource "<host>"), what Docker keeps on its disk (#5304).
+    Swarm-wide (resource "swarm"), from any manager's report: services short
+    of their replicas, nodes down and managers unreachable, drained nodes
+    aside (#5788, swarm_health.py)."""
     from .disk_view import DISK_METRICS
-    return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy", *DISK_METRICS]}
+    from .swarm_health import SWARM_METRICS
+    return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy", *DISK_METRICS, *SWARM_METRICS]}

plugins/docker/ingest.py

@@ -292,6 +292,20 @@ async def _persist_status(session, host, status: dict) -> None:
     row.reported_at = status["at"]
 
 
+async def _record_swarm_health(session, swarm: dict) -> None:
+    """The swarm-wide figures alert rules watch (#5788), at most once every
+    RECORD_EVERY whichever manager reported."""
+    from datetime import timezone
+    from steward.core.alerts import MetricBatch
+    from . import swarm_health
+    if not swarm_health.due(datetime.now(timezone.utc)):
+        return
+    batch = MetricBatch()
+    for metric, value in swarm_health.figures(swarm).items():
+        batch.add(source_module="docker", resource_name=swarm_health.RESOURCE, metric_name=metric, value=value)
+    await batch.record(session)
+
+
 async def _expire_swarm(session, host) -> None:
     """Drop the swarm rows a host reported while it was a manager."""
     from .models import DockerSwarmNode, DockerSwarmService
@@ -338,6 +352,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
         await mark("status")
     if swarm is not None:
         await _persist_swarm(session, host, swarm)
+        await _record_swarm_health(session, swarm)
         await mark("swarm")
     if disk is not None:
         await _persist_disk(session, host, disk)

plugins/docker/swarm_health.py

@@ -0,0 +1,57 @@
+"""The swarm's health as three figures an alert rule can watch (#5788).
+
+Every manager's agent reports the whole swarm, so the figures are swarm-wide,
+on one resource, `swarm`, and are recorded at most once every RECORD_EVERY
+whichever manager sent them: three managers then read as one series, and one
+manager going silent doesn't silence it.
+
+- swarm_services_short: services running fewer tasks than they want. A
+  service scaled to 0 wants none and is never short.
+- swarm_nodes_down: nodes whose status isn't ready, other than drained ones.
+  A drained node is one taken out of service on purpose (VDNT-Util01 always;
+  any node during its maintenance), so its going away is expected. Its host's
+  own "Agent offline" rule still covers it.
+- swarm_managers_unreachable: managers the swarm can't reach (agent 1.19.0
+  sends ManagerStatus.Reachability; before that, a manager whose status
+  isn't ready), drained ones aside for the same reason. With three managers
+  one can be lost and the swarm keeps working; two, and it can't change.
+
+A maintenance job on a swarm node holds these rules' notifications in its
+window (host_agent maintenance._swarm_window), since draining and rebooting a
+node is exactly what moves them.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+RESOURCE = "swarm"
+SWARM_METRICS = ("swarm_services_short", "swarm_nodes_down", "swarm_managers_unreachable")
+RECORD_EVERY = timedelta(seconds=30)
+
+_last_recorded: datetime | None = None
+
+
+def figures(swarm: dict) -> dict[str, float]:
+    """The three figures from one manager's swarm report."""
+    services = swarm.get("services") or []
+    nodes = swarm.get("nodes") or []
+    short = sum(1 for s in services
+                if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0))
+    in_service = [n for n in nodes if (n.get("availability") or "active") != "drain"]
+    down = sum(1 for n in in_service if (n.get("status") or "unknown") != "ready")
+    managers = [n for n in in_service if n.get("role") == "manager"]
+    unreachable = sum(1 for n in managers
+                      if (n["reachability"] != "reachable" if n.get("reachability")
+                          else (n.get("status") or "unknown") != "ready"))
+    return {"swarm_services_short": float(short), "swarm_nodes_down": float(down),
+            "swarm_managers_unreachable": float(unreachable)}
+
+
+def due(now: datetime) -> bool:
+    """Whether this report should be recorded: the first in RECORD_EVERY,
+    whichever manager it came from."""
+    global _last_recorded
+    if _last_recorded is not None and now - _last_recorded < RECORD_EVERY:
+        return False
+    _last_recorded = now
+    return True

plugins/host_agent/agent.py

@@ -22,7 +22,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.18.0"
+AGENT_VERSION = "1.19.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -1170,7 +1170,10 @@ def _swarm_services(services: list, tasks: list) -> list:
 
 
 def _swarm_nodes(nodes: list) -> list:
-    """Normalise node role / availability / status (+ manager leader flag)."""
+    """Normalise node role / availability / status (+ manager leader flag and,
+    from 1.19.0, whether the swarm can reach a manager: "reachable",
+    "unreachable" or "unknown"; "" for a worker), so Steward can count the
+    managers the swarm has lost (#5788)."""
     out = []
     for n in (nodes or []):
         spec = n.get("Spec") or {}
@@ -1184,6 +1187,7 @@ def _swarm_nodes(nodes: list) -> list:
             "availability": (spec.get("Availability") or "active").lower(),
             "status": (status.get("State") or "unknown").lower(),
             "leader": bool(mgr.get("Leader", False)),
+            "reachability": (mgr.get("Reachability") or "").lower(),
         })
     return out
 

plugins/host_agent/maintenance.py

@@ -188,21 +188,47 @@ def utcnow() -> datetime:
 
 async def _window(session, job, host_name: str, now: datetime) -> None:
     """Keep the job's maintenance window over the host until the job ends: one
-    scoped to this host's agent metrics, so an alert the job causes waits."""
+    scoped to this host's agent metrics, so an alert the job causes waits.
+    A swarm node's job keeps a second over the swarm-wide figures (#5788):
+    draining and rebooting a node takes it down and can leave a service
+    pinned to it short, which is the job, not news."""
     from steward.models.alerts import MaintenanceWindow
     if job.window_id:
         w = await session.get(MaintenanceWindow, job.window_id)
         if w is not None:
             w.end_at = max(w.start_at, window_end(job, now))
-        return
-    if job.created_by is None or job.state in FINISHED:
+    elif job.created_by is None or job.state in FINISHED:
         return                          # a window needs an owner; none for a finished job
-    w = MaintenanceWindow(name=f"Maintenance: {host_name}"[:128], start_at=now,
-                          end_at=window_end(job, now), scope_source="host_agent",
-                          scope_resource=host_name, created_by=job.created_by)
-    session.add(w)
-    await session.flush()
-    job.window_id = w.id
+    else:
+        w = MaintenanceWindow(name=f"Maintenance: {host_name}"[:128], start_at=now,
+                              end_at=window_end(job, now), scope_source="host_agent",
+                              scope_resource=host_name, created_by=job.created_by)
+        session.add(w)
+        await session.flush()
+        job.window_id = w.id
+    if job.node_id:
+        await _swarm_window(session, job, host_name, now)
+
+
+SWARM_SCOPE = ("docker", "swarm")       # the swarm-wide figures' source and resource
+
+
+async def _swarm_window(session, job, host_name: str, now: datetime) -> None:
+    """The swarm half of a swarm node's job window, found again by its name
+    and the job's start, so it needs no column of its own."""
+    from sqlalchemy import select
+    from steward.models.alerts import MaintenanceWindow
+    name = f"Maintenance: {host_name} (swarm)"[:128]
+    w = (await session.execute(select(MaintenanceWindow).where(
+        MaintenanceWindow.name == name, MaintenanceWindow.scope_source == SWARM_SCOPE[0],
+        MaintenanceWindow.scope_resource == SWARM_SCOPE[1],
+        MaintenanceWindow.start_at >= job.created_at))).scalars().first()
+    if w is not None:
+        w.end_at = max(w.start_at, window_end(job, now))
+    elif job.created_by is not None and job.state not in FINISHED:
+        session.add(MaintenanceWindow(name=name, start_at=now, end_at=window_end(job, now),
+                                      scope_source=SWARM_SCOPE[0], scope_resource=SWARM_SCOPE[1],
+                                      created_by=job.created_by))
 
 
 async def _host_name(session, host_id: str) -> str: