feat(swarm): services short, nodes down and managers lost are alertable (#5788)

On 2026-10-14 nine swarm services sat at 0/1 and nothing fired. From each
manager's swarm report the docker plugin now records three figures on
resource "swarm": swarm_services_short, swarm_nodes_down and
swarm_managers_unreachable. Drained nodes are left out of the last two,
since a drain is deliberate. The figures are recorded at most once every
30 s whichever manager reported, and are offered to alert rules.

Agent 1.19.0 sends each manager's ManagerStatus.Reachability. Older agents
fall back to the node's status.

A maintenance job on a swarm node also keeps a window over docker/swarm,
so its drain and reboot hold those alerts until the job ends.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/__init__.py

@@ -83,6 +83,10 @@ def get_alert_metrics() -> dict[str, list[str]]:
     """What ingest.py records per container (resource "<host>/<container>"):
     restart_count is cumulative (alert on crash-looping); is_healthy is 1.0 /
     0.0 from the container's HEALTHCHECK, only for containers that define one.
-    Per host (resource "<host>"), what Docker keeps on its disk (#5304)."""
+    Per host (resource "<host>"), what Docker keeps on its disk (#5304).
+    Swarm-wide (resource "swarm"), from any manager's report: services short
+    of their replicas, nodes down and managers unreachable, drained nodes
+    aside (#5788, swarm_health.py)."""
     from .disk_view import DISK_METRICS
-    return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy", *DISK_METRICS]}
+    from .swarm_health import SWARM_METRICS
+    return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy", *DISK_METRICS, *SWARM_METRICS]}

plugins/docker/ingest.py

@@ -292,6 +292,20 @@ async def _persist_status(session, host, status: dict) -> None:
     row.reported_at = status["at"]
 
 
+async def _record_swarm_health(session, swarm: dict) -> None:
+    """The swarm-wide figures alert rules watch (#5788), at most once every
+    RECORD_EVERY whichever manager reported."""
+    from datetime import timezone
+    from steward.core.alerts import MetricBatch
+    from . import swarm_health
+    if not swarm_health.due(datetime.now(timezone.utc)):
+        return
+    batch = MetricBatch()
+    for metric, value in swarm_health.figures(swarm).items():
+        batch.add(source_module="docker", resource_name=swarm_health.RESOURCE, metric_name=metric, value=value)
+    await batch.record(session)
+
+
 async def _expire_swarm(session, host) -> None:
     """Drop the swarm rows a host reported while it was a manager."""
     from .models import DockerSwarmNode, DockerSwarmService
@@ -338,6 +352,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
         await mark("status")
     if swarm is not None:
         await _persist_swarm(session, host, swarm)
+        await _record_swarm_health(session, swarm)
         await mark("swarm")
     if disk is not None:
         await _persist_disk(session, host, disk)

plugins/docker/swarm_health.py

@@ -0,0 +1,57 @@
+"""The swarm's health as three figures an alert rule can watch (#5788).
+
+Every manager's agent reports the whole swarm, so the figures are swarm-wide,
+on one resource, `swarm`, and are recorded at most once every RECORD_EVERY
+whichever manager sent them: three managers then read as one series, and one
+manager going silent doesn't silence it.
+
+- swarm_services_short: services running fewer tasks than they want. A
+  service scaled to 0 wants none and is never short.
+- swarm_nodes_down: nodes whose status isn't ready, other than drained ones.
+  A drained node is one taken out of service on purpose (VDNT-Util01 always;
+  any node during its maintenance), so its going away is expected. Its host's
+  own "Agent offline" rule still covers it.
+- swarm_managers_unreachable: managers the swarm can't reach (agent 1.19.0
+  sends ManagerStatus.Reachability; before that, a manager whose status
+  isn't ready), drained ones aside for the same reason. With three managers
+  one can be lost and the swarm keeps working; two, and it can't change.
+
+A maintenance job on a swarm node holds these rules' notifications in its
+window (host_agent maintenance._swarm_window), since draining and rebooting a
+node is exactly what moves them.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+RESOURCE = "swarm"
+SWARM_METRICS = ("swarm_services_short", "swarm_nodes_down", "swarm_managers_unreachable")
+RECORD_EVERY = timedelta(seconds=30)
+
+_last_recorded: datetime | None = None
+
+
+def figures(swarm: dict) -> dict[str, float]:
+    """The three figures from one manager's swarm report."""
+    services = swarm.get("services") or []
+    nodes = swarm.get("nodes") or []
+    short = sum(1 for s in services
+                if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0))
+    in_service = [n for n in nodes if (n.get("availability") or "active") != "drain"]
+    down = sum(1 for n in in_service if (n.get("status") or "unknown") != "ready")
+    managers = [n for n in in_service if n.get("role") == "manager"]
+    unreachable = sum(1 for n in managers
+                      if (n["reachability"] != "reachable" if n.get("reachability")
+                          else (n.get("status") or "unknown") != "ready"))
+    return {"swarm_services_short": float(short), "swarm_nodes_down": float(down),
+            "swarm_managers_unreachable": float(unreachable)}
+
+
+def due(now: datetime) -> bool:
+    """Whether this report should be recorded: the first in RECORD_EVERY,
+    whichever manager it came from."""
+    global _last_recorded
+    if _last_recorded is not None and now - _last_recorded < RECORD_EVERY:
+        return False
+    _last_recorded = now
+    return True