feat(swarm): services short, nodes down and managers lost are alertable (#5788)

On 2026-10-14 nine swarm services sat at 0/1 and nothing fired. From each
manager's swarm report the docker plugin now records three figures on
resource "swarm": swarm_services_short, swarm_nodes_down and
swarm_managers_unreachable. Drained nodes are left out of the last two,
since a drain is deliberate. The figures are recorded at most once every
30 s whichever manager reported, and are offered to alert rules.

Agent 1.19.0 sends each manager's ManagerStatus.Reachability. Older agents
fall back to the node's status.

A maintenance job on a swarm node also keeps a window over docker/swarm,
so its drain and reboot hold those alerts until the job ends.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/plugins/host-agent-design.md

@@ -252,6 +252,20 @@ Maintenance** (`/plugins/host_agent/maintenance`).
   last week, then each plan with its schedule, next slot and last five runs.
   Each host's Agent panel shows its last maintenance job.
 
+### The swarm's health, alertable (1.19.0, #5788)
+
+Every manager's agent reports the whole swarm. From each report the docker plugin (`plugins/docker/swarm_health.py`) records three swarm-wide figures on resource `swarm`, source `docker`:
+
+- `swarm_services_short`: services running fewer tasks than they want. A service scaled to 0 is never short.
+- `swarm_nodes_down`: nodes not `ready`.
+- `swarm_managers_unreachable`: managers the swarm reports as not `reachable`. Agent 1.19.0 sends `ManagerStatus.Reachability`; before that, a manager whose status isn't `ready` counts.
+
+Drained nodes are left out of the last two. A node is drained on purpose (VDNT-Util01 always, any node during its maintenance), so its going away is expected, and its host's own "Agent offline" rule still covers it.
+
+The figures are recorded at most once every 30 s, whichever manager sent them, so three managers make one series and losing one doesn't silence it.
+
+A maintenance job on a swarm node opens a second window over `docker` / `swarm` beside its host's (`maintenance._swarm_window`). An alert the drain or reboot causes then waits for the job, and one still firing when the window ends is sent then.
+
 ### What happened to the host (1.14.0, #5756)
 
 A post may carry `events`, said once each, with the next post that gets through:

plugins/docker/__init__.py

@@ -83,6 +83,10 @@ def get_alert_metrics() -> dict[str, list[str]]:
     """What ingest.py records per container (resource "<host>/<container>"):
     restart_count is cumulative (alert on crash-looping); is_healthy is 1.0 /
     0.0 from the container's HEALTHCHECK, only for containers that define one.
-    Per host (resource "<host>"), what Docker keeps on its disk (#5304)."""
+    Per host (resource "<host>"), what Docker keeps on its disk (#5304).
+    Swarm-wide (resource "swarm"), from any manager's report: services short
+    of their replicas, nodes down and managers unreachable, drained nodes
+    aside (#5788, swarm_health.py)."""
     from .disk_view import DISK_METRICS
-    return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy", *DISK_METRICS]}
+    from .swarm_health import SWARM_METRICS
+    return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy", *DISK_METRICS, *SWARM_METRICS]}

plugins/docker/ingest.py

@@ -292,6 +292,20 @@ async def _persist_status(session, host, status: dict) -> None:
     row.reported_at = status["at"]
 
 
+async def _record_swarm_health(session, swarm: dict) -> None:
+    """The swarm-wide figures alert rules watch (#5788), at most once every
+    RECORD_EVERY whichever manager reported."""
+    from datetime import timezone
+    from steward.core.alerts import MetricBatch
+    from . import swarm_health
+    if not swarm_health.due(datetime.now(timezone.utc)):
+        return
+    batch = MetricBatch()
+    for metric, value in swarm_health.figures(swarm).items():
+        batch.add(source_module="docker", resource_name=swarm_health.RESOURCE, metric_name=metric, value=value)
+    await batch.record(session)
+
+
 async def _expire_swarm(session, host) -> None:
     """Drop the swarm rows a host reported while it was a manager."""
     from .models import DockerSwarmNode, DockerSwarmService
@@ -338,6 +352,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
         await mark("status")
     if swarm is not None:
         await _persist_swarm(session, host, swarm)
+        await _record_swarm_health(session, swarm)
         await mark("swarm")
     if disk is not None:
         await _persist_disk(session, host, disk)

plugins/docker/swarm_health.py

@@ -0,0 +1,57 @@
+"""The swarm's health as three figures an alert rule can watch (#5788).
+
+Every manager's agent reports the whole swarm, so the figures are swarm-wide,
+on one resource, `swarm`, and are recorded at most once every RECORD_EVERY
+whichever manager sent them: three managers then read as one series, and one
+manager going silent doesn't silence it.
+
+- swarm_services_short: services running fewer tasks than they want. A
+  service scaled to 0 wants none and is never short.
+- swarm_nodes_down: nodes whose status isn't ready, other than drained ones.
+  A drained node is one taken out of service on purpose (VDNT-Util01 always;
+  any node during its maintenance), so its going away is expected. Its host's
+  own "Agent offline" rule still covers it.
+- swarm_managers_unreachable: managers the swarm can't reach (agent 1.19.0
+  sends ManagerStatus.Reachability; before that, a manager whose status
+  isn't ready), drained ones aside for the same reason. With three managers
+  one can be lost and the swarm keeps working; two, and it can't change.
+
+A maintenance job on a swarm node holds these rules' notifications in its
+window (host_agent maintenance._swarm_window), since draining and rebooting a
+node is exactly what moves them.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+RESOURCE = "swarm"
+SWARM_METRICS = ("swarm_services_short", "swarm_nodes_down", "swarm_managers_unreachable")
+RECORD_EVERY = timedelta(seconds=30)
+
+_last_recorded: datetime | None = None
+
+
+def figures(swarm: dict) -> dict[str, float]:
+    """The three figures from one manager's swarm report."""
+    services = swarm.get("services") or []
+    nodes = swarm.get("nodes") or []
+    short = sum(1 for s in services
+                if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0))
+    in_service = [n for n in nodes if (n.get("availability") or "active") != "drain"]
+    down = sum(1 for n in in_service if (n.get("status") or "unknown") != "ready")
+    managers = [n for n in in_service if n.get("role") == "manager"]
+    unreachable = sum(1 for n in managers
+                      if (n["reachability"] != "reachable" if n.get("reachability")
+                          else (n.get("status") or "unknown") != "ready"))
+    return {"swarm_services_short": float(short), "swarm_nodes_down": float(down),
+            "swarm_managers_unreachable": float(unreachable)}
+
+
+def due(now: datetime) -> bool:
+    """Whether this report should be recorded: the first in RECORD_EVERY,
+    whichever manager it came from."""
+    global _last_recorded
+    if _last_recorded is not None and now - _last_recorded < RECORD_EVERY:
+        return False
+    _last_recorded = now
+    return True

plugins/host_agent/agent.py

@@ -22,7 +22,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.18.0"
+AGENT_VERSION = "1.19.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -1170,7 +1170,10 @@ def _swarm_services(services: list, tasks: list) -> list:
 
 
 def _swarm_nodes(nodes: list) -> list:
-    """Normalise node role / availability / status (+ manager leader flag)."""
+    """Normalise node role / availability / status (+ manager leader flag and,
+    from 1.19.0, whether the swarm can reach a manager: "reachable",
+    "unreachable" or "unknown"; "" for a worker), so Steward can count the
+    managers the swarm has lost (#5788)."""
     out = []
     for n in (nodes or []):
         spec = n.get("Spec") or {}
@@ -1184,6 +1187,7 @@ def _swarm_nodes(nodes: list) -> list:
             "availability": (spec.get("Availability") or "active").lower(),
             "status": (status.get("State") or "unknown").lower(),
             "leader": bool(mgr.get("Leader", False)),
+            "reachability": (mgr.get("Reachability") or "").lower(),
         })
     return out
 

plugins/host_agent/maintenance.py

@@ -188,21 +188,47 @@ def utcnow() -> datetime:
 
 async def _window(session, job, host_name: str, now: datetime) -> None:
     """Keep the job's maintenance window over the host until the job ends: one
-    scoped to this host's agent metrics, so an alert the job causes waits."""
+    scoped to this host's agent metrics, so an alert the job causes waits.
+    A swarm node's job keeps a second over the swarm-wide figures (#5788):
+    draining and rebooting a node takes it down and can leave a service
+    pinned to it short, which is the job, not news."""
     from steward.models.alerts import MaintenanceWindow
     if job.window_id:
         w = await session.get(MaintenanceWindow, job.window_id)
         if w is not None:
             w.end_at = max(w.start_at, window_end(job, now))
-        return
-    if job.created_by is None or job.state in FINISHED:
+    elif job.created_by is None or job.state in FINISHED:
         return                          # a window needs an owner; none for a finished job
-    w = MaintenanceWindow(name=f"Maintenance: {host_name}"[:128], start_at=now,
-                          end_at=window_end(job, now), scope_source="host_agent",
-                          scope_resource=host_name, created_by=job.created_by)
-    session.add(w)
-    await session.flush()
-    job.window_id = w.id
+    else:
+        w = MaintenanceWindow(name=f"Maintenance: {host_name}"[:128], start_at=now,
+                              end_at=window_end(job, now), scope_source="host_agent",
+                              scope_resource=host_name, created_by=job.created_by)
+        session.add(w)
+        await session.flush()
+        job.window_id = w.id
+    if job.node_id:
+        await _swarm_window(session, job, host_name, now)
+
+
+SWARM_SCOPE = ("docker", "swarm")       # the swarm-wide figures' source and resource
+
+
+async def _swarm_window(session, job, host_name: str, now: datetime) -> None:
+    """The swarm half of a swarm node's job window, found again by its name
+    and the job's start, so it needs no column of its own."""
+    from sqlalchemy import select
+    from steward.models.alerts import MaintenanceWindow
+    name = f"Maintenance: {host_name} (swarm)"[:128]
+    w = (await session.execute(select(MaintenanceWindow).where(
+        MaintenanceWindow.name == name, MaintenanceWindow.scope_source == SWARM_SCOPE[0],
+        MaintenanceWindow.scope_resource == SWARM_SCOPE[1],
+        MaintenanceWindow.start_at >= job.created_at))).scalars().first()
+    if w is not None:
+        w.end_at = max(w.start_at, window_end(job, now))
+    elif job.created_by is not None and job.state not in FINISHED:
+        session.add(MaintenanceWindow(name=name, start_at=now, end_at=window_end(job, now),
+                                      scope_source=SWARM_SCOPE[0], scope_resource=SWARM_SCOPE[1],
+                                      created_by=job.created_by))
 
 
 async def _host_name(session, host_id: str) -> str:

tests/integration/test_docker.py

@@ -575,17 +575,26 @@ def test_swarm_topology_persisted(app):
                 s.add(h)
                 await s.flush()
                 # No containers in this sample — swarm must still persist.
+                from steward.core.plugin_manager import loaded_plugin_submodule
+                loaded_plugin_submodule("docker", "swarm_health")._last_recorded = None
+                since = datetime.now(timezone.utc)
                 await persist(s, h, [], swarm)
                 hid = h.id
+            # The swarm-wide figures alert rules watch (#5788): web is 2 of 3.
+            figures = dict((await s.execute(text(
+                "SELECT metric_name, value FROM plugin_metrics WHERE source_module = 'docker' "
+                "AND resource_name = 'swarm' AND recorded_at >= :t"), {"t": since})).all())
             svc = (await s.execute(text(
                 "SELECT mode, desired, running, image, placement_json, tasks_json "
                 "FROM docker_swarm_services WHERE host_id = :h"), {"h": hid})).first()
             node = (await s.execute(text(
                 "SELECT role, availability, status, leader "
                 "FROM docker_swarm_nodes WHERE host_id = :h"), {"h": hid})).first()
-            return svc, node
+            return svc, node, figures
 
-    svc, node = asyncio.run(_go())
+    svc, node, figures = asyncio.run(_go())
+    assert figures == {"swarm_services_short": 1.0, "swarm_nodes_down": 0.0,
+                       "swarm_managers_unreachable": 0.0}
     assert svc[0] == "replicated" and svc[1] == 3 and svc[2] == 2 and svc[3] == "nginx"
     assert "n1" in svc[4]  # placement JSON carries the node id
     (task,) = json.loads(svc[5])  # docker_010: the manager's task records survive

tests/integration/test_host_maintenance_job.py

@@ -116,3 +116,39 @@ def test_a_job_is_followed_through_a_steward_restart_and_the_host_reboot():
         assert window_end <= datetime.now(timezone.utc)
 
     asyncio.run(_second_half())
+
+
+def test_a_swarm_node_s_job_also_holds_the_swarm_wide_alerts():
+    """Draining and rebooting a node moves the swarm figures; that is the job,
+    not news (#5788). The window over them follows the job's own, and ends
+    with it."""
+    from types import SimpleNamespace
+
+    from sqlalchemy import text
+    from steward.core.plugin_manager import loaded_plugin_submodule
+    from steward.models.users import User
+    app = _app()
+    maintenance = loaded_plugin_submodule("host_agent", "maintenance")
+    tag = uuid.uuid4().hex[:8]
+    host_name = f"swarm-node-{tag}"
+
+    async def _go():
+        now = datetime.now(timezone.utc)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                user = User(username=f"sw-{tag}", email=f"sw-{tag}@test", password_hash="x")
+                s.add(user)
+                await s.flush()
+                job = SimpleNamespace(window_id=None, created_by=user.id, state="draining",
+                                      node_id="node-1", created_at=now - timedelta(seconds=1))
+                await maintenance._window(s, job, host_name, now)
+                await maintenance._window(s, job, host_name, now + timedelta(minutes=1))   # the next tick
+                job.state = "done"
+                await maintenance._window(s, job, host_name, now + timedelta(minutes=2))   # finished
+            return (await s.execute(text(
+                "SELECT scope_source, scope_resource, end_at FROM maintenance_windows "
+                "WHERE name LIKE :n ORDER BY scope_source"), {"n": f"Maintenance: {host_name}%"})).all(), now
+
+    rows, now = asyncio.run(_go())
+    assert [(r[0], r[1]) for r in rows] == [("docker", "swarm"), ("host_agent", host_name)]
+    assert all(r[2] == now + timedelta(minutes=2) for r in rows)     # ended with the job, not before

tests/plugins/docker/test_swarm_health.py

@@ -0,0 +1,61 @@
+"""The swarm-wide figures alert rules watch (#5788). On 2026-10-14 nine
+services sat at 0/1 after VDNT-Docker03 became a manager, and nothing fired."""
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+from plugins.docker import swarm_health as sh
+
+T0 = datetime(2026, 10, 15, tzinfo=timezone.utc)
+
+
+def _node(name, role="manager", availability="active", status="ready", reachability="reachable"):
+    return {"node_id": name, "hostname": name, "role": role, "availability": availability,
+            "status": status, "leader": False, "reachability": reachability if role == "manager" else ""}
+
+
+def test_services_short_of_their_replicas_are_counted():
+    swarm = {"services": [{"service_name": "a", "desired": 1, "running": 0},     # the 2026-10-14 shape
+                          {"service_name": "b", "desired": 3, "running": 2},
+                          {"service_name": "c", "desired": 2, "running": 2},
+                          {"service_name": "d", "desired": 0, "running": 0},     # scaled to 0: wants none
+                          {"service_name": "e", "desired": 1, "running": 2}]}    # an update overlapping
+    assert sh.figures(swarm)["swarm_services_short"] == 2
+
+
+def test_a_drained_node_going_away_is_expected():
+    """VDNT-Util01 is always drained, and maintenance drains a node before it
+    reboots: neither is news. Its host's own Agent offline rule covers it."""
+    nodes = [_node("util01", availability="drain", status="down", reachability="unreachable"),
+             _node("docker02"), _node("docker03")]
+    got = sh.figures({"nodes": nodes})
+    assert got["swarm_nodes_down"] == 0 and got["swarm_managers_unreachable"] == 0
+
+
+def test_a_node_in_service_going_down_is_counted_and_a_manager_lost():
+    nodes = [_node("util01", availability="drain"), _node("docker02", status="down", reachability="unreachable"),
+             _node("docker03"), _node("worker1", role="worker", status="disconnected")]
+    got = sh.figures({"nodes": nodes})
+    assert got["swarm_nodes_down"] == 2 and got["swarm_managers_unreachable"] == 1
+
+
+@pytest.mark.parametrize("node, unreachable", [
+    (_node("m", status="ready", reachability="unreachable"), 1),   # the swarm lost it, though Docker says ready
+    (_node("m", status="down", reachability=""), 1),               # agent < 1.19.0: its status decides
+    (_node("m", status="ready", reachability=""), 0),
+])
+def test_a_manager_is_unreachable_by_the_swarm_s_own_word_when_the_agent_sends_it(node, unreachable):
+    assert sh.figures({"nodes": [node]})["swarm_managers_unreachable"] == unreachable
+
+
+def test_three_managers_reporting_record_once_per_window(monkeypatch):
+    monkeypatch.setattr(sh, "_last_recorded", None)
+    assert sh.due(T0)
+    assert not sh.due(T0 + timedelta(seconds=7))           # the next manager's report
+    assert not sh.due(T0 + timedelta(seconds=29))
+    assert sh.due(T0 + timedelta(seconds=30))
+
+
+def test_the_figures_are_offered_to_alert_rules():
+    from plugins.docker import get_alert_metrics
+    assert set(sh.SWARM_METRICS) <= set(get_alert_metrics()["docker"])

tests/plugins/host_agent/test_agent_docker.py

@@ -298,16 +298,18 @@ def test_swarm_nodes_normalises_fields():
     nodes = [
         {"ID": "n1", "Description": {"Hostname": "mgr"},
          "Spec": {"Role": "manager", "Availability": "active"},
-         "Status": {"State": "ready"}, "ManagerStatus": {"Leader": True}},
+         "Status": {"State": "ready"}, "ManagerStatus": {"Leader": True, "Reachability": "Reachable"}},
         {"ID": "n2", "Description": {"Hostname": "wrk"},
          "Spec": {"Role": "worker", "Availability": "drain"},
          "Status": {"State": "down"}},
     ]
     out = {n["node_id"]: n for n in a._swarm_nodes(nodes)}
     assert out["n1"] == {"node_id": "n1", "hostname": "mgr", "role": "manager",
-                         "availability": "active", "status": "ready", "leader": True}
+                         "availability": "active", "status": "ready", "leader": True,
+                         "reachability": "reachable"}
     assert out["n2"] == {"node_id": "n2", "hostname": "wrk", "role": "worker",
-                         "availability": "drain", "status": "down", "leader": False}
+                         "availability": "drain", "status": "down", "leader": False,
+                         "reachability": ""}
 
 
 def test_collect_swarm_skips_on_worker(monkeypatch):