feat(swarm): services short, nodes down and managers lost are alertable (#5788)

On 2026-10-14 nine swarm services sat at 0/1 and nothing fired. From each
manager's swarm report the docker plugin now records three figures on
resource "swarm": swarm_services_short, swarm_nodes_down and
swarm_managers_unreachable. Drained nodes are left out of the last two,
since a drain is deliberate. The figures are recorded at most once every
30 s whichever manager reported, and are offered to alert rules.

Agent 1.19.0 sends each manager's ManagerStatus.Reachability. Older agents
fall back to the node's status.

A maintenance job on a swarm node also keeps a window over docker/swarm,
so its drain and reboot hold those alerts until the job ends.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_docker.py

@@ -575,17 +575,26 @@ def test_swarm_topology_persisted(app):
                 s.add(h)
                 await s.flush()
                 # No containers in this sample — swarm must still persist.
+                from steward.core.plugin_manager import loaded_plugin_submodule
+                loaded_plugin_submodule("docker", "swarm_health")._last_recorded = None
+                since = datetime.now(timezone.utc)
                 await persist(s, h, [], swarm)
                 hid = h.id
+            # The swarm-wide figures alert rules watch (#5788): web is 2 of 3.
+            figures = dict((await s.execute(text(
+                "SELECT metric_name, value FROM plugin_metrics WHERE source_module = 'docker' "
+                "AND resource_name = 'swarm' AND recorded_at >= :t"), {"t": since})).all())
             svc = (await s.execute(text(
                 "SELECT mode, desired, running, image, placement_json, tasks_json "
                 "FROM docker_swarm_services WHERE host_id = :h"), {"h": hid})).first()
             node = (await s.execute(text(
                 "SELECT role, availability, status, leader "
                 "FROM docker_swarm_nodes WHERE host_id = :h"), {"h": hid})).first()
-            return svc, node
+            return svc, node, figures
 
-    svc, node = asyncio.run(_go())
+    svc, node, figures = asyncio.run(_go())
+    assert figures == {"swarm_services_short": 1.0, "swarm_nodes_down": 0.0,
+                       "swarm_managers_unreachable": 0.0}
     assert svc[0] == "replicated" and svc[1] == 3 and svc[2] == 2 and svc[3] == "nginx"
     assert "n1" in svc[4]  # placement JSON carries the node id
     (task,) = json.loads(svc[5])  # docker_010: the manager's task records survive

tests/integration/test_host_maintenance_job.py

@@ -116,3 +116,39 @@ def test_a_job_is_followed_through_a_steward_restart_and_the_host_reboot():
         assert window_end <= datetime.now(timezone.utc)
 
     asyncio.run(_second_half())
+
+
+def test_a_swarm_node_s_job_also_holds_the_swarm_wide_alerts():
+    """Draining and rebooting a node moves the swarm figures; that is the job,
+    not news (#5788). The window over them follows the job's own, and ends
+    with it."""
+    from types import SimpleNamespace
+
+    from sqlalchemy import text
+    from steward.core.plugin_manager import loaded_plugin_submodule
+    from steward.models.users import User
+    app = _app()
+    maintenance = loaded_plugin_submodule("host_agent", "maintenance")
+    tag = uuid.uuid4().hex[:8]
+    host_name = f"swarm-node-{tag}"
+
+    async def _go():
+        now = datetime.now(timezone.utc)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                user = User(username=f"sw-{tag}", email=f"sw-{tag}@test", password_hash="x")
+                s.add(user)
+                await s.flush()
+                job = SimpleNamespace(window_id=None, created_by=user.id, state="draining",
+                                      node_id="node-1", created_at=now - timedelta(seconds=1))
+                await maintenance._window(s, job, host_name, now)
+                await maintenance._window(s, job, host_name, now + timedelta(minutes=1))   # the next tick
+                job.state = "done"
+                await maintenance._window(s, job, host_name, now + timedelta(minutes=2))   # finished
+            return (await s.execute(text(
+                "SELECT scope_source, scope_resource, end_at FROM maintenance_windows "
+                "WHERE name LIKE :n ORDER BY scope_source"), {"n": f"Maintenance: {host_name}%"})).all(), now
+
+    rows, now = asyncio.run(_go())
+    assert [(r[0], r[1]) for r in rows] == [("docker", "swarm"), ("host_agent", host_name)]
+    assert all(r[2] == now + timedelta(minutes=2) for r in rows)     # ended with the job, not before

tests/plugins/docker/test_swarm_health.py

@@ -0,0 +1,61 @@
+"""The swarm-wide figures alert rules watch (#5788). On 2026-10-14 nine
+services sat at 0/1 after VDNT-Docker03 became a manager, and nothing fired."""
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+from plugins.docker import swarm_health as sh
+
+T0 = datetime(2026, 10, 15, tzinfo=timezone.utc)
+
+
+def _node(name, role="manager", availability="active", status="ready", reachability="reachable"):
+    return {"node_id": name, "hostname": name, "role": role, "availability": availability,
+            "status": status, "leader": False, "reachability": reachability if role == "manager" else ""}
+
+
+def test_services_short_of_their_replicas_are_counted():
+    swarm = {"services": [{"service_name": "a", "desired": 1, "running": 0},     # the 2026-10-14 shape
+                          {"service_name": "b", "desired": 3, "running": 2},
+                          {"service_name": "c", "desired": 2, "running": 2},
+                          {"service_name": "d", "desired": 0, "running": 0},     # scaled to 0: wants none
+                          {"service_name": "e", "desired": 1, "running": 2}]}    # an update overlapping
+    assert sh.figures(swarm)["swarm_services_short"] == 2
+
+
+def test_a_drained_node_going_away_is_expected():
+    """VDNT-Util01 is always drained, and maintenance drains a node before it
+    reboots: neither is news. Its host's own Agent offline rule covers it."""
+    nodes = [_node("util01", availability="drain", status="down", reachability="unreachable"),
+             _node("docker02"), _node("docker03")]
+    got = sh.figures({"nodes": nodes})
+    assert got["swarm_nodes_down"] == 0 and got["swarm_managers_unreachable"] == 0
+
+
+def test_a_node_in_service_going_down_is_counted_and_a_manager_lost():
+    nodes = [_node("util01", availability="drain"), _node("docker02", status="down", reachability="unreachable"),
+             _node("docker03"), _node("worker1", role="worker", status="disconnected")]
+    got = sh.figures({"nodes": nodes})
+    assert got["swarm_nodes_down"] == 2 and got["swarm_managers_unreachable"] == 1
+
+
+@pytest.mark.parametrize("node, unreachable", [
+    (_node("m", status="ready", reachability="unreachable"), 1),   # the swarm lost it, though Docker says ready
+    (_node("m", status="down", reachability=""), 1),               # agent < 1.19.0: its status decides
+    (_node("m", status="ready", reachability=""), 0),
+])
+def test_a_manager_is_unreachable_by_the_swarm_s_own_word_when_the_agent_sends_it(node, unreachable):
+    assert sh.figures({"nodes": [node]})["swarm_managers_unreachable"] == unreachable
+
+
+def test_three_managers_reporting_record_once_per_window(monkeypatch):
+    monkeypatch.setattr(sh, "_last_recorded", None)
+    assert sh.due(T0)
+    assert not sh.due(T0 + timedelta(seconds=7))           # the next manager's report
+    assert not sh.due(T0 + timedelta(seconds=29))
+    assert sh.due(T0 + timedelta(seconds=30))
+
+
+def test_the_figures_are_offered_to_alert_rules():
+    from plugins.docker import get_alert_metrics
+    assert set(sh.SWARM_METRICS) <= set(get_alert_metrics()["docker"])

tests/plugins/host_agent/test_agent_docker.py

@@ -298,16 +298,18 @@ def test_swarm_nodes_normalises_fields():
     nodes = [
         {"ID": "n1", "Description": {"Hostname": "mgr"},
          "Spec": {"Role": "manager", "Availability": "active"},
-         "Status": {"State": "ready"}, "ManagerStatus": {"Leader": True}},
+         "Status": {"State": "ready"}, "ManagerStatus": {"Leader": True, "Reachability": "Reachable"}},
         {"ID": "n2", "Description": {"Hostname": "wrk"},
          "Spec": {"Role": "worker", "Availability": "drain"},
          "Status": {"State": "down"}},
     ]
     out = {n["node_id"]: n for n in a._swarm_nodes(nodes)}
     assert out["n1"] == {"node_id": "n1", "hostname": "mgr", "role": "manager",
-                         "availability": "active", "status": "ready", "leader": True}
+                         "availability": "active", "status": "ready", "leader": True,
+                         "reachability": "reachable"}
     assert out["n2"] == {"node_id": "n2", "hostname": "wrk", "role": "worker",
-                         "availability": "drain", "status": "down", "leader": False}
+                         "availability": "drain", "status": "down", "leader": False,
+                         "reachability": ""}
 
 
 def test_collect_swarm_skips_on_worker(monkeypatch):