feat(swarm): services short, nodes down and managers lost are alertable (#5788)

On 2026-10-14 nine swarm services sat at 0/1 and nothing fired. From each
manager's swarm report the docker plugin now records three figures on
resource "swarm": swarm_services_short, swarm_nodes_down and
swarm_managers_unreachable. Drained nodes are left out of the last two,
since a drain is deliberate. The figures are recorded at most once every
30 s whichever manager reported, and are offered to alert rules.

Agent 1.19.0 sends each manager's ManagerStatus.Reachability. Older agents
fall back to the node's status.

A maintenance job on a swarm node also keeps a window over docker/swarm,
so its drain and reboot hold those alerts until the job ends.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/plugins/docker/test_swarm_health.py

@@ -0,0 +1,61 @@
+"""The swarm-wide figures alert rules watch (#5788). On 2026-10-14 nine
+services sat at 0/1 after VDNT-Docker03 became a manager, and nothing fired."""
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+from plugins.docker import swarm_health as sh
+
+T0 = datetime(2026, 10, 15, tzinfo=timezone.utc)
+
+
+def _node(name, role="manager", availability="active", status="ready", reachability="reachable"):
+    return {"node_id": name, "hostname": name, "role": role, "availability": availability,
+            "status": status, "leader": False, "reachability": reachability if role == "manager" else ""}
+
+
+def test_services_short_of_their_replicas_are_counted():
+    swarm = {"services": [{"service_name": "a", "desired": 1, "running": 0},     # the 2026-10-14 shape
+                          {"service_name": "b", "desired": 3, "running": 2},
+                          {"service_name": "c", "desired": 2, "running": 2},
+                          {"service_name": "d", "desired": 0, "running": 0},     # scaled to 0: wants none
+                          {"service_name": "e", "desired": 1, "running": 2}]}    # an update overlapping
+    assert sh.figures(swarm)["swarm_services_short"] == 2
+
+
+def test_a_drained_node_going_away_is_expected():
+    """VDNT-Util01 is always drained, and maintenance drains a node before it
+    reboots: neither is news. Its host's own Agent offline rule covers it."""
+    nodes = [_node("util01", availability="drain", status="down", reachability="unreachable"),
+             _node("docker02"), _node("docker03")]
+    got = sh.figures({"nodes": nodes})
+    assert got["swarm_nodes_down"] == 0 and got["swarm_managers_unreachable"] == 0
+
+
+def test_a_node_in_service_going_down_is_counted_and_a_manager_lost():
+    nodes = [_node("util01", availability="drain"), _node("docker02", status="down", reachability="unreachable"),
+             _node("docker03"), _node("worker1", role="worker", status="disconnected")]
+    got = sh.figures({"nodes": nodes})
+    assert got["swarm_nodes_down"] == 2 and got["swarm_managers_unreachable"] == 1
+
+
+@pytest.mark.parametrize("node, unreachable", [
+    (_node("m", status="ready", reachability="unreachable"), 1),   # the swarm lost it, though Docker says ready
+    (_node("m", status="down", reachability=""), 1),               # agent < 1.19.0: its status decides
+    (_node("m", status="ready", reachability=""), 0),
+])
+def test_a_manager_is_unreachable_by_the_swarm_s_own_word_when_the_agent_sends_it(node, unreachable):
+    assert sh.figures({"nodes": [node]})["swarm_managers_unreachable"] == unreachable
+
+
+def test_three_managers_reporting_record_once_per_window(monkeypatch):
+    monkeypatch.setattr(sh, "_last_recorded", None)
+    assert sh.due(T0)
+    assert not sh.due(T0 + timedelta(seconds=7))           # the next manager's report
+    assert not sh.due(T0 + timedelta(seconds=29))
+    assert sh.due(T0 + timedelta(seconds=30))
+
+
+def test_the_figures_are_offered_to_alert_rules():
+    from plugins.docker import get_alert_metrics
+    assert set(sh.SWARM_METRICS) <= set(get_alert_metrics()["docker"])