bvandeusenandClaude Opus 5.5
fdc5c07183
On 2026-10-14 nine swarm services sat at 0/1 and nothing fired. From each manager's swarm report the docker plugin now records three figures on resource "swarm": swarm_services_short, swarm_nodes_down and swarm_managers_unreachable. Drained nodes are left out of the last two, since a drain is deliberate. The figures are recorded at most once every 30 s whichever manager reported, and are offered to alert rules. Agent 1.19.0 sends each manager's ManagerStatus.Reachability. Older agents fall back to the node's status. A maintenance job on a swarm node also keeps a window over docker/swarm, so its drain and reboot hold those alerts until the job ends. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
62 lines
3.0 KiB
Python
62 lines
3.0 KiB
Python
"""The swarm-wide figures alert rules watch (#5788). On 2026-10-14 nine
|
|
services sat at 0/1 after VDNT-Docker03 became a manager, and nothing fired."""
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
from plugins.docker import swarm_health as sh
|
|
|
|
T0 = datetime(2026, 10, 15, tzinfo=timezone.utc)
|
|
|
|
|
|
def _node(name, role="manager", availability="active", status="ready", reachability="reachable"):
|
|
return {"node_id": name, "hostname": name, "role": role, "availability": availability,
|
|
"status": status, "leader": False, "reachability": reachability if role == "manager" else ""}
|
|
|
|
|
|
def test_services_short_of_their_replicas_are_counted():
|
|
swarm = {"services": [{"service_name": "a", "desired": 1, "running": 0}, # the 2026-10-14 shape
|
|
{"service_name": "b", "desired": 3, "running": 2},
|
|
{"service_name": "c", "desired": 2, "running": 2},
|
|
{"service_name": "d", "desired": 0, "running": 0}, # scaled to 0: wants none
|
|
{"service_name": "e", "desired": 1, "running": 2}]} # an update overlapping
|
|
assert sh.figures(swarm)["swarm_services_short"] == 2
|
|
|
|
|
|
def test_a_drained_node_going_away_is_expected():
|
|
"""VDNT-Util01 is always drained, and maintenance drains a node before it
|
|
reboots: neither is news. Its host's own Agent offline rule covers it."""
|
|
nodes = [_node("util01", availability="drain", status="down", reachability="unreachable"),
|
|
_node("docker02"), _node("docker03")]
|
|
got = sh.figures({"nodes": nodes})
|
|
assert got["swarm_nodes_down"] == 0 and got["swarm_managers_unreachable"] == 0
|
|
|
|
|
|
def test_a_node_in_service_going_down_is_counted_and_a_manager_lost():
|
|
nodes = [_node("util01", availability="drain"), _node("docker02", status="down", reachability="unreachable"),
|
|
_node("docker03"), _node("worker1", role="worker", status="disconnected")]
|
|
got = sh.figures({"nodes": nodes})
|
|
assert got["swarm_nodes_down"] == 2 and got["swarm_managers_unreachable"] == 1
|
|
|
|
|
|
@pytest.mark.parametrize("node, unreachable", [
|
|
(_node("m", status="ready", reachability="unreachable"), 1), # the swarm lost it, though Docker says ready
|
|
(_node("m", status="down", reachability=""), 1), # agent < 1.19.0: its status decides
|
|
(_node("m", status="ready", reachability=""), 0),
|
|
])
|
|
def test_a_manager_is_unreachable_by_the_swarm_s_own_word_when_the_agent_sends_it(node, unreachable):
|
|
assert sh.figures({"nodes": [node]})["swarm_managers_unreachable"] == unreachable
|
|
|
|
|
|
def test_three_managers_reporting_record_once_per_window(monkeypatch):
|
|
monkeypatch.setattr(sh, "_last_recorded", None)
|
|
assert sh.due(T0)
|
|
assert not sh.due(T0 + timedelta(seconds=7)) # the next manager's report
|
|
assert not sh.due(T0 + timedelta(seconds=29))
|
|
assert sh.due(T0 + timedelta(seconds=30))
|
|
|
|
|
|
def test_the_figures_are_offered_to_alert_rules():
|
|
from plugins.docker import get_alert_metrics
|
|
assert set(sh.SWARM_METRICS) <= set(get_alert_metrics()["docker"])
|