bvandeusenandClaude Opus 5.5
0b787a0676
"swarm services short on swarm is 6." becomes "6 swarm services are short of their replicas.", with a Which line naming each service and how many of its tasks run. The swarm figures get labels, and the counting figures (swarm, mounts, drives, updates) read as sentences in the email, its subject and the Alerts page. A plugin names what its count counts through core.alerts.names_behind; the Docker plugin does for the three swarm figures, from the stored swarm. Webhooks get alert.which too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
72 lines
3.6 KiB
Python
72 lines
3.6 KiB
Python
"""The swarm-wide figures alert rules watch (#5788). On 2026-10-14 nine
|
|
services sat at 0/1 after VDNT-Docker03 became a manager, and nothing fired."""
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
from plugins.docker import swarm_health as sh
|
|
|
|
T0 = datetime(2026, 10, 15, tzinfo=timezone.utc)
|
|
|
|
|
|
def _node(name, role="manager", availability="active", status="ready", reachability="reachable"):
|
|
return {"node_id": name, "hostname": name, "role": role, "availability": availability,
|
|
"status": status, "leader": False, "reachability": reachability if role == "manager" else ""}
|
|
|
|
|
|
def test_services_short_of_their_replicas_are_counted():
|
|
swarm = {"services": [{"service_name": "a", "desired": 1, "running": 0}, # the 2026-10-14 shape
|
|
{"service_name": "b", "desired": 3, "running": 2},
|
|
{"service_name": "c", "desired": 2, "running": 2},
|
|
{"service_name": "d", "desired": 0, "running": 0}, # scaled to 0: wants none
|
|
{"service_name": "e", "desired": 1, "running": 2}]} # an update overlapping
|
|
assert sh.figures(swarm)["swarm_services_short"] == 2
|
|
# An alert on it names them, with how many of each run (#6400).
|
|
assert sh.short_services(swarm["services"]) == ["a (0 of 1 running)", "b (2 of 3 running)"]
|
|
|
|
|
|
def test_the_nodes_and_managers_counted_are_named():
|
|
nodes = [_node("util01", availability="drain", status="down"), _node("docker02", status="down",
|
|
reachability="unreachable"), _node("docker03"), _node("worker1", role="worker", status="disconnected")]
|
|
assert sh.down_nodes(nodes) == ["docker02 (down)", "worker1 (disconnected)"]
|
|
assert sh.unreachable_managers(nodes) == ["docker02"]
|
|
assert set(sh.NAMERS) == set(sh.SWARM_METRICS)
|
|
|
|
|
|
def test_a_drained_node_going_away_is_expected():
|
|
"""VDNT-Util01 is always drained, and maintenance drains a node before it
|
|
reboots: neither is news. Its host's own Agent offline rule covers it."""
|
|
nodes = [_node("util01", availability="drain", status="down", reachability="unreachable"),
|
|
_node("docker02"), _node("docker03")]
|
|
got = sh.figures({"nodes": nodes})
|
|
assert got["swarm_nodes_down"] == 0 and got["swarm_managers_unreachable"] == 0
|
|
|
|
|
|
def test_a_node_in_service_going_down_is_counted_and_a_manager_lost():
|
|
nodes = [_node("util01", availability="drain"), _node("docker02", status="down", reachability="unreachable"),
|
|
_node("docker03"), _node("worker1", role="worker", status="disconnected")]
|
|
got = sh.figures({"nodes": nodes})
|
|
assert got["swarm_nodes_down"] == 2 and got["swarm_managers_unreachable"] == 1
|
|
|
|
|
|
@pytest.mark.parametrize("node, unreachable", [
|
|
(_node("m", status="ready", reachability="unreachable"), 1), # the swarm lost it, though Docker says ready
|
|
(_node("m", status="down", reachability=""), 1), # agent < 1.19.0: its status decides
|
|
(_node("m", status="ready", reachability=""), 0),
|
|
])
|
|
def test_a_manager_is_unreachable_by_the_swarm_s_own_word_when_the_agent_sends_it(node, unreachable):
|
|
assert sh.figures({"nodes": [node]})["swarm_managers_unreachable"] == unreachable
|
|
|
|
|
|
def test_three_managers_reporting_record_once_per_window(monkeypatch):
|
|
monkeypatch.setattr(sh, "_last_recorded", None)
|
|
assert sh.due(T0)
|
|
assert not sh.due(T0 + timedelta(seconds=7)) # the next manager's report
|
|
assert not sh.due(T0 + timedelta(seconds=29))
|
|
assert sh.due(T0 + timedelta(seconds=30))
|
|
|
|
|
|
def test_the_figures_are_offered_to_alert_rules():
|
|
from plugins.docker import get_alert_metrics
|
|
assert set(sh.SWARM_METRICS) <= set(get_alert_metrics()["docker"])
|