"""The swarm's health as three figures an alert rule can watch (#5788). Every manager's agent reports the whole swarm, so the figures are swarm-wide, on one resource, `swarm`, and are recorded at most once every RECORD_EVERY whichever manager sent them: three managers then read as one series, and one manager going silent doesn't silence it. - swarm_services_short: services running fewer tasks than they want. A service scaled to 0 wants none and is never short. - swarm_nodes_down: nodes whose status isn't ready, other than drained ones. A drained node is one taken out of service on purpose (VDNT-Util01 always; any node during its maintenance), so its going away is expected. Its host's own "Agent offline" rule still covers it. - swarm_managers_unreachable: managers the swarm can't reach (agent 1.19.0 sends ManagerStatus.Reachability; before that, a manager whose status isn't ready), drained ones aside for the same reason. With three managers one can be lost and the swarm keeps working; two, and it can't change. A maintenance job on a swarm node holds these rules' notifications in its window (host_agent maintenance._swarm_window), since draining and rebooting a node is exactly what moves them. """ from __future__ import annotations from datetime import datetime, timedelta RESOURCE = "swarm" SWARM_METRICS = ("swarm_services_short", "swarm_nodes_down", "swarm_managers_unreachable") RECORD_EVERY = timedelta(seconds=30) _last_recorded: datetime | None = None def figures(swarm: dict) -> dict[str, float]: """The three figures from one manager's swarm report.""" services = swarm.get("services") or [] nodes = swarm.get("nodes") or [] short = sum(1 for s in services if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0)) in_service = [n for n in nodes if (n.get("availability") or "active") != "drain"] down = sum(1 for n in in_service if (n.get("status") or "unknown") != "ready") managers = [n for n in in_service if n.get("role") == "manager"] unreachable = sum(1 for n in managers if (n["reachability"] != "reachable" if n.get("reachability") else (n.get("status") or "unknown") != "ready")) return {"swarm_services_short": float(short), "swarm_nodes_down": float(down), "swarm_managers_unreachable": float(unreachable)} def due(now: datetime) -> bool: """Whether this report should be recorded: the first in RECORD_EVERY, whichever manager it came from.""" global _last_recorded if _last_recorded is not None and now - _last_recorded < RECORD_EVERY: return False _last_recorded = now return True