bvandeusenandClaude Opus 5.5
0b787a0676
"swarm services short on swarm is 6." becomes "6 swarm services are short of their replicas.", with a Which line naming each service and how many of its tasks run. The swarm figures get labels, and the counting figures (swarm, mounts, drives, updates) read as sentences in the email, its subject and the Alerts page. A plugin names what its count counts through core.alerts.names_behind; the Docker plugin does for the three swarm figures, from the stored swarm. Webhooks get alert.which too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
112 lines
5.1 KiB
Python
112 lines
5.1 KiB
Python
"""The swarm's health as three figures an alert rule can watch (#5788).
|
|
|
|
Every manager's agent reports the whole swarm, so the figures are swarm-wide,
|
|
on one resource, `swarm`, and are recorded at most once every RECORD_EVERY
|
|
whichever manager sent them: three managers then read as one series, and one
|
|
manager going silent doesn't silence it.
|
|
|
|
- swarm_services_short: services running fewer tasks than they want. A
|
|
service scaled to 0 wants none and is never short.
|
|
- swarm_nodes_down: nodes whose status isn't ready, other than drained ones.
|
|
A drained node is one taken out of service on purpose (VDNT-Util01 always;
|
|
any node during its maintenance), so its going away is expected. Its host's
|
|
own "Agent offline" rule still covers it.
|
|
- swarm_managers_unreachable: managers the swarm can't reach (agent 1.19.0
|
|
sends ManagerStatus.Reachability; before that, a manager whose status
|
|
isn't ready), drained ones aside for the same reason. With three managers
|
|
one can be lost and the swarm keeps working; two, and it can't change.
|
|
|
|
A maintenance job on a swarm node holds these rules' notifications in its
|
|
window (host_agent maintenance._swarm_window), since draining and rebooting a
|
|
node is exactly what moves them.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timedelta
|
|
|
|
RESOURCE = "swarm"
|
|
SWARM_METRICS = ("swarm_services_short", "swarm_nodes_down", "swarm_managers_unreachable")
|
|
RECORD_EVERY = timedelta(seconds=30)
|
|
|
|
_last_recorded: datetime | None = None
|
|
|
|
|
|
def short_services(services: list[dict]) -> list[str]:
|
|
"""The services swarm_services_short counts, each with how many of its
|
|
tasks run: "web_app (0 of 2 running)"."""
|
|
return [f"{s.get('service_name')} ({int(s.get('running') or 0)} of {int(s.get('desired') or 0)} running)"
|
|
for s in sorted(services, key=lambda s: s.get("service_name") or "")
|
|
if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0)]
|
|
|
|
|
|
def _in_service(nodes: list[dict]) -> list[dict]:
|
|
return sorted((n for n in nodes if (n.get("availability") or "active") != "drain"),
|
|
key=lambda n: n.get("hostname") or "")
|
|
|
|
|
|
def down_nodes(nodes: list[dict]) -> list[str]:
|
|
"""The nodes swarm_nodes_down counts, by hostname, with their status."""
|
|
return [f"{n.get('hostname') or n.get('node_id')} ({n.get('status') or 'unknown'})"
|
|
for n in _in_service(nodes) if (n.get("status") or "unknown") != "ready"]
|
|
|
|
|
|
def unreachable_managers(nodes: list[dict]) -> list[str]:
|
|
"""The managers swarm_managers_unreachable counts, by hostname."""
|
|
return [n.get("hostname") or n.get("node_id") for n in _in_service(nodes)
|
|
if n.get("role") == "manager"
|
|
and (n["reachability"] != "reachable" if n.get("reachability")
|
|
else (n.get("status") or "unknown") != "ready")]
|
|
|
|
|
|
def figures(swarm: dict) -> dict[str, float]:
|
|
"""The three figures from one manager's swarm report."""
|
|
services = swarm.get("services") or []
|
|
nodes = swarm.get("nodes") or []
|
|
return {"swarm_services_short": float(len(short_services(services))),
|
|
"swarm_nodes_down": float(len(down_nodes(nodes))),
|
|
"swarm_managers_unreachable": float(len(unreachable_managers(nodes)))}
|
|
|
|
|
|
async def _stored(session) -> tuple[list[dict], list[dict]]:
|
|
"""The swarm's services and nodes as stored, the newest report of each
|
|
when more than one manager reported it, in the shape figures() reads.
|
|
A stored node has no reachability, so its status stands in for it."""
|
|
from sqlalchemy import select
|
|
from .mcp_tools import unique_nodes
|
|
from .models import DockerSwarmNode, DockerSwarmService
|
|
services: dict[str, DockerSwarmService] = {}
|
|
for s in (await session.execute(select(DockerSwarmService))).scalars():
|
|
if s.service_name not in services or s.updated_at > services[s.service_name].updated_at:
|
|
services[s.service_name] = s
|
|
nodes = unique_nodes(list((await session.execute(select(DockerSwarmNode))).scalars()))
|
|
return ([{"service_name": s.service_name, "desired": s.desired, "running": s.running}
|
|
for s in services.values()],
|
|
[{"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
|
|
"availability": n.availability, "status": n.status} for n in nodes])
|
|
|
|
|
|
async def name_short_services(session, resource: str) -> list[str]:
|
|
return short_services((await _stored(session))[0])
|
|
|
|
|
|
async def name_down_nodes(session, resource: str) -> list[str]:
|
|
return down_nodes((await _stored(session))[1])
|
|
|
|
|
|
async def name_unreachable_managers(session, resource: str) -> list[str]:
|
|
return unreachable_managers((await _stored(session))[1])
|
|
|
|
|
|
NAMERS = {"swarm_services_short": name_short_services, "swarm_nodes_down": name_down_nodes,
|
|
"swarm_managers_unreachable": name_unreachable_managers}
|
|
|
|
|
|
def due(now: datetime) -> bool:
|
|
"""Whether this report should be recorded: the first in RECORD_EVERY,
|
|
whichever manager it came from."""
|
|
global _last_recorded
|
|
if _last_recorded is not None and now - _last_recorded < RECORD_EVERY:
|
|
return False
|
|
_last_recorded = now
|
|
return True
|