fix(alerts): an alert on a count says how many of what, and names them (#6400)
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 1m36s
CI / integration (push) Successful in 1m48s
CI / publish (push) Successful in 1m8s

"swarm services short on swarm is 6." becomes "6 swarm services are
short of their replicas.", with a Which line naming each service and how
many of its tasks run. The swarm figures get labels, and the counting
figures (swarm, mounts, drives, updates) read as sentences in the email,
its subject and the Alerts page. A plugin names what its count counts
through core.alerts.names_behind; the Docker plugin does for the three
swarm figures, from the stored swarm. Webhooks get alert.which too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-25 13:00:56 -04:00
co-authored by Claude Opus 5.5
parent ae789bc0f1
commit 0b787a0676
17 changed files with 308 additions and 24 deletions
+64 -10
View File
@@ -31,20 +31,74 @@ RECORD_EVERY = timedelta(seconds=30)
_last_recorded: datetime | None = None
def short_services(services: list[dict]) -> list[str]:
"""The services swarm_services_short counts, each with how many of its
tasks run: "web_app (0 of 2 running)"."""
return [f"{s.get('service_name')} ({int(s.get('running') or 0)} of {int(s.get('desired') or 0)} running)"
for s in sorted(services, key=lambda s: s.get("service_name") or "")
if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0)]
def _in_service(nodes: list[dict]) -> list[dict]:
return sorted((n for n in nodes if (n.get("availability") or "active") != "drain"),
key=lambda n: n.get("hostname") or "")
def down_nodes(nodes: list[dict]) -> list[str]:
"""The nodes swarm_nodes_down counts, by hostname, with their status."""
return [f"{n.get('hostname') or n.get('node_id')} ({n.get('status') or 'unknown'})"
for n in _in_service(nodes) if (n.get("status") or "unknown") != "ready"]
def unreachable_managers(nodes: list[dict]) -> list[str]:
"""The managers swarm_managers_unreachable counts, by hostname."""
return [n.get("hostname") or n.get("node_id") for n in _in_service(nodes)
if n.get("role") == "manager"
and (n["reachability"] != "reachable" if n.get("reachability")
else (n.get("status") or "unknown") != "ready")]
def figures(swarm: dict) -> dict[str, float]:
"""The three figures from one manager's swarm report."""
services = swarm.get("services") or []
nodes = swarm.get("nodes") or []
short = sum(1 for s in services
if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0))
in_service = [n for n in nodes if (n.get("availability") or "active") != "drain"]
down = sum(1 for n in in_service if (n.get("status") or "unknown") != "ready")
managers = [n for n in in_service if n.get("role") == "manager"]
unreachable = sum(1 for n in managers
if (n["reachability"] != "reachable" if n.get("reachability")
else (n.get("status") or "unknown") != "ready"))
return {"swarm_services_short": float(short), "swarm_nodes_down": float(down),
"swarm_managers_unreachable": float(unreachable)}
return {"swarm_services_short": float(len(short_services(services))),
"swarm_nodes_down": float(len(down_nodes(nodes))),
"swarm_managers_unreachable": float(len(unreachable_managers(nodes)))}
async def _stored(session) -> tuple[list[dict], list[dict]]:
"""The swarm's services and nodes as stored, the newest report of each
when more than one manager reported it, in the shape figures() reads.
A stored node has no reachability, so its status stands in for it."""
from sqlalchemy import select
from .mcp_tools import unique_nodes
from .models import DockerSwarmNode, DockerSwarmService
services: dict[str, DockerSwarmService] = {}
for s in (await session.execute(select(DockerSwarmService))).scalars():
if s.service_name not in services or s.updated_at > services[s.service_name].updated_at:
services[s.service_name] = s
nodes = unique_nodes(list((await session.execute(select(DockerSwarmNode))).scalars()))
return ([{"service_name": s.service_name, "desired": s.desired, "running": s.running}
for s in services.values()],
[{"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
"availability": n.availability, "status": n.status} for n in nodes])
async def name_short_services(session, resource: str) -> list[str]:
return short_services((await _stored(session))[0])
async def name_down_nodes(session, resource: str) -> list[str]:
return down_nodes((await _stored(session))[1])
async def name_unreachable_managers(session, resource: str) -> list[str]:
return unreachable_managers((await _stored(session))[1])
NAMERS = {"swarm_services_short": name_short_services, "swarm_nodes_down": name_down_nodes,
"swarm_managers_unreachable": name_unreachable_managers}
def due(now: datetime) -> bool: