fix(alerts): an alert on a count says how many of what, and names them (#6400)
"swarm services short on swarm is 6." becomes "6 swarm services are short of their replicas.", with a Which line naming each service and how many of its tasks run. The swarm figures get labels, and the counting figures (swarm, mounts, drives, updates) read as sentences in the email, its subject and the Alerts page. A plugin names what its count counts through core.alerts.names_behind; the Docker plugin does for the three swarm figures, from the stored swarm. Webhooks get alert.which too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -31,20 +31,74 @@ RECORD_EVERY = timedelta(seconds=30)
|
||||
_last_recorded: datetime | None = None
|
||||
|
||||
|
||||
def short_services(services: list[dict]) -> list[str]:
|
||||
"""The services swarm_services_short counts, each with how many of its
|
||||
tasks run: "web_app (0 of 2 running)"."""
|
||||
return [f"{s.get('service_name')} ({int(s.get('running') or 0)} of {int(s.get('desired') or 0)} running)"
|
||||
for s in sorted(services, key=lambda s: s.get("service_name") or "")
|
||||
if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0)]
|
||||
|
||||
|
||||
def _in_service(nodes: list[dict]) -> list[dict]:
|
||||
return sorted((n for n in nodes if (n.get("availability") or "active") != "drain"),
|
||||
key=lambda n: n.get("hostname") or "")
|
||||
|
||||
|
||||
def down_nodes(nodes: list[dict]) -> list[str]:
|
||||
"""The nodes swarm_nodes_down counts, by hostname, with their status."""
|
||||
return [f"{n.get('hostname') or n.get('node_id')} ({n.get('status') or 'unknown'})"
|
||||
for n in _in_service(nodes) if (n.get("status") or "unknown") != "ready"]
|
||||
|
||||
|
||||
def unreachable_managers(nodes: list[dict]) -> list[str]:
|
||||
"""The managers swarm_managers_unreachable counts, by hostname."""
|
||||
return [n.get("hostname") or n.get("node_id") for n in _in_service(nodes)
|
||||
if n.get("role") == "manager"
|
||||
and (n["reachability"] != "reachable" if n.get("reachability")
|
||||
else (n.get("status") or "unknown") != "ready")]
|
||||
|
||||
|
||||
def figures(swarm: dict) -> dict[str, float]:
|
||||
"""The three figures from one manager's swarm report."""
|
||||
services = swarm.get("services") or []
|
||||
nodes = swarm.get("nodes") or []
|
||||
short = sum(1 for s in services
|
||||
if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0))
|
||||
in_service = [n for n in nodes if (n.get("availability") or "active") != "drain"]
|
||||
down = sum(1 for n in in_service if (n.get("status") or "unknown") != "ready")
|
||||
managers = [n for n in in_service if n.get("role") == "manager"]
|
||||
unreachable = sum(1 for n in managers
|
||||
if (n["reachability"] != "reachable" if n.get("reachability")
|
||||
else (n.get("status") or "unknown") != "ready"))
|
||||
return {"swarm_services_short": float(short), "swarm_nodes_down": float(down),
|
||||
"swarm_managers_unreachable": float(unreachable)}
|
||||
return {"swarm_services_short": float(len(short_services(services))),
|
||||
"swarm_nodes_down": float(len(down_nodes(nodes))),
|
||||
"swarm_managers_unreachable": float(len(unreachable_managers(nodes)))}
|
||||
|
||||
|
||||
async def _stored(session) -> tuple[list[dict], list[dict]]:
|
||||
"""The swarm's services and nodes as stored, the newest report of each
|
||||
when more than one manager reported it, in the shape figures() reads.
|
||||
A stored node has no reachability, so its status stands in for it."""
|
||||
from sqlalchemy import select
|
||||
from .mcp_tools import unique_nodes
|
||||
from .models import DockerSwarmNode, DockerSwarmService
|
||||
services: dict[str, DockerSwarmService] = {}
|
||||
for s in (await session.execute(select(DockerSwarmService))).scalars():
|
||||
if s.service_name not in services or s.updated_at > services[s.service_name].updated_at:
|
||||
services[s.service_name] = s
|
||||
nodes = unique_nodes(list((await session.execute(select(DockerSwarmNode))).scalars()))
|
||||
return ([{"service_name": s.service_name, "desired": s.desired, "running": s.running}
|
||||
for s in services.values()],
|
||||
[{"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
|
||||
"availability": n.availability, "status": n.status} for n in nodes])
|
||||
|
||||
|
||||
async def name_short_services(session, resource: str) -> list[str]:
|
||||
return short_services((await _stored(session))[0])
|
||||
|
||||
|
||||
async def name_down_nodes(session, resource: str) -> list[str]:
|
||||
return down_nodes((await _stored(session))[1])
|
||||
|
||||
|
||||
async def name_unreachable_managers(session, resource: str) -> list[str]:
|
||||
return unreachable_managers((await _stored(session))[1])
|
||||
|
||||
|
||||
NAMERS = {"swarm_services_short": name_short_services, "swarm_nodes_down": name_down_nodes,
|
||||
"swarm_managers_unreachable": name_unreachable_managers}
|
||||
|
||||
|
||||
def due(now: datetime) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user