"""The swarm's health as three figures an alert rule can watch (#5788). Every manager's agent reports the whole swarm, so the figures are swarm-wide, on one resource, `swarm`, and are recorded at most once every RECORD_EVERY whichever manager sent them: three managers then read as one series, and one manager going silent doesn't silence it. - swarm_services_short: services running fewer tasks than they want. A service scaled to 0 wants none and is never short. - swarm_nodes_down: nodes whose status isn't ready, other than drained ones. A drained node is one taken out of service on purpose (VDNT-Util01 always; any node during its maintenance), so its going away is expected. Its host's own "Agent offline" rule still covers it. - swarm_managers_unreachable: managers the swarm can't reach (agent 1.19.0 sends ManagerStatus.Reachability; before that, a manager whose status isn't ready), drained ones aside for the same reason. With three managers one can be lost and the swarm keeps working; two, and it can't change. A maintenance job on a swarm node holds these rules' notifications in its window (host_agent maintenance._swarm_window), since draining and rebooting a node is exactly what moves them. """ from __future__ import annotations from datetime import datetime, timedelta RESOURCE = "swarm" SWARM_METRICS = ("swarm_services_short", "swarm_nodes_down", "swarm_managers_unreachable") RECORD_EVERY = timedelta(seconds=30) _last_recorded: datetime | None = None def short_services(services: list[dict]) -> list[str]: """The services swarm_services_short counts, each with how many of its tasks run: "web_app (0 of 2 running)".""" return [f"{s.get('service_name')} ({int(s.get('running') or 0)} of {int(s.get('desired') or 0)} running)" for s in sorted(services, key=lambda s: s.get("service_name") or "") if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0)] def _in_service(nodes: list[dict]) -> list[dict]: return sorted((n for n in nodes if (n.get("availability") or "active") != "drain"), key=lambda n: n.get("hostname") or "") def down_nodes(nodes: list[dict]) -> list[str]: """The nodes swarm_nodes_down counts, by hostname, with their status.""" return [f"{n.get('hostname') or n.get('node_id')} ({n.get('status') or 'unknown'})" for n in _in_service(nodes) if (n.get("status") or "unknown") != "ready"] def unreachable_managers(nodes: list[dict]) -> list[str]: """The managers swarm_managers_unreachable counts, by hostname.""" return [n.get("hostname") or n.get("node_id") for n in _in_service(nodes) if n.get("role") == "manager" and (n["reachability"] != "reachable" if n.get("reachability") else (n.get("status") or "unknown") != "ready")] def figures(swarm: dict) -> dict[str, float]: """The three figures from one manager's swarm report.""" services = swarm.get("services") or [] nodes = swarm.get("nodes") or [] return {"swarm_services_short": float(len(short_services(services))), "swarm_nodes_down": float(len(down_nodes(nodes))), "swarm_managers_unreachable": float(len(unreachable_managers(nodes)))} async def _stored(session) -> tuple[list[dict], list[dict]]: """The swarm's services and nodes as stored, the newest report of each when more than one manager reported it, in the shape figures() reads. A stored node has no reachability, so its status stands in for it.""" from sqlalchemy import select from .mcp_tools import unique_nodes from .models import DockerSwarmNode, DockerSwarmService services: dict[str, DockerSwarmService] = {} for s in (await session.execute(select(DockerSwarmService))).scalars(): if s.service_name not in services or s.updated_at > services[s.service_name].updated_at: services[s.service_name] = s nodes = unique_nodes(list((await session.execute(select(DockerSwarmNode))).scalars())) return ([{"service_name": s.service_name, "desired": s.desired, "running": s.running} for s in services.values()], [{"node_id": n.node_id, "hostname": n.hostname, "role": n.role, "availability": n.availability, "status": n.status} for n in nodes]) async def name_short_services(session, resource: str) -> list[str]: return short_services((await _stored(session))[0]) async def name_down_nodes(session, resource: str) -> list[str]: return down_nodes((await _stored(session))[1]) async def name_unreachable_managers(session, resource: str) -> list[str]: return unreachable_managers((await _stored(session))[1]) NAMERS = {"swarm_services_short": name_short_services, "swarm_nodes_down": name_down_nodes, "swarm_managers_unreachable": name_unreachable_managers} def due(now: datetime) -> bool: """Whether this report should be recorded: the first in RECORD_EVERY, whichever manager it came from.""" global _last_recorded if _last_recorded is not None and now - _last_recorded < RECORD_EVERY: return False _last_recorded = now return True