diff --git a/docs/core/alerting.md b/docs/core/alerting.md index 77b5790..3948cdc 100644 --- a/docs/core/alerting.md +++ b/docs/core/alerting.md @@ -213,6 +213,7 @@ Configured at `/settings/` under Webhook. The body is a Jinja2 template rendered | `alert.threshold` | float | Configured threshold | | `alert.resource` | str | Resource name | | `alert.source_module` | str | `ping`, `dns`, `traefik`, etc. | +| `alert.which` | str | For a figure that counts things, what it counts when it fires, joined by ", " (`web_app (0 of 2 running), …`); `raw.which` is the list. Empty for other figures. | | `alert.timestamp` | str | ISO 8601 UTC | Default template (Discord-compatible): diff --git a/plugins/docker/__init__.py b/plugins/docker/__init__.py index 655f2b3..14a4e3f 100644 --- a/plugins/docker/__init__.py +++ b/plugins/docker/__init__.py @@ -52,6 +52,11 @@ def setup(app: "Quart") -> None: # A host's agent carries its containers' figures: they go silent with it (#5721). from steward.core.alerts import agent_fed agent_fed("docker", lambda resource: resource.split("/", 1)[0]) + # An alert on a swarm figure names the services or nodes it counts (#6400). + from steward.core.alerts import names_behind + from .swarm_health import NAMERS + for metric, names in NAMERS.items(): + names_behind("docker", metric, names) # Deploys and crash-loops go on the timeline. from .timeline import register as register_timeline register_timeline() diff --git a/plugins/docker/swarm_health.py b/plugins/docker/swarm_health.py index bc6ad49..18a967a 100644 --- a/plugins/docker/swarm_health.py +++ b/plugins/docker/swarm_health.py @@ -31,20 +31,74 @@ RECORD_EVERY = timedelta(seconds=30) _last_recorded: datetime | None = None +def short_services(services: list[dict]) -> list[str]: + """The services swarm_services_short counts, each with how many of its + tasks run: "web_app (0 of 2 running)".""" + return [f"{s.get('service_name')} ({int(s.get('running') or 0)} of {int(s.get('desired') or 0)} running)" + for s in sorted(services, key=lambda s: s.get("service_name") or "") + if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0)] + + +def _in_service(nodes: list[dict]) -> list[dict]: + return sorted((n for n in nodes if (n.get("availability") or "active") != "drain"), + key=lambda n: n.get("hostname") or "") + + +def down_nodes(nodes: list[dict]) -> list[str]: + """The nodes swarm_nodes_down counts, by hostname, with their status.""" + return [f"{n.get('hostname') or n.get('node_id')} ({n.get('status') or 'unknown'})" + for n in _in_service(nodes) if (n.get("status") or "unknown") != "ready"] + + +def unreachable_managers(nodes: list[dict]) -> list[str]: + """The managers swarm_managers_unreachable counts, by hostname.""" + return [n.get("hostname") or n.get("node_id") for n in _in_service(nodes) + if n.get("role") == "manager" + and (n["reachability"] != "reachable" if n.get("reachability") + else (n.get("status") or "unknown") != "ready")] + + def figures(swarm: dict) -> dict[str, float]: """The three figures from one manager's swarm report.""" services = swarm.get("services") or [] nodes = swarm.get("nodes") or [] - short = sum(1 for s in services - if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0)) - in_service = [n for n in nodes if (n.get("availability") or "active") != "drain"] - down = sum(1 for n in in_service if (n.get("status") or "unknown") != "ready") - managers = [n for n in in_service if n.get("role") == "manager"] - unreachable = sum(1 for n in managers - if (n["reachability"] != "reachable" if n.get("reachability") - else (n.get("status") or "unknown") != "ready")) - return {"swarm_services_short": float(short), "swarm_nodes_down": float(down), - "swarm_managers_unreachable": float(unreachable)} + return {"swarm_services_short": float(len(short_services(services))), + "swarm_nodes_down": float(len(down_nodes(nodes))), + "swarm_managers_unreachable": float(len(unreachable_managers(nodes)))} + + +async def _stored(session) -> tuple[list[dict], list[dict]]: + """The swarm's services and nodes as stored, the newest report of each + when more than one manager reported it, in the shape figures() reads. + A stored node has no reachability, so its status stands in for it.""" + from sqlalchemy import select + from .mcp_tools import unique_nodes + from .models import DockerSwarmNode, DockerSwarmService + services: dict[str, DockerSwarmService] = {} + for s in (await session.execute(select(DockerSwarmService))).scalars(): + if s.service_name not in services or s.updated_at > services[s.service_name].updated_at: + services[s.service_name] = s + nodes = unique_nodes(list((await session.execute(select(DockerSwarmNode))).scalars())) + return ([{"service_name": s.service_name, "desired": s.desired, "running": s.running} + for s in services.values()], + [{"node_id": n.node_id, "hostname": n.hostname, "role": n.role, + "availability": n.availability, "status": n.status} for n in nodes]) + + +async def name_short_services(session, resource: str) -> list[str]: + return short_services((await _stored(session))[0]) + + +async def name_down_nodes(session, resource: str) -> list[str]: + return down_nodes((await _stored(session))[1]) + + +async def name_unreachable_managers(session, resource: str) -> list[str]: + return unreachable_managers((await _stored(session))[1]) + + +NAMERS = {"swarm_services_short": name_short_services, "swarm_nodes_down": name_down_nodes, + "swarm_managers_unreachable": name_unreachable_managers} def due(now: datetime) -> bool: diff --git a/steward/alerts/service.py b/steward/alerts/service.py index ec84c79..ebad864 100644 --- a/steward/alerts/service.py +++ b/steward/alerts/service.py @@ -163,7 +163,8 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None, "tail"}, read lead + resource + tail, e.g. "CPU on " + "web-1" + " is 94% (> 90%), firing for 12m". label is alert_label(state); since is when that label began (no_data_since for no_data).""" - from steward.core.formatting import human_duration, metric_label, metric_value + from steward.core.formatting import (WHOLE_RESOURCES, count_words, human_duration, metric_label, + metric_value) metric = rule.metric_name said = metric_label(metric) held = human_duration((now - since).total_seconds()) if since else None @@ -174,6 +175,15 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None, # not how long ago someone acknowledged it (the page says that apart). word = "breaching" if label == "acknowledged" else label for_how_long = f", {word} for {held}" if held else f", {word}" + condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})" + counted = count_words(metric, value) + if counted and rule.resource_name in WHOLE_RESOURCES: + # The count is the whole of it, and links where the swarm is shown. + return {"lead": "", "resource": counted[:1].upper() + counted[1:], + "tail": f"{condition}{for_how_long}"} + if counted: + return {"lead": f"{counted[:1].upper() + counted[1:]} on ", "resource": rule.resource_name, + "tail": f"{condition}{for_how_long}"} if value is None: reading = " has no reading yet" elif metric in _BINARY: @@ -181,7 +191,6 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None, "tail": f" is {metric_value(metric, value)}{for_how_long}"} else: reading = f" is {metric_value(metric, value)}" - condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})" return {"lead": f"{said} on ", "resource": rule.resource_name, "tail": f"{reading}{condition}{for_how_long}"} @@ -787,11 +796,14 @@ async def test_alert(db, rule: AlertRule) -> dict: target = (await db.execute( select(AlertRule).where(AlertRule.parent_id == rule.id) .order_by(AlertRule.resource_name).limit(1))).scalar_one_or_none() or rule + from steward.core.alerts import which return {"rule_name": rule.name, "state": "TEST", "metric": rule.metric_name, "value": await latest_value(db, target) if target.resource_name else None, "operator": rule.operator.value, "threshold": rule.threshold, "resource": target.resource_name or covers_text(rule), "source_module": rule.source_module, + "which": await which(db, rule.source_module, rule.metric_name, target.resource_name) + if target.resource_name else [], "note": "A test of this rule's notifications. Nothing has fired.", "timestamp": datetime.now(timezone.utc).isoformat()} diff --git a/steward/core/alerts.py b/steward/core/alerts.py index cbeb64e..615da3f 100644 --- a/steward/core/alerts.py +++ b/steward/core/alerts.py @@ -54,6 +54,7 @@ class Heartbeat: _HEARTBEAT: Heartbeat | None = None _AGENT_FED: dict[str, Callable[[str], str]] = {} _AWAY: Callable[[AsyncSession], Awaitable[set[str]]] | None = None +_NAMED: dict[tuple[str, str], Callable[[AsyncSession, str], Awaitable[list[str]]]] = {} def agent_heartbeat(source_module: str, metric_name: str, @@ -78,6 +79,28 @@ def agents_away(hosts_away: Callable[[AsyncSession], Awaitable[set[str]]]) -> No _AWAY = hosts_away +def names_behind(source_module: str, metric_name: str, + names: Callable[[AsyncSession, str], Awaitable[list[str]]]) -> None: + """A figure that counts things: `await names(session, resource)` names + the things it counts right now ("web_app (0 of 1 running)"), so the + alert about it can say which. The plugin recording the figure registers it.""" + _NAMED[(source_module, metric_name)] = names + + +async def which(session: AsyncSession, source_module: str, metric_name: str, + resource: str) -> list[str]: + """What a counting figure counts right now, by name; [] for a figure no + plugin names, or when naming fails (the alert still goes out).""" + names = _NAMED.get((source_module, metric_name)) + if names is None: + return [] + try: + return list(await names(session, resource)) + except Exception: + logger.exception("Naming what %s on %s counts failed", metric_name, resource) + return [] + + def is_heartbeat(rule) -> bool: """Whether `rule` watches the agent heartbeat ("agent offline").""" return _HEARTBEAT is not None and (rule.source_module, rule.metric_name) == ( @@ -672,8 +695,14 @@ async def _dispatch_notification( "resource": rule.resource_name, "source_module": rule.source_module, "note": note, + "which": [], "timestamp": datetime.now(timezone.utc).isoformat(), } + # A firing count says what it counts, as it stands when it fires. + if notif_type == "firing" and (rule.source_module, rule.metric_name) in _NAMED: + async with app.db_sessionmaker() as db: + alert_data["which"] = await which(db, rule.source_module, rule.metric_name, + rule.resource_name) # Email waits so alerts that arrive together go out as one message per # recipient (#5465, #5507); the webhook posts this alert now. diff --git a/steward/core/formatting.py b/steward/core/formatting.py index 12aa794..859b971 100644 --- a/steward/core/formatting.py +++ b/steward/core/formatting.py @@ -376,8 +376,45 @@ METRICS: dict[str, tuple[str, str]] = { "drive_life_used_pct": ("Drive life used", "how much of its rated write life this drive has used"), "drive_bad_sectors": ("Bad sectors", "media errors, or reallocated, pending and uncorrectable sectors, as the drive counts them"), "drive_temp_c": ("Drive temperature", "the drive's own temperature sensor"), + "swarm_services_short": ("Swarm services short of replicas", "services running fewer tasks than they want; one scaled to 0 is never short"), + "swarm_nodes_down": ("Swarm nodes down", "nodes that aren't ready, drained ones aside"), + "swarm_managers_unreachable": ("Swarm managers unreachable", "managers the swarm can't reach, drained ones aside; with three, losing two stops the swarm changing"), } +# Figures that count things. An alert on one says how many of what, not +# "