From 0b787a0676b397147ea8a1d74eacea07bba7da39 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 25 Oct 2026 13:00:56 -0400 Subject: [PATCH] fix(alerts): an alert on a count says how many of what, and names them (#6400) "swarm services short on swarm is 6." becomes "6 swarm services are short of their replicas.", with a Which line naming each service and how many of its tasks run. The swarm figures get labels, and the counting figures (swarm, mounts, drives, updates) read as sentences in the email, its subject and the Alerts page. A plugin names what its count counts through core.alerts.names_behind; the Docker plugin does for the three swarm figures, from the stored swarm. Webhooks get alert.which too. Co-Authored-By: Claude Opus 5.5 --- docs/core/alerting.md | 1 + plugins/docker/__init__.py | 5 ++ plugins/docker/swarm_health.py | 74 ++++++++++++++++++++--- steward/alerts/service.py | 16 ++++- steward/core/alerts.py | 29 +++++++++ steward/core/formatting.py | 37 ++++++++++++ steward/core/mail.py | 54 ++++++++++++++--- steward/core/notifications.py | 10 +-- steward/templates/email/alert.html | 1 + steward/templates/email/alert.txt | 2 + steward/templates/email/burst.html | 1 + steward/templates/email/burst.txt | 3 + tests/core/test_alert_names_behind.py | 25 ++++++++ tests/core/test_alerts_page.py | 12 ++++ tests/core/test_mail.py | 46 ++++++++++++++ tests/core/test_notifications.py | 6 ++ tests/plugins/docker/test_swarm_health.py | 10 +++ 17 files changed, 308 insertions(+), 24 deletions(-) create mode 100644 tests/core/test_alert_names_behind.py diff --git a/docs/core/alerting.md b/docs/core/alerting.md index 77b5790..3948cdc 100644 --- a/docs/core/alerting.md +++ b/docs/core/alerting.md @@ -213,6 +213,7 @@ Configured at `/settings/` under Webhook. The body is a Jinja2 template rendered | `alert.threshold` | float | Configured threshold | | `alert.resource` | str | Resource name | | `alert.source_module` | str | `ping`, `dns`, `traefik`, etc. | +| `alert.which` | str | For a figure that counts things, what it counts when it fires, joined by ", " (`web_app (0 of 2 running), …`); `raw.which` is the list. Empty for other figures. | | `alert.timestamp` | str | ISO 8601 UTC | Default template (Discord-compatible): diff --git a/plugins/docker/__init__.py b/plugins/docker/__init__.py index 655f2b3..14a4e3f 100644 --- a/plugins/docker/__init__.py +++ b/plugins/docker/__init__.py @@ -52,6 +52,11 @@ def setup(app: "Quart") -> None: # A host's agent carries its containers' figures: they go silent with it (#5721). from steward.core.alerts import agent_fed agent_fed("docker", lambda resource: resource.split("/", 1)[0]) + # An alert on a swarm figure names the services or nodes it counts (#6400). + from steward.core.alerts import names_behind + from .swarm_health import NAMERS + for metric, names in NAMERS.items(): + names_behind("docker", metric, names) # Deploys and crash-loops go on the timeline. from .timeline import register as register_timeline register_timeline() diff --git a/plugins/docker/swarm_health.py b/plugins/docker/swarm_health.py index bc6ad49..18a967a 100644 --- a/plugins/docker/swarm_health.py +++ b/plugins/docker/swarm_health.py @@ -31,20 +31,74 @@ RECORD_EVERY = timedelta(seconds=30) _last_recorded: datetime | None = None +def short_services(services: list[dict]) -> list[str]: + """The services swarm_services_short counts, each with how many of its + tasks run: "web_app (0 of 2 running)".""" + return [f"{s.get('service_name')} ({int(s.get('running') or 0)} of {int(s.get('desired') or 0)} running)" + for s in sorted(services, key=lambda s: s.get("service_name") or "") + if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0)] + + +def _in_service(nodes: list[dict]) -> list[dict]: + return sorted((n for n in nodes if (n.get("availability") or "active") != "drain"), + key=lambda n: n.get("hostname") or "") + + +def down_nodes(nodes: list[dict]) -> list[str]: + """The nodes swarm_nodes_down counts, by hostname, with their status.""" + return [f"{n.get('hostname') or n.get('node_id')} ({n.get('status') or 'unknown'})" + for n in _in_service(nodes) if (n.get("status") or "unknown") != "ready"] + + +def unreachable_managers(nodes: list[dict]) -> list[str]: + """The managers swarm_managers_unreachable counts, by hostname.""" + return [n.get("hostname") or n.get("node_id") for n in _in_service(nodes) + if n.get("role") == "manager" + and (n["reachability"] != "reachable" if n.get("reachability") + else (n.get("status") or "unknown") != "ready")] + + def figures(swarm: dict) -> dict[str, float]: """The three figures from one manager's swarm report.""" services = swarm.get("services") or [] nodes = swarm.get("nodes") or [] - short = sum(1 for s in services - if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0)) - in_service = [n for n in nodes if (n.get("availability") or "active") != "drain"] - down = sum(1 for n in in_service if (n.get("status") or "unknown") != "ready") - managers = [n for n in in_service if n.get("role") == "manager"] - unreachable = sum(1 for n in managers - if (n["reachability"] != "reachable" if n.get("reachability") - else (n.get("status") or "unknown") != "ready")) - return {"swarm_services_short": float(short), "swarm_nodes_down": float(down), - "swarm_managers_unreachable": float(unreachable)} + return {"swarm_services_short": float(len(short_services(services))), + "swarm_nodes_down": float(len(down_nodes(nodes))), + "swarm_managers_unreachable": float(len(unreachable_managers(nodes)))} + + +async def _stored(session) -> tuple[list[dict], list[dict]]: + """The swarm's services and nodes as stored, the newest report of each + when more than one manager reported it, in the shape figures() reads. + A stored node has no reachability, so its status stands in for it.""" + from sqlalchemy import select + from .mcp_tools import unique_nodes + from .models import DockerSwarmNode, DockerSwarmService + services: dict[str, DockerSwarmService] = {} + for s in (await session.execute(select(DockerSwarmService))).scalars(): + if s.service_name not in services or s.updated_at > services[s.service_name].updated_at: + services[s.service_name] = s + nodes = unique_nodes(list((await session.execute(select(DockerSwarmNode))).scalars())) + return ([{"service_name": s.service_name, "desired": s.desired, "running": s.running} + for s in services.values()], + [{"node_id": n.node_id, "hostname": n.hostname, "role": n.role, + "availability": n.availability, "status": n.status} for n in nodes]) + + +async def name_short_services(session, resource: str) -> list[str]: + return short_services((await _stored(session))[0]) + + +async def name_down_nodes(session, resource: str) -> list[str]: + return down_nodes((await _stored(session))[1]) + + +async def name_unreachable_managers(session, resource: str) -> list[str]: + return unreachable_managers((await _stored(session))[1]) + + +NAMERS = {"swarm_services_short": name_short_services, "swarm_nodes_down": name_down_nodes, + "swarm_managers_unreachable": name_unreachable_managers} def due(now: datetime) -> bool: diff --git a/steward/alerts/service.py b/steward/alerts/service.py index ec84c79..ebad864 100644 --- a/steward/alerts/service.py +++ b/steward/alerts/service.py @@ -163,7 +163,8 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None, "tail"}, read lead + resource + tail, e.g. "CPU on " + "web-1" + " is 94% (> 90%), firing for 12m". label is alert_label(state); since is when that label began (no_data_since for no_data).""" - from steward.core.formatting import human_duration, metric_label, metric_value + from steward.core.formatting import (WHOLE_RESOURCES, count_words, human_duration, metric_label, + metric_value) metric = rule.metric_name said = metric_label(metric) held = human_duration((now - since).total_seconds()) if since else None @@ -174,6 +175,15 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None, # not how long ago someone acknowledged it (the page says that apart). word = "breaching" if label == "acknowledged" else label for_how_long = f", {word} for {held}" if held else f", {word}" + condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})" + counted = count_words(metric, value) + if counted and rule.resource_name in WHOLE_RESOURCES: + # The count is the whole of it, and links where the swarm is shown. + return {"lead": "", "resource": counted[:1].upper() + counted[1:], + "tail": f"{condition}{for_how_long}"} + if counted: + return {"lead": f"{counted[:1].upper() + counted[1:]} on ", "resource": rule.resource_name, + "tail": f"{condition}{for_how_long}"} if value is None: reading = " has no reading yet" elif metric in _BINARY: @@ -181,7 +191,6 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None, "tail": f" is {metric_value(metric, value)}{for_how_long}"} else: reading = f" is {metric_value(metric, value)}" - condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})" return {"lead": f"{said} on ", "resource": rule.resource_name, "tail": f"{reading}{condition}{for_how_long}"} @@ -787,11 +796,14 @@ async def test_alert(db, rule: AlertRule) -> dict: target = (await db.execute( select(AlertRule).where(AlertRule.parent_id == rule.id) .order_by(AlertRule.resource_name).limit(1))).scalar_one_or_none() or rule + from steward.core.alerts import which return {"rule_name": rule.name, "state": "TEST", "metric": rule.metric_name, "value": await latest_value(db, target) if target.resource_name else None, "operator": rule.operator.value, "threshold": rule.threshold, "resource": target.resource_name or covers_text(rule), "source_module": rule.source_module, + "which": await which(db, rule.source_module, rule.metric_name, target.resource_name) + if target.resource_name else [], "note": "A test of this rule's notifications. Nothing has fired.", "timestamp": datetime.now(timezone.utc).isoformat()} diff --git a/steward/core/alerts.py b/steward/core/alerts.py index cbeb64e..615da3f 100644 --- a/steward/core/alerts.py +++ b/steward/core/alerts.py @@ -54,6 +54,7 @@ class Heartbeat: _HEARTBEAT: Heartbeat | None = None _AGENT_FED: dict[str, Callable[[str], str]] = {} _AWAY: Callable[[AsyncSession], Awaitable[set[str]]] | None = None +_NAMED: dict[tuple[str, str], Callable[[AsyncSession, str], Awaitable[list[str]]]] = {} def agent_heartbeat(source_module: str, metric_name: str, @@ -78,6 +79,28 @@ def agents_away(hosts_away: Callable[[AsyncSession], Awaitable[set[str]]]) -> No _AWAY = hosts_away +def names_behind(source_module: str, metric_name: str, + names: Callable[[AsyncSession, str], Awaitable[list[str]]]) -> None: + """A figure that counts things: `await names(session, resource)` names + the things it counts right now ("web_app (0 of 1 running)"), so the + alert about it can say which. The plugin recording the figure registers it.""" + _NAMED[(source_module, metric_name)] = names + + +async def which(session: AsyncSession, source_module: str, metric_name: str, + resource: str) -> list[str]: + """What a counting figure counts right now, by name; [] for a figure no + plugin names, or when naming fails (the alert still goes out).""" + names = _NAMED.get((source_module, metric_name)) + if names is None: + return [] + try: + return list(await names(session, resource)) + except Exception: + logger.exception("Naming what %s on %s counts failed", metric_name, resource) + return [] + + def is_heartbeat(rule) -> bool: """Whether `rule` watches the agent heartbeat ("agent offline").""" return _HEARTBEAT is not None and (rule.source_module, rule.metric_name) == ( @@ -672,8 +695,14 @@ async def _dispatch_notification( "resource": rule.resource_name, "source_module": rule.source_module, "note": note, + "which": [], "timestamp": datetime.now(timezone.utc).isoformat(), } + # A firing count says what it counts, as it stands when it fires. + if notif_type == "firing" and (rule.source_module, rule.metric_name) in _NAMED: + async with app.db_sessionmaker() as db: + alert_data["which"] = await which(db, rule.source_module, rule.metric_name, + rule.resource_name) # Email waits so alerts that arrive together go out as one message per # recipient (#5465, #5507); the webhook posts this alert now. diff --git a/steward/core/formatting.py b/steward/core/formatting.py index 12aa794..859b971 100644 --- a/steward/core/formatting.py +++ b/steward/core/formatting.py @@ -376,8 +376,45 @@ METRICS: dict[str, tuple[str, str]] = { "drive_life_used_pct": ("Drive life used", "how much of its rated write life this drive has used"), "drive_bad_sectors": ("Bad sectors", "media errors, or reallocated, pending and uncorrectable sectors, as the drive counts them"), "drive_temp_c": ("Drive temperature", "the drive's own temperature sensor"), + "swarm_services_short": ("Swarm services short of replicas", "services running fewer tasks than they want; one scaled to 0 is never short"), + "swarm_nodes_down": ("Swarm nodes down", "nodes that aren't ready, drained ones aside"), + "swarm_managers_unreachable": ("Swarm managers unreachable", "managers the swarm can't reach, drained ones aside; with three, losing two stops the swarm changing"), } +# Figures that count things. An alert on one says how many of what, not +# "