fix(alerts): an alert on a count says how many of what, and names them (#6400)

"swarm services short on swarm is 6." becomes "6 swarm services are
short of their replicas.", with a Which line naming each service and how
many of its tasks run. The swarm figures get labels, and the counting
figures (swarm, mounts, drives, updates) read as sentences in the email,
its subject and the Alerts page. A plugin names what its count counts
through core.alerts.names_behind; the Docker plugin does for the three
swarm figures, from the stored swarm. Webhooks get alert.which too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/service.py

@@ -163,7 +163,8 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None,
     "tail"}, read lead + resource + tail, e.g. "CPU on " + "web-1" +
     " is 94% (> 90%), firing for 12m". label is alert_label(state); since is
     when that label began (no_data_since for no_data)."""
-    from steward.core.formatting import human_duration, metric_label, metric_value
+    from steward.core.formatting import (WHOLE_RESOURCES, count_words, human_duration, metric_label,
+                                         metric_value)
     metric = rule.metric_name
     said = metric_label(metric)
     held = human_duration((now - since).total_seconds()) if since else None
@@ -174,6 +175,15 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None,
     # not how long ago someone acknowledged it (the page says that apart).
     word = "breaching" if label == "acknowledged" else label
     for_how_long = f", {word} for {held}" if held else f", {word}"
+    condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})"
+    counted = count_words(metric, value)
+    if counted and rule.resource_name in WHOLE_RESOURCES:
+        # The count is the whole of it, and links where the swarm is shown.
+        return {"lead": "", "resource": counted[:1].upper() + counted[1:],
+                "tail": f"{condition}{for_how_long}"}
+    if counted:
+        return {"lead": f"{counted[:1].upper() + counted[1:]} on ", "resource": rule.resource_name,
+                "tail": f"{condition}{for_how_long}"}
     if value is None:
         reading = " has no reading yet"
     elif metric in _BINARY:
@@ -181,7 +191,6 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None,
                 "tail": f" is {metric_value(metric, value)}{for_how_long}"}
     else:
         reading = f" is {metric_value(metric, value)}"
-    condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})"
     return {"lead": f"{said} on ", "resource": rule.resource_name,
             "tail": f"{reading}{condition}{for_how_long}"}
 
@@ -787,11 +796,14 @@ async def test_alert(db, rule: AlertRule) -> dict:
         target = (await db.execute(
             select(AlertRule).where(AlertRule.parent_id == rule.id)
             .order_by(AlertRule.resource_name).limit(1))).scalar_one_or_none() or rule
+    from steward.core.alerts import which
     return {"rule_name": rule.name, "state": "TEST", "metric": rule.metric_name,
             "value": await latest_value(db, target) if target.resource_name else None,
             "operator": rule.operator.value, "threshold": rule.threshold,
             "resource": target.resource_name or covers_text(rule),
             "source_module": rule.source_module,
+            "which": await which(db, rule.source_module, rule.metric_name, target.resource_name)
+            if target.resource_name else [],
             "note": "A test of this rule's notifications. Nothing has fired.",
             "timestamp": datetime.now(timezone.utc).isoformat()}
 

steward/core/alerts.py

@@ -54,6 +54,7 @@ class Heartbeat:
 _HEARTBEAT: Heartbeat | None = None
 _AGENT_FED: dict[str, Callable[[str], str]] = {}
 _AWAY: Callable[[AsyncSession], Awaitable[set[str]]] | None = None
+_NAMED: dict[tuple[str, str], Callable[[AsyncSession, str], Awaitable[list[str]]]] = {}
 
 
 def agent_heartbeat(source_module: str, metric_name: str,
@@ -78,6 +79,28 @@ def agents_away(hosts_away: Callable[[AsyncSession], Awaitable[set[str]]]) -> No
     _AWAY = hosts_away
 
 
+def names_behind(source_module: str, metric_name: str,
+                 names: Callable[[AsyncSession, str], Awaitable[list[str]]]) -> None:
+    """A figure that counts things: `await names(session, resource)` names
+    the things it counts right now ("web_app (0 of 1 running)"), so the
+    alert about it can say which. The plugin recording the figure registers it."""
+    _NAMED[(source_module, metric_name)] = names
+
+
+async def which(session: AsyncSession, source_module: str, metric_name: str,
+                resource: str) -> list[str]:
+    """What a counting figure counts right now, by name; [] for a figure no
+    plugin names, or when naming fails (the alert still goes out)."""
+    names = _NAMED.get((source_module, metric_name))
+    if names is None:
+        return []
+    try:
+        return list(await names(session, resource))
+    except Exception:
+        logger.exception("Naming what %s on %s counts failed", metric_name, resource)
+        return []
+
+
 def is_heartbeat(rule) -> bool:
     """Whether `rule` watches the agent heartbeat ("agent offline")."""
     return _HEARTBEAT is not None and (rule.source_module, rule.metric_name) == (
@@ -672,8 +695,14 @@ async def _dispatch_notification(
         "resource": rule.resource_name,
         "source_module": rule.source_module,
         "note": note,
+        "which": [],
         "timestamp": datetime.now(timezone.utc).isoformat(),
     }
+    # A firing count says what it counts, as it stands when it fires.
+    if notif_type == "firing" and (rule.source_module, rule.metric_name) in _NAMED:
+        async with app.db_sessionmaker() as db:
+            alert_data["which"] = await which(db, rule.source_module, rule.metric_name,
+                                              rule.resource_name)
 
     # Email waits so alerts that arrive together go out as one message per
     # recipient (#5465, #5507); the webhook posts this alert now.

steward/core/formatting.py

@@ -376,8 +376,45 @@ METRICS: dict[str, tuple[str, str]] = {
     "drive_life_used_pct": ("Drive life used", "how much of its rated write life this drive has used"),
     "drive_bad_sectors": ("Bad sectors", "media errors, or reallocated, pending and uncorrectable sectors, as the drive counts them"),
     "drive_temp_c": ("Drive temperature", "the drive's own temperature sensor"),
+    "swarm_services_short": ("Swarm services short of replicas", "services running fewer tasks than they want; one scaled to 0 is never short"),
+    "swarm_nodes_down": ("Swarm nodes down", "nodes that aren't ready, drained ones aside"),
+    "swarm_managers_unreachable": ("Swarm managers unreachable", "managers the swarm can't reach, drained ones aside; with three, losing two stops the swarm changing"),
 }
 
+# Figures that count things. An alert on one says how many of what, not
+# "<label> is 6": (for one, for several, for none), each read after the
+# count ("6 swarm services are short of their replicas") or alone.
+COUNTED: dict[str, tuple[str, str, str]] = {
+    "swarm_services_short": ("swarm service is short of its replicas",
+                             "swarm services are short of their replicas",
+                             "every swarm service has its replicas"),
+    "swarm_nodes_down": ("swarm node is down", "swarm nodes are down", "every swarm node is up"),
+    "swarm_managers_unreachable": ("swarm manager can't be reached", "swarm managers can't be reached",
+                                   "every swarm manager can be reached"),
+    "mounts_stalled": ("mount isn't answering", "mounts aren't answering", "every mount is answering"),
+    "drives_failing": ("drive is failing", "drives are failing", "no drive is failing"),
+    "updates_pending": ("update is waiting", "updates are waiting", "no updates are waiting"),
+    "updates_security": ("security update is waiting", "security updates are waiting",
+                         "no security updates are waiting"),
+}
+
+# Resources that are the whole of what their figures measure: a sentence
+# about the swarm's figures says nothing by adding "on swarm".
+WHOLE_RESOURCES = frozenset({"swarm"})
+
+
+def count_words(name: str, value: float | None) -> str | None:
+    """A counting figure's reading in words: "6 swarm services are short of
+    their replicas", "every swarm node is up"; None for a figure that isn't
+    a count, or no reading."""
+    phrases = COUNTED.get(name or "")
+    if phrases is None or value is None:
+        return None
+    n = int(value)
+    if n <= 0:
+        return phrases[2]
+    return f"{n} {phrases[0] if n == 1 else phrases[1]}"
+
 # Where a rule's metric comes from, as the picker and the rules list say it.
 SOURCES: dict[str, str] = {
     "icmp": "Ping monitors", "tcp": "TCP monitors", "dns": "DNS monitors",

steward/core/mail.py

@@ -201,39 +201,77 @@ def alert_facts(alert: dict, tz: tzinfo, show_utc: bool = True) -> dict:
     """What an alert email says, in words, from the dict alerts dispatch, its
     time in `tz` (and in UTC beside it when `show_utc` and they differ)."""
     from steward.alerts.service import _BINARY, condition_text
-    from steward.core.formatting import metric_label, metric_value, status_info
+    from steward.core.formatting import (WHOLE_RESOURCES, count_words, metric_label, metric_value,
+                                         status_info)
     tone, label = status_info(alert["state"])
     metric, resource, value = alert["metric"], alert["resource"], alert.get("value")
     reading = metric_value(metric, value) if value is not None else None
+    counted = count_words(metric, value)
     what = _lower_first(metric_label(metric))
     state = alert["state"].lower()
+    whole = resource in WHOLE_RESOURCES
+    # The resource mid-sentence: "the swarm", or a host's own name.
+    named = f"the {resource}" if whole else resource
+    on = "" if whole else f" on {resource}"
     if state == "no_data":
-        headline = f"Steward has had no {what} reading from {resource}."
+        headline = f"Steward has had no {what} reading from {named}."
     elif state == "data_resumed":
-        headline = f"{resource} is reporting {what} again."
+        headline = f"{_upper_first(named) if whole else named} is reporting {what} again."
     elif metric in _BINARY:
         headline = f"{resource} is {reading or 'back to normal'}."
+    elif counted and state == "resolved":
+        headline = _upper_first(f"{counted}{on}, back within its limit.") if int(value) > 0 \
+            else _upper_first(f"{counted}{on} again.")
+    elif counted:
+        headline = _upper_first(f"{counted}{on}.")
     elif state == "resolved":
-        headline = f"{metric_label(metric)} on {resource} is back within its limit" + (
+        headline = f"{metric_label(metric)}{on} is back within its limit" + (
             f", at {reading}." if reading else ".")
     elif reading is None:
-        headline = f"{metric_label(metric)} on {resource} has no reading yet."
+        headline = f"{metric_label(metric)}{on} has no reading yet."
     else:
-        headline = f"{metric_label(metric)} on {resource} is {reading}."
+        headline = f"{metric_label(metric)}{on} is {reading}."
+    which = list(alert.get("which") or [])
     at = datetime.fromisoformat(alert["timestamp"])
     local, utc = moment(at, tz), moment(at, timezone.utc)
+    # The subject's tail is the reading where it stands ("web01 94%", "6"),
+    # then the first of what a count names.
+    tail = (reading if whole else f"{resource} {reading}") if reading else resource
+    if which:
+        tail += f" ({', '.join(which[:SUBJECT_NAMES])}{', …' if len(which) > SUBJECT_NAMES else ''})"
     return {
         "state": label, "tone": tone, "headline": headline,
         "rule": alert["rule_name"], "resource": resource,
         "reading": reading or "no data",
+        "which": _listed(which),
         "condition": condition_text(metric, alert["operator"], alert["threshold"]),
         "when": local, "when_utc": utc if show_utc and utc != local else "",
         "note": alert.get("note") or "",
-        "subject": f"[Steward] {label}: {alert['rule_name']} — {resource}"
-                   + (f" {reading}" if reading and state in ("firing", "test") else ""),
+        "subject": f"[Steward] {label}: {alert['rule_name']} — "
+                   + (tail if state in ("firing", "test") else resource),
     }
 
 
+# How many names a subject carries before "…", and the email body before
+# "and N more": enough to say which, short enough to read at a glance.
+SUBJECT_NAMES = 3
+LISTED_NAMES = 12
+
+
+def _listed(names: list[str]) -> str:
+    """Names in a sentence: "a, b and c"; past LISTED_NAMES, "a, b, … and 4 more"."""
+    if not names:
+        return ""
+    shown, more = names[:LISTED_NAMES], len(names) - LISTED_NAMES
+    if more > 0:
+        return f"{', '.join(shown)} and {more} more"
+    return shown[0] if len(shown) == 1 else f"{', '.join(shown[:-1])} and {shown[-1]}"
+
+
+def _upper_first(sentence: str) -> str:
+    return sentence[:1].upper() + sentence[1:]
+
+
 def alert_email(cfg: dict, alert: dict, to: Recipient) -> tuple[str, str, str]:
     """(subject, text, html) for an alert, written for one recipient in their
     zone, linking to Steward when its address is set."""

steward/core/notifications.py

@@ -65,11 +65,13 @@ def render_webhook(template_str: str, alert: dict) -> str:
 
     In the template, `alert.<field>` is escaped to sit inside a JSON string
     ("{{ alert.rule_name }}" stays valid whatever the name holds), with a
-    missing value reading "no data"; `raw.<field>` is the plain value for use
-    with |tojson ({{ raw.value | tojson }} gives a number or null). The
-    template runs sandboxed. ValueError says why a body could not be made.
+    missing value reading "no data" and a list (`which`) its items joined by
+    ", "; `raw.<field>` is the plain value for use with |tojson
+    ({{ raw.value | tojson }} gives a number or null, {{ raw.which | tojson }}
+    an array). The template runs sandboxed. ValueError says why a body could
+    not be made.
     """
-    shown = dict(alert)
+    shown = {k: ", ".join(map(str, v)) if isinstance(v, list) else v for k, v in alert.items()}
     if shown.get("value") is None:
         shown["value"] = "no data"
     env = SandboxedEnvironment()

steward/templates/email/alert.html

@@ -10,6 +10,7 @@
 {{ fact("Rule", a.rule) }}
 {{ fact("Resource", a.resource) }}
 {{ fact("Reading", a.reading) }}
+{% if a.which %}{{ fact("Which", a.which) }}{% endif %}
 {{ fact("Fires when", a.condition) }}
 {% call fact("When") %}{{ a.when }}{% if a.when_utc %}<br><span style="color:{{ c.muted }};">{{ a.when_utc }}</span>{% endif %}{% endcall %}
 {% if a.note %}{{ fact("Note", a.note) }}{% endif %}

steward/templates/email/alert.txt

@@ -4,6 +4,8 @@
 Rule: {{ a.rule }}
 Resource: {{ a.resource }}
 Reading: {{ a.reading }}
+{% if a.which %}Which: {{ a.which }}
+{% endif %}
 Fires when: {{ a.condition }}
 When: {{ a.when }}{% if a.when_utc %} ({{ a.when_utc }}){% endif %}
 

steward/templates/email/burst.html

@@ -13,6 +13,7 @@
 <tr><td style="padding:10px 0;border-bottom:1px solid {{ c.border }};">
 <div>{{ badge(a.tone, a.state) }}</div>
 <div style="padding-top:6px;font:600 15px/1.4 {{ c.font }};color:{{ c.text }};">{{ a.headline }}</div>
+{% if a.which %}<div style="padding-top:2px;font:14px/1.5 {{ c.font }};color:{{ c.text }};">Which: {{ a.which }}</div>{% endif %}
 <div style="padding-top:2px;font:13px/1.5 {{ c.font }};color:{{ c.muted }};">{{ a.rule }} · fires when {{ a.condition }} · {{ a.when }}</div>
 {% if a.note %}<div style="padding-top:2px;font:13px/1.5 {{ c.font }};color:{{ c.muted }};">{{ a.note }}</div>{% endif %}
 </td></tr>

steward/templates/email/burst.txt

@@ -3,6 +3,9 @@
 {% for a in alerts %}
 
 {{ a.state }}: {{ a.headline }}
+{%- if a.which %}
+  Which: {{ a.which }}
+{%- endif %}
   {{ a.rule }} · fires when {{ a.condition }} · {{ a.when }}
 {%- if a.note %}
   {{ a.note }}