fix(alerts): an alert on a count says how many of what, and names them (#6400)

"swarm services short on swarm is 6." becomes "6 swarm services are
short of their replicas.", with a Which line naming each service and how
many of its tasks run. The swarm figures get labels, and the counting
figures (swarm, mounts, drives, updates) read as sentences in the email,
its subject and the Alerts page. A plugin names what its count counts
through core.alerts.names_behind; the Docker plugin does for the three
swarm figures, from the stored swarm. Webhooks get alert.which too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/core/alerting.md

@@ -213,6 +213,7 @@ Configured at `/settings/` under Webhook. The body is a Jinja2 template rendered
 | `alert.threshold` | float | Configured threshold |
 | `alert.resource` | str | Resource name |
 | `alert.source_module` | str | `ping`, `dns`, `traefik`, etc. |
+| `alert.which` | str | For a figure that counts things, what it counts when it fires, joined by ", " (`web_app (0 of 2 running), …`); `raw.which` is the list. Empty for other figures. |
 | `alert.timestamp` | str | ISO 8601 UTC |
 
 Default template (Discord-compatible):

plugins/docker/__init__.py

@@ -52,6 +52,11 @@ def setup(app: "Quart") -> None:
     # A host's agent carries its containers' figures: they go silent with it (#5721).
     from steward.core.alerts import agent_fed
     agent_fed("docker", lambda resource: resource.split("/", 1)[0])
+    # An alert on a swarm figure names the services or nodes it counts (#6400).
+    from steward.core.alerts import names_behind
+    from .swarm_health import NAMERS
+    for metric, names in NAMERS.items():
+        names_behind("docker", metric, names)
     # Deploys and crash-loops go on the timeline.
     from .timeline import register as register_timeline
     register_timeline()

plugins/docker/swarm_health.py

@@ -31,20 +31,74 @@ RECORD_EVERY = timedelta(seconds=30)
 _last_recorded: datetime | None = None
 
 
+def short_services(services: list[dict]) -> list[str]:
+    """The services swarm_services_short counts, each with how many of its
+    tasks run: "web_app (0 of 2 running)"."""
+    return [f"{s.get('service_name')} ({int(s.get('running') or 0)} of {int(s.get('desired') or 0)} running)"
+            for s in sorted(services, key=lambda s: s.get("service_name") or "")
+            if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0)]
+
+
+def _in_service(nodes: list[dict]) -> list[dict]:
+    return sorted((n for n in nodes if (n.get("availability") or "active") != "drain"),
+                  key=lambda n: n.get("hostname") or "")
+
+
+def down_nodes(nodes: list[dict]) -> list[str]:
+    """The nodes swarm_nodes_down counts, by hostname, with their status."""
+    return [f"{n.get('hostname') or n.get('node_id')} ({n.get('status') or 'unknown'})"
+            for n in _in_service(nodes) if (n.get("status") or "unknown") != "ready"]
+
+
+def unreachable_managers(nodes: list[dict]) -> list[str]:
+    """The managers swarm_managers_unreachable counts, by hostname."""
+    return [n.get("hostname") or n.get("node_id") for n in _in_service(nodes)
+            if n.get("role") == "manager"
+            and (n["reachability"] != "reachable" if n.get("reachability")
+                 else (n.get("status") or "unknown") != "ready")]
+
+
 def figures(swarm: dict) -> dict[str, float]:
     """The three figures from one manager's swarm report."""
     services = swarm.get("services") or []
     nodes = swarm.get("nodes") or []
-    short = sum(1 for s in services
+    return {"swarm_services_short": float(len(short_services(services))),
-                if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0))
+            "swarm_nodes_down": float(len(down_nodes(nodes))),
-    in_service = [n for n in nodes if (n.get("availability") or "active") != "drain"]
+            "swarm_managers_unreachable": float(len(unreachable_managers(nodes)))}
-    down = sum(1 for n in in_service if (n.get("status") or "unknown") != "ready")
+
-    managers = [n for n in in_service if n.get("role") == "manager"]
+
-    unreachable = sum(1 for n in managers
+async def _stored(session) -> tuple[list[dict], list[dict]]:
-                      if (n["reachability"] != "reachable" if n.get("reachability")
+    """The swarm's services and nodes as stored, the newest report of each
-                          else (n.get("status") or "unknown") != "ready"))
+    when more than one manager reported it, in the shape figures() reads.
-    return {"swarm_services_short": float(short), "swarm_nodes_down": float(down),
+    A stored node has no reachability, so its status stands in for it."""
-            "swarm_managers_unreachable": float(unreachable)}
+    from sqlalchemy import select
+    from .mcp_tools import unique_nodes
+    from .models import DockerSwarmNode, DockerSwarmService
+    services: dict[str, DockerSwarmService] = {}
+    for s in (await session.execute(select(DockerSwarmService))).scalars():
+        if s.service_name not in services or s.updated_at > services[s.service_name].updated_at:
+            services[s.service_name] = s
+    nodes = unique_nodes(list((await session.execute(select(DockerSwarmNode))).scalars()))
+    return ([{"service_name": s.service_name, "desired": s.desired, "running": s.running}
+             for s in services.values()],
+            [{"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
+              "availability": n.availability, "status": n.status} for n in nodes])
+
+
+async def name_short_services(session, resource: str) -> list[str]:
+    return short_services((await _stored(session))[0])
+
+
+async def name_down_nodes(session, resource: str) -> list[str]:
+    return down_nodes((await _stored(session))[1])
+
+
+async def name_unreachable_managers(session, resource: str) -> list[str]:
+    return unreachable_managers((await _stored(session))[1])
+
+
+NAMERS = {"swarm_services_short": name_short_services, "swarm_nodes_down": name_down_nodes,
+          "swarm_managers_unreachable": name_unreachable_managers}
 
 
 def due(now: datetime) -> bool:

steward/alerts/service.py

@@ -163,7 +163,8 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None,
     "tail"}, read lead + resource + tail, e.g. "CPU on " + "web-1" +
     " is 94% (> 90%), firing for 12m". label is alert_label(state); since is
     when that label began (no_data_since for no_data)."""
-    from steward.core.formatting import human_duration, metric_label, metric_value
+    from steward.core.formatting import (WHOLE_RESOURCES, count_words, human_duration, metric_label,
+                                         metric_value)
     metric = rule.metric_name
     said = metric_label(metric)
     held = human_duration((now - since).total_seconds()) if since else None
@@ -174,6 +175,15 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None,
     # not how long ago someone acknowledged it (the page says that apart).
     word = "breaching" if label == "acknowledged" else label
     for_how_long = f", {word} for {held}" if held else f", {word}"
+    condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})"
+    counted = count_words(metric, value)
+    if counted and rule.resource_name in WHOLE_RESOURCES:
+        # The count is the whole of it, and links where the swarm is shown.
+        return {"lead": "", "resource": counted[:1].upper() + counted[1:],
+                "tail": f"{condition}{for_how_long}"}
+    if counted:
+        return {"lead": f"{counted[:1].upper() + counted[1:]} on ", "resource": rule.resource_name,
+                "tail": f"{condition}{for_how_long}"}
     if value is None:
         reading = " has no reading yet"
     elif metric in _BINARY:
@@ -181,7 +191,6 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None,
                 "tail": f" is {metric_value(metric, value)}{for_how_long}"}
     else:
         reading = f" is {metric_value(metric, value)}"
-    condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})"
     return {"lead": f"{said} on ", "resource": rule.resource_name,
             "tail": f"{reading}{condition}{for_how_long}"}
 
@@ -787,11 +796,14 @@ async def test_alert(db, rule: AlertRule) -> dict:
         target = (await db.execute(
             select(AlertRule).where(AlertRule.parent_id == rule.id)
             .order_by(AlertRule.resource_name).limit(1))).scalar_one_or_none() or rule
+    from steward.core.alerts import which
     return {"rule_name": rule.name, "state": "TEST", "metric": rule.metric_name,
             "value": await latest_value(db, target) if target.resource_name else None,
             "operator": rule.operator.value, "threshold": rule.threshold,
             "resource": target.resource_name or covers_text(rule),
             "source_module": rule.source_module,
+            "which": await which(db, rule.source_module, rule.metric_name, target.resource_name)
+            if target.resource_name else [],
             "note": "A test of this rule's notifications. Nothing has fired.",
             "timestamp": datetime.now(timezone.utc).isoformat()}
 

steward/core/alerts.py

@@ -54,6 +54,7 @@ class Heartbeat:
 _HEARTBEAT: Heartbeat | None = None
 _AGENT_FED: dict[str, Callable[[str], str]] = {}
 _AWAY: Callable[[AsyncSession], Awaitable[set[str]]] | None = None
+_NAMED: dict[tuple[str, str], Callable[[AsyncSession, str], Awaitable[list[str]]]] = {}
 
 
 def agent_heartbeat(source_module: str, metric_name: str,
@@ -78,6 +79,28 @@ def agents_away(hosts_away: Callable[[AsyncSession], Awaitable[set[str]]]) -> No
     _AWAY = hosts_away
 
 
+def names_behind(source_module: str, metric_name: str,
+                 names: Callable[[AsyncSession, str], Awaitable[list[str]]]) -> None:
+    """A figure that counts things: `await names(session, resource)` names
+    the things it counts right now ("web_app (0 of 1 running)"), so the
+    alert about it can say which. The plugin recording the figure registers it."""
+    _NAMED[(source_module, metric_name)] = names
+
+
+async def which(session: AsyncSession, source_module: str, metric_name: str,
+                resource: str) -> list[str]:
+    """What a counting figure counts right now, by name; [] for a figure no
+    plugin names, or when naming fails (the alert still goes out)."""
+    names = _NAMED.get((source_module, metric_name))
+    if names is None:
+        return []
+    try:
+        return list(await names(session, resource))
+    except Exception:
+        logger.exception("Naming what %s on %s counts failed", metric_name, resource)
+        return []
+
+
 def is_heartbeat(rule) -> bool:
     """Whether `rule` watches the agent heartbeat ("agent offline")."""
     return _HEARTBEAT is not None and (rule.source_module, rule.metric_name) == (
@@ -672,8 +695,14 @@ async def _dispatch_notification(
         "resource": rule.resource_name,
         "source_module": rule.source_module,
         "note": note,
+        "which": [],
         "timestamp": datetime.now(timezone.utc).isoformat(),
     }
+    # A firing count says what it counts, as it stands when it fires.
+    if notif_type == "firing" and (rule.source_module, rule.metric_name) in _NAMED:
+        async with app.db_sessionmaker() as db:
+            alert_data["which"] = await which(db, rule.source_module, rule.metric_name,
+                                              rule.resource_name)
 
     # Email waits so alerts that arrive together go out as one message per
     # recipient (#5465, #5507); the webhook posts this alert now.

steward/core/formatting.py

@@ -376,8 +376,45 @@ METRICS: dict[str, tuple[str, str]] = {
     "drive_life_used_pct": ("Drive life used", "how much of its rated write life this drive has used"),
     "drive_bad_sectors": ("Bad sectors", "media errors, or reallocated, pending and uncorrectable sectors, as the drive counts them"),
     "drive_temp_c": ("Drive temperature", "the drive's own temperature sensor"),
+    "swarm_services_short": ("Swarm services short of replicas", "services running fewer tasks than they want; one scaled to 0 is never short"),
+    "swarm_nodes_down": ("Swarm nodes down", "nodes that aren't ready, drained ones aside"),
+    "swarm_managers_unreachable": ("Swarm managers unreachable", "managers the swarm can't reach, drained ones aside; with three, losing two stops the swarm changing"),
 }
 
+# Figures that count things. An alert on one says how many of what, not
+# "<label> is 6": (for one, for several, for none), each read after the
+# count ("6 swarm services are short of their replicas") or alone.
+COUNTED: dict[str, tuple[str, str, str]] = {
+    "swarm_services_short": ("swarm service is short of its replicas",
+                             "swarm services are short of their replicas",
+                             "every swarm service has its replicas"),
+    "swarm_nodes_down": ("swarm node is down", "swarm nodes are down", "every swarm node is up"),
+    "swarm_managers_unreachable": ("swarm manager can't be reached", "swarm managers can't be reached",
+                                   "every swarm manager can be reached"),
+    "mounts_stalled": ("mount isn't answering", "mounts aren't answering", "every mount is answering"),
+    "drives_failing": ("drive is failing", "drives are failing", "no drive is failing"),
+    "updates_pending": ("update is waiting", "updates are waiting", "no updates are waiting"),
+    "updates_security": ("security update is waiting", "security updates are waiting",
+                         "no security updates are waiting"),
+}
+
+# Resources that are the whole of what their figures measure: a sentence
+# about the swarm's figures says nothing by adding "on swarm".
+WHOLE_RESOURCES = frozenset({"swarm"})
+
+
+def count_words(name: str, value: float | None) -> str | None:
+    """A counting figure's reading in words: "6 swarm services are short of
+    their replicas", "every swarm node is up"; None for a figure that isn't
+    a count, or no reading."""
+    phrases = COUNTED.get(name or "")
+    if phrases is None or value is None:
+        return None
+    n = int(value)
+    if n <= 0:
+        return phrases[2]
+    return f"{n} {phrases[0] if n == 1 else phrases[1]}"
+
 # Where a rule's metric comes from, as the picker and the rules list say it.
 SOURCES: dict[str, str] = {
     "icmp": "Ping monitors", "tcp": "TCP monitors", "dns": "DNS monitors",

steward/core/mail.py

@@ -201,39 +201,77 @@ def alert_facts(alert: dict, tz: tzinfo, show_utc: bool = True) -> dict:
     """What an alert email says, in words, from the dict alerts dispatch, its
     time in `tz` (and in UTC beside it when `show_utc` and they differ)."""
     from steward.alerts.service import _BINARY, condition_text
-    from steward.core.formatting import metric_label, metric_value, status_info
+    from steward.core.formatting import (WHOLE_RESOURCES, count_words, metric_label, metric_value,
+                                         status_info)
     tone, label = status_info(alert["state"])
     metric, resource, value = alert["metric"], alert["resource"], alert.get("value")
     reading = metric_value(metric, value) if value is not None else None
+    counted = count_words(metric, value)
     what = _lower_first(metric_label(metric))
     state = alert["state"].lower()
+    whole = resource in WHOLE_RESOURCES
+    # The resource mid-sentence: "the swarm", or a host's own name.
+    named = f"the {resource}" if whole else resource
+    on = "" if whole else f" on {resource}"
     if state == "no_data":
-        headline = f"Steward has had no {what} reading from {resource}."
+        headline = f"Steward has had no {what} reading from {named}."
     elif state == "data_resumed":
-        headline = f"{resource} is reporting {what} again."
+        headline = f"{_upper_first(named) if whole else named} is reporting {what} again."
     elif metric in _BINARY:
         headline = f"{resource} is {reading or 'back to normal'}."
+    elif counted and state == "resolved":
+        headline = _upper_first(f"{counted}{on}, back within its limit.") if int(value) > 0 \
+            else _upper_first(f"{counted}{on} again.")
+    elif counted:
+        headline = _upper_first(f"{counted}{on}.")
     elif state == "resolved":
-        headline = f"{metric_label(metric)} on {resource} is back within its limit" + (
+        headline = f"{metric_label(metric)}{on} is back within its limit" + (
             f", at {reading}." if reading else ".")
     elif reading is None:
-        headline = f"{metric_label(metric)} on {resource} has no reading yet."
+        headline = f"{metric_label(metric)}{on} has no reading yet."
     else:
-        headline = f"{metric_label(metric)} on {resource} is {reading}."
+        headline = f"{metric_label(metric)}{on} is {reading}."
+    which = list(alert.get("which") or [])
     at = datetime.fromisoformat(alert["timestamp"])
     local, utc = moment(at, tz), moment(at, timezone.utc)
+    # The subject's tail is the reading where it stands ("web01 94%", "6"),
+    # then the first of what a count names.
+    tail = (reading if whole else f"{resource} {reading}") if reading else resource
+    if which:
+        tail += f" ({', '.join(which[:SUBJECT_NAMES])}{', …' if len(which) > SUBJECT_NAMES else ''})"
     return {
         "state": label, "tone": tone, "headline": headline,
         "rule": alert["rule_name"], "resource": resource,
         "reading": reading or "no data",
+        "which": _listed(which),
         "condition": condition_text(metric, alert["operator"], alert["threshold"]),
         "when": local, "when_utc": utc if show_utc and utc != local else "",
         "note": alert.get("note") or "",
-        "subject": f"[Steward] {label}: {alert['rule_name']} — {resource}"
+        "subject": f"[Steward] {label}: {alert['rule_name']} — "
-                   + (f" {reading}" if reading and state in ("firing", "test") else ""),
+                   + (tail if state in ("firing", "test") else resource),
     }
 
 
+# How many names a subject carries before "…", and the email body before
+# "and N more": enough to say which, short enough to read at a glance.
+SUBJECT_NAMES = 3
+LISTED_NAMES = 12
+
+
+def _listed(names: list[str]) -> str:
+    """Names in a sentence: "a, b and c"; past LISTED_NAMES, "a, b, … and 4 more"."""
+    if not names:
+        return ""
+    shown, more = names[:LISTED_NAMES], len(names) - LISTED_NAMES
+    if more > 0:
+        return f"{', '.join(shown)} and {more} more"
+    return shown[0] if len(shown) == 1 else f"{', '.join(shown[:-1])} and {shown[-1]}"
+
+
+def _upper_first(sentence: str) -> str:
+    return sentence[:1].upper() + sentence[1:]
+
+
 def alert_email(cfg: dict, alert: dict, to: Recipient) -> tuple[str, str, str]:
     """(subject, text, html) for an alert, written for one recipient in their
     zone, linking to Steward when its address is set."""

steward/core/notifications.py

@@ -65,11 +65,13 @@ def render_webhook(template_str: str, alert: dict) -> str:
 
     In the template, `alert.<field>` is escaped to sit inside a JSON string
     ("{{ alert.rule_name }}" stays valid whatever the name holds), with a
-    missing value reading "no data"; `raw.<field>` is the plain value for use
+    missing value reading "no data" and a list (`which`) its items joined by
-    with |tojson ({{ raw.value | tojson }} gives a number or null). The
+    ", "; `raw.<field>` is the plain value for use with |tojson
-    template runs sandboxed. ValueError says why a body could not be made.
+    ({{ raw.value | tojson }} gives a number or null, {{ raw.which | tojson }}
+    an array). The template runs sandboxed. ValueError says why a body could
+    not be made.
     """
-    shown = dict(alert)
+    shown = {k: ", ".join(map(str, v)) if isinstance(v, list) else v for k, v in alert.items()}
     if shown.get("value") is None:
         shown["value"] = "no data"
     env = SandboxedEnvironment()

steward/templates/email/alert.html

@@ -10,6 +10,7 @@
 {{ fact("Rule", a.rule) }}
 {{ fact("Resource", a.resource) }}
 {{ fact("Reading", a.reading) }}
+{% if a.which %}{{ fact("Which", a.which) }}{% endif %}
 {{ fact("Fires when", a.condition) }}
 {% call fact("When") %}{{ a.when }}{% if a.when_utc %}<br><span style="color:{{ c.muted }};">{{ a.when_utc }}</span>{% endif %}{% endcall %}
 {% if a.note %}{{ fact("Note", a.note) }}{% endif %}

steward/templates/email/alert.txt

@@ -4,6 +4,8 @@
 Rule: {{ a.rule }}
 Resource: {{ a.resource }}
 Reading: {{ a.reading }}
+{% if a.which %}Which: {{ a.which }}
+{% endif %}
 Fires when: {{ a.condition }}
 When: {{ a.when }}{% if a.when_utc %} ({{ a.when_utc }}){% endif %}
 

steward/templates/email/burst.html

@@ -13,6 +13,7 @@
 <tr><td style="padding:10px 0;border-bottom:1px solid {{ c.border }};">
 <div>{{ badge(a.tone, a.state) }}</div>
 <div style="padding-top:6px;font:600 15px/1.4 {{ c.font }};color:{{ c.text }};">{{ a.headline }}</div>
+{% if a.which %}<div style="padding-top:2px;font:14px/1.5 {{ c.font }};color:{{ c.text }};">Which: {{ a.which }}</div>{% endif %}
 <div style="padding-top:2px;font:13px/1.5 {{ c.font }};color:{{ c.muted }};">{{ a.rule }} · fires when {{ a.condition }} · {{ a.when }}</div>
 {% if a.note %}<div style="padding-top:2px;font:13px/1.5 {{ c.font }};color:{{ c.muted }};">{{ a.note }}</div>{% endif %}
 </td></tr>

steward/templates/email/burst.txt

@@ -3,6 +3,9 @@
 {% for a in alerts %}
 
 {{ a.state }}: {{ a.headline }}
+{%- if a.which %}
+  Which: {{ a.which }}
+{%- endif %}
   {{ a.rule }} · fires when {{ a.condition }} · {{ a.when }}
 {%- if a.note %}
   {{ a.note }}

tests/core/test_alert_names_behind.py

@@ -0,0 +1,25 @@
+"""A figure that counts things names what it counts in the alert about it
+(#6400): the plugin recording it registers how, and a lookup that fails
+leaves the alert to go out without names."""
+from steward.core import alerts
+
+
+async def test_a_registered_count_is_named_and_others_are_not(monkeypatch):
+    monkeypatch.setattr(alerts, "_NAMED", {})
+
+    async def short(session, resource):
+        return [f"{resource}: api (0 of 1 running)"]
+
+    alerts.names_behind("docker", "swarm_services_short", short)
+    assert await alerts.which(None, "docker", "swarm_services_short", "swarm") == ["swarm: api (0 of 1 running)"]
+    assert await alerts.which(None, "host_agent", "cpu_pct", "web-1") == []
+
+
+async def test_a_lookup_that_fails_names_nothing(monkeypatch):
+    monkeypatch.setattr(alerts, "_NAMED", {})
+
+    async def broken(session, resource):
+        raise RuntimeError("database away")
+
+    alerts.names_behind("docker", "swarm_nodes_down", broken)
+    assert await alerts.which(None, "docker", "swarm_nodes_down", "swarm") == []

tests/core/test_alerts_page.py

@@ -62,6 +62,18 @@ def test_no_reading_yet_is_said_not_shown_as_a_number():
     assert _said(says) == "CPU on web-1 has no reading yet (> 90%), firing for 3m"
 
 
+def test_a_count_says_how_many_of_what():
+    """#6400: "swarm services short on swarm is 6" named nothing and wasn't a sentence."""
+    now = datetime.now(timezone.utc)
+    swarm = alert_sentence(_rule("swarm_services_short", threshold=0.0, resource="swarm"), "firing", 6.0,
+                           now - timedelta(minutes=4), now)
+    assert swarm["resource"] == "6 swarm services are short of their replicas"
+    assert _said(swarm) == "6 swarm services are short of their replicas (> 0), firing for 4m"
+    host = alert_sentence(_rule("mounts_stalled", threshold=0.0, resource="nas"), "firing", 1.0,
+                          now - timedelta(minutes=4), now)
+    assert (host["resource"], _said(host)) == ("nas", "1 mount isn't answering on nas (> 0), firing for 4m")
+
+
 def test_a_silent_series_says_how_long_it_has_been_quiet():
     now = datetime.now(timezone.utc)
     says = alert_sentence(_rule(), "no_data", 50.0, now - timedelta(minutes=20), now)

tests/core/test_mail.py

@@ -106,6 +106,52 @@ def test_each_state_reads_as_a_sentence():
     assert (down["headline"], down["condition"]) == ("web01 is down.", "is down")
 
 
+SHORT = {**ALERT, "rule_name": "Swarm services short of replicas", "metric": "swarm_services_short",
+         "value": 6.0, "operator": ">", "threshold": 0.0, "resource": "swarm", "source_module": "docker",
+         "which": ["api_app (0 of 1 running)", "db (0 of 1 running)", "proxy (1 of 2 running)",
+                   "web_app (0 of 1 running)"]}
+
+
+def test_a_count_on_the_swarm_says_how_many_of_what_and_names_them():
+    a = mail.alert_facts(SHORT, timezone.utc)
+    assert a["headline"] == "6 swarm services are short of their replicas."
+    assert a["which"] == ("api_app (0 of 1 running), db (0 of 1 running), proxy (1 of 2 running)"
+                          " and web_app (0 of 1 running)")
+    assert a["subject"] == ("[Steward] Firing: Swarm services short of replicas — 6 (api_app (0 of 1 running),"
+                            " db (0 of 1 running), proxy (1 of 2 running), …)")
+    one = mail.alert_facts({**SHORT, "value": 1.0, "which": ["db (0 of 1 running)"]}, timezone.utc)
+    assert one["headline"] == "1 swarm service is short of its replicas."
+    clear = mail.alert_facts({**SHORT, "state": "RESOLVED", "value": 0.0, "which": []}, timezone.utc)
+    assert (clear["headline"], clear["subject"], clear["which"]) == (
+        "Every swarm service has its replicas again.",
+        "[Steward] Resolved: Swarm services short of replicas — swarm", "")
+    silent = mail.alert_facts({**SHORT, "state": "NO_DATA", "value": None}, timezone.utc)
+    assert silent["headline"] == "Steward has had no swarm services short of replicas reading from the swarm."
+
+
+def test_a_count_on_a_host_says_where():
+    stalled = {**ALERT, "metric": "mounts_stalled", "value": 2.0, "operator": ">", "threshold": 0.0}
+    assert mail.alert_facts(stalled, timezone.utc)["headline"] == "2 mounts aren't answering on web01."
+    assert mail.alert_facts({**stalled, "state": "RESOLVED", "value": 1.0}, timezone.utc)["headline"] == \
+        "1 mount isn't answering on web01, back within its limit."
+
+
+async def test_the_names_reach_both_parts_and_a_burst(monkeypatch):
+    msg = await _sent(monkeypatch, CFG, SHORT)
+    for part in (msg.get_body(("plain",)).get_content(), msg.get_body(("html",)).get_content()):
+        assert "Which" in part and "web_app (0 of 1 running)" in part, part
+    plain = await _sent(monkeypatch, CFG, ALERT)
+    assert "Which" not in plain.get_body(("plain",)).get_content() + plain.get_body(("html",)).get_content()
+    _, text, html = mail.burst_email(CFG, [SHORT, ALERT], EXTRA_NY)
+    assert text.count("Which: ") == 1 and html.count("Which: ") == 1
+
+
+def test_a_long_list_is_cut_short():
+    names = [f"svc{i:02d}" for i in range(15)]
+    said = mail.alert_facts({**SHORT, "value": 15.0, "which": names}, timezone.utc)["which"]
+    assert said.endswith("svc11 and 3 more") and "svc12" not in said
+
+
 def test_the_report_has_its_sections_in_both_parts():
     subject, text, html = mail.report_email(CFG, REPORT, PERSON_NY)
     assert subject == "[Steward] Weekly report, 4 Oct – 11 Oct 2026"

tests/core/test_notifications.py

@@ -34,6 +34,12 @@ def test_raw_fields_give_bare_json_values():
     assert json.loads(n.render_webhook(tpl, {**ALERT, "value": None}))["value"] is None
 
 
+def test_the_names_behind_a_count_read_as_a_list_or_as_an_array():
+    alert = {**ALERT, "which": ["a (0 of 1 running)", 'b "q"']}
+    body = json.loads(n.render_webhook('{"w": "{{ alert.which }}", "raw": {{ raw.which | tojson }}}', alert))
+    assert body == {"w": 'a (0 of 1 running), b "q"', "raw": ["a (0 of 1 running)", 'b "q"']}
+
+
 def test_a_missing_value_reads_no_data():
     body = n.render_webhook('{"v": "{{ alert.value }}", "note": "{{ alert.note }}"}', {**ALERT, "value": None})
     assert json.loads(body) == {"v": "no data", "note": ""}

tests/plugins/docker/test_swarm_health.py

@@ -21,6 +21,16 @@ def test_services_short_of_their_replicas_are_counted():
                           {"service_name": "d", "desired": 0, "running": 0},     # scaled to 0: wants none
                           {"service_name": "e", "desired": 1, "running": 2}]}    # an update overlapping
     assert sh.figures(swarm)["swarm_services_short"] == 2
+    # An alert on it names them, with how many of each run (#6400).
+    assert sh.short_services(swarm["services"]) == ["a (0 of 1 running)", "b (2 of 3 running)"]
+
+
+def test_the_nodes_and_managers_counted_are_named():
+    nodes = [_node("util01", availability="drain", status="down"), _node("docker02", status="down",
+             reachability="unreachable"), _node("docker03"), _node("worker1", role="worker", status="disconnected")]
+    assert sh.down_nodes(nodes) == ["docker02 (down)", "worker1 (disconnected)"]
+    assert sh.unreachable_managers(nodes) == ["docker02"]
+    assert set(sh.NAMERS) == set(sh.SWARM_METRICS)
 
 
 def test_a_drained_node_going_away_is_expected():