fix(alerts): an alert on a count says how many of what, and names them (#6400)

"swarm services short on swarm is 6." becomes "6 swarm services are
short of their replicas.", with a Which line naming each service and how
many of its tasks run. The swarm figures get labels, and the counting
figures (swarm, mounts, drives, updates) read as sentences in the email,
its subject and the Alerts page. A plugin names what its count counts
through core.alerts.names_behind; the Docker plugin does for the three
swarm figures, from the stored swarm. Webhooks get alert.which too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/alerts.py

@@ -54,6 +54,7 @@ class Heartbeat:
 _HEARTBEAT: Heartbeat | None = None
 _AGENT_FED: dict[str, Callable[[str], str]] = {}
 _AWAY: Callable[[AsyncSession], Awaitable[set[str]]] | None = None
+_NAMED: dict[tuple[str, str], Callable[[AsyncSession, str], Awaitable[list[str]]]] = {}
 
 
 def agent_heartbeat(source_module: str, metric_name: str,
@@ -78,6 +79,28 @@ def agents_away(hosts_away: Callable[[AsyncSession], Awaitable[set[str]]]) -> No
     _AWAY = hosts_away
 
 
+def names_behind(source_module: str, metric_name: str,
+                 names: Callable[[AsyncSession, str], Awaitable[list[str]]]) -> None:
+    """A figure that counts things: `await names(session, resource)` names
+    the things it counts right now ("web_app (0 of 1 running)"), so the
+    alert about it can say which. The plugin recording the figure registers it."""
+    _NAMED[(source_module, metric_name)] = names
+
+
+async def which(session: AsyncSession, source_module: str, metric_name: str,
+                resource: str) -> list[str]:
+    """What a counting figure counts right now, by name; [] for a figure no
+    plugin names, or when naming fails (the alert still goes out)."""
+    names = _NAMED.get((source_module, metric_name))
+    if names is None:
+        return []
+    try:
+        return list(await names(session, resource))
+    except Exception:
+        logger.exception("Naming what %s on %s counts failed", metric_name, resource)
+        return []
+
+
 def is_heartbeat(rule) -> bool:
     """Whether `rule` watches the agent heartbeat ("agent offline")."""
     return _HEARTBEAT is not None and (rule.source_module, rule.metric_name) == (
@@ -672,8 +695,14 @@ async def _dispatch_notification(
         "resource": rule.resource_name,
         "source_module": rule.source_module,
         "note": note,
+        "which": [],
         "timestamp": datetime.now(timezone.utc).isoformat(),
     }
+    # A firing count says what it counts, as it stands when it fires.
+    if notif_type == "firing" and (rule.source_module, rule.metric_name) in _NAMED:
+        async with app.db_sessionmaker() as db:
+            alert_data["which"] = await which(db, rule.source_module, rule.metric_name,
+                                              rule.resource_name)
 
     # Email waits so alerts that arrive together go out as one message per
     # recipient (#5465, #5507); the webhook posts this alert now.