fix(alerts): one silent agent is one alert and one finding (#5721)

VDNT-Docker03's silent agent read as three overview findings (agent offline,
agent stale, the disk rule's no-data) and every drop emailed the disk rule's
silence and return on top of agent offline.

- The host agent registers its heartbeat (seconds_since_seen) and the
  sources it carries (host_agent, docker). While a host's agent is past its
  offline window, its other rules' no_data is recorded "while its agent is
  offline" and not emailed, nor is their quiet return; a return past a
  threshold still fires.
- The overview folds those rules under the host's agent-offline finding
  ("also silent with it: ..."), and a stale agent whose agent-offline alert
  is active is that alert's finding, not a second one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/alerts.py

@@ -4,7 +4,7 @@ import logging
 import uuid
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Callable
 
 from sqlalchemy import event as sa_event, select
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -35,6 +35,75 @@ def init_alerts(app: "Quart") -> None:
     _app = app
 
 
+# ── One silent agent is one alert (#5721) ───────────────────────────────────
+# A host's agent carries more than its own figures (its containers' too), so
+# when it goes quiet every rule it feeds goes silent with it. That is one
+# cause: "agent offline" says it, and the other rules' silence is recorded but
+# not emailed, nor is their quiet return. A return past a threshold still fires.
+
+AGENT_OFFLINE = "while its agent is offline"
+
+
+@dataclass(frozen=True)
+class Heartbeat:
+    source_module: str
+    metric_name: str
+    offline_after: Callable[[], float]     # seconds; read at each use
+
+
+_HEARTBEAT: Heartbeat | None = None
+_AGENT_FED: dict[str, Callable[[str], str]] = {}
+
+
+def agent_heartbeat(source_module: str, metric_name: str,
+                    offline_after: Callable[[], float]) -> None:
+    """The metric that says how long each host's agent has been quiet; past
+    `offline_after()` the agent is offline. The host agent plugin registers it."""
+    global _HEARTBEAT
+    _HEARTBEAT = Heartbeat(source_module, metric_name, offline_after)
+
+
+def agent_fed(source_module: str, host_of: Callable[[str], str]) -> None:
+    """A source whose samples a host's agent carries; `host_of(resource)`
+    names the host ("nas:/" → "nas", "nas/web" → "nas")."""
+    _AGENT_FED[source_module] = host_of
+
+
+def is_heartbeat(rule) -> bool:
+    """Whether `rule` watches the agent heartbeat ("agent offline")."""
+    return _HEARTBEAT is not None and (rule.source_module, rule.metric_name) == (
+        _HEARTBEAT.source_module, _HEARTBEAT.metric_name)
+
+
+def host_behind(rule) -> str | None:
+    """The host whose agent feeds `rule`, or None: a rule on another source,
+    or the heartbeat itself (agent offline is the one that speaks)."""
+    host_of = _AGENT_FED.get(rule.source_module)
+    if host_of is None or is_heartbeat(rule):
+        return None
+    return host_of(rule.resource_name)
+
+
+async def offline_agents(session) -> set[str]:
+    """Hosts whose agent's last heartbeat was past its offline window."""
+    if _HEARTBEAT is None:
+        return set()
+    return set((await session.execute(
+        select(PluginMetricLatest.resource_name).where(
+            PluginMetricLatest.source_module == _HEARTBEAT.source_module,
+            PluginMetricLatest.metric_name == _HEARTBEAT.metric_name,
+            PluginMetricLatest.value > _HEARTBEAT.offline_after()))).scalars())
+
+
+async def _went_quiet_with_its_agent(session, rule_id: str) -> bool:
+    """Whether a rule's latest silence was recorded while its agent was offline."""
+    note = (await session.execute(
+        select(AlertEvent.note).where(AlertEvent.rule_id == rule_id,
+                                      AlertEvent.to_state == "no_data")
+        .order_by(AlertEvent.transitioned_at.desc()).limit(1))).scalar_one_or_none()
+    return bool(note and AGENT_OFFLINE in note)
+
+
 def alert_extra_vars(rule: AlertRule, value: float) -> dict:
     """Ansible extra-vars describing the alert that fired.
 
@@ -264,6 +333,10 @@ async def record_metrics(session: AsyncSession, rows: list[PluginMetric], phases
             for notif_type, event_id, note in notifications:
                 if notif_type == "data_resumed" and not rule.notify_no_data:
                     continue
+                # Back with its agent: the agent's return is the news (#5721).
+                if notif_type == "data_resumed" and host_behind(rule) is not None \
+                        and await _went_quiet_with_its_agent(session, rule.id):
+                    continue
                 _after_commit(session, _dispatch_notification, notif_type, rule, sample.value,
                               event_id, note)
                 # Admin-configured Ansible action runs only on transition-to-firing.
@@ -670,6 +743,7 @@ async def check_silence(app: "Quart") -> None:
                 select(MaintenanceWindow).where(
                     MaintenanceWindow.start_at <= now, MaintenanceWindow.end_at >= now)
             )).scalars().all()
+            offline = await offline_agents(session) if _AGENT_FED else set()
             for state, rule in rows:
                 watching_since = max(rule.created_at, state.last_transition_at)
                 since = silent_since(state.last_sample_at, state.sample_interval_s,
@@ -685,9 +759,11 @@ async def check_silence(app: "Quart") -> None:
                     if state.sample_interval_s:
                         note += f" (usually every {human_duration(state.sample_interval_s)})"
                 muted = _muted_by(windows, rule)
-                event_id = _add_event(session, rule.id, state.state.value, "no_data", None,
-                                      now, note=f"{note}; {muted}" if muted else note)
-                if rule.notify_no_data and not muted:
+                # Silent with its agent: recorded, and "agent offline" is the email (#5721).
+                hushed = AGENT_OFFLINE if host_behind(rule) in offline else None
+                event_id = _add_event(session, rule.id, state.state.value, "no_data", None, now,
+                                      note="; ".join(n for n in (note, hushed, muted) if n))
+                if rule.notify_no_data and not muted and not hushed:
                     notify.append((rule, event_id, note))
     for rule, event_id, note in notify:
         _spawn(_dispatch_notification(app, "no_data", rule, None, event_id, note))

steward/mcp/tools/overview.py

@@ -7,6 +7,7 @@ from datetime import timedelta
 from sqlalchemy import select
 
 from steward.alerts import service as alert_svc
+from steward.core import alerts
 from steward.core import anomalies as anom
 from steward.hosts import service as host_svc
 from steward.mcp._context import mcp_app
@@ -70,8 +71,37 @@ def monitor_findings(monitors, states: dict, host_names: dict, now,
     return out
 
 
-def alert_findings(active) -> list[dict]:
+def fold_agent_silence(active, agent: dict) -> tuple[list, dict[str, list[str]], set[str]]:
+    """One silent agent is one finding (#5721). Returns (the active alerts left
+    to list, {host: names of its rules silent with its agent}, the hosts
+    whose "agent offline" alert is active and so speaks for the agent).
+
+    A rule the agent feeds that has only gone quiet, on a host whose agent is
+    stale, is folded under that host; one firing or pending stays listed,
+    since what it measured before going quiet is news of its own."""
+    offline = {name for name, a in agent.items() if a and not a.get("fresh")}
+    kept, silent, speaking = [], {}, set()
+    for state, rule in active:
+        if alerts.is_heartbeat(rule):
+            speaking.add(rule.resource_name)
+            kept.append((state, rule))
+            continue
+        host = alerts.host_behind(rule)
+        if host in offline and state.no_data_since is not None \
+                and state.state.value not in ("firing", "pending", "acknowledged"):
+            silent.setdefault(host, []).append(rule.name)
+            continue
+        kept.append((state, rule))
+    return kept, silent, speaking
+
+
+def _also_silent(names: list[str] | None) -> str:
+    return f"; also silent with it: {', '.join(sorted(names))}" if names else ""
+
+
+def alert_findings(active, silent: dict[str, list[str]] | None = None) -> list[dict]:
     out = []
+    silent = silent or {}
     for state, rule in active:
         s = state.state.value
         if s == "firing":
@@ -84,6 +114,8 @@ def alert_findings(active) -> list[dict]:
             s, sev, what = "no_data", "warning", "has stopped reporting"
         if state.no_data_since is not None and s != "no_data":
             what += f" and has stopped reporting (no data since {iso(state.no_data_since)})"
+        if alerts.is_heartbeat(rule):
+            what += _also_silent(silent.get(rule.resource_name))
         out.append(finding(
             sev, f"alert_{s}", rule.name,
             f"{rule.metric_name} {rule.operator.value} {rule.threshold:g} on "
@@ -98,18 +130,26 @@ _AGENT_CHECKS = (("cpu_pct", "cpu", "CPU"), ("mem_used_pct", "mem", "memory"),
                  ("disk_used_pct_worst", "disk", "fullest disk"))
 
 
-def agent_findings(hosts, agent: dict, thresholds: dict) -> list[dict]:
+def agent_findings(hosts, agent: dict, thresholds: dict,
+                   silent: dict[str, list[str]] | None = None,
+                   speaking: set[str] | frozenset = frozenset()) -> list[dict]:
+    """Hosts past a CPU, memory or disk cutoff, and agents that stopped
+    reporting. A stale agent whose "agent offline" alert is active (`speaking`)
+    is that alert's finding, not a second one (#5721)."""
     out = []
+    silent = silent or {}
     for h in hosts:
         a = agent.get(h.name)
         if not a:
             continue
         ids = {"host_id": h.id}
         if not a.get("fresh"):
-            out.append(finding(
-                "warning", "agent_stale", h.name,
-                "host agent has stopped reporting; its metrics are out of date",
-                since=a.get("last_at"), ids=ids, next=f'get_host(host="{h.id}")'))
+            if h.name not in speaking:
+                out.append(finding(
+                    "warning", "agent_stale", h.name,
+                    "host agent has stopped reporting; its metrics are out of date"
+                    + _also_silent(silent.get(h.name)),
+                    since=a.get("last_at"), ids=ids, next=f'get_host(host="{h.id}")'))
             continue  # stale values would only add noise
         # CPU and memory within the host's own normal for the hour aren't
         # findings (#5742); the fullest disk keeps its cutoff.
@@ -216,9 +256,10 @@ async def get_overview() -> dict:
         to_review = await anom.to_review_count(db)
 
     thresholds = app.config.get("THRESHOLDS") or {}
+    listed, silent, speaking = fold_agent_silence(active, agent)
     findings = (monitor_findings(monitors, states, host_names, now, thresholds.get("tls_days"))
-                + alert_findings(active)
-                + agent_findings(hosts, agent, thresholds)
+                + alert_findings(listed, silent)
+                + agent_findings(hosts, agent, thresholds, silent, speaking)
                 + run_findings(runs)
                 + anomaly_findings(open_anomalies))
     plugin_found, unavailable = await _plugin_findings(getattr(app, "mcp_server", None))