fix(alerts): one silent agent is one alert and one finding (#5721)

VDNT-Docker03's silent agent read as three overview findings (agent offline,
agent stale, the disk rule's no-data) and every drop emailed the disk rule's
silence and return on top of agent offline.

- The host agent registers its heartbeat (seconds_since_seen) and the
  sources it carries (host_agent, docker). While a host's agent is past its
  offline window, its other rules' no_data is recorded "while its agent is
  offline" and not emailed, nor is their quiet return; a return past a
  threshold still fires.
- The overview folds those rules under the host's agent-offline finding
  ("also silent with it: ..."), and a stale agent whose agent-offline alert
  is active is that alert's finding, not a second one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/alerts.py

@@ -4,7 +4,7 @@ import logging
 import uuid
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Callable
 
 from sqlalchemy import event as sa_event, select
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -35,6 +35,75 @@ def init_alerts(app: "Quart") -> None:
     _app = app
 
 
+# ── One silent agent is one alert (#5721) ───────────────────────────────────
+# A host's agent carries more than its own figures (its containers' too), so
+# when it goes quiet every rule it feeds goes silent with it. That is one
+# cause: "agent offline" says it, and the other rules' silence is recorded but
+# not emailed, nor is their quiet return. A return past a threshold still fires.
+
+AGENT_OFFLINE = "while its agent is offline"
+
+
+@dataclass(frozen=True)
+class Heartbeat:
+    source_module: str
+    metric_name: str
+    offline_after: Callable[[], float]     # seconds; read at each use
+
+
+_HEARTBEAT: Heartbeat | None = None
+_AGENT_FED: dict[str, Callable[[str], str]] = {}
+
+
+def agent_heartbeat(source_module: str, metric_name: str,
+                    offline_after: Callable[[], float]) -> None:
+    """The metric that says how long each host's agent has been quiet; past
+    `offline_after()` the agent is offline. The host agent plugin registers it."""
+    global _HEARTBEAT
+    _HEARTBEAT = Heartbeat(source_module, metric_name, offline_after)
+
+
+def agent_fed(source_module: str, host_of: Callable[[str], str]) -> None:
+    """A source whose samples a host's agent carries; `host_of(resource)`
+    names the host ("nas:/" → "nas", "nas/web" → "nas")."""
+    _AGENT_FED[source_module] = host_of
+
+
+def is_heartbeat(rule) -> bool:
+    """Whether `rule` watches the agent heartbeat ("agent offline")."""
+    return _HEARTBEAT is not None and (rule.source_module, rule.metric_name) == (
+        _HEARTBEAT.source_module, _HEARTBEAT.metric_name)
+
+
+def host_behind(rule) -> str | None:
+    """The host whose agent feeds `rule`, or None: a rule on another source,
+    or the heartbeat itself (agent offline is the one that speaks)."""
+    host_of = _AGENT_FED.get(rule.source_module)
+    if host_of is None or is_heartbeat(rule):
+        return None
+    return host_of(rule.resource_name)
+
+
+async def offline_agents(session) -> set[str]:
+    """Hosts whose agent's last heartbeat was past its offline window."""
+    if _HEARTBEAT is None:
+        return set()
+    return set((await session.execute(
+        select(PluginMetricLatest.resource_name).where(
+            PluginMetricLatest.source_module == _HEARTBEAT.source_module,
+            PluginMetricLatest.metric_name == _HEARTBEAT.metric_name,
+            PluginMetricLatest.value > _HEARTBEAT.offline_after()))).scalars())
+
+
+async def _went_quiet_with_its_agent(session, rule_id: str) -> bool:
+    """Whether a rule's latest silence was recorded while its agent was offline."""
+    note = (await session.execute(
+        select(AlertEvent.note).where(AlertEvent.rule_id == rule_id,
+                                      AlertEvent.to_state == "no_data")
+        .order_by(AlertEvent.transitioned_at.desc()).limit(1))).scalar_one_or_none()
+    return bool(note and AGENT_OFFLINE in note)
+
+
 def alert_extra_vars(rule: AlertRule, value: float) -> dict:
     """Ansible extra-vars describing the alert that fired.
 
@@ -264,6 +333,10 @@ async def record_metrics(session: AsyncSession, rows: list[PluginMetric], phases
             for notif_type, event_id, note in notifications:
                 if notif_type == "data_resumed" and not rule.notify_no_data:
                     continue
+                # Back with its agent: the agent's return is the news (#5721).
+                if notif_type == "data_resumed" and host_behind(rule) is not None \
+                        and await _went_quiet_with_its_agent(session, rule.id):
+                    continue
                 _after_commit(session, _dispatch_notification, notif_type, rule, sample.value,
                               event_id, note)
                 # Admin-configured Ansible action runs only on transition-to-firing.
@@ -670,6 +743,7 @@ async def check_silence(app: "Quart") -> None:
                 select(MaintenanceWindow).where(
                     MaintenanceWindow.start_at <= now, MaintenanceWindow.end_at >= now)
             )).scalars().all()
+            offline = await offline_agents(session) if _AGENT_FED else set()
             for state, rule in rows:
                 watching_since = max(rule.created_at, state.last_transition_at)
                 since = silent_since(state.last_sample_at, state.sample_interval_s,
@@ -685,9 +759,11 @@ async def check_silence(app: "Quart") -> None:
                     if state.sample_interval_s:
                         note += f" (usually every {human_duration(state.sample_interval_s)})"
                 muted = _muted_by(windows, rule)
-                event_id = _add_event(session, rule.id, state.state.value, "no_data", None,
-                                      now, note=f"{note}; {muted}" if muted else note)
-                if rule.notify_no_data and not muted:
+                # Silent with its agent: recorded, and "agent offline" is the email (#5721).
+                hushed = AGENT_OFFLINE if host_behind(rule) in offline else None
+                event_id = _add_event(session, rule.id, state.state.value, "no_data", None, now,
+                                      note="; ".join(n for n in (note, hushed, muted) if n))
+                if rule.notify_no_data and not muted and not hushed:
                     notify.append((rule, event_id, note))
     for rule, event_id, note in notify:
         _spawn(_dispatch_notification(app, "no_data", rule, None, event_id, note))