fix(alerts): one silent agent is one alert and one finding (#5721)

VDNT-Docker03's silent agent read as three overview findings (agent offline,
agent stale, the disk rule's no-data) and every drop emailed the disk rule's
silence and return on top of agent offline.

- The host agent registers its heartbeat (seconds_since_seen) and the
  sources it carries (host_agent, docker). While a host's agent is past its
  offline window, its other rules' no_data is recorded "while its agent is
  offline" and not emailed, nor is their quiet return; a return past a
  threshold still fires.
- The overview folds those rules under the host's agent-offline finding
  ("also silent with it: ..."), and a stale agent whose agent-offline alert
  is active is that alert's finding, not a second one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_alert_no_data.py

@@ -189,3 +189,70 @@ def test_a_series_quiet_only_while_steward_was_down_is_not_silent(app):
     with patch.object(alerts, "_dispatch_notification", fake_dispatch):
         asyncio.run(_go())
 
+
+
+def test_a_rule_silent_with_its_agent_says_nothing_of_its_own(app):
+    """#5721: while a host's agent is offline, "agent offline" is the one
+    email. The host's other rules going silent is recorded, marked as with
+    its agent, and not emailed; their quiet return isn't either, but a
+    return past the threshold still fires."""
+    from sqlalchemy import select
+    from steward.alerts import service as alert_svc
+    from steward.core import alerts
+    from steward.core.watch import watch
+    from steward.models.alerts import AlertEvent, AlertState
+    from steward.models.metrics import PluginMetric, PluginMetricLatest
+    from steward.models.users import User, UserRole
+
+    alerts.agent_heartbeat("host_agent", "seconds_since_seen", lambda: 180.0)
+    alerts.agent_fed("host_agent", lambda resource: resource.split(":", 1)[0])
+    sent = []
+
+    async def fake_dispatch(app_, notif_type, rule, value, event_id, note=None):
+        sent.append((rule.id, notif_type))
+
+    async def _go():
+        tag = uuid.uuid4().hex[:8]
+        host = f"gone-{tag}"
+        now = datetime.now(timezone.utc)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                user = User(username=f"ag-{tag}", email=f"ag-{tag}@test", password_hash="x",
+                            role=UserRole.admin)
+                s.add(user)
+                await alerts.record_metrics(s, [PluginMetric(
+                    source_module="host_agent", resource_name=f"{host}:/", metric_name="disk_used_pct",
+                    value=50.0, recorded_at=now - timedelta(minutes=20))])
+                await s.flush()
+                fields = await alert_svc.rule_fields(
+                    s, name=f"disk {tag}", source_module="host_agent", resource_name=f"{host}:/",
+                    metric_name="disk_used_pct", operator=">", threshold=90, notify_no_data=True)
+                rule_id = (await alert_svc.add_rule(s, fields, created_by=user.id)).id
+                await s.flush()
+                st = (await s.execute(select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one()
+                st.last_sample_at, st.sample_interval_s = now - timedelta(minutes=20), 60.0
+                # The agent has been quiet for 20 minutes: offline.
+                await s.merge(PluginMetricLatest(source_module="host_agent", resource_name=host,
+                                                 metric_name="seconds_since_seen", value=1200.0,
+                                                 recorded_at=now))
+
+        w = watch(app, "alert_silence")
+        w.since, w.last = now - timedelta(days=1), datetime.now(timezone.utc)
+        await alerts.check_silence(app)
+        await asyncio.sleep(0)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                [note] = (await s.execute(select(AlertEvent.note).where(
+                    AlertEvent.rule_id == rule_id, AlertEvent.to_state == "no_data"))).scalars().all()
+        assert alerts.AGENT_OFFLINE in note
+        assert sent == []                         # agent offline is the email
+
+        # The agent comes back and the disk is fine: back quietly.
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await alerts.record_metric(s, "host_agent", f"{host}:/", "disk_used_pct", 50.0)
+        await asyncio.sleep(0)
+        assert sent == []
+
+    with patch.object(alerts, "_dispatch_notification", fake_dispatch):
+        asyncio.run(_go())

tests/mcp/test_tools.py

@@ -283,3 +283,37 @@ def test_get_host_groups_parts_by_kind_and_summarises_cores():
     assert got["temperatures"]["items"][0] == {"sensor": "Package id 0", "part": "temp:Package id 0",
                                                "temp_c": 61.0}
     assert "other" not in got
+
+
+def test_one_silent_agent_is_one_finding():
+    """#5721: VDNT-Docker03's silent agent read as three findings. Its agent-
+    offline alert speaks for it, the stale-agent finding goes, and its rules
+    that only went quiet are listed under it; one still firing stays listed."""
+    from steward.core import alerts
+    alerts.agent_heartbeat("host_agent", "seconds_since_seen", lambda: 180.0)
+    alerts.agent_fed("host_agent", lambda resource: resource.split(":", 1)[0])
+
+    def rule(name, metric, resource):
+        return NS(id=name, name=name, metric_name=metric, operator=AlertOperator.gt, threshold=90.0,
+                  source_module="host_agent", resource_name=resource)
+
+    gone = NOW - timedelta(hours=2)
+    active = [
+        (NS(state=NS(value="firing"), last_transition_at=gone, no_data_since=None),
+         rule("Agent offline: d3", "seconds_since_seen", "d3")),
+        (NS(state=NS(value="inactive"), last_transition_at=gone, no_data_since=gone),
+         rule("Disk nearly full: d3", "disk_used_pct", "d3:/")),
+        (NS(state=NS(value="firing"), last_transition_at=gone, no_data_since=gone),
+         rule("Memory high: d3", "mem_used_pct", "d3")),
+    ]
+    agent = {"d3": {"fresh": False, "last_at": gone}}
+    listed, silent, speaking = overview.fold_agent_silence(active, agent)
+    assert [r.name for _s, r in listed] == ["Agent offline: d3", "Memory high: d3"]
+    assert silent == {"d3": ["Disk nearly full: d3"]} and speaking == {"d3"}
+    found = overview.alert_findings(listed, silent) + overview.agent_findings(
+        [NS(id="h3", name="d3")], agent, THRESHOLDS, silent, speaking)
+    assert [f["subject"] for f in found] == ["Agent offline: d3", "Memory high: d3"]
+    assert "also silent with it: Disk nearly full: d3" in found[0]["summary"]
+    # Without its alert, the stale agent is the finding, and says the same.
+    [stale] = overview.agent_findings([NS(id="h3", name="d3")], agent, THRESHOLDS, silent)
+    assert stale["kind"] == "agent_stale" and "Disk nearly full: d3" in stale["summary"]