feat(alerts): silence is alertable — agent offline and no-data (#5062)

Agent offline is an ordinary rule now. The host_agent scheduler records
host_agent/<host>/seconds_since_seen every minute (from last_seen_at, or
registration time for an agent that never reported), replacing the task
that only logged stale agents. The catalog offers the metric, and picking
it on a new rule fills "> 180".

No data: each rule's state tracks its last sample and its usual gap,
smoothed so that an outage counts as at most three gaps. A core task,
alert_silence_check, runs every 60s and marks a rule no_data_since once its
series has had no sample for three usual gaps (at least 5 minutes; an hour
while the cadence is unknown). A never-sampled rule counts from when it
started watching. The mark sits beside the state, so a firing alert that
goes silent stays firing. Each silence records one event and sends one
notification (rule option notify_no_data, default on; muted in maintenance
windows). The next sample clears it with a "data resumed" event and
notification. Creating, changing, disabling or re-enabling a rule restarts
its watch.

Migration 0029 adds the columns and seeds each existing rule's last sample
and gap from its last day of data, so an upgrade does not flag every rule.

Shown on the Alerts page (a NO DATA badge with its start time), in MCP
list_alerts, list_alert_rules and get_host (state "no_data", no_data_since),
in get_overview findings (alert_no_data, or a firing alert marked silent)
and in the weekly report. The rule form has the notify option, and its
metric reference now matches what traefik records and lists host_agent.
Emails say "no data" and carry the event's note.

The metric-catalog guard also reads positional MetricBatch.add arguments.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_alert_no_data.py

@@ -0,0 +1,98 @@
+"""Silence is alertable (#5062): a series' usual gap is learned from its
+samples, an outage does not become its cadence, and a series counts as silent
+after three usual gaps (at least five minutes; an hour while its cadence is
+unknown)."""
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace as NS
+
+import pytest
+
+from steward.core.alerts import (
+    NO_DATA_FLOOR_S, NO_DATA_UNKNOWN_S, _track_samples, next_interval, no_data_after, silent_since,
+)
+from steward.models.alerts import AlertStateEnum
+
+NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc)
+
+
+@pytest.mark.parametrize("interval,gap,want", [
+    (None, 60, 60),             # the first gap is the cadence
+    (0.0, 30, 30),
+    (60, 60, 60),               # steady
+    (60, 80, 65),               # a late sample barely moves it
+    (60, 3600, 90),             # an outage counts as at most three gaps
+    (60, -5, 45),               # never a negative gap
+])
+def test_next_interval(interval, gap, want):
+    assert next_interval(interval, gap) == pytest.approx(want)
+
+
+def test_a_new_cadence_is_learned_in_steps():
+    interval, steps = 60.0, 0
+    while no_data_after(interval) < 600:  # samples now arrive every 10 minutes
+        interval, steps = next_interval(interval, 600), steps + 1
+    assert steps == 3
+
+
+@pytest.mark.parametrize("interval,want", [
+    (None, NO_DATA_UNKNOWN_S), (0.0, NO_DATA_UNKNOWN_S),
+    (30, NO_DATA_FLOOR_S), (100, 300), (600, 1800),
+])
+def test_no_data_after(interval, want):
+    assert no_data_after(interval) == want
+
+
+@pytest.mark.parametrize("last_ago,interval,watch_ago,silent_after", [
+    (60, 60, 9999, None),            # reporting
+    (300, 60, 9999, None),           # exactly at the limit is not yet silent
+    (301, 60, 9999, 300),            # silent since last + 300s
+    (2000, 600, 9999, 1800),         # slow series: three of its gaps
+    (None, None, 3000, None),        # never sampled, watched under an hour
+    (None, None, 4000, 3600),        # never sampled, watched over an hour
+])
+def test_silent_since(last_ago, interval, watch_ago, silent_after):
+    last = NOW - timedelta(seconds=last_ago) if last_ago is not None else None
+    watch = NOW - timedelta(seconds=watch_ago)
+    got = silent_since(last, interval, watch, NOW)
+    if silent_after is None:
+        assert got is None
+    else:
+        assert got == (last or watch) + timedelta(seconds=silent_after)
+
+
+class _Session:
+    def __init__(self):
+        self.added = []
+
+    def add(self, row):
+        self.added.append(row)
+
+
+def _state(**kw):
+    base = dict(state=AlertStateEnum.firing, last_sample_at=None, sample_interval_s=None,
+                no_data_since=None)
+    return NS(**{**base, **kw})
+
+
+def _samples(*ago_values):
+    return [NS(recorded_at=NOW - timedelta(seconds=a), value=v) for a, v in ago_values]
+
+
+def test_tracking_learns_the_gap_and_skips_late_samples():
+    st, session = _state(), _Session()
+    assert _track_samples(session, NS(id="r"), st, _samples((180, 1), (120, 1), (60, 1)), NOW) is None
+    assert (st.last_sample_at, st.sample_interval_s) == (NOW - timedelta(seconds=60), 60)
+    _track_samples(session, NS(id="r"), st, _samples((90, 1)), NOW)  # older than the last
+    assert (st.last_sample_at, st.sample_interval_s) == (NOW - timedelta(seconds=60), 60)
+    assert session.added == []
+
+
+def test_a_sample_after_silence_clears_it_and_records_the_resume():
+    st, session = _state(last_sample_at=NOW - timedelta(minutes=20), sample_interval_s=60,
+                         no_data_since=NOW - timedelta(minutes=15)), _Session()
+    event_id, value, note = _track_samples(session, NS(id="r"), st, _samples((0, 42.0)), NOW)
+    assert st.no_data_since is None and st.state == AlertStateEnum.firing  # still firing
+    assert st.sample_interval_s == 90       # the outage counted as three gaps, at most
+    assert value == 42.0 and note == "data resumed after 15m silent"
+    [event] = session.added
+    assert (event.id, event.from_state, event.to_state) == (event_id, "no_data", "firing")

tests/core/test_metric_catalog.py

@@ -55,7 +55,9 @@ def recorded(source: str) -> set[tuple[str, str]]:
                 or (isinstance(callee, ast.Name) and callee.id == "record_metric")
             if not is_writer:
                 continue
-            kw = {k.arg: k.value for k in call.keywords}
+            # Positional or keyword: (source_module, resource_name, metric_name, value).
+            kw = dict(zip(("source_module", "resource_name", "metric_name"), call.args))
+            kw.update({k.arg: k.value for k in call.keywords})
             src = _const(kw.get("source_module"))
             metric = kw.get("metric_name")
             if src is None or metric is None:
@@ -79,19 +81,22 @@ async def poll(session):
     batch.add(source_module="traefik", resource_name="r", metric_name="error_rate", value=1)
     for metric_name, value in [("request_rate", 1), ("made_up", 2)]:
         batch.add(source_module="traefik", resource_name="r", metric_name=metric_name, value=value)
+    batch.add("traefik", "r", "by_position", 3)
 '''
     pairs = recorded(code)
-    assert pairs == {("traefik", "error_rate"), ("traefik", "request_rate"), ("traefik", "made_up")}
+    assert pairs == {("traefik", "error_rate"), ("traefik", "request_rate"), ("traefik", "made_up"),
+                     ("traefik", "by_position")}
     cat = catalog()
     assert {p for p in pairs if p[1] not in cat.get(p[0], set())} == {
-        ("traefik", "error_rate"), ("traefik", "made_up")}
+        ("traefik", "error_rate"), ("traefik", "made_up"), ("traefik", "by_position")}
 
 
 def test_every_recorded_metric_is_in_the_catalog():
     files = _writers()
-    assert len(files) >= 6, files  # docker, traefik, unifi, snmp, monitors, self_monitor
+    assert len(files) >= 7, files  # docker, traefik, unifi, snmp, monitors, self_monitor, host_agent
     pairs = {p for f in files for p in recorded(f.read_text())}
     assert ("traefik", "error_rate_5xx_pct") in pairs and ("docker", "is_healthy") in pairs
+    assert ("host_agent", "seconds_since_seen") in pairs
     cat = catalog()
     missing = sorted(p for p in pairs if p[1] not in cat.get(p[0], set()))
     assert not missing, f"recorded but not in the alert catalog: {missing}"

tests/integration/test_alert_no_data.py

@@ -0,0 +1,125 @@
+"""Integration (#5062): against real Postgres, a rule whose series stops is
+marked no_data once, notified once, and cleared by the next sample; a firing
+alert that goes silent stays firing."""
+from __future__ import annotations
+
+import asyncio
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+from unittest.mock import patch
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def test_silence_is_marked_once_and_cleared_by_data(app):
+    from sqlalchemy import select, update
+    from steward.alerts import service as alert_svc
+    from steward.core import alerts
+    from steward.models.alerts import AlertEvent, AlertRule, AlertState, AlertStateEnum
+    from steward.models.metrics import PluginMetric
+    from steward.models.users import User, UserRole
+
+    sent: list[tuple[str, str, str | None]] = []
+
+    async def fake_dispatch(app_, notif_type, rule, value, event_id, note=None):
+        sent.append((rule.name, notif_type, note))
+
+    async def _go():
+        tag = uuid.uuid4().hex[:8]
+        res = f"quiet-{tag}"
+        now = datetime.now(timezone.utc)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                user = User(username=f"nd-{tag}", email=f"nd-{tag}@test", password_hash="x",
+                            role=UserRole.admin)
+                s.add(user)
+                s.add(PluginMetric(source_module="host_agent", resource_name=res,
+                                   metric_name="cpu_pct", value=1.0,
+                                   recorded_at=now - timedelta(hours=1)))
+                await s.flush()
+                ids = {}
+                for key, notify in (("loud", True), ("hushed", False)):
+                    fields = await alert_svc.rule_fields(
+                        s, name=f"{key} {tag}", source_module="host_agent", resource_name=res,
+                        metric_name="cpu_pct", operator=">", threshold=90,
+                        notify_no_data=notify)
+                    ids[key] = (await alert_svc.add_rule(s, fields, created_by=user.id)).id
+                # Every minute, from 20 to 18 minutes ago; then nothing.
+                batch = alerts.MetricBatch()
+                for ago in (20, 19, 18):
+                    batch.add(source_module="host_agent", resource_name=res, metric_name="cpu_pct",
+                              value=10.0, recorded_at=now - timedelta(minutes=ago))
+                await batch.record(s)
+
+        async def state(key):
+            async with app.db_sessionmaker() as s:
+                return (await s.execute(select(AlertState).where(
+                    AlertState.rule_id == ids[key]))).scalar_one()
+
+        async def events(key):
+            async with app.db_sessionmaker() as s:
+                rows = (await s.execute(select(AlertEvent).where(AlertEvent.rule_id == ids[key])
+                                        .order_by(AlertEvent.transitioned_at))).scalars().all()
+            return [(e.from_state, e.to_state, e.note) for e in rows]
+
+        st = await state("loud")
+        assert st.sample_interval_s == pytest.approx(60)
+        assert st.last_sample_at == now - timedelta(minutes=18)
+
+        await alerts.check_silence(app)
+        await alerts.check_silence(app)  # already marked: nothing more
+        await asyncio.sleep(0)
+        st = await state("loud")
+        assert st.state == AlertStateEnum.inactive
+        assert st.no_data_since == now - timedelta(minutes=18) + timedelta(seconds=300)
+        [(frm, to, note)] = await events("loud")
+        assert (frm, to) == ("inactive", "no_data")
+        assert note == "no sample for 18m (usually every 60s)"
+        assert len(await events("hushed")) == 1
+        assert sent == [(f"loud {tag}", "no_data", note)]  # the hushed rule notifies nobody
+
+        async with app.db_sessionmaker() as s:
+            active = {r.id: alert_svc.alert_label(st_) for st_, r in await alert_svc.active_alerts(s)}
+        assert active[ids["loud"]] == active[ids["hushed"]] == "no_data"
+
+        # Data resumes, breaching: the mark clears and the rule fires.
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await alerts.record_metric(s, "host_agent", res, "cpu_pct", 95.0)
+        await asyncio.sleep(0)
+        st = await state("loud")
+        assert st.no_data_since is None and st.state == AlertStateEnum.firing
+        first, *same_instant = await events("loud")
+        assert first[:2] == ("inactive", "no_data")
+        # The resume and the firing it leads to share the sample's timestamp.
+        assert sorted(e[:2] for e in same_instant) == [("inactive", "firing"),
+                                                       ("no_data", "inactive")]
+        assert any((e[2] or "").startswith("data resumed after ") for e in same_instant)
+        assert [t for _, t, _ in sent if t != "firing"] == ["no_data", "data_resumed"]
+
+        # A firing alert that goes silent stays firing, marked.
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await s.execute(update(AlertState).where(AlertState.rule_id == ids["loud"])
+                                .values(last_sample_at=now - timedelta(hours=2)))
+                await s.execute(update(AlertRule).where(AlertRule.id == ids["hushed"])
+                                .values(enabled=False))
+        await alerts.check_silence(app)
+        st = await state("loud")
+        assert st.state == AlertStateEnum.firing and st.no_data_since is not None
+        assert (await events("loud"))[-1][:2] == ("firing", "no_data")
+        assert len(await events("hushed")) == 3  # disabled: not watched
+
+    with patch.object(alerts, "_dispatch_notification", fake_dispatch):
+        asyncio.run(_go())

tests/integration/test_alert_states.py

@@ -104,4 +104,13 @@ def test_acknowledge_holds_clear_after_and_reset(app):
         assert (await state()).state == AlertStateEnum.inactive
         assert (await events())[-1] == ("firing", "resolved", "rule disabled")
 
+        # Re-enabling starts the watch over: silence is measured from now.
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                r = await s.get(AlertRule, rule_id)
+                await alert_svc.apply_rule_change(s, r, {}, enabled=True)
+        st = await state()
+        assert (st.state, st.last_sample_at, st.no_data_since) == (AlertStateEnum.inactive, None, None)
+        assert (await events())[-1] == ("firing", "resolved", "rule disabled")
+
     asyncio.run(_go())

tests/mcp/test_tools.py

@@ -143,13 +143,30 @@ def test_monitor_findings():
 def test_alert_findings_severity_by_state():
     rule = NS(id="r1", name="cpu", metric_name="cpu_pct", operator=AlertOperator.gt,
               threshold=90.0, source_module="host_agent", resource_name="nas")
-    rows = [(NS(state=NS(value=s), last_transition_at=NOW), rule)
+    rows = [(NS(state=NS(value=s), last_transition_at=NOW, no_data_since=None), rule)
             for s in ("firing", "pending", "acknowledged")]
     sev = {f["kind"]: f["severity"] for f in overview.alert_findings(rows)}
     assert sev == {"alert_firing": "critical", "alert_pending": "warning",
                    "alert_acknowledged": "warning"}
 
 
+def test_alert_findings_say_when_a_metric_went_silent():
+    rule = NS(id="r1", name="cpu", metric_name="cpu_pct", operator=AlertOperator.gt,
+              threshold=90.0, source_module="host_agent", resource_name="nas")
+    silent = NOW - timedelta(minutes=10)
+    quiet, firing = overview.alert_findings([
+        (NS(state=NS(value="inactive"), last_transition_at=NOW - timedelta(days=1),
+            no_data_since=silent), rule),
+        (NS(state=NS(value="firing"), last_transition_at=NOW - timedelta(hours=1),
+            no_data_since=silent), rule),
+    ])
+    assert (quiet["kind"], quiet["severity"]) == ("alert_no_data", "warning")
+    assert quiet["since"] == "2026-10-06T09:50:00Z" and "stopped reporting" in quiet["summary"]
+    # A firing alert that goes silent stays a firing finding, marked.
+    assert (firing["kind"], firing["severity"]) == ("alert_firing", "critical")
+    assert "no data since 2026-10-06T09:50:00Z" in firing["summary"]
+
+
 THRESHOLDS = {"cpu": {"warn": 80, "crit": 90}, "mem": {"warn": 80, "crit": 90},
               "disk": {"warn": 80, "crit": 90}}
 

tests/plugins/host_agent/test_scheduler.py

@@ -1,48 +1,41 @@
-"""Pure-function tests for host_agent staleness filter."""
+"""Pure-function tests for the host_agent seconds_since_seen reconciler."""
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 
-from plugins.host_agent.scheduler import _filter_stale
+from plugins.host_agent.scheduler import seconds_since_seen
 
 
 @dataclass
 class FakeReg:
     host_id: str
     last_seen_at: datetime | None
+    created_at: datetime | None
 
 
 NOW = datetime(2026, 4, 15, 12, 0, 0, tzinfo=timezone.utc)
+NAMES = {"h1": "nas", "h2": "web", "h3": "db"}
 
 
-def test_filter_stale_returns_only_old_rows():
-    regs = [
-        FakeReg("fresh", NOW - timedelta(seconds=30)),
-        FakeReg("stale", NOW - timedelta(seconds=600)),
-        FakeReg("never", None),
-    ]
-    stale = _filter_stale(regs, now=NOW, stale_after_seconds=180)
-    ids = {r.host_id for r in stale}
-    assert ids == {"stale"}
+def test_seconds_since_each_agent_last_reported():
+    regs = [FakeReg("h1", NOW - timedelta(seconds=30), NOW - timedelta(days=9)),
+            FakeReg("h2", NOW - timedelta(minutes=10), NOW - timedelta(days=9))]
+    assert seconds_since_seen(regs, NAMES, NOW) == [("nas", 30.0), ("web", 600.0)]
 
 
-def test_filter_stale_empty_input():
-    assert _filter_stale([], now=NOW, stale_after_seconds=180) == []
+def test_an_agent_that_never_reported_counts_from_registration():
+    regs = [FakeReg("h3", None, NOW - timedelta(minutes=5))]
+    assert seconds_since_seen(regs, NAMES, NOW) == [("db", 300.0)]
 
 
-def test_filter_stale_all_fresh():
-    regs = [
-        FakeReg("a", NOW - timedelta(seconds=10)),
-        FakeReg("b", NOW - timedelta(seconds=30)),
-    ]
-    assert _filter_stale(regs, now=NOW, stale_after_seconds=60) == []
+def test_a_registration_without_its_host_is_skipped():
+    regs = [FakeReg("gone", NOW, NOW)]
+    assert seconds_since_seen(regs, NAMES, NOW) == []
 
 
-def test_filter_stale_boundary_exact_cutoff_is_not_stale():
-    # A row exactly at the cutoff should not be flagged (strict less-than).
-    regs = [FakeReg("edge", NOW - timedelta(seconds=180))]
-    assert _filter_stale(regs, now=NOW, stale_after_seconds=180) == []
+def test_a_clock_ahead_of_ours_reads_as_just_seen():
+    regs = [FakeReg("h1", NOW + timedelta(seconds=5), NOW)]
+    assert seconds_since_seen(regs, NAMES, NOW) == [("nas", 0.0)]
 
 
-def test_filter_stale_skips_none_last_seen():
-    regs = [FakeReg("never", None)]
-    assert _filter_stale(regs, now=NOW, stale_after_seconds=1) == []
+def test_empty_input():
+    assert seconds_since_seen([], NAMES, NOW) == []