feat(alerts): silence is alertable — agent offline and no-data (#5062)

Agent offline is an ordinary rule now. The host_agent scheduler records
host_agent/<host>/seconds_since_seen every minute (from last_seen_at, or
registration time for an agent that never reported), replacing the task
that only logged stale agents. The catalog offers the metric, and picking
it on a new rule fills "> 180".

No data: each rule's state tracks its last sample and its usual gap,
smoothed so that an outage counts as at most three gaps. A core task,
alert_silence_check, runs every 60s and marks a rule no_data_since once its
series has had no sample for three usual gaps (at least 5 minutes; an hour
while the cadence is unknown). A never-sampled rule counts from when it
started watching. The mark sits beside the state, so a firing alert that
goes silent stays firing. Each silence records one event and sends one
notification (rule option notify_no_data, default on; muted in maintenance
windows). The next sample clears it with a "data resumed" event and
notification. Creating, changing, disabling or re-enabling a rule restarts
its watch.

Migration 0029 adds the columns and seeds each existing rule's last sample
and gap from its last day of data, so an upgrade does not flag every rule.

Shown on the Alerts page (a NO DATA badge with its start time), in MCP
list_alerts, list_alert_rules and get_host (state "no_data", no_data_since),
in get_overview findings (alert_no_data, or a firing alert marked silent)
and in the weekly report. The rule form has the notify option, and its
metric reference now matches what traefik records and lists host_agent.
Emails say "no data" and carry the event's note.

The metric-catalog guard also reads positional MetricBatch.add arguments.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/alerts.py

@@ -3,7 +3,7 @@ import asyncio
 import logging
 import uuid
 from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING
 
 from sqlalchemy import select
@@ -141,14 +141,20 @@ async def record_metrics(session: AsyncSession, rows: list[PluginMetric]) -> Non
     )).scalars()}
 
     for rule in rules:
-        if any(_in_window(w, rule.source_module, rule.resource_name) for w in windows):
-            continue  # suppressed by maintenance window
         alert_state = states.get(rule.id)
         if alert_state is None:
             logger.warning("No alert_state row for rule %s, skipping", rule.id)
             continue
         samples = sorted(by_key[(rule.source_module, rule.resource_name, rule.metric_name)],
                          key=lambda r: r.recorded_at)
+        muted = any(_in_window(w, rule.source_module, rule.resource_name) for w in windows)
+        resumed = _track_samples(session, rule, alert_state, samples, now)
+        if resumed and rule.notify_no_data and not muted and _app is not None:
+            event_id, value, note = resumed
+            asyncio.create_task(
+                _dispatch_notification(_app, "data_resumed", rule, value, event_id, note))
+        if muted:
+            continue  # suppressed by maintenance window
         for sample in samples:
             notifications = await _evaluate_rule(session, rule, alert_state, sample.value,
                                                  sample.recorded_at)
@@ -162,6 +168,78 @@ async def record_metrics(session: AsyncSession, rows: list[PluginMetric]) -> Non
                             _run_ansible_action(_app, rule, sample.value, event_id))
 
 
+# A series is silent once it has gone this long without a sample, or three of
+# its usual gaps if that is longer. Until its cadence is known (fewer than two
+# samples seen), it gets an hour: a monitor may check only every ten minutes.
+NO_DATA_FLOOR_S = 300.0
+NO_DATA_GAPS = 3
+NO_DATA_UNKNOWN_S = 3600.0
+
+
+def next_interval(interval: float | None, gap: float) -> float:
+    """A series' usual gap between samples, updated with one more gap.
+
+    A smoothed average, so one late sample barely moves it. A gap longer than
+    the silence threshold counts only as that threshold: an outage is not the
+    series' cadence. A real change of cadence still gets learned, a step at a
+    time (each step at most 1.5x).
+    """
+    gap = max(0.0, gap)
+    if interval is None or interval <= 0:
+        return gap
+    return 0.75 * interval + 0.25 * min(gap, NO_DATA_GAPS * interval)
+
+
+def no_data_after(interval: float | None) -> float:
+    """Seconds without a sample before a series counts as silent."""
+    if not interval:
+        return NO_DATA_UNKNOWN_S
+    return max(NO_DATA_FLOOR_S, NO_DATA_GAPS * interval)
+
+
+def silent_since(last_sample_at: datetime | None, interval: float | None,
+                 watching_since: datetime, now: datetime) -> datetime | None:
+    """When a series went silent, or None while it is still reporting.
+
+    A series that has never had a sample counts from when the rule started
+    watching it (created, changed or re-enabled).
+    """
+    last = last_sample_at or watching_since
+    limit = timedelta(seconds=no_data_after(interval))
+    return last + limit if now - last > limit else None
+
+
+def _duration(seconds: float) -> str:
+    seconds = int(seconds)
+    if seconds < 120:
+        return f"{seconds}s"
+    if seconds < 7200:
+        return f"{seconds // 60}m"
+    return f"{seconds // 3600}h{(seconds % 3600) // 60:02d}m"
+
+
+def _track_samples(session: AsyncSession, rule: AlertRule, state: AlertState,
+                   samples: list[PluginMetric], now: datetime
+                   ) -> tuple[str, float, str] | None:
+    """Note a rule's new samples (oldest first) for silence tracking. When the
+    series had been marked no_data, clear it and record a resume event,
+    returned as (event_id, value, note) for the caller to notify."""
+    for sample in samples:
+        if state.last_sample_at is not None:
+            if sample.recorded_at <= state.last_sample_at:
+                continue  # late or duplicate: not a gap
+            gap = (sample.recorded_at - state.last_sample_at).total_seconds()
+            state.sample_interval_s = next_interval(state.sample_interval_s, gap)
+        state.last_sample_at = sample.recorded_at
+    if state.no_data_since is None or not samples:
+        return None
+    note = f"data resumed after {_duration((now - state.no_data_since).total_seconds())} silent"
+    value = samples[-1].value
+    event_id = _add_event(session, rule.id, "no_data", state.state.value, value, now, note=note)
+    state.no_data_since = None
+    return event_id, value, note
+
+
 def _is_breached(value: float, operator: AlertOperator, threshold: float) -> bool:
     op = operator.value
     if op == ">":
@@ -247,7 +325,8 @@ async def _evaluate_rule(
 
 async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str) -> None:
     """Put a rule's alert back to inactive because the rule itself changed
-    (what it watches, its threshold, or it was disabled). An alert that was
+    (what it watches, its threshold, or it was disabled or re-enabled), and
+    start its silence watch over. An alert that was
     firing or acknowledged gets a resolved event carrying `note`; nobody is
     notified, since nothing about the watched resource changed."""
     state = (await session.execute(
@@ -257,8 +336,12 @@ async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str) ->
     now = datetime.now(timezone.utc)
     if state.state in (AlertStateEnum.firing, AlertStateEnum.acknowledged):
         _add_event(session, rule.id, state.state.value, "resolved", None, now, note=note)
-    if state.state != AlertStateEnum.inactive:
-        state.last_transition_at = now
+    # The watch starts over now: silence is measured from here, against the
+    # cadence of whatever the rule watches from now on.
+    state.last_transition_at = now
+    state.last_sample_at = None
+    state.sample_interval_s = None
+    state.no_data_since = None
     state.state = AlertStateEnum.inactive
     state.consecutive_failure_count = 0
     state.consecutive_ok_count = 0
@@ -294,8 +377,9 @@ async def _dispatch_notification(
     app: "Quart",
     notif_type: str,
     rule: AlertRule,
-    value: float,
+    value: float | None,
     event_id: str,
+    note: str | None = None,
 ) -> None:
     """Run after the transaction commits. Sends notifications and updates the event row."""
     from steward.core.notifications import dispatch_notifications
@@ -308,6 +392,7 @@ async def _dispatch_notification(
         "threshold": rule.threshold,
         "resource": rule.resource_name,
         "source_module": rule.source_module,
+        "note": note,
         "timestamp": datetime.now(timezone.utc).isoformat(),
     }
 
@@ -395,3 +480,51 @@ async def _run_ansible_action(
                 event.ansible_run_id = run_id
 
     await executor.start_run(app, run_id, playbook, inventory, source["path"], params)
+
+
+async def check_silence(app: "Quart") -> None:
+    """Mark the enabled rules whose series has gone silent (no_data).
+
+    Run every minute by the core scheduler. A rule is silent once its series
+    has had no sample for no_data_after(its usual gap); one that never had a
+    sample counts from when the rule started watching. The mark sits beside
+    the state, so a firing alert that goes silent stays firing. Each silence
+    is recorded once, as an event, and notified once (rule.notify_no_data,
+    outside maintenance windows), after the transaction commits.
+    """
+    now = datetime.now(timezone.utc)
+    notify: list[tuple[AlertRule, str, str]] = []
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            rows = (await session.execute(
+                select(AlertState, AlertRule)
+                .join(AlertRule, AlertState.rule_id == AlertRule.id)
+                .where(AlertRule.enabled.is_(True), AlertState.no_data_since.is_(None))
+            )).all()
+            if not rows:
+                return
+            windows = (await session.execute(
+                select(MaintenanceWindow).where(
+                    MaintenanceWindow.start_at <= now, MaintenanceWindow.end_at >= now)
+            )).scalars().all()
+            for state, rule in rows:
+                watching_since = max(rule.created_at, state.last_transition_at)
+                since = silent_since(state.last_sample_at, state.sample_interval_s,
+                                     watching_since, now)
+                if since is None:
+                    continue
+                state.no_data_since = since
+                if state.last_sample_at is None:
+                    note = (f"no sample yet, {_duration((now - watching_since).total_seconds())}"
+                            " after the rule started watching")
+                else:
+                    note = f"no sample for {_duration((now - state.last_sample_at).total_seconds())}"
+                    if state.sample_interval_s:
+                        note += f" (usually every {_duration(state.sample_interval_s)})"
+                event_id = _add_event(session, rule.id, state.state.value, "no_data", None,
+                                      now, note=note)
+                if rule.notify_no_data and not any(
+                        _in_window(w, rule.source_module, rule.resource_name) for w in windows):
+                    notify.append((rule, event_id, note))
+    for rule, event_id, note in notify:
+        asyncio.create_task(_dispatch_notification(app, "no_data", rule, None, event_id, note))