feat(alerts): silence is alertable — agent offline and no-data (#5062)

Agent offline is an ordinary rule now. The host_agent scheduler records
host_agent/<host>/seconds_since_seen every minute (from last_seen_at, or
registration time for an agent that never reported), replacing the task
that only logged stale agents. The catalog offers the metric, and picking
it on a new rule fills "> 180".

No data: each rule's state tracks its last sample and its usual gap,
smoothed so that an outage counts as at most three gaps. A core task,
alert_silence_check, runs every 60s and marks a rule no_data_since once its
series has had no sample for three usual gaps (at least 5 minutes; an hour
while the cadence is unknown). A never-sampled rule counts from when it
started watching. The mark sits beside the state, so a firing alert that
goes silent stays firing. Each silence records one event and sends one
notification (rule option notify_no_data, default on; muted in maintenance
windows). The next sample clears it with a "data resumed" event and
notification. Creating, changing, disabling or re-enabling a rule restarts
its watch.

Migration 0029 adds the columns and seeds each existing rule's last sample
and gap from its last day of data, so an upgrade does not flag every rule.

Shown on the Alerts page (a NO DATA badge with its start time), in MCP
list_alerts, list_alert_rules and get_host (state "no_data", no_data_since),
in get_overview findings (alert_no_data, or a firing alert marked silent)
and in the weekly report. The rule form has the notify option, and its
metric reference now matches what traefik records and lists host_agent.
Emails say "no data" and carry the event's note.

The metric-catalog guard also reads positional MetricBatch.add arguments.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/mcp/tools/alerts.py

@@ -41,24 +41,30 @@ async def _usernames(db, ids) -> dict[str, str]:
 
 
 async def list_alerts() -> dict:
-    """Alerts that need attention now: firing, pending, or acknowledged but still breaching.
+    """Alerts that need attention now: firing, pending, acknowledged but still
+    breaching, or gone silent (no_data).
 
     pending: the condition holds but not yet for the rule's required number of
     consecutive checks. firing: it has. acknowledged: someone has seen it and
-    it still holds. Each carries the rule's condition, the resource and metric
-    it watches, and `since` (its latest state change). Firing, unacknowledged
+    it still holds. no_data: the metric the rule watches has stopped reporting
+    (no sample for three of its usual gaps, at least 5 minutes); a firing
+    alert that goes silent stays "firing" with `no_data_since` set. Each
+    carries the rule's condition, the resource and metric it watches, `since`
+    (its latest state change) and `last_sample_at`. Firing, unacknowledged
     alerts come first. An alert that cleared is not here; get_alert_history
     shows the transitions.
     """
     async with db_session() as db:
         active = await alert_svc.active_alerts(db)
         names = await _usernames(db, [s.acknowledged_by for s, _ in active])
-    order = {"firing": 0, "pending": 1, "acknowledged": 2}
+    order = {"firing": 0, "no_data": 1, "pending": 2, "acknowledged": 3}
     rows = [
         {
             "rule_id": rule.id,
             "rule_name": rule.name,
-            "state": state.state.value,
+            "state": alert_svc.alert_label(state),
+            "no_data_since": iso(state.no_data_since),
+            "last_sample_at": iso(state.last_sample_at),
             "source_module": rule.source_module,
             "resource": rule.resource_name,
             "metric": rule.metric_name,
@@ -70,7 +76,7 @@ async def list_alerts() -> dict:
         }
         for state, rule in active
     ]
-    rows.sort(key=lambda r: (order.get(r["state"], 3), -(r["age_s"] or 0)))
+    rows.sort(key=lambda r: (order.get(r["state"], 4), -(r["age_s"] or 0)))
     return {"alerts": rows, "total": len(rows)}
 
 
@@ -135,7 +141,9 @@ async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict
 
     `state` is the rule's current state: "inactive" (condition not met),
     "pending", "firing", "acknowledged", "resolved" (cleared since it last
-    fired), or null if it has never been evaluated. `ansible_action` is the
+    fired), "no_data" (inactive and its metric has stopped reporting), or null
+    if it has no state row. `no_data_since` is set while its metric is silent,
+    whatever the state. `ansible_action` is the
     playbook the rule runs when it fires, if any (`check: true` = dry run).
     """
     limit = clamp_limit(limit, 200, 500)
@@ -153,8 +161,9 @@ async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict
             "resource": r.resource_name,
             "metric": r.metric_name,
             "condition": _condition(r),
-            "state": states[r.id].state.value if r.id in states else None,
+            "state": alert_svc.alert_label(states[r.id]) if r.id in states else None,
             "state_since": iso(states[r.id].last_transition_at) if r.id in states else None,
+            "no_data_since": iso(states[r.id].no_data_since) if r.id in states else None,
             "ansible_action": _action(r),
         }
         for r in rules
@@ -224,12 +233,14 @@ async def _rule(db, rule: str) -> AlertRule:
 def _rule_row(r: AlertRule) -> dict:
     return {"rule_id": r.id, "rule_name": r.name, "enabled": r.enabled,
             "source_module": r.source_module, "resource": r.resource_name,
-            "metric": r.metric_name, "condition": _condition(r), "ansible_action": _action(r)}
+            "metric": r.metric_name, "condition": _condition(r),
+            "notify_no_data": r.notify_no_data, "ansible_action": _action(r)}
 
 
 async def create_alert_rule(name: str, source_module: str, resource: str, metric: str,
                             operator: str, threshold: float, consecutive_checks: int = 1,
-                            clear_after_checks: int = 1, enabled: bool = True) -> dict:
+                            clear_after_checks: int = 1, notify_no_data: bool = True,
+                            enabled: bool = True) -> dict:
     """Add an alert rule: notify when `metric` of `resource` crosses `threshold`.
 
     source_module, resource, metric: get_alert_rule_options lists the valid
@@ -240,6 +251,9 @@ async def create_alert_rule(name: str, source_module: str, resource: str, metric
     clear_after_checks: how many checks in a row must be OK before a firing or
       acknowledged alert resolves (default 1). An acknowledgement holds until
       then, however long the breach lasts.
+    notify_no_data: also notify when the metric stops reporting and when it
+      resumes (default true). The silence shows in list_alerts either way.
+      "Agent offline" is source host_agent, metric seconds_since_seen, "> 180".
 
     Notifications go to the channels set in Steward's Settings. The playbook a
     rule runs when it fires is set separately (set_alert_action, admin).
@@ -251,7 +265,7 @@ async def create_alert_rule(name: str, source_module: str, resource: str, metric
             db, name=name, source_module=source_module, resource_name=resource,
             metric_name=metric, operator=operator, threshold=threshold,
             consecutive_failures_required=consecutive_checks,
-            consecutive_ok_required=clear_after_checks)
+            consecutive_ok_required=clear_after_checks, notify_no_data=notify_no_data)
         r = await alert_svc.add_rule(db, fields, created_by=caller.user_id, enabled=enabled)
     await audit_mcp("alert_rule.created", "alert_rule", r.name,
                     {"rule_id": r.id, "source": r.source_module, "resource": r.resource_name,
@@ -264,14 +278,16 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s
                             operator: str | None = None, threshold: float | None = None,
                             consecutive_checks: int | None = None,
                             clear_after_checks: int | None = None,
+                            notify_no_data: bool | None = None,
                             enabled: bool | None = None) -> dict:
     """Change an alert rule. Only the arguments you pass change; the rest stay.
 
     rule: its name or id (list_alert_rules). Pass enabled=false to silence it
     without deleting it. The changed rule is validated as create_alert_rule
     validates a new one. Changing what it watches (source, resource, metric,
-    operator, threshold) or disabling it puts its alert back to inactive; a
-    firing alert records a resolved event saying why. Recorded in the audit log.
+    operator, threshold), or disabling or re-enabling it, puts its alert back
+    to inactive; a firing alert records a resolved event saying why. Recorded
+    in the audit log.
     """
     async with db_session() as db, db.begin():
         r = await _rule(db, rule)
@@ -286,12 +302,14 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s
             consecutive_failures_required=(r.consecutive_failures_required
                                            if consecutive_checks is None else consecutive_checks),
             consecutive_ok_required=(r.consecutive_ok_required
-                                     if clear_after_checks is None else clear_after_checks))
+                                     if clear_after_checks is None else clear_after_checks),
+            notify_no_data=r.notify_no_data if notify_no_data is None else notify_no_data)
         await alert_svc.apply_rule_change(db, r, fields, enabled=enabled)
     changed = sorted(k for k, v in {"name": name, "source_module": source_module,
                                     "resource": resource, "metric": metric, "operator": operator,
                                     "threshold": threshold, "consecutive_checks": consecutive_checks,
                                     "clear_after_checks": clear_after_checks,
+                                    "notify_no_data": notify_no_data,
                                     "enabled": enabled}.items() if v is not None)
     await audit_mcp("alert_rule.updated", "alert_rule", r.name, {"rule_id": r.id, "changed": changed})
     return _rule_row(r)

steward/mcp/tools/hosts.py

@@ -183,7 +183,7 @@ async def get_host(host: str, parts_limit: int = 20) -> dict:
     prefix = h.name + ":"
     alerts = [
         {
-            "rule_id": rule.id, "rule_name": rule.name, "state": state.state.value,
+            "rule_id": rule.id, "rule_name": rule.name, "state": alert_svc.alert_label(state),
             "resource": rule.resource_name, "metric": rule.metric_name,
             "condition": f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}",
             "since": iso(state.last_transition_at), "age_s": age_s(state.last_transition_at),

steward/mcp/tools/overview.py

@@ -77,13 +77,18 @@ def alert_findings(active) -> list[dict]:
             sev, what = "critical", "is firing"
         elif s == "pending":
             sev, what = "warning", "is pending (breaching, not yet for long enough to fire)"
-        else:
+        elif s == "acknowledged":
             sev, what = "warning", "is acknowledged but still breaching"
+        else:
+            s, sev, what = "no_data", "warning", "has stopped reporting"
+        if state.no_data_since is not None and s != "no_data":
+            what += f" and has stopped reporting (no data since {iso(state.no_data_since)})"
         out.append(finding(
             sev, f"alert_{s}", rule.name,
             f"{rule.metric_name} {rule.operator.value} {rule.threshold:g} on "
             f"{rule.source_module}/{rule.resource_name} {what}",
-            since=state.last_transition_at, ids={"rule_id": rule.id},
+            since=state.no_data_since if s == "no_data" else state.last_transition_at,
+            ids={"rule_id": rule.id},
             next=f'get_alert_history(rule="{rule.id}")'))
     return out