feat(alerts): silence is alertable — agent offline and no-data (#5062)

Agent offline is an ordinary rule now. The host_agent scheduler records
host_agent/<host>/seconds_since_seen every minute (from last_seen_at, or
registration time for an agent that never reported), replacing the task
that only logged stale agents. The catalog offers the metric, and picking
it on a new rule fills "> 180".

No data: each rule's state tracks its last sample and its usual gap,
smoothed so that an outage counts as at most three gaps. A core task,
alert_silence_check, runs every 60s and marks a rule no_data_since once its
series has had no sample for three usual gaps (at least 5 minutes; an hour
while the cadence is unknown). A never-sampled rule counts from when it
started watching. The mark sits beside the state, so a firing alert that
goes silent stays firing. Each silence records one event and sends one
notification (rule option notify_no_data, default on; muted in maintenance
windows). The next sample clears it with a "data resumed" event and
notification. Creating, changing, disabling or re-enabling a rule restarts
its watch.

Migration 0029 adds the columns and seeds each existing rule's last sample
and gap from its last day of data, so an upgrade does not flag every rule.

Shown on the Alerts page (a NO DATA badge with its start time), in MCP
list_alerts, list_alert_rules and get_host (state "no_data", no_data_since),
in get_overview findings (alert_no_data, or a firing alert marked silent)
and in the weekly report. The rule form has the notify option, and its
metric reference now matches what traefik records and lists host_agent.
Emails say "no data" and carry the event's note.

The metric-catalog guard also reads positional MetricBatch.add arguments.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/routes.py

@@ -24,6 +24,7 @@ def _rule_form(form) -> dict:
         "threshold": form.get("threshold"),
         "consecutive_failures_required": form.get("consecutive_failures_required"),
         "consecutive_ok_required": form.get("consecutive_ok_required"),
+        "notify_no_data": "notify_no_data" in form,
     }
 
 

steward/alerts/service.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 import logging
 from datetime import datetime, timezone
 
-from sqlalchemy import select
+from sqlalchemy import or_, select
 
 from steward.models.alerts import (
     AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
@@ -66,15 +66,25 @@ ACTIVE_STATES = (AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateE
 
 
 async def active_alerts(db) -> list:
-    """(AlertState, AlertRule) rows not yet resolved, latest transition first."""
+    """(AlertState, AlertRule) rows that need attention, latest transition
+    first: not yet resolved, or with a series that has gone silent (no_data)."""
     return list((await db.execute(
         select(AlertState, AlertRule)
         .join(AlertRule, AlertState.rule_id == AlertRule.id)
-        .where(AlertState.state.in_(ACTIVE_STATES))
+        .where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
         .order_by(AlertState.last_transition_at.desc())
     )).all())
 
 
+def alert_label(state: AlertState) -> str:
+    """What an alert is, in one word: its state, or "no_data" for a quiet rule
+    whose series has gone silent (a firing alert that goes silent stays
+    "firing"; check no_data_since for the mark)."""
+    if state.no_data_since is not None and state.state == AlertStateEnum.inactive:
+        return "no_data"
+    return state.state.value
+
+
 async def list_rules(db) -> list[AlertRule]:
     return list((await db.execute(select(AlertRule).order_by(AlertRule.name))).scalars())
 
@@ -129,7 +139,7 @@ def _choices(values: list[str], limit: int = 20) -> str:
 
 async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
                       operator: str, threshold, consecutive_failures_required=1,
-                      consecutive_ok_required=1) -> dict:
+                      consecutive_ok_required=1, notify_no_data=True) -> dict:
     """Validated column values for a new or edited alert rule.
 
     The same choices the rule form offers: a known source, a resource that
@@ -179,7 +189,7 @@ async def rule_fields(db, *, name: str, source_module: str, resource_name: str,
     return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
             "metric_name": metric_name, "operator": op, "threshold": threshold,
             "consecutive_failures_required": consecutive,
-            "consecutive_ok_required": ok_checks}
+            "consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data)}
 
 
 # Changing any of these changes what the rule is watching, so whatever state
@@ -189,17 +199,21 @@ WATCHED_FIELDS = ("source_module", "resource_name", "metric_name", "operator", "
 
 async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool | None = None) -> None:
     """Apply validated fields (and optionally enabled) to a rule, in the
-    caller's transaction. A change to what it watches, or disabling it, puts
-    its alert back to inactive (core.alerts.reset_rule_state)."""
+    caller's transaction. A change to what it watches, or disabling or
+    re-enabling it, puts its alert back to inactive and starts its silence
+    watch over (core.alerts.reset_rule_state)."""
     from steward.core.alerts import reset_rule_state
     changed = [k for k in WATCHED_FIELDS if k in fields and getattr(rule, k) != fields[k]]
     for k, v in fields.items():
         setattr(rule, k, v)
     disabling = enabled is False and rule.enabled
+    enabling = enabled is True and not rule.enabled
     if enabled is not None:
         rule.enabled = bool(enabled)
     if disabling:
         await reset_rule_state(db, rule, "rule disabled")
+    elif enabling:
+        await reset_rule_state(db, rule, "rule enabled")
     elif changed:
         await reset_rule_state(db, rule, "rule changed: " + ", ".join(changed))