feat(alerts): silence is alertable — agent offline and no-data (#5062)

Agent offline is an ordinary rule now. The host_agent scheduler records
host_agent/<host>/seconds_since_seen every minute (from last_seen_at, or
registration time for an agent that never reported), replacing the task
that only logged stale agents. The catalog offers the metric, and picking
it on a new rule fills "> 180".

No data: each rule's state tracks its last sample and its usual gap,
smoothed so that an outage counts as at most three gaps. A core task,
alert_silence_check, runs every 60s and marks a rule no_data_since once its
series has had no sample for three usual gaps (at least 5 minutes; an hour
while the cadence is unknown). A never-sampled rule counts from when it
started watching. The mark sits beside the state, so a firing alert that
goes silent stays firing. Each silence records one event and sends one
notification (rule option notify_no_data, default on; muted in maintenance
windows). The next sample clears it with a "data resumed" event and
notification. Creating, changing, disabling or re-enabling a rule restarts
its watch.

Migration 0029 adds the columns and seeds each existing rule's last sample
and gap from its last day of data, so an upgrade does not flag every rule.

Shown on the Alerts page (a NO DATA badge with its start time), in MCP
list_alerts, list_alert_rules and get_host (state "no_data", no_data_since),
in get_overview findings (alert_no_data, or a firing alert marked silent)
and in the weekly report. The rule form has the notify option, and its
metric reference now matches what traefik records and lists host_agent.
Emails say "no data" and carry the event's note.

The metric-catalog guard also reads positional MetricBatch.add arguments.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/routes.py

@@ -24,6 +24,7 @@ def _rule_form(form) -> dict:
         "threshold": form.get("threshold"),
         "consecutive_failures_required": form.get("consecutive_failures_required"),
         "consecutive_ok_required": form.get("consecutive_ok_required"),
+        "notify_no_data": "notify_no_data" in form,
     }
 
 

steward/alerts/service.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 import logging
 from datetime import datetime, timezone
 
-from sqlalchemy import select
+from sqlalchemy import or_, select
 
 from steward.models.alerts import (
     AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
@@ -66,15 +66,25 @@ ACTIVE_STATES = (AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateE
 
 
 async def active_alerts(db) -> list:
-    """(AlertState, AlertRule) rows not yet resolved, latest transition first."""
+    """(AlertState, AlertRule) rows that need attention, latest transition
+    first: not yet resolved, or with a series that has gone silent (no_data)."""
     return list((await db.execute(
         select(AlertState, AlertRule)
         .join(AlertRule, AlertState.rule_id == AlertRule.id)
-        .where(AlertState.state.in_(ACTIVE_STATES))
+        .where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
         .order_by(AlertState.last_transition_at.desc())
     )).all())
 
 
+def alert_label(state: AlertState) -> str:
+    """What an alert is, in one word: its state, or "no_data" for a quiet rule
+    whose series has gone silent (a firing alert that goes silent stays
+    "firing"; check no_data_since for the mark)."""
+    if state.no_data_since is not None and state.state == AlertStateEnum.inactive:
+        return "no_data"
+    return state.state.value
+
+
 async def list_rules(db) -> list[AlertRule]:
     return list((await db.execute(select(AlertRule).order_by(AlertRule.name))).scalars())
 
@@ -129,7 +139,7 @@ def _choices(values: list[str], limit: int = 20) -> str:
 
 async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
                       operator: str, threshold, consecutive_failures_required=1,
-                      consecutive_ok_required=1) -> dict:
+                      consecutive_ok_required=1, notify_no_data=True) -> dict:
     """Validated column values for a new or edited alert rule.
 
     The same choices the rule form offers: a known source, a resource that
@@ -179,7 +189,7 @@ async def rule_fields(db, *, name: str, source_module: str, resource_name: str,
     return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
             "metric_name": metric_name, "operator": op, "threshold": threshold,
             "consecutive_failures_required": consecutive,
-            "consecutive_ok_required": ok_checks}
+            "consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data)}
 
 
 # Changing any of these changes what the rule is watching, so whatever state
@@ -189,17 +199,21 @@ WATCHED_FIELDS = ("source_module", "resource_name", "metric_name", "operator", "
 
 async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool | None = None) -> None:
     """Apply validated fields (and optionally enabled) to a rule, in the
-    caller's transaction. A change to what it watches, or disabling it, puts
-    its alert back to inactive (core.alerts.reset_rule_state)."""
+    caller's transaction. A change to what it watches, or disabling or
+    re-enabling it, puts its alert back to inactive and starts its silence
+    watch over (core.alerts.reset_rule_state)."""
     from steward.core.alerts import reset_rule_state
     changed = [k for k in WATCHED_FIELDS if k in fields and getattr(rule, k) != fields[k]]
     for k, v in fields.items():
         setattr(rule, k, v)
     disabling = enabled is False and rule.enabled
+    enabling = enabled is True and not rule.enabled
     if enabled is not None:
         rule.enabled = bool(enabled)
     if disabling:
         await reset_rule_state(db, rule, "rule disabled")
+    elif enabling:
+        await reset_rule_state(db, rule, "rule enabled")
     elif changed:
         await reset_rule_state(db, rule, "rule changed: " + ", ".join(changed))
 

steward/app.py

@@ -344,6 +344,10 @@ def _register_core_tasks(app: Quart) -> None:
         from .core.cleanup import run_cleanup as _cleanup
         await _cleanup(app)
 
+    async def run_silence_check():
+        from .core.alerts import check_silence
+        await check_silence(app)
+
     async def run_report_check():
         from .core.reports import check_and_send
         await check_and_send(app)
@@ -368,6 +372,13 @@ def _register_core_tasks(app: Quart) -> None:
             interval_seconds=60,
             run_on_startup=True,
         ),
+        # Rules whose series has gone quiet get a visible no_data mark.
+        ScheduledTask(
+            name="alert_silence_check",
+            coro_factory=run_silence_check,
+            interval_seconds=60,
+            run_on_startup=False,
+        ),
         ScheduledTask(
             name="monitor_check",
             coro_factory=run_monitor_checks,

steward/core/alerts.py

@@ -3,7 +3,7 @@ import asyncio
 import logging
 import uuid
 from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING
 
 from sqlalchemy import select
@@ -141,14 +141,20 @@ async def record_metrics(session: AsyncSession, rows: list[PluginMetric]) -> Non
     )).scalars()}
 
     for rule in rules:
-        if any(_in_window(w, rule.source_module, rule.resource_name) for w in windows):
-            continue  # suppressed by maintenance window
         alert_state = states.get(rule.id)
         if alert_state is None:
             logger.warning("No alert_state row for rule %s, skipping", rule.id)
             continue
         samples = sorted(by_key[(rule.source_module, rule.resource_name, rule.metric_name)],
                          key=lambda r: r.recorded_at)
+        muted = any(_in_window(w, rule.source_module, rule.resource_name) for w in windows)
+        resumed = _track_samples(session, rule, alert_state, samples, now)
+        if resumed and rule.notify_no_data and not muted and _app is not None:
+            event_id, value, note = resumed
+            asyncio.create_task(
+                _dispatch_notification(_app, "data_resumed", rule, value, event_id, note))
+        if muted:
+            continue  # suppressed by maintenance window
         for sample in samples:
             notifications = await _evaluate_rule(session, rule, alert_state, sample.value,
                                                  sample.recorded_at)
@@ -162,6 +168,78 @@ async def record_metrics(session: AsyncSession, rows: list[PluginMetric]) -> Non
                             _run_ansible_action(_app, rule, sample.value, event_id))
 
 
+# A series is silent once it has gone this long without a sample, or three of
+# its usual gaps if that is longer. Until its cadence is known (fewer than two
+# samples seen), it gets an hour: a monitor may check only every ten minutes.
+NO_DATA_FLOOR_S = 300.0
+NO_DATA_GAPS = 3
+NO_DATA_UNKNOWN_S = 3600.0
+
+
+def next_interval(interval: float | None, gap: float) -> float:
+    """A series' usual gap between samples, updated with one more gap.
+
+    A smoothed average, so one late sample barely moves it. A gap longer than
+    the silence threshold counts only as that threshold: an outage is not the
+    series' cadence. A real change of cadence still gets learned, a step at a
+    time (each step at most 1.5x).
+    """
+    gap = max(0.0, gap)
+    if interval is None or interval <= 0:
+        return gap
+    return 0.75 * interval + 0.25 * min(gap, NO_DATA_GAPS * interval)
+
+
+def no_data_after(interval: float | None) -> float:
+    """Seconds without a sample before a series counts as silent."""
+    if not interval:
+        return NO_DATA_UNKNOWN_S
+    return max(NO_DATA_FLOOR_S, NO_DATA_GAPS * interval)
+
+
+def silent_since(last_sample_at: datetime | None, interval: float | None,
+                 watching_since: datetime, now: datetime) -> datetime | None:
+    """When a series went silent, or None while it is still reporting.
+
+    A series that has never had a sample counts from when the rule started
+    watching it (created, changed or re-enabled).
+    """
+    last = last_sample_at or watching_since
+    limit = timedelta(seconds=no_data_after(interval))
+    return last + limit if now - last > limit else None
+
+
+def _duration(seconds: float) -> str:
+    seconds = int(seconds)
+    if seconds < 120:
+        return f"{seconds}s"
+    if seconds < 7200:
+        return f"{seconds // 60}m"
+    return f"{seconds // 3600}h{(seconds % 3600) // 60:02d}m"
+
+
+def _track_samples(session: AsyncSession, rule: AlertRule, state: AlertState,
+                   samples: list[PluginMetric], now: datetime
+                   ) -> tuple[str, float, str] | None:
+    """Note a rule's new samples (oldest first) for silence tracking. When the
+    series had been marked no_data, clear it and record a resume event,
+    returned as (event_id, value, note) for the caller to notify."""
+    for sample in samples:
+        if state.last_sample_at is not None:
+            if sample.recorded_at <= state.last_sample_at:
+                continue  # late or duplicate: not a gap
+            gap = (sample.recorded_at - state.last_sample_at).total_seconds()
+            state.sample_interval_s = next_interval(state.sample_interval_s, gap)
+        state.last_sample_at = sample.recorded_at
+    if state.no_data_since is None or not samples:
+        return None
+    note = f"data resumed after {_duration((now - state.no_data_since).total_seconds())} silent"
+    value = samples[-1].value
+    event_id = _add_event(session, rule.id, "no_data", state.state.value, value, now, note=note)
+    state.no_data_since = None
+    return event_id, value, note
+
+
 def _is_breached(value: float, operator: AlertOperator, threshold: float) -> bool:
     op = operator.value
     if op == ">":
@@ -247,7 +325,8 @@ async def _evaluate_rule(
 
 async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str) -> None:
     """Put a rule's alert back to inactive because the rule itself changed
-    (what it watches, its threshold, or it was disabled). An alert that was
+    (what it watches, its threshold, or it was disabled or re-enabled), and
+    start its silence watch over. An alert that was
     firing or acknowledged gets a resolved event carrying `note`; nobody is
     notified, since nothing about the watched resource changed."""
     state = (await session.execute(
@@ -257,8 +336,12 @@ async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str) ->
     now = datetime.now(timezone.utc)
     if state.state in (AlertStateEnum.firing, AlertStateEnum.acknowledged):
         _add_event(session, rule.id, state.state.value, "resolved", None, now, note=note)
-    if state.state != AlertStateEnum.inactive:
-        state.last_transition_at = now
+    # The watch starts over now: silence is measured from here, against the
+    # cadence of whatever the rule watches from now on.
+    state.last_transition_at = now
+    state.last_sample_at = None
+    state.sample_interval_s = None
+    state.no_data_since = None
     state.state = AlertStateEnum.inactive
     state.consecutive_failure_count = 0
     state.consecutive_ok_count = 0
@@ -294,8 +377,9 @@ async def _dispatch_notification(
     app: "Quart",
     notif_type: str,
     rule: AlertRule,
-    value: float,
+    value: float | None,
     event_id: str,
+    note: str | None = None,
 ) -> None:
     """Run after the transaction commits. Sends notifications and updates the event row."""
     from steward.core.notifications import dispatch_notifications
@@ -308,6 +392,7 @@ async def _dispatch_notification(
         "threshold": rule.threshold,
         "resource": rule.resource_name,
         "source_module": rule.source_module,
+        "note": note,
         "timestamp": datetime.now(timezone.utc).isoformat(),
     }
 
@@ -395,3 +480,51 @@ async def _run_ansible_action(
                 event.ansible_run_id = run_id
 
     await executor.start_run(app, run_id, playbook, inventory, source["path"], params)
+
+
+async def check_silence(app: "Quart") -> None:
+    """Mark the enabled rules whose series has gone silent (no_data).
+
+    Run every minute by the core scheduler. A rule is silent once its series
+    has had no sample for no_data_after(its usual gap); one that never had a
+    sample counts from when the rule started watching. The mark sits beside
+    the state, so a firing alert that goes silent stays firing. Each silence
+    is recorded once, as an event, and notified once (rule.notify_no_data,
+    outside maintenance windows), after the transaction commits.
+    """
+    now = datetime.now(timezone.utc)
+    notify: list[tuple[AlertRule, str, str]] = []
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            rows = (await session.execute(
+                select(AlertState, AlertRule)
+                .join(AlertRule, AlertState.rule_id == AlertRule.id)
+                .where(AlertRule.enabled.is_(True), AlertState.no_data_since.is_(None))
+            )).all()
+            if not rows:
+                return
+            windows = (await session.execute(
+                select(MaintenanceWindow).where(
+                    MaintenanceWindow.start_at <= now, MaintenanceWindow.end_at >= now)
+            )).scalars().all()
+            for state, rule in rows:
+                watching_since = max(rule.created_at, state.last_transition_at)
+                since = silent_since(state.last_sample_at, state.sample_interval_s,
+                                     watching_since, now)
+                if since is None:
+                    continue
+                state.no_data_since = since
+                if state.last_sample_at is None:
+                    note = (f"no sample yet, {_duration((now - watching_since).total_seconds())}"
+                            " after the rule started watching")
+                else:
+                    note = f"no sample for {_duration((now - state.last_sample_at).total_seconds())}"
+                    if state.sample_interval_s:
+                        note += f" (usually every {_duration(state.sample_interval_s)})"
+                event_id = _add_event(session, rule.id, state.state.value, "no_data", None,
+                                      now, note=note)
+                if rule.notify_no_data and not any(
+                        _in_window(w, rule.source_module, rule.resource_name) for w in windows):
+                    notify.append((rule, event_id, note))
+    for rule, event_id, note in notify:
+        asyncio.create_task(_dispatch_notification(app, "no_data", rule, None, event_id, note))

steward/core/notifications.py

@@ -36,13 +36,15 @@ async def _send_email(cfg: dict, alert: dict) -> dict:
         msg["Subject"] = f"[Steward] {alert['state']} — {alert['rule_name']}"
         msg["From"] = cfg.get("username", "steward@localhost")
         msg["To"] = ", ".join(cfg["recipients"])
+        value = "no data" if alert.get("value") is None else alert["value"]
         msg.set_content(
             f"Alert: {alert['state']}\n"
             f"Rule: {alert['rule_name']}\n"
             f"Resource: {alert['resource']}\n"
-            f"Metric: {alert['metric']} = {alert['value']}\n"
+            f"Metric: {alert['metric']} = {value}\n"
             f"Threshold: {alert['threshold']}\n"
-            f"Time: {alert['timestamp']}\n"
+            + (f"Note: {alert['note']}\n" if alert.get("note") else "")
+            + f"Time: {alert['timestamp']}\n"
         )
         loop = asyncio.get_running_loop()
         await loop.run_in_executor(None, _smtp_send_sync, cfg, msg)

steward/core/reports.py

@@ -55,7 +55,7 @@ async def build_report_data(app) -> dict:
         active_rows = await alert_svc.active_alerts(db)
         active_alerts = [
             {
-                "state": state.state.value.upper(),
+                "state": alert_svc.alert_label(state).upper(),
                 "name": rule.name,
                 "resource": rule.resource_name,
                 "metric": rule.metric_name,

steward/mcp/tools/alerts.py

@@ -41,24 +41,30 @@ async def _usernames(db, ids) -> dict[str, str]:
 
 
 async def list_alerts() -> dict:
-    """Alerts that need attention now: firing, pending, or acknowledged but still breaching.
+    """Alerts that need attention now: firing, pending, acknowledged but still
+    breaching, or gone silent (no_data).
 
     pending: the condition holds but not yet for the rule's required number of
     consecutive checks. firing: it has. acknowledged: someone has seen it and
-    it still holds. Each carries the rule's condition, the resource and metric
-    it watches, and `since` (its latest state change). Firing, unacknowledged
+    it still holds. no_data: the metric the rule watches has stopped reporting
+    (no sample for three of its usual gaps, at least 5 minutes); a firing
+    alert that goes silent stays "firing" with `no_data_since` set. Each
+    carries the rule's condition, the resource and metric it watches, `since`
+    (its latest state change) and `last_sample_at`. Firing, unacknowledged
     alerts come first. An alert that cleared is not here; get_alert_history
     shows the transitions.
     """
     async with db_session() as db:
         active = await alert_svc.active_alerts(db)
         names = await _usernames(db, [s.acknowledged_by for s, _ in active])
-    order = {"firing": 0, "pending": 1, "acknowledged": 2}
+    order = {"firing": 0, "no_data": 1, "pending": 2, "acknowledged": 3}
     rows = [
         {
             "rule_id": rule.id,
             "rule_name": rule.name,
-            "state": state.state.value,
+            "state": alert_svc.alert_label(state),
+            "no_data_since": iso(state.no_data_since),
+            "last_sample_at": iso(state.last_sample_at),
             "source_module": rule.source_module,
             "resource": rule.resource_name,
             "metric": rule.metric_name,
@@ -70,7 +76,7 @@ async def list_alerts() -> dict:
         }
         for state, rule in active
     ]
-    rows.sort(key=lambda r: (order.get(r["state"], 3), -(r["age_s"] or 0)))
+    rows.sort(key=lambda r: (order.get(r["state"], 4), -(r["age_s"] or 0)))
     return {"alerts": rows, "total": len(rows)}
 
 
@@ -135,7 +141,9 @@ async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict
 
     `state` is the rule's current state: "inactive" (condition not met),
     "pending", "firing", "acknowledged", "resolved" (cleared since it last
-    fired), or null if it has never been evaluated. `ansible_action` is the
+    fired), "no_data" (inactive and its metric has stopped reporting), or null
+    if it has no state row. `no_data_since` is set while its metric is silent,
+    whatever the state. `ansible_action` is the
     playbook the rule runs when it fires, if any (`check: true` = dry run).
     """
     limit = clamp_limit(limit, 200, 500)
@@ -153,8 +161,9 @@ async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict
             "resource": r.resource_name,
             "metric": r.metric_name,
             "condition": _condition(r),
-            "state": states[r.id].state.value if r.id in states else None,
+            "state": alert_svc.alert_label(states[r.id]) if r.id in states else None,
             "state_since": iso(states[r.id].last_transition_at) if r.id in states else None,
+            "no_data_since": iso(states[r.id].no_data_since) if r.id in states else None,
             "ansible_action": _action(r),
         }
         for r in rules
@@ -224,12 +233,14 @@ async def _rule(db, rule: str) -> AlertRule:
 def _rule_row(r: AlertRule) -> dict:
     return {"rule_id": r.id, "rule_name": r.name, "enabled": r.enabled,
             "source_module": r.source_module, "resource": r.resource_name,
-            "metric": r.metric_name, "condition": _condition(r), "ansible_action": _action(r)}
+            "metric": r.metric_name, "condition": _condition(r),
+            "notify_no_data": r.notify_no_data, "ansible_action": _action(r)}
 
 
 async def create_alert_rule(name: str, source_module: str, resource: str, metric: str,
                             operator: str, threshold: float, consecutive_checks: int = 1,
-                            clear_after_checks: int = 1, enabled: bool = True) -> dict:
+                            clear_after_checks: int = 1, notify_no_data: bool = True,
+                            enabled: bool = True) -> dict:
     """Add an alert rule: notify when `metric` of `resource` crosses `threshold`.
 
     source_module, resource, metric: get_alert_rule_options lists the valid
@@ -240,6 +251,9 @@ async def create_alert_rule(name: str, source_module: str, resource: str, metric
     clear_after_checks: how many checks in a row must be OK before a firing or
       acknowledged alert resolves (default 1). An acknowledgement holds until
       then, however long the breach lasts.
+    notify_no_data: also notify when the metric stops reporting and when it
+      resumes (default true). The silence shows in list_alerts either way.
+      "Agent offline" is source host_agent, metric seconds_since_seen, "> 180".
 
     Notifications go to the channels set in Steward's Settings. The playbook a
     rule runs when it fires is set separately (set_alert_action, admin).
@@ -251,7 +265,7 @@ async def create_alert_rule(name: str, source_module: str, resource: str, metric
             db, name=name, source_module=source_module, resource_name=resource,
             metric_name=metric, operator=operator, threshold=threshold,
             consecutive_failures_required=consecutive_checks,
-            consecutive_ok_required=clear_after_checks)
+            consecutive_ok_required=clear_after_checks, notify_no_data=notify_no_data)
         r = await alert_svc.add_rule(db, fields, created_by=caller.user_id, enabled=enabled)
     await audit_mcp("alert_rule.created", "alert_rule", r.name,
                     {"rule_id": r.id, "source": r.source_module, "resource": r.resource_name,
@@ -264,14 +278,16 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s
                             operator: str | None = None, threshold: float | None = None,
                             consecutive_checks: int | None = None,
                             clear_after_checks: int | None = None,
+                            notify_no_data: bool | None = None,
                             enabled: bool | None = None) -> dict:
     """Change an alert rule. Only the arguments you pass change; the rest stay.
 
     rule: its name or id (list_alert_rules). Pass enabled=false to silence it
     without deleting it. The changed rule is validated as create_alert_rule
     validates a new one. Changing what it watches (source, resource, metric,
-    operator, threshold) or disabling it puts its alert back to inactive; a
-    firing alert records a resolved event saying why. Recorded in the audit log.
+    operator, threshold), or disabling or re-enabling it, puts its alert back
+    to inactive; a firing alert records a resolved event saying why. Recorded
+    in the audit log.
     """
     async with db_session() as db, db.begin():
         r = await _rule(db, rule)
@@ -286,12 +302,14 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s
             consecutive_failures_required=(r.consecutive_failures_required
                                            if consecutive_checks is None else consecutive_checks),
             consecutive_ok_required=(r.consecutive_ok_required
-                                     if clear_after_checks is None else clear_after_checks))
+                                     if clear_after_checks is None else clear_after_checks),
+            notify_no_data=r.notify_no_data if notify_no_data is None else notify_no_data)
         await alert_svc.apply_rule_change(db, r, fields, enabled=enabled)
     changed = sorted(k for k, v in {"name": name, "source_module": source_module,
                                     "resource": resource, "metric": metric, "operator": operator,
                                     "threshold": threshold, "consecutive_checks": consecutive_checks,
                                     "clear_after_checks": clear_after_checks,
+                                    "notify_no_data": notify_no_data,
                                     "enabled": enabled}.items() if v is not None)
     await audit_mcp("alert_rule.updated", "alert_rule", r.name, {"rule_id": r.id, "changed": changed})
     return _rule_row(r)

steward/mcp/tools/hosts.py

@@ -183,7 +183,7 @@ async def get_host(host: str, parts_limit: int = 20) -> dict:
     prefix = h.name + ":"
     alerts = [
         {
-            "rule_id": rule.id, "rule_name": rule.name, "state": state.state.value,
+            "rule_id": rule.id, "rule_name": rule.name, "state": alert_svc.alert_label(state),
             "resource": rule.resource_name, "metric": rule.metric_name,
             "condition": f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}",
             "since": iso(state.last_transition_at), "age_s": age_s(state.last_transition_at),

steward/mcp/tools/overview.py

@@ -77,13 +77,18 @@ def alert_findings(active) -> list[dict]:
             sev, what = "critical", "is firing"
         elif s == "pending":
             sev, what = "warning", "is pending (breaching, not yet for long enough to fire)"
-        else:
+        elif s == "acknowledged":
             sev, what = "warning", "is acknowledged but still breaching"
+        else:
+            s, sev, what = "no_data", "warning", "has stopped reporting"
+        if state.no_data_since is not None and s != "no_data":
+            what += f" and has stopped reporting (no data since {iso(state.no_data_since)})"
         out.append(finding(
             sev, f"alert_{s}", rule.name,
             f"{rule.metric_name} {rule.operator.value} {rule.threshold:g} on "
             f"{rule.source_module}/{rule.resource_name} {what}",
-            since=state.last_transition_at, ids={"rule_id": rule.id},
+            since=state.no_data_since if s == "no_data" else state.last_transition_at,
+            ids={"rule_id": rule.id},
             next=f'get_alert_history(rule="{rule.id}")'))
     return out
 

steward/migrations/versions/0029_alert_no_data.py

@@ -0,0 +1,49 @@
+"""Alerts notice silence: per-rule sample tracking and a no-data mark
+
+Revision ID: 0029_alert_no_data
+Revises: 0028_alert_resolve_and_reset
+Create Date: 2026-10-06
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "0029_alert_no_data"
+down_revision: Union[str, None] = "0028_alert_resolve_and_reset"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("alert_rules", sa.Column("notify_no_data", sa.Boolean,
+                                           nullable=False, server_default=sa.true()))
+    op.add_column("alert_states", sa.Column("last_sample_at", sa.DateTime(timezone=True),
+                                            nullable=True))
+    op.add_column("alert_states", sa.Column("sample_interval_s", sa.Float, nullable=True))
+    op.add_column("alert_states", sa.Column("no_data_since", sa.DateTime(timezone=True),
+                                            nullable=True))
+    # Seed each existing rule's last sample and usual gap from the last day of
+    # its series, so the first silence check after the upgrade judges every
+    # rule against what it has actually been receiving.
+    op.execute("""
+        UPDATE alert_states s
+        SET last_sample_at = k.last_at, sample_interval_s = k.gap
+        FROM (
+            SELECT r.id AS rule_id, max(m.recorded_at) AS last_at,
+                   EXTRACT(EPOCH FROM max(m.recorded_at) - min(m.recorded_at))
+                       / NULLIF(count(*) - 1, 0) AS gap
+            FROM alert_rules r
+            JOIN plugin_metrics m
+              ON m.source_module = r.source_module AND m.resource_name = r.resource_name
+             AND m.metric_name = r.metric_name AND m.recorded_at > now() - interval '1 day'
+            GROUP BY r.id
+        ) k
+        WHERE s.rule_id = k.rule_id
+    """)
+
+
+def downgrade() -> None:
+    op.drop_column("alert_states", "no_data_since")
+    op.drop_column("alert_states", "sample_interval_s")
+    op.drop_column("alert_states", "last_sample_at")
+    op.drop_column("alert_rules", "notify_no_data")

steward/models/alerts.py

@@ -45,6 +45,9 @@ class AlertRule(Base):
     consecutive_ok_required: Mapped[int] = mapped_column(Integer, nullable=False, default=1,
                                                          server_default="1")
     enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+    # Notify when the watched series goes silent (no_data) and when it resumes.
+    notify_no_data: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True,
+                                                 server_default="true")
     # Optional admin-only action: run an Ansible playbook on transition-to-firing.
     # Shape: {source, playbook, inventory, extra_vars, limit, tags, check}. NULL = none.
     ansible_action: Mapped[dict | None] = mapped_column(JSON, nullable=True)
@@ -74,6 +77,13 @@ class AlertState(Base):
         String(36), ForeignKey("users.id", ondelete="SET NULL"), nullable=True
     )
     acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    # Silence tracking (core.alerts.check_silence): when the watched series last
+    # had a sample, its usual gap between samples, and since when it has had
+    # none for too long. no_data_since is a mark beside the state, not a state:
+    # a firing alert that goes silent stays firing.
+    last_sample_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    sample_interval_s: Mapped[float | None] = mapped_column(Float, nullable=True)
+    no_data_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
 
 
 class MaintenanceWindow(Base):

steward/templates/alerts/list.html

@@ -24,9 +24,16 @@
                     <span class="badge badge-red">FIRING</span>
                     {% elif state.state.value == 'acknowledged' %}
                     <span class="badge badge-yellow">ACK</span>
-                    {% else %}
+                    {% elif state.state.value == 'pending' %}
                     <span class="badge badge-dim">PENDING</span>
                     {% endif %}
+                    {% if state.no_data_since %}
+                    <span class="badge badge-yellow"
+                          title="No sample since {{ state.last_sample_at.strftime('%Y-%m-%d %H:%M UTC') if state.last_sample_at else 'the rule started watching' }}">NO DATA</span>
+                    <div style="font-size:0.72rem;color:var(--text-dim);margin-top:0.2rem;">
+                      since {{ state.no_data_since.strftime("%Y-%m-%d %H:%M") }} UTC
+                    </div>
+                    {% endif %}
                 </td>
                 <td>{{ rule.name }}</td>
                 <td style="color:var(--text-muted);">{{ rule.resource_name }}</td>

steward/templates/alerts/rules_form.html

@@ -99,6 +99,18 @@
           within the threshold. An acknowledgement holds until then.</p>
       </div>
 
+      <div class="form-group">
+        <label style="display:flex;align-items:center;gap:0.5rem;font-weight:normal;">
+          <input type="checkbox" name="notify_no_data"
+                 {% if not rule or rule.notify_no_data %}checked{% endif %}>
+          Notify when this metric stops reporting
+        </label>
+        <p style="font-size:0.75rem;color:var(--text-muted);margin-top:0.2rem;">
+          Silence is always shown as NO DATA on the Alerts page: no sample for three of the
+          metric's usual gaps (at least 5 minutes). This also sends a notification when it starts
+          and when data resumes. A firing alert that goes silent stays firing.</p>
+      </div>
+
       {# ── Metric reference ────────────────────────────────────────────────── #}
       <details style="margin-bottom:1rem;">
         <summary style="cursor:pointer;font-size:0.82rem;color:var(--text-muted);user-select:none;">
@@ -113,7 +125,8 @@
             "dns":     [("is_up", "1.0 = resolved (and matches expected IP), 0.0 = failed"),
                         ("response_ms", "resolution time in milliseconds")],
             "traefik": [("request_rate", "requests/sec for this router"),
-                        ("error_rate", "fraction of 5xx responses (0–1)"),
+                        ("error_rate_4xx_pct", "% of responses that are 4xx"),
+                        ("error_rate_5xx_pct", "% of responses that are 5xx"),
                         ("latency_p50_ms", "median response time ms"),
                         ("latency_p95_ms", "p95 response time ms"),
                         ("latency_p99_ms", "p99 response time ms"),
@@ -126,6 +139,11 @@
                         ("mem_pct", "container memory usage %")],
             "http":    [("is_up", "1.0 = check passed, 0.0 = failed"),
                         ("response_ms", "HTTP response time in ms")],
+            "host_agent": [("seconds_since_seen", "seconds since the agent last reported; > 180 = offline"),
+                           ("cpu_pct", "host CPU usage %"),
+                           ("mem_used_pct", "host memory usage %"),
+                           ("disk_used_pct_worst", "fullest disk, % used"),
+                           ("disk_used_pct", "one mount, % used (resource host:mount)")],
           } %}
           {% for src, pairs in descs.items() %}
           <div>
@@ -192,6 +210,20 @@
       </div>
       {% endif %}
 
+      {% if not rule %}
+      <script>
+        // "Agent offline" is seconds_since_seen > 180 (the agent's stale window).
+        document.addEventListener("change", function (e) {
+          if (e.target.name !== "metric_name" || e.target.value !== "seconds_since_seen") return;
+          var form = e.target.form, threshold = form.elements["threshold"];
+          if (threshold.value === "") {
+            form.elements["operator"].value = ">";
+            threshold.value = "180";
+          }
+        });
+      </script>
+      {% endif %}
+
       <div style="display:flex;gap:1rem;margin-top:1.5rem;">
         <button type="submit" class="btn">{% if rule %}Save Changes{% else %}Create Rule{% endif %}</button>
         <a href="/alerts/" class="btn btn-ghost">Cancel</a>