feat(alerts): silence is alertable — agent offline and no-data (#5062)

Agent offline is an ordinary rule now. The host_agent scheduler records
host_agent/<host>/seconds_since_seen every minute (from last_seen_at, or
registration time for an agent that never reported), replacing the task
that only logged stale agents. The catalog offers the metric, and picking
it on a new rule fills "> 180".

No data: each rule's state tracks its last sample and its usual gap,
smoothed so that an outage counts as at most three gaps. A core task,
alert_silence_check, runs every 60s and marks a rule no_data_since once its
series has had no sample for three usual gaps (at least 5 minutes; an hour
while the cadence is unknown). A never-sampled rule counts from when it
started watching. The mark sits beside the state, so a firing alert that
goes silent stays firing. Each silence records one event and sends one
notification (rule option notify_no_data, default on; muted in maintenance
windows). The next sample clears it with a "data resumed" event and
notification. Creating, changing, disabling or re-enabling a rule restarts
its watch.

Migration 0029 adds the columns and seeds each existing rule's last sample
and gap from its last day of data, so an upgrade does not flag every rule.

Shown on the Alerts page (a NO DATA badge with its start time), in MCP
list_alerts, list_alert_rules and get_host (state "no_data", no_data_since),
in get_overview findings (alert_no_data, or a firing alert marked silent)
and in the weekly report. The rule form has the notify option, and its
metric reference now matches what traefik records and lists host_agent.
Emails say "no data" and carry the event's note.

The metric-catalog guard also reads positional MetricBatch.add arguments.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/__init__.py

@@ -35,9 +35,10 @@ async def mcp_overview(db) -> list[dict]:
 
 def get_alert_metrics() -> dict[str, list[str]]:
     """Host metrics ingest records for alert rules (resource = the host name;
-    disk_used_pct per mount, resource "<host>:<mount>")."""
+    disk_used_pct per mount, resource "<host>:<mount>"). seconds_since_seen
+    comes from the scheduler, not the agent: "agent offline" is > 180."""
     return {"host_agent": [
-        "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
+        "seconds_since_seen", "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
         "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m",
         "uptime_secs",
     ]}

plugins/host_agent/scheduler.py

@@ -1,11 +1,12 @@
 # plugins/host_agent/scheduler.py
 from __future__ import annotations
 import logging
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timezone
 from typing import Iterable
 
 from sqlalchemy import select
 
+from steward.core.alerts import MetricBatch
 from steward.core.scheduler import ScheduledTask
 from steward.models.hosts import Host
 from .models import HostAgentRegistration
@@ -13,66 +14,52 @@ from .models import HostAgentRegistration
 logger = logging.getLogger(__name__)
 
 
-def _filter_stale(
-    regs: Iterable,
-    *,
-    now: datetime,
-    stale_after_seconds: int,
-) -> list:
-    """Pure staleness filter: returns the subset with last_seen_at strictly
-    older than (now - stale_after_seconds). Rows with last_seen_at=None are
-    never stale (they are unregistered-in-practice)."""
-    cutoff = now - timedelta(seconds=stale_after_seconds)
-    return [
-        r for r in regs
-        if r.last_seen_at is not None and r.last_seen_at < cutoff
-    ]
+def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
+                       now: datetime) -> list[tuple[str, float]]:
+    """(host name, seconds since its agent last reported) per registration.
+
+    An agent that has never reported counts from when it was registered. A
+    registration whose host is gone is skipped.
+    """
+    out = []
+    for r in regs:
+        name = host_names.get(r.host_id)
+        seen = r.last_seen_at or r.created_at
+        if name is None or seen is None:
+            continue
+        out.append((name, max(0.0, (now - seen).total_seconds())))
+    return out
 
 
-async def find_stale_registrations(app, stale_after_seconds: int = 180) -> list[dict]:
+async def record_seconds_since_seen(app) -> None:
+    """Record host_agent/<host>/seconds_since_seen for every registered agent,
+    so "agent offline" is an ordinary alert rule (seconds_since_seen > 180)."""
+    now = datetime.now(timezone.utc)
     async with app.db_sessionmaker() as session:
-        all_regs = (await session.execute(
-            select(HostAgentRegistration)
-        )).scalars().all()
-        stale_rows = _filter_stale(
-            all_regs,
-            now=datetime.now(timezone.utc),
-            stale_after_seconds=stale_after_seconds,
-        )
-        if not stale_rows:
-            return []
-        hosts = {
-            h.id: h for h in (await session.execute(
-                select(Host).where(Host.id.in_([r.host_id for r in stale_rows]))
-            )).scalars().all()
-        }
-    return [
-        {
-            "host_id": r.host_id,
-            "host_name": hosts[r.host_id].name if r.host_id in hosts else "?",
-            "last_seen_at": r.last_seen_at,
-        }
-        for r in stale_rows
-    ]
+        async with session.begin():
+            regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
+            if not regs:
+                return
+            host_names = dict((await session.execute(
+                select(Host.id, Host.name).where(Host.id.in_([r.host_id for r in regs]))
+            )).all())
+            batch = MetricBatch()
+            for name, seconds in seconds_since_seen(regs, host_names, now):
+                batch.add(source_module="host_agent", resource_name=name,
+                          metric_name="seconds_since_seen", value=seconds, recorded_at=now)
+            await batch.record(session)
 
 
 def make_task(app) -> ScheduledTask:
-    async def _check_stale():
+    async def _record():
         try:
-            stale = await find_stale_registrations(app)
+            await record_seconds_since_seen(app)
         except Exception:
-            logger.exception("host_agent stale check failed")
-            return
-        if stale:
-            logger.info(
-                "host_agent: %d stale agent(s): %s",
-                len(stale),
-                [s["host_name"] for s in stale],
-            )
+            logger.exception("host_agent: recording seconds_since_seen failed")
 
     return ScheduledTask(
-        name="host_agent_stale_check",
-        coro_factory=_check_stale,
+        name="host_agent_seconds_since_seen",
+        coro_factory=_record,
         interval_seconds=60,
         run_on_startup=False,
     )