feat(alerts): one evaluation path; host-agent ingest fires rules (#5060)

#5060 "One evaluation path: record_metrics batch API; host-agent ingest
evaluates rules".

- record_metrics(session, rows) writes metric rows and evaluates the rules
  that watch them, reading rules, states and maintenance windows once per
  batch. Each rule sees its key's samples oldest first, so an agent catching
  up after an outage counts consecutive checks as they happened.
- Host-agent ingest now goes through it. Before, it wrote plugin_metrics
  directly and no host_agent rule could ever fire. The catalog gains the
  per-mount disk_used_pct ("<host>:<mount>").
- MetricBatch gathers a transaction's samples; the docker ingest, traefik,
  unifi, snmp, monitor and self-monitor writers record one batch each instead
  of a rule query per sample.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/ingest.py

@@ -266,7 +266,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
     log batches (m79) — appended one row per line when present, independent of
     whether this sample also carried container metrics.
     """
-    from steward.core.alerts import record_metric
+    from steward.core.alerts import MetricBatch
     from .models import DockerContainer, DockerEvent, DockerMetric
 
     if swarm is not None:
@@ -313,6 +313,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
             ))
 
     # Current state + alerts from the newest snapshot only.
+    batch = MetricBatch()
     for c in latest_containers:
         name = c.get("name")
         if not name:
@@ -350,19 +351,19 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
         # on different hosts don't collide in the metric/alert namespace.
         resource = f"{host.name}/{name}"
         if c.get("status") == "running" and c.get("cpu_pct") is not None:
-            await record_metric(
-                session=session, source_module="docker",
+            batch.add(
+                source_module="docker",
                 resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"],
             )
             if c.get("mem_pct") is not None:
-                await record_metric(
-                    session=session, source_module="docker",
+                batch.add(
+                    source_module="docker",
                     resource_name=resource, metric_name="mem_pct", value=c["mem_pct"],
                 )
         # Restart count is alertable regardless of state (crash-looping matters
         # most while the container is down/restarting).
-        await record_metric(
-            session=session, source_module="docker",
+        batch.add(
+            source_module="docker",
             resource_name=resource, metric_name="restart_count",
             value=float(c.get("restart_count", 0) or 0),
         )
@@ -371,12 +372,14 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
         # container).
         health = c.get("health")
         if health in ("healthy", "unhealthy", "starting"):
-            await record_metric(
-                session=session, source_module="docker",
+            batch.add(
+                source_module="docker",
                 resource_name=resource, metric_name="is_healthy",
                 value=1.0 if health == "healthy" else 0.0,
             )
 
+    await batch.record(session)
+
     # Reap containers that vanished from the host's latest listing. Without this,
     # removed one-shot containers (CI job runners, buildkit builders, codex jobs)
     # accumulate forever as permanent "stopped" rows and inflate every count.

plugins/host_agent/routes.py

@@ -201,6 +201,7 @@ async def ingest():
             return _error(401, "invalid_token")
 
         accepted = 0
+        metric_rows: list[PluginMetric] = []
         latest_ts: datetime | None = None
         docker_snapshots: list[tuple[datetime, list]] = []
         docker_log_batches: list[tuple[datetime, list]] = []
@@ -213,9 +214,7 @@ async def ingest():
                 recorded_at = _parse_ts(sample["ts"])
             except (KeyError, ValueError):
                 continue
-            metrics = _expand_sample_to_metrics(sample, host.name, recorded_at)
-            for m in metrics:
-                session.add(m)
+            metric_rows.extend(_expand_sample_to_metrics(sample, host.name, recorded_at))
             docker = sample.get("docker")
             if isinstance(docker, list) and docker:
                 docker_snapshots.append((recorded_at, docker))
@@ -243,6 +242,10 @@ async def ingest():
             await session.rollback()
             return _error(400, "malformed_payload", "no valid samples")
 
+        # Through the alert pipeline, so rules on host metrics can fire.
+        from steward.core.alerts import record_metrics
+        await record_metrics(session, metric_rows)
+
         # Hand host-scoped container data to the docker plugin if it's enabled
         # (opportunistic synergy via the capability registry — no hard import,
         # no-op when docker is disabled). A failure here must never sink the

plugins/snmp/scheduler.py

@@ -27,7 +27,7 @@ def make_poll_task(app: "Quart") -> ScheduledTask:
 
 async def _do_poll(app: "Quart") -> None:
     from .poller import poll_device
-    from steward.core.alerts import record_metric
+    from steward.core.alerts import MetricBatch
 
     devices: list[dict] = app.config["PLUGINS"]["snmp"].get("devices", [])
     if not devices:
@@ -35,6 +35,7 @@ async def _do_poll(app: "Quart") -> None:
 
     async with app.db_sessionmaker() as session:
         async with session.begin():
+            batch = MetricBatch()
             for device in devices:
                 if not isinstance(device, dict):
                     continue
@@ -55,8 +56,7 @@ async def _do_poll(app: "Quart") -> None:
                     continue
 
                 for label, value in readings.items():
-                    await record_metric(
-                        session=session,
+                    batch.add(
                         source_module="snmp",
                         resource_name=name,
                         metric_name=label,
@@ -67,3 +67,4 @@ async def _do_poll(app: "Quart") -> None:
                     logger.debug("SNMP polled %s (%s): %d OID(s)", name, host, len(readings))
                 else:
                     logger.debug("SNMP polled %s (%s): no readings", name, host)
+            await batch.record(session)

plugins/traefik/scheduler.py

@@ -48,7 +48,7 @@ async def _do_scrape(app: "Quart") -> None:
         extract_certs,
         fetch_metrics,
     )
-    from steward.core.alerts import record_metric
+    from steward.core.alerts import MetricBatch
 
     metrics_url: str = app.config["PLUGINS"]["traefik"]["metrics_url"]
 
@@ -71,6 +71,7 @@ async def _do_scrape(app: "Quart") -> None:
 
     async with app.db_sessionmaker() as session:
         async with session.begin():
+            batch = MetricBatch()
             # ── Per-service metrics ───────────────────────────────────────────
             for router_name, metrics in router_metrics.items():
                 row = TraefikMetric(
@@ -95,8 +96,7 @@ async def _do_scrape(app: "Quart") -> None:
                     ("latency_p99_ms", metrics["latency_p99_ms"]),
                     ("response_bytes_rate", metrics["response_bytes_rate"]),
                 ]:
-                    await record_metric(
-                        session=session,
+                    batch.add(
                         source_module="traefik",
                         resource_name=router_name,
                         metric_name=metric_name,
@@ -125,14 +125,15 @@ async def _do_scrape(app: "Quart") -> None:
 
                 # Emit cert_expiry_days for alert rules
                 days_remaining = (cert["not_after"] - scraped_at).total_seconds() / 86400.0
-                await record_metric(
-                    session=session,
+                batch.add(
                     source_module="traefik",
                     resource_name=cert["cn"] or cert["serial"],
                     metric_name="cert_expiry_days",
                     value=max(0.0, days_remaining),
                 )
 
+            await batch.record(session)
+
     logger.debug(
         "Traefik scrape: %d router(s), %d cert(s)",
         len(router_metrics),

plugins/unifi/scheduler.py

@@ -54,7 +54,7 @@ async def _do_poll(app: "Quart") -> None:
 
     from .client import UnifiClient
     from .models import UnifiClientSnapshot, UnifiDevice, UnifiWanStat
-    from steward.core.alerts import record_metric
+    from steward.core.alerts import MetricBatch
 
     cfg = app.config["PLUGINS"]["unifi"]
 
@@ -171,28 +171,27 @@ async def _do_poll(app: "Quart") -> None:
                     ))
 
             # ── Alert metrics ─────────────────────────────────────────────────
+            batch = MetricBatch()
             if wan_stat.latency_ms is not None:
-                await record_metric(
-                    session=session,
+                batch.add(
                     source_module="unifi",
                     resource_name="wan",
                     metric_name="latency_ms",
                     value=wan_stat.latency_ms,
                 )
-            await record_metric(
-                session=session,
+            batch.add(
                 source_module="unifi",
                 resource_name="wan",
                 metric_name="is_up",
                 value=1.0 if wan_stat.is_up else 0.0,
             )
-            await record_metric(
-                session=session,
+            batch.add(
                 source_module="unifi",
                 resource_name="clients",
                 metric_name="total_clients",
                 value=float(len(clients)),
             )
+            await batch.record(session)
 
     logger.debug(
         "UniFi poll: wan=%s, clients=%d, devices=%d",