feat(alerts): one evaluation path; host-agent ingest fires rules (#5060)

#5060 "One evaluation path: record_metrics batch API; host-agent ingest
evaluates rules".

- record_metrics(session, rows) writes metric rows and evaluates the rules
  that watch them, reading rules, states and maintenance windows once per
  batch. Each rule sees its key's samples oldest first, so an agent catching
  up after an outage counts consecutive checks as they happened.
- Host-agent ingest now goes through it. Before, it wrote plugin_metrics
  directly and no host_agent rule could ever fire. The catalog gains the
  per-mount disk_used_pct ("<host>:<mount>").
- MetricBatch gathers a transaction's samples; the docker ingest, traefik,
  unifi, snmp, monitor and self-monitor writers record one batch each instead
  of a rule query per sample.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/alerts.py

@@ -6,7 +6,7 @@ from dataclasses import dataclass
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING
 
-from sqlalchemy import and_, or_, select
+from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from steward.models.alerts import (
@@ -57,82 +57,109 @@ async def record_metric(
     metric_name: str,
     value: float,
 ) -> None:
-    """Write a metric row and evaluate alert rules inline.
+    """One metric sample: record_metrics with a single row."""
+    await record_metrics(session, [PluginMetric(
+        source_module=source_module, resource_name=resource_name,
+        metric_name=metric_name, value=value)])
 
-    Must be called inside an active transaction:
-        async with session.begin():
-            await record_metric(session, ...)
 
-    Notification I/O is deferred via asyncio.create_task() so no network
-    calls occur while the transaction is open.
+class MetricBatch:
+    """Gathers one transaction's samples so they are recorded, and their alert
+    rules evaluated, in a single record_metrics call:
+
+        batch = MetricBatch()
+        batch.add(source_module="unifi", resource_name="wan", metric_name="is_up", value=1.0)
+        await batch.record(session)
     """
-    now = datetime.now(timezone.utc)
 
-    # Write metric row
-    metric = PluginMetric(
-        source_module=source_module,
-        resource_name=resource_name,
-        metric_name=metric_name,
-        value=value,
-        recorded_at=now,
-    )
-    session.add(metric)
+    def __init__(self) -> None:
+        self.rows: list[PluginMetric] = []
+
+    def add(self, source_module: str, resource_name: str, metric_name: str, value: float,
+            recorded_at: datetime | None = None) -> None:
+        self.rows.append(PluginMetric(source_module=source_module, resource_name=resource_name,
+                                      metric_name=metric_name, value=float(value),
+                                      recorded_at=recorded_at))
+
+    async def record(self, session: AsyncSession) -> None:
+        rows, self.rows = self.rows, []
+        await record_metrics(session, rows)
+
+
+def _in_window(w: MaintenanceWindow, source_module: str, resource_name: str) -> bool:
+    if w.scope_source is None:
+        return True
+    return w.scope_source == source_module and w.scope_resource in (None, resource_name)
+
+
+async def record_metrics(session: AsyncSession, rows: list[PluginMetric]) -> None:
+    """Write metric rows and evaluate the alert rules that watch them.
+
+    Every ingest path comes through here, so a rule on any metric Steward
+    records can fire. Must be called inside an active transaction:
+        async with session.begin():
+            await record_metrics(session, rows)
+
+    Maintenance windows, rules and their states are read once per batch, not
+    once per row. Each rule sees its key's samples oldest first, so a backlog
+    (an agent catching up after an outage) counts consecutive checks as they
+    happened. Notification I/O is deferred via asyncio.create_task() so no
+    network calls occur while the transaction is open.
+    """
+    if not rows:
+        return
+    now = datetime.now(timezone.utc)
+    for row in rows:
+        if row.recorded_at is None:
+            row.recorded_at = now
+    session.add_all(rows)
     await session.flush()
 
-    # Check active maintenance windows — skip rule evaluation if suppressed
-    mw_result = await session.execute(
-        select(MaintenanceWindow).where(
-            MaintenanceWindow.start_at <= now,
-            MaintenanceWindow.end_at >= now,
-            or_(
-                MaintenanceWindow.scope_source.is_(None),
-                and_(
-                    MaintenanceWindow.scope_source == source_module,
-                    or_(
-                        MaintenanceWindow.scope_resource.is_(None),
-                        MaintenanceWindow.scope_resource == resource_name,
-                    ),
-                ),
-            ),
-        ).limit(1)
-    )
-    if mw_result.scalar_one_or_none() is not None:
-        return  # suppressed by maintenance window
+    by_key: dict[tuple[str, str, str], list[PluginMetric]] = {}
+    for row in rows:
+        by_key.setdefault((row.source_module, row.resource_name, row.metric_name), []).append(row)
+    sources = {k[0] for k in by_key}
+    metric_names = {k[2] for k in by_key}
 
-    # Load matching enabled rules
-    result = await session.execute(
+    rules = (await session.execute(
         select(AlertRule).where(
-            AlertRule.source_module == source_module,
-            AlertRule.resource_name == resource_name,
-            AlertRule.metric_name == metric_name,
+            AlertRule.source_module.in_(sources),
+            AlertRule.metric_name.in_(metric_names),
             AlertRule.enabled.is_(True),
         )
-    )
-    rules = result.scalars().all()
+    )).scalars().all()
+    rules = [r for r in rules if (r.source_module, r.resource_name, r.metric_name) in by_key]
+    if not rules:
+        return
+
+    windows = (await session.execute(
+        select(MaintenanceWindow).where(
+            MaintenanceWindow.start_at <= now, MaintenanceWindow.end_at >= now)
+    )).scalars().all()
+    states = {st.rule_id: st for st in (await session.execute(
+        select(AlertState).where(AlertState.rule_id.in_([r.id for r in rules]))
+    )).scalars()}
 
     for rule in rules:
-        state_result = await session.execute(
-            select(AlertState).where(AlertState.rule_id == rule.id)
-        )
-        alert_state = state_result.scalar_one_or_none()
+        if any(_in_window(w, rule.source_module, rule.resource_name) for w in windows):
+            continue  # suppressed by maintenance window
+        alert_state = states.get(rule.id)
         if alert_state is None:
-            logger.warning(f"No alert_state row for rule {rule.id}, skipping")
+            logger.warning("No alert_state row for rule %s, skipping", rule.id)
             continue
-
-        notifications = await _evaluate_rule(session, rule, alert_state, value, now)
-
-        for notif_type, event_id in notifications:
-            if _app is not None:
-                asyncio.create_task(
-                    _dispatch_notification(
-                        _app, notif_type, rule, value, event_id
-                    )
-                )
-                # Admin-configured Ansible action runs only on transition-to-firing.
-                if notif_type == "firing" and rule.ansible_action:
+        samples = sorted(by_key[(rule.source_module, rule.resource_name, rule.metric_name)],
+                         key=lambda r: r.recorded_at)
+        for sample in samples:
+            notifications = await _evaluate_rule(session, rule, alert_state, sample.value,
+                                                 sample.recorded_at)
+            for notif_type, event_id in notifications:
+                if _app is not None:
                     asyncio.create_task(
-                        _run_ansible_action(_app, rule, value, event_id)
-                    )
+                        _dispatch_notification(_app, notif_type, rule, sample.value, event_id))
+                    # Admin-configured Ansible action runs only on transition-to-firing.
+                    if notif_type == "firing" and rule.ansible_action:
+                        asyncio.create_task(
+                            _run_ansible_action(_app, rule, sample.value, event_id))
 
 
 def _is_breached(value: float, operator: AlertOperator, threshold: float) -> bool:

steward/core/self_monitor.py

@@ -80,28 +80,28 @@ async def record_self_metrics(app) -> None:
         logger.debug("self_monitor: /proc/self/fd unavailable — skipping fd metrics")
         return
 
-    from .alerts import record_metric
+    from .alerts import MetricBatch
 
     soft = fd_soft_limit()
     pct = (fds / soft * 100.0) if soft else None
 
     async with app.db_sessionmaker() as session:
         async with session.begin():
-            await record_metric(
-                session=session,
+            batch = MetricBatch()
+            batch.add(
                 source_module="steward",
                 resource_name="process",
                 metric_name="open_fds",
                 value=float(fds),
             )
             if pct is not None:
-                await record_metric(
-                    session=session,
+                batch.add(
                     source_module="steward",
                     resource_name="process",
                     metric_name="open_fds_pct",
                     value=pct,
                 )
+            await batch.record(session)
 
     if pct is not None and pct >= FD_WARN_PCT:
         if not _warned: