feat(alerts): one evaluation path; host-agent ingest fires rules (#5060)

#5060 "One evaluation path: record_metrics batch API; host-agent ingest
evaluates rules".

- record_metrics(session, rows) writes metric rows and evaluates the rules
  that watch them, reading rules, states and maintenance windows once per
  batch. Each rule sees its key's samples oldest first, so an agent catching
  up after an outage counts consecutive checks as they happened.
- Host-agent ingest now goes through it. Before, it wrote plugin_metrics
  directly and no host_agent rule could ever fire. The catalog gains the
  per-mount disk_used_pct ("<host>:<mount>").
- MetricBatch gathers a transaction's samples; the docker ingest, traefik,
  unifi, snmp, monitor and self-monitor writers record one batch each instead
  of a rule query per sample.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_self_monitor.py

@@ -3,7 +3,7 @@
 Exercises the fd accounting, the percentage/warning floor, and that the
 recorder degrades to a no-op where fd accounting isn't available. The DB session
 and the alert pipeline are faked — we only assert that the right metrics get
-handed to record_metric.
+handed to record_metrics.
 """
 import asyncio
 import logging
@@ -48,10 +48,10 @@ class _FakeApp:
 def _patch_recorder(monkeypatch):
     recorded: list[tuple] = []
 
-    async def fake_record_metric(session, source_module, resource_name, metric_name, value):
-        recorded.append((source_module, resource_name, metric_name, value))
+    async def fake_record_metrics(session, rows):
+        recorded.extend((r.source_module, r.resource_name, r.metric_name, r.value) for r in rows)
 
-    monkeypatch.setattr(alerts, "record_metric", fake_record_metric)
+    monkeypatch.setattr(alerts, "record_metrics", fake_record_metrics)
     return recorded
 
 

tests/integration/test_host_agent_alerts.py

@@ -0,0 +1,75 @@
+"""Integration: a host-agent ingest POST evaluates alert rules on the host's
+metrics (#5060). Before, ingest wrote rows straight to plugin_metrics and no
+host_agent rule could ever fire."""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def test_ingest_fires_a_host_rule_from_a_backlog(app):
+    from sqlalchemy import select
+    from plugins.host_agent.models import HostAgentRegistration
+    from steward.alerts import service as alert_svc
+    from steward.models.alerts import AlertEvent, AlertState, AlertStateEnum
+    from steward.models.hosts import Host
+    from steward.models.metrics import PluginMetric
+    from steward.models.users import User, UserRole
+
+    async def _go():
+        tag = uuid.uuid4().hex[:8]
+        hostname, token = f"agent-{tag}", f"tok-{tag}-{uuid.uuid4().hex}"
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                user = User(username=f"ha-{tag}", email=f"ha-{tag}@test", password_hash="x",
+                            role=UserRole.admin)
+                host = Host(name=hostname, address="10.9.9.9")
+                s.add_all([user, host])
+                await s.flush()
+                s.add(HostAgentRegistration(
+                    host_id=host.id, token_hash=hashlib.sha256(token.encode()).hexdigest()))
+                s.add(PluginMetric(source_module="host_agent", resource_name=hostname,
+                                   metric_name="cpu_pct", value=5.0))
+                await s.flush()
+                fields = await alert_svc.rule_fields(
+                    s, name=f"cpu {tag}", source_module="host_agent", resource_name=hostname,
+                    metric_name="cpu_pct", operator=">", threshold=90,
+                    consecutive_failures_required=2)
+                rule = await alert_svc.add_rule(s, fields, created_by=user.id)
+            rule_id = rule.id
+
+        now = datetime.now(timezone.utc)
+        # Two buffered samples in one POST, newest first: evaluated oldest
+        # first, they make the two breaching checks the rule needs.
+        samples = [{"ts": (now - timedelta(seconds=s)).isoformat(), "cpu_pct": v}
+                   for s, v in ((0, 97.0), (30, 95.0))]
+        client = app.test_client()
+        resp = await client.post("/plugins/host_agent/ingest", json={"samples": samples},
+                                 headers={"Authorization": f"Bearer {token}"})
+        assert resp.status_code == 200, await resp.get_data(as_text=True)
+
+        async with app.db_sessionmaker() as s:
+            st = (await s.execute(select(AlertState).where(
+                AlertState.rule_id == rule_id))).scalar_one()
+            events = (await s.execute(select(AlertEvent).where(
+                AlertEvent.rule_id == rule_id))).scalars().all()
+        assert st.state == AlertStateEnum.firing
+        assert [(e.from_state, e.to_state, e.metric_value) for e in events] == [
+            ("pending", "firing", 97.0)]
+
+    asyncio.run(_go())