bvandeusenandClaude Opus 5.5
58bfc7c361
#5060 "One evaluation path: record_metrics batch API; host-agent ingest evaluates rules". - record_metrics(session, rows) writes metric rows and evaluates the rules that watch them, reading rules, states and maintenance windows once per batch. Each rule sees its key's samples oldest first, so an agent catching up after an outage counts consecutive checks as they happened. - Host-agent ingest now goes through it. Before, it wrote plugin_metrics directly and no host_agent rule could ever fire. The catalog gains the per-mount disk_used_pct ("<host>:<mount>"). - MetricBatch gathers a transaction's samples; the docker ingest, traefik, unifi, snmp, monitor and self-monitor writers record one batch each instead of a rule query per sample. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
76 lines
3.2 KiB
Python
76 lines
3.2 KiB
Python
"""Integration: a host-agent ingest POST evaluates alert rules on the host's
|
|
metrics (#5060). Before, ingest wrote rows straight to plugin_metrics and no
|
|
host_agent rule could ever fire."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import os
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
def test_ingest_fires_a_host_rule_from_a_backlog(app):
|
|
from sqlalchemy import select
|
|
from plugins.host_agent.models import HostAgentRegistration
|
|
from steward.alerts import service as alert_svc
|
|
from steward.models.alerts import AlertEvent, AlertState, AlertStateEnum
|
|
from steward.models.hosts import Host
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.users import User, UserRole
|
|
|
|
async def _go():
|
|
tag = uuid.uuid4().hex[:8]
|
|
hostname, token = f"agent-{tag}", f"tok-{tag}-{uuid.uuid4().hex}"
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
user = User(username=f"ha-{tag}", email=f"ha-{tag}@test", password_hash="x",
|
|
role=UserRole.admin)
|
|
host = Host(name=hostname, address="10.9.9.9")
|
|
s.add_all([user, host])
|
|
await s.flush()
|
|
s.add(HostAgentRegistration(
|
|
host_id=host.id, token_hash=hashlib.sha256(token.encode()).hexdigest()))
|
|
s.add(PluginMetric(source_module="host_agent", resource_name=hostname,
|
|
metric_name="cpu_pct", value=5.0))
|
|
await s.flush()
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=f"cpu {tag}", source_module="host_agent", resource_name=hostname,
|
|
metric_name="cpu_pct", operator=">", threshold=90,
|
|
consecutive_failures_required=2)
|
|
rule = await alert_svc.add_rule(s, fields, created_by=user.id)
|
|
rule_id = rule.id
|
|
|
|
now = datetime.now(timezone.utc)
|
|
# Two buffered samples in one POST, newest first: evaluated oldest
|
|
# first, they make the two breaching checks the rule needs.
|
|
samples = [{"ts": (now - timedelta(seconds=s)).isoformat(), "cpu_pct": v}
|
|
for s, v in ((0, 97.0), (30, 95.0))]
|
|
client = app.test_client()
|
|
resp = await client.post("/plugins/host_agent/ingest", json={"samples": samples},
|
|
headers={"Authorization": f"Bearer {token}"})
|
|
assert resp.status_code == 200, await resp.get_data(as_text=True)
|
|
|
|
async with app.db_sessionmaker() as s:
|
|
st = (await s.execute(select(AlertState).where(
|
|
AlertState.rule_id == rule_id))).scalar_one()
|
|
events = (await s.execute(select(AlertEvent).where(
|
|
AlertEvent.rule_id == rule_id))).scalars().all()
|
|
assert st.state == AlertStateEnum.firing
|
|
assert [(e.from_state, e.to_state, e.metric_value) for e in events] == [
|
|
("pending", "firing", 97.0)]
|
|
|
|
asyncio.run(_go())
|