bvandeusenandClaude Opus 5.5
c368e93e2c
Agent offline is an ordinary rule now. The host_agent scheduler records host_agent/<host>/seconds_since_seen every minute (from last_seen_at, or registration time for an agent that never reported), replacing the task that only logged stale agents. The catalog offers the metric, and picking it on a new rule fills "> 180". No data: each rule's state tracks its last sample and its usual gap, smoothed so that an outage counts as at most three gaps. A core task, alert_silence_check, runs every 60s and marks a rule no_data_since once its series has had no sample for three usual gaps (at least 5 minutes; an hour while the cadence is unknown). A never-sampled rule counts from when it started watching. The mark sits beside the state, so a firing alert that goes silent stays firing. Each silence records one event and sends one notification (rule option notify_no_data, default on; muted in maintenance windows). The next sample clears it with a "data resumed" event and notification. Creating, changing, disabling or re-enabling a rule restarts its watch. Migration 0029 adds the columns and seeds each existing rule's last sample and gap from its last day of data, so an upgrade does not flag every rule. Shown on the Alerts page (a NO DATA badge with its start time), in MCP list_alerts, list_alert_rules and get_host (state "no_data", no_data_since), in get_overview findings (alert_no_data, or a firing alert marked silent) and in the weekly report. The rule form has the notify option, and its metric reference now matches what traefik records and lists host_agent. Emails say "no data" and carry the event's note. The metric-catalog guard also reads positional MetricBatch.add arguments. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
117 lines
5.2 KiB
Python
117 lines
5.2 KiB
Python
"""Integration: alert state against real Postgres. An acknowledgement holds
|
|
while the breach lasts, an alert clears after its OK run, and changing what a
|
|
rule watches (or disabling it) resets its state with a note on the event."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import uuid
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
def test_acknowledge_holds_clear_after_and_reset(app):
|
|
from sqlalchemy import select
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.alerts import record_metric
|
|
from steward.models.alerts import AlertEvent, AlertRule, AlertState, AlertStateEnum
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.users import User, UserRole
|
|
|
|
async def _go():
|
|
tag = uuid.uuid4().hex[:8]
|
|
res = f"box-{tag}"
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
user = User(username=f"al-{tag}", email=f"al-{tag}@test", password_hash="x",
|
|
role=UserRole.admin)
|
|
s.add(user)
|
|
s.add(PluginMetric(source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", value=1.0))
|
|
await s.flush()
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=f"hot {tag}", source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", operator=">", threshold=90,
|
|
consecutive_failures_required=1, consecutive_ok_required=2)
|
|
rule = await alert_svc.add_rule(s, fields, created_by=user.id)
|
|
rule_id = rule.id
|
|
|
|
async def sample(v):
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await record_metric(s, "host_agent", res, "cpu_pct", v)
|
|
|
|
async def state():
|
|
async with app.db_sessionmaker() as s:
|
|
return (await s.execute(select(AlertState).where(
|
|
AlertState.rule_id == rule_id))).scalar_one()
|
|
|
|
async def events():
|
|
async with app.db_sessionmaker() as s:
|
|
rows = (await s.execute(select(AlertEvent).where(AlertEvent.rule_id == rule_id)
|
|
.order_by(AlertEvent.transitioned_at))).scalars().all()
|
|
return [(e.from_state, e.to_state, e.note) for e in rows]
|
|
|
|
await sample(95)
|
|
assert (await state()).state == AlertStateEnum.firing
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await alert_svc.acknowledge(s, rule_id, None)
|
|
for _ in range(3):
|
|
await sample(97) # still breached: the acknowledgement holds
|
|
assert (await state()).state == AlertStateEnum.acknowledged
|
|
await sample(10) # one OK check of the two it needs
|
|
assert (await state()).state == AlertStateEnum.acknowledged
|
|
await sample(10)
|
|
st = await state()
|
|
assert st.state == AlertStateEnum.inactive and st.acknowledged_at is None
|
|
assert await events() == [("inactive", "firing", None), ("acknowledged", "resolved", None)]
|
|
|
|
# Changing the threshold of a firing rule resets it, saying why.
|
|
await sample(95)
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
r = await s.get(AlertRule, rule_id)
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=r.name, source_module=r.source_module, resource_name=r.resource_name,
|
|
metric_name=r.metric_name, operator=">", threshold=99,
|
|
consecutive_failures_required=1, consecutive_ok_required=2)
|
|
await alert_svc.apply_rule_change(s, r, fields)
|
|
assert (await state()).state == AlertStateEnum.inactive
|
|
assert (await events())[-1] == ("firing", "resolved", "rule changed: threshold")
|
|
|
|
# Renaming alone does not reset; disabling does.
|
|
await sample(100)
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
r = await s.get(AlertRule, rule_id)
|
|
await alert_svc.apply_rule_change(s, r, {"name": f"renamed {tag}"})
|
|
assert (await state()).state == AlertStateEnum.firing
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
r = await s.get(AlertRule, rule_id)
|
|
await alert_svc.apply_rule_change(s, r, {}, enabled=False)
|
|
assert (await state()).state == AlertStateEnum.inactive
|
|
assert (await events())[-1] == ("firing", "resolved", "rule disabled")
|
|
|
|
# Re-enabling starts the watch over: silence is measured from now.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
r = await s.get(AlertRule, rule_id)
|
|
await alert_svc.apply_rule_change(s, r, {}, enabled=True)
|
|
st = await state()
|
|
assert (st.state, st.last_sample_at, st.no_data_since) == (AlertStateEnum.inactive, None, None)
|
|
assert (await events())[-1] == ("firing", "resolved", "rule disabled")
|
|
|
|
asyncio.run(_go())
|