bvandeusenandClaude Opus 5.5
a6c24299a2
#5059 "Alert state machine: acknowledgement holds until resolve; N consecutive OK to resolve; editing, disabling or deleting a rule resets it". - The state machine is now next_step(), a pure function of one sample. An acknowledged alert stays acknowledged while breached; before, the next breaching sample re-fired it. Firing and acknowledged alerts resolve after consecutive_ok_required OK checks in a row (new rule column, default 1; rule form, rules list and MCP create/update_alert_rule clear_after_checks). - Changing what a rule watches (source, resource, metric, operator, threshold) or disabling it resets its alert to inactive; a firing or acknowledged one records a resolved event whose new note says why, and nobody is notified. Web and MCP edits share alerts.service.apply_rule_change. - Migration 0028: consecutive_ok_required, consecutive_ok_count, alert_events.note, and metric_value nullable (a reset has no sample). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
108 lines
4.8 KiB
Python
108 lines
4.8 KiB
Python
"""Integration: alert state against real Postgres. An acknowledgement holds
|
|
while the breach lasts, an alert clears after its OK run, and changing what a
|
|
rule watches (or disabling it) resets its state with a note on the event."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import uuid
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
def test_acknowledge_holds_clear_after_and_reset(app):
|
|
from sqlalchemy import select
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.alerts import record_metric
|
|
from steward.models.alerts import AlertEvent, AlertRule, AlertState, AlertStateEnum
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.users import User, UserRole
|
|
|
|
async def _go():
|
|
tag = uuid.uuid4().hex[:8]
|
|
res = f"box-{tag}"
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
user = User(username=f"al-{tag}", email=f"al-{tag}@test", password_hash="x",
|
|
role=UserRole.admin)
|
|
s.add(user)
|
|
s.add(PluginMetric(source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", value=1.0))
|
|
await s.flush()
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=f"hot {tag}", source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", operator=">", threshold=90,
|
|
consecutive_failures_required=1, consecutive_ok_required=2)
|
|
rule = await alert_svc.add_rule(s, fields, created_by=user.id)
|
|
rule_id = rule.id
|
|
|
|
async def sample(v):
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await record_metric(s, "host_agent", res, "cpu_pct", v)
|
|
|
|
async def state():
|
|
async with app.db_sessionmaker() as s:
|
|
return (await s.execute(select(AlertState).where(
|
|
AlertState.rule_id == rule_id))).scalar_one()
|
|
|
|
async def events():
|
|
async with app.db_sessionmaker() as s:
|
|
rows = (await s.execute(select(AlertEvent).where(AlertEvent.rule_id == rule_id)
|
|
.order_by(AlertEvent.transitioned_at))).scalars().all()
|
|
return [(e.from_state, e.to_state, e.note) for e in rows]
|
|
|
|
await sample(95)
|
|
assert (await state()).state == AlertStateEnum.firing
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await alert_svc.acknowledge(s, rule_id, None)
|
|
for _ in range(3):
|
|
await sample(97) # still breached: the acknowledgement holds
|
|
assert (await state()).state == AlertStateEnum.acknowledged
|
|
await sample(10) # one OK check of the two it needs
|
|
assert (await state()).state == AlertStateEnum.acknowledged
|
|
await sample(10)
|
|
st = await state()
|
|
assert st.state == AlertStateEnum.inactive and st.acknowledged_at is None
|
|
assert await events() == [("inactive", "firing", None), ("acknowledged", "resolved", None)]
|
|
|
|
# Changing the threshold of a firing rule resets it, saying why.
|
|
await sample(95)
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
r = await s.get(AlertRule, rule_id)
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=r.name, source_module=r.source_module, resource_name=r.resource_name,
|
|
metric_name=r.metric_name, operator=">", threshold=99,
|
|
consecutive_failures_required=1, consecutive_ok_required=2)
|
|
await alert_svc.apply_rule_change(s, r, fields)
|
|
assert (await state()).state == AlertStateEnum.inactive
|
|
assert (await events())[-1] == ("firing", "resolved", "rule changed: threshold")
|
|
|
|
# Renaming alone does not reset; disabling does.
|
|
await sample(100)
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
r = await s.get(AlertRule, rule_id)
|
|
await alert_svc.apply_rule_change(s, r, {"name": f"renamed {tag}"})
|
|
assert (await state()).state == AlertStateEnum.firing
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
r = await s.get(AlertRule, rule_id)
|
|
await alert_svc.apply_rule_change(s, r, {}, enabled=False)
|
|
assert (await state()).state == AlertStateEnum.inactive
|
|
assert (await events())[-1] == ("firing", "resolved", "rule disabled")
|
|
|
|
asyncio.run(_go())
|