bvandeusenandClaude Opus 5.5
e51d969c43
A rule compared a reading with a fixed number, so Synology Disk 1's bad sectors going from 3,160 to 3,392 could say nothing: a count that only grows is soon past any line. "rose by" compares each reading with the one before it, kept on the alert's state and only for such a rule. A fall (a replaced disk) doesn't fire. On the rule form, through MCP, and worded in emails, the alert sentence, history and the timeline. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
281 lines
13 KiB
Python
281 lines
13 KiB
Python
"""Integration: alert state against real Postgres. An acknowledgement holds
|
|
while the breach lasts and is on the record with who made it (#5463), an alert
|
|
clears after its OK run, and changing what a rule watches (or disabling it)
|
|
resets its state with a note and the person on the event."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import uuid
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
def test_acknowledge_holds_clear_after_and_reset(app):
|
|
from sqlalchemy import select
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.alerts import record_metric, record_metrics
|
|
from steward.models.alerts import AlertEvent, AlertRule, AlertState, AlertStateEnum
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.users import User, UserRole
|
|
|
|
async def _go():
|
|
tag = uuid.uuid4().hex[:8]
|
|
res = f"box-{tag}"
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
user = User(username=f"al-{tag}", email=f"al-{tag}@test", password_hash="x",
|
|
role=UserRole.admin)
|
|
s.add(user)
|
|
await record_metrics(s, [PluginMetric(source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", value=1.0)])
|
|
await s.flush()
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=f"hot {tag}", source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", operator=">", threshold=90,
|
|
consecutive_failures_required=1, consecutive_ok_required=2)
|
|
rule = await alert_svc.add_rule(s, fields, created_by=user.id)
|
|
rule_id, user_id = rule.id, user.id
|
|
|
|
async def sample(v):
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await record_metric(s, "host_agent", res, "cpu_pct", v)
|
|
|
|
async def state():
|
|
async with app.db_sessionmaker() as s:
|
|
return (await s.execute(select(AlertState).where(
|
|
AlertState.rule_id == rule_id))).scalar_one()
|
|
|
|
async def events():
|
|
async with app.db_sessionmaker() as s:
|
|
rows = (await s.execute(select(AlertEvent).where(AlertEvent.rule_id == rule_id)
|
|
.order_by(AlertEvent.transitioned_at))).scalars().all()
|
|
return [(e.from_state, e.to_state, e.note) for e in rows]
|
|
|
|
async def last_actor():
|
|
async with app.db_sessionmaker() as s:
|
|
return (await s.execute(select(AlertEvent.actor_id).where(
|
|
AlertEvent.rule_id == rule_id).order_by(
|
|
AlertEvent.transitioned_at.desc()).limit(1))).scalar_one()
|
|
|
|
await sample(95)
|
|
assert (await state()).state == AlertStateEnum.firing
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await alert_svc.acknowledge(s, rule_id, user_id)
|
|
assert await last_actor() == user_id
|
|
for _ in range(3):
|
|
await sample(97) # still breached: the acknowledgement holds
|
|
assert (await state()).state == AlertStateEnum.acknowledged
|
|
await sample(10) # one OK check of the two it needs
|
|
assert (await state()).state == AlertStateEnum.acknowledged
|
|
await sample(10)
|
|
st = await state()
|
|
assert st.state == AlertStateEnum.inactive and st.acknowledged_at is None
|
|
assert await events() == [("inactive", "firing", None), ("firing", "acknowledged", None),
|
|
("acknowledged", "resolved", None)]
|
|
assert await last_actor() is None # the readings resolved it, not a person
|
|
|
|
# Changing the threshold of a firing rule resets it, saying why.
|
|
await sample(95)
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
r = await s.get(AlertRule, rule_id)
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=r.name, source_module=r.source_module, resource_name=r.resource_name,
|
|
metric_name=r.metric_name, operator=">", threshold=99,
|
|
consecutive_failures_required=1, consecutive_ok_required=2)
|
|
await alert_svc.apply_rule_change(s, r, fields)
|
|
assert (await state()).state == AlertStateEnum.inactive
|
|
assert (await events())[-1] == ("firing", "resolved", "rule changed: threshold")
|
|
|
|
# Renaming alone does not reset; disabling does.
|
|
await sample(100)
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
r = await s.get(AlertRule, rule_id)
|
|
await alert_svc.apply_rule_change(s, r, {"name": f"renamed {tag}"})
|
|
assert (await state()).state == AlertStateEnum.firing
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
r = await s.get(AlertRule, rule_id)
|
|
await alert_svc.apply_rule_change(s, r, {}, enabled=False, actor_id=user_id)
|
|
assert (await state()).state == AlertStateEnum.inactive
|
|
assert (await events())[-1] == ("firing", "resolved", "rule disabled")
|
|
assert await last_actor() == user_id
|
|
|
|
# Re-enabling starts the watch over: silence is measured from now.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
r = await s.get(AlertRule, rule_id)
|
|
await alert_svc.apply_rule_change(s, r, {}, enabled=True)
|
|
st = await state()
|
|
assert (st.state, st.last_sample_at, st.no_data_since) == (AlertStateEnum.inactive, None, None)
|
|
assert (await events())[-1] == ("firing", "resolved", "rule disabled")
|
|
|
|
asyncio.run(_go())
|
|
|
|
|
|
def test_record_metrics_marks_each_step(app):
|
|
"""A caller timing an ingest can tell the row write from the rule reads
|
|
from the evaluation (#5149)."""
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.alerts import record_metrics
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.users import User, UserRole
|
|
|
|
class _Marks:
|
|
def __init__(self):
|
|
self.names = []
|
|
|
|
def mark(self, name):
|
|
self.names.append(name)
|
|
|
|
def rows(res):
|
|
return [PluginMetric(source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", value=5.0)]
|
|
|
|
async def _go():
|
|
tag = uuid.uuid4().hex[:8]
|
|
watched, unwatched = f"box-{tag}", f"other-{tag}"
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
user = User(username=f"mk-{tag}", email=f"mk-{tag}@test", password_hash="x",
|
|
role=UserRole.admin)
|
|
s.add(user)
|
|
await record_metrics(s, rows(watched))
|
|
await s.flush()
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=f"hot {tag}", source_module="host_agent", resource_name=watched,
|
|
metric_name="cpu_pct", operator=">", threshold=90)
|
|
await alert_svc.add_rule(s, fields, created_by=user.id)
|
|
|
|
seen = {}
|
|
for res in (watched, unwatched):
|
|
marks = _Marks()
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await record_metrics(s, rows(res), marks)
|
|
seen[res] = marks.names
|
|
return seen[watched], seen[unwatched]
|
|
|
|
watched, unwatched = asyncio.run(_go())
|
|
assert watched == ["flush", "rules", "evaluate"]
|
|
assert unwatched == ["flush", "rules"] # no rule watches it: nothing to evaluate
|
|
|
|
|
|
def test_a_rule_without_state_gets_one_and_is_watched_again(app):
|
|
"""A backup leaves alert state out (System 180, 2026-10-15): a restored
|
|
rule has none, and evaluation skips it until alert_reconcile adds it."""
|
|
from sqlalchemy import delete, func, select
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.alerts import record_metric
|
|
from steward.models.alerts import AlertOperator, AlertRule, AlertState, AlertStateEnum
|
|
from steward.models.users import User, UserRole
|
|
|
|
async def _go():
|
|
tag = uuid.uuid4().hex[:8]
|
|
res = f"restored-{tag}"
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
user = User(username=f"rs-{tag}", email=f"rs-{tag}@test", password_hash="x",
|
|
role=UserRole.admin)
|
|
s.add(user)
|
|
await s.flush()
|
|
rule = AlertRule(name=f"restored {tag}", source_module="host_agent",
|
|
resource_name=res, metric_name="cpu_pct",
|
|
operator=AlertOperator.gt, threshold=90.0,
|
|
consecutive_failures_required=1, enabled=True,
|
|
created_by=user.id)
|
|
cover = AlertRule(name=f"all hosts {tag}", source_module="host_agent",
|
|
resource_name="", metric_name="cpu_pct", covers="all_hosts",
|
|
operator=AlertOperator.gt, threshold=90.0,
|
|
consecutive_failures_required=1, enabled=True,
|
|
created_by=user.id)
|
|
s.add_all([rule, cover])
|
|
await s.flush()
|
|
await s.execute(delete(AlertState).where(AlertState.rule_id == rule.id))
|
|
ids = (rule.id, cover.id)
|
|
|
|
async def states(rule_id):
|
|
async with app.db_sessionmaker() as s:
|
|
return (await s.execute(select(func.count()).select_from(AlertState).where(
|
|
AlertState.rule_id == rule_id))).scalar_one()
|
|
|
|
added = await alert_svc.ensure_rule_states(app)
|
|
first = (await states(ids[0]), await states(ids[1]))
|
|
await alert_svc.ensure_rule_states(app)
|
|
second = await states(ids[0])
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await record_metric(s, "host_agent", res, "cpu_pct", 99.0)
|
|
async with app.db_sessionmaker() as s:
|
|
state = (await s.execute(select(AlertState.state).where(
|
|
AlertState.rule_id == ids[0]))).scalar_one()
|
|
return added, first, second, state
|
|
|
|
added, first, second, state = asyncio.run(_go())
|
|
assert added >= 1
|
|
assert first == (1, 0) # the covering rule has no state of its own
|
|
assert second == 1 # a second pass adds nothing
|
|
assert state == AlertStateEnum.firing
|
|
|
|
|
|
def test_a_rule_on_a_rise_fires_on_the_rise_and_clears_after_its_ok_run(app):
|
|
from sqlalchemy import select
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.alerts import record_metric, record_metrics
|
|
from steward.models.alerts import AlertState, AlertStateEnum
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.users import User, UserRole
|
|
|
|
async def _go():
|
|
tag = uuid.uuid4().hex[:8]
|
|
res = f"box-{tag}"
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
user = User(username=f"rs-{tag}", email=f"rs-{tag}@test", password_hash="x",
|
|
role=UserRole.admin)
|
|
s.add(user)
|
|
await record_metrics(s, [PluginMetric(source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", value=100.0)])
|
|
await s.flush()
|
|
with pytest.raises(ValueError, match="above 0"):
|
|
await alert_svc.rule_fields(
|
|
s, name=f"flat {tag}", source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", operator="rose", threshold=0)
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=f"rise {tag}", source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", operator="rose", threshold=1,
|
|
consecutive_failures_required=1, consecutive_ok_required=2)
|
|
rule = await alert_svc.add_rule(s, fields, created_by=user.id)
|
|
rule_id = rule.id
|
|
|
|
async def after(v):
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await record_metric(s, "host_agent", res, "cpu_pct", v)
|
|
async with app.db_sessionmaker() as s:
|
|
state = (await s.execute(select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one()
|
|
return state.state
|
|
|
|
assert await after(100.0) == AlertStateEnum.inactive # the first has nothing to have risen from
|
|
assert await after(100.0) == AlertStateEnum.inactive # flat
|
|
assert await after(105.0) == AlertStateEnum.firing
|
|
assert await after(105.0) == AlertStateEnum.firing # one good check of two
|
|
assert await after(105.0) == AlertStateEnum.inactive
|
|
assert await after(40.0) == AlertStateEnum.inactive # a fall is no rise
|
|
|
|
asyncio.run(_go())
|