bvandeusenandClaude Opus 5.5
e6da0ff299
#5822 "Backup contents: settings and keys only". The operator's ruling (System 180, 2026-10-15) overturns 2026-10-12's "Config + audit + alerts": a backup is for system recovery and carries what would otherwise have to be set up again, with no logs or history. - contents.py: audit_events, alert_events, alert_states, metric_anomalies, metric_anomaly_spans and user_invites move to EXCLUDED. - Alert state is no longer restored, and evaluation skips a rule with no state row. ensure_rule_states (alerts/service.py) gives every single-resource rule lacking one an inactive row; alert_reconcile runs it each minute. - Docs, the Backups page and the CLI help say what a backup holds now. CI's image check also finds the audit log empty after a restore. - tests/integration/test_backup_restore_fixes.py is removed: its case (an alert event naming a left-out Ansible run) no longer arises. The SET NULL fix-up is covered by a unit test on synthetic tables. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
233 lines
11 KiB
Python
233 lines
11 KiB
Python
"""Integration: alert state against real Postgres. An acknowledgement holds
|
|
while the breach lasts and is on the record with who made it (#5463), an alert
|
|
clears after its OK run, and changing what a rule watches (or disabling it)
|
|
resets its state with a note and the person on the event."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import uuid
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
def test_acknowledge_holds_clear_after_and_reset(app):
|
|
from sqlalchemy import select
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.alerts import record_metric, record_metrics
|
|
from steward.models.alerts import AlertEvent, AlertRule, AlertState, AlertStateEnum
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.users import User, UserRole
|
|
|
|
async def _go():
|
|
tag = uuid.uuid4().hex[:8]
|
|
res = f"box-{tag}"
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
user = User(username=f"al-{tag}", email=f"al-{tag}@test", password_hash="x",
|
|
role=UserRole.admin)
|
|
s.add(user)
|
|
await record_metrics(s, [PluginMetric(source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", value=1.0)])
|
|
await s.flush()
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=f"hot {tag}", source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", operator=">", threshold=90,
|
|
consecutive_failures_required=1, consecutive_ok_required=2)
|
|
rule = await alert_svc.add_rule(s, fields, created_by=user.id)
|
|
rule_id, user_id = rule.id, user.id
|
|
|
|
async def sample(v):
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await record_metric(s, "host_agent", res, "cpu_pct", v)
|
|
|
|
async def state():
|
|
async with app.db_sessionmaker() as s:
|
|
return (await s.execute(select(AlertState).where(
|
|
AlertState.rule_id == rule_id))).scalar_one()
|
|
|
|
async def events():
|
|
async with app.db_sessionmaker() as s:
|
|
rows = (await s.execute(select(AlertEvent).where(AlertEvent.rule_id == rule_id)
|
|
.order_by(AlertEvent.transitioned_at))).scalars().all()
|
|
return [(e.from_state, e.to_state, e.note) for e in rows]
|
|
|
|
async def last_actor():
|
|
async with app.db_sessionmaker() as s:
|
|
return (await s.execute(select(AlertEvent.actor_id).where(
|
|
AlertEvent.rule_id == rule_id).order_by(
|
|
AlertEvent.transitioned_at.desc()).limit(1))).scalar_one()
|
|
|
|
await sample(95)
|
|
assert (await state()).state == AlertStateEnum.firing
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await alert_svc.acknowledge(s, rule_id, user_id)
|
|
assert await last_actor() == user_id
|
|
for _ in range(3):
|
|
await sample(97) # still breached: the acknowledgement holds
|
|
assert (await state()).state == AlertStateEnum.acknowledged
|
|
await sample(10) # one OK check of the two it needs
|
|
assert (await state()).state == AlertStateEnum.acknowledged
|
|
await sample(10)
|
|
st = await state()
|
|
assert st.state == AlertStateEnum.inactive and st.acknowledged_at is None
|
|
assert await events() == [("inactive", "firing", None), ("firing", "acknowledged", None),
|
|
("acknowledged", "resolved", None)]
|
|
assert await last_actor() is None # the readings resolved it, not a person
|
|
|
|
# Changing the threshold of a firing rule resets it, saying why.
|
|
await sample(95)
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
r = await s.get(AlertRule, rule_id)
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=r.name, source_module=r.source_module, resource_name=r.resource_name,
|
|
metric_name=r.metric_name, operator=">", threshold=99,
|
|
consecutive_failures_required=1, consecutive_ok_required=2)
|
|
await alert_svc.apply_rule_change(s, r, fields)
|
|
assert (await state()).state == AlertStateEnum.inactive
|
|
assert (await events())[-1] == ("firing", "resolved", "rule changed: threshold")
|
|
|
|
# Renaming alone does not reset; disabling does.
|
|
await sample(100)
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
r = await s.get(AlertRule, rule_id)
|
|
await alert_svc.apply_rule_change(s, r, {"name": f"renamed {tag}"})
|
|
assert (await state()).state == AlertStateEnum.firing
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
r = await s.get(AlertRule, rule_id)
|
|
await alert_svc.apply_rule_change(s, r, {}, enabled=False, actor_id=user_id)
|
|
assert (await state()).state == AlertStateEnum.inactive
|
|
assert (await events())[-1] == ("firing", "resolved", "rule disabled")
|
|
assert await last_actor() == user_id
|
|
|
|
# Re-enabling starts the watch over: silence is measured from now.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
r = await s.get(AlertRule, rule_id)
|
|
await alert_svc.apply_rule_change(s, r, {}, enabled=True)
|
|
st = await state()
|
|
assert (st.state, st.last_sample_at, st.no_data_since) == (AlertStateEnum.inactive, None, None)
|
|
assert (await events())[-1] == ("firing", "resolved", "rule disabled")
|
|
|
|
asyncio.run(_go())
|
|
|
|
|
|
def test_record_metrics_marks_each_step(app):
|
|
"""A caller timing an ingest can tell the row write from the rule reads
|
|
from the evaluation (#5149)."""
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.alerts import record_metrics
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.users import User, UserRole
|
|
|
|
class _Marks:
|
|
def __init__(self):
|
|
self.names = []
|
|
|
|
def mark(self, name):
|
|
self.names.append(name)
|
|
|
|
def rows(res):
|
|
return [PluginMetric(source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", value=5.0)]
|
|
|
|
async def _go():
|
|
tag = uuid.uuid4().hex[:8]
|
|
watched, unwatched = f"box-{tag}", f"other-{tag}"
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
user = User(username=f"mk-{tag}", email=f"mk-{tag}@test", password_hash="x",
|
|
role=UserRole.admin)
|
|
s.add(user)
|
|
await record_metrics(s, rows(watched))
|
|
await s.flush()
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=f"hot {tag}", source_module="host_agent", resource_name=watched,
|
|
metric_name="cpu_pct", operator=">", threshold=90)
|
|
await alert_svc.add_rule(s, fields, created_by=user.id)
|
|
|
|
seen = {}
|
|
for res in (watched, unwatched):
|
|
marks = _Marks()
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await record_metrics(s, rows(res), marks)
|
|
seen[res] = marks.names
|
|
return seen[watched], seen[unwatched]
|
|
|
|
watched, unwatched = asyncio.run(_go())
|
|
assert watched == ["flush", "rules", "evaluate"]
|
|
assert unwatched == ["flush", "rules"] # no rule watches it: nothing to evaluate
|
|
|
|
|
|
def test_a_rule_without_state_gets_one_and_is_watched_again(app):
|
|
"""A backup leaves alert state out (System 180, 2026-10-15): a restored
|
|
rule has none, and evaluation skips it until alert_reconcile adds it."""
|
|
from sqlalchemy import delete, func, select
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.alerts import record_metric
|
|
from steward.models.alerts import AlertOperator, AlertRule, AlertState, AlertStateEnum
|
|
from steward.models.users import User, UserRole
|
|
|
|
async def _go():
|
|
tag = uuid.uuid4().hex[:8]
|
|
res = f"restored-{tag}"
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
user = User(username=f"rs-{tag}", email=f"rs-{tag}@test", password_hash="x",
|
|
role=UserRole.admin)
|
|
s.add(user)
|
|
await s.flush()
|
|
rule = AlertRule(name=f"restored {tag}", source_module="host_agent",
|
|
resource_name=res, metric_name="cpu_pct",
|
|
operator=AlertOperator.gt, threshold=90.0,
|
|
consecutive_failures_required=1, enabled=True,
|
|
created_by=user.id)
|
|
cover = AlertRule(name=f"all hosts {tag}", source_module="host_agent",
|
|
resource_name="", metric_name="cpu_pct", covers="all_hosts",
|
|
operator=AlertOperator.gt, threshold=90.0,
|
|
consecutive_failures_required=1, enabled=True,
|
|
created_by=user.id)
|
|
s.add_all([rule, cover])
|
|
await s.flush()
|
|
await s.execute(delete(AlertState).where(AlertState.rule_id == rule.id))
|
|
ids = (rule.id, cover.id)
|
|
|
|
async def states(rule_id):
|
|
async with app.db_sessionmaker() as s:
|
|
return (await s.execute(select(func.count()).select_from(AlertState).where(
|
|
AlertState.rule_id == rule_id))).scalar_one()
|
|
|
|
added = await alert_svc.ensure_rule_states(app)
|
|
first = (await states(ids[0]), await states(ids[1]))
|
|
await alert_svc.ensure_rule_states(app)
|
|
second = await states(ids[0])
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await record_metric(s, "host_agent", res, "cpu_pct", 99.0)
|
|
async with app.db_sessionmaker() as s:
|
|
state = (await s.execute(select(AlertState.state).where(
|
|
AlertState.rule_id == ids[0]))).scalar_one()
|
|
return added, first, second, state
|
|
|
|
added, first, second, state = asyncio.run(_go())
|
|
assert added >= 1
|
|
assert first == (1, 0) # the covering rule has no state of its own
|
|
assert second == 1 # a second pass adds nothing
|
|
assert state == AlertStateEnum.firing
|