"""Integration (#5062): against real Postgres, a rule whose series stops is marked no_data once, notified once, and cleared by the next sample; a firing alert that goes silent stays firing.""" from __future__ import annotations import asyncio import os import uuid from datetime import datetime, timedelta, timezone from unittest.mock import patch import pytest pytestmark = pytest.mark.integration @pytest.fixture def app(): if not os.environ.get("STEWARD_DATABASE_URL"): pytest.skip("needs Postgres") from steward.app import create_app return create_app(testing=False) def test_silence_is_marked_once_and_cleared_by_data(app): from sqlalchemy import select, update from steward.alerts import service as alert_svc from steward.core import alerts from steward.models.alerts import AlertEvent, AlertRule, AlertState, AlertStateEnum from steward.models.metrics import PluginMetric from steward.models.users import User, UserRole sent: list[tuple[str, str, str | None]] = [] async def fake_dispatch(app_, notif_type, rule, value, event_id, note=None): sent.append((rule.name, notif_type, note)) async def _go(): tag = uuid.uuid4().hex[:8] res = f"quiet-{tag}" now = datetime.now(timezone.utc) async with app.db_sessionmaker() as s: async with s.begin(): user = User(username=f"nd-{tag}", email=f"nd-{tag}@test", password_hash="x", role=UserRole.admin) s.add(user) s.add(PluginMetric(source_module="host_agent", resource_name=res, metric_name="cpu_pct", value=1.0, recorded_at=now - timedelta(hours=1))) await s.flush() ids = {} for key, notify in (("loud", True), ("hushed", False)): fields = await alert_svc.rule_fields( s, name=f"{key} {tag}", source_module="host_agent", resource_name=res, metric_name="cpu_pct", operator=">", threshold=90, notify_no_data=notify) ids[key] = (await alert_svc.add_rule(s, fields, created_by=user.id)).id # Every minute, from 20 to 18 minutes ago; then nothing. batch = alerts.MetricBatch() for ago in (20, 19, 18): batch.add(source_module="host_agent", resource_name=res, metric_name="cpu_pct", value=10.0, recorded_at=now - timedelta(minutes=ago)) await batch.record(s) async def state(key): async with app.db_sessionmaker() as s: return (await s.execute(select(AlertState).where( AlertState.rule_id == ids[key]))).scalar_one() async def events(key): async with app.db_sessionmaker() as s: rows = (await s.execute(select(AlertEvent).where(AlertEvent.rule_id == ids[key]) .order_by(AlertEvent.transitioned_at))).scalars().all() return [(e.from_state, e.to_state, e.note) for e in rows] st = await state("loud") assert st.sample_interval_s == pytest.approx(60) assert st.last_sample_at == now - timedelta(minutes=18) await alerts.check_silence(app) await alerts.check_silence(app) # already marked: nothing more await asyncio.sleep(0) st = await state("loud") assert st.state == AlertStateEnum.inactive assert st.no_data_since == now - timedelta(minutes=18) + timedelta(seconds=300) [(frm, to, note)] = await events("loud") assert (frm, to) == ("inactive", "no_data") assert note == "no sample for 18m (usually every 60s)" assert len(await events("hushed")) == 1 assert sent == [(f"loud {tag}", "no_data", note)] # the hushed rule notifies nobody async with app.db_sessionmaker() as s: active = {r.id: alert_svc.alert_label(st_) for st_, r in await alert_svc.active_alerts(s)} assert active[ids["loud"]] == active[ids["hushed"]] == "no_data" # Data resumes, breaching: the mark clears and the rule fires. async with app.db_sessionmaker() as s: async with s.begin(): await alerts.record_metric(s, "host_agent", res, "cpu_pct", 95.0) await asyncio.sleep(0) st = await state("loud") assert st.no_data_since is None and st.state == AlertStateEnum.firing first, *same_instant = await events("loud") assert first[:2] == ("inactive", "no_data") # The resume and the firing it leads to share the sample's timestamp. assert sorted(e[:2] for e in same_instant) == [("inactive", "firing"), ("no_data", "inactive")] assert any((e[2] or "").startswith("data resumed after ") for e in same_instant) assert [t for _, t, _ in sent if t != "firing"] == ["no_data", "data_resumed"] # A firing alert that goes silent stays firing, marked. async with app.db_sessionmaker() as s: async with s.begin(): await s.execute(update(AlertState).where(AlertState.rule_id == ids["loud"]) .values(last_sample_at=now - timedelta(hours=2))) await s.execute(update(AlertRule).where(AlertRule.id == ids["hushed"]) .values(enabled=False)) await alerts.check_silence(app) st = await state("loud") assert st.state == AlertStateEnum.firing and st.no_data_since is not None assert (await events("loud"))[-1][:2] == ("firing", "no_data") assert len(await events("hushed")) == 3 # disabled: not watched with patch.object(alerts, "_dispatch_notification", fake_dispatch): asyncio.run(_go())