"""Integration (#5063): against real Postgres, notifications and actions start only after the evaluating transaction commits; a maintenance window mutes them without stopping the state from moving, and an alert still firing when the window ends notifies then; an alert action is held to what its source lists.""" from __future__ import annotations import asyncio import os import uuid from datetime import datetime, timedelta, timezone from unittest.mock import patch import pytest pytestmark = pytest.mark.integration @pytest.fixture def app(): if not os.environ.get("STEWARD_DATABASE_URL"): pytest.skip("needs Postgres") from steward.app import create_app return create_app(testing=False) async def _rule(app, tag, **extra): from steward.alerts import service as alert_svc from steward.core.alerts import record_metrics from steward.models.metrics import PluginMetric from steward.models.users import User, UserRole res = f"box-{tag}" async with app.db_sessionmaker() as s: async with s.begin(): user = User(username=f"dp-{tag}", email=f"dp-{tag}@test", password_hash="x", role=UserRole.admin) s.add(user) await record_metrics(s, [PluginMetric(source_module="host_agent", resource_name=res, metric_name="cpu_pct", value=1.0)]) await s.flush() fields = await alert_svc.rule_fields( s, name=f"hot {tag}", source_module="host_agent", resource_name=res, metric_name="cpu_pct", operator=">", threshold=90) rule = await alert_svc.add_rule(s, fields, created_by=user.id, **extra) return rule.id, res, user.id async def _events(app, rule_id): from sqlalchemy import select from steward.models.alerts import AlertEvent async with app.db_sessionmaker() as s: rows = (await s.execute(select(AlertEvent).where(AlertEvent.rule_id == rule_id) .order_by(AlertEvent.transitioned_at))).scalars().all() return [(e.from_state, e.to_state, e.note, e.ansible_run_id) for e in rows] async def _settle(): for _ in range(5): await asyncio.sleep(0) def test_a_rolled_back_evaluation_notifies_nobody(app): from steward.core import alerts sent = [] async def fake_dispatch(app_, notif_type, rule, value, event_id, note=None): sent.append((notif_type, value)) async def _go(): rule_id, res, _ = await _rule(app, uuid.uuid4().hex[:8]) class Boom(Exception): pass with pytest.raises(Boom): async with app.db_sessionmaker() as s: async with s.begin(): await alerts.record_metric(s, "host_agent", res, "cpu_pct", 95.0) raise Boom await _settle() assert sent == [] and await _events(app, rule_id) == [] async with app.db_sessionmaker() as s: async with s.begin(): await alerts.record_metric(s, "host_agent", res, "cpu_pct", 96.0) await _settle() assert sent == [] # not before the commit await _settle() assert sent == [("firing", 96.0)] with patch.object(alerts, "_dispatch_notification", fake_dispatch): asyncio.run(_go()) def test_maintenance_mutes_but_the_state_moves_and_notifies_when_it_ends(app): from sqlalchemy import select, update from steward.core import alerts from steward.models.alerts import AlertState, AlertStateEnum, MaintenanceWindow sent = [] async def fake_dispatch(app_, notif_type, rule, value, event_id, note=None): sent.append((notif_type, value)) async def _go(): tag = uuid.uuid4().hex[:8] rule_id, res, user_id = await _rule(app, tag) now = datetime.now(timezone.utc) async with app.db_sessionmaker() as s: async with s.begin(): window = MaintenanceWindow( name=f"patching {tag}", start_at=now - timedelta(minutes=1), end_at=now + timedelta(hours=1), scope_source="host_agent", scope_resource=res, created_by=user_id) s.add(window) window_id = window.id async with app.db_sessionmaker() as s: async with s.begin(): await alerts.record_metric(s, "host_agent", res, "cpu_pct", 95.0) await _settle() async with app.db_sessionmaker() as s: st = (await s.execute(select(AlertState).where( AlertState.rule_id == rule_id))).scalar_one() assert st.state == AlertStateEnum.firing # evaluated inside the window [(frm, to, note, _)] = await _events(app, rule_id) assert (frm, to, note) == ("inactive", "firing", f"notification muted: maintenance window patching {tag}") assert sent == [] await alerts.notify_after_maintenance(app) # window still open: held await _settle() assert sent == [] async with app.db_sessionmaker() as s: async with s.begin(): await s.execute(update(MaintenanceWindow).where(MaintenanceWindow.id == window_id) .values(end_at=datetime.now(timezone.utc))) await alerts.notify_after_maintenance(app) await alerts.notify_after_maintenance(app) # released once await _settle() assert sent == [("firing", 95.0)] [(_, _, note, _)] = await _events(app, rule_id) assert note.endswith("; notified when the window ended") with patch.object(alerts, "_dispatch_notification", fake_dispatch): asyncio.run(_go()) def test_an_alert_that_fired_and_ended_unheard_is_not_mailed_resolved(app): """#5769: an agent offline for its own maintenance reboot fired inside the job's window (muted) and resolved just after the window closed, before the minute's release. Nobody heard it fire, so its end isn't mailed.""" from sqlalchemy import update from steward.core import alerts from steward.models.alerts import MaintenanceWindow sent = [] async def fake_dispatch(app_, notif_type, rule, value, event_id, note=None): sent.append((notif_type, value)) async def _go(): tag = uuid.uuid4().hex[:8] rule_id, res, user_id = await _rule(app, tag) now = datetime.now(timezone.utc) async with app.db_sessionmaker() as s: async with s.begin(): window = MaintenanceWindow( name=f"reboot {tag}", start_at=now - timedelta(minutes=1), end_at=now + timedelta(hours=1), scope_source="host_agent", scope_resource=res, created_by=user_id) s.add(window) window_id = window.id async with app.db_sessionmaker() as s: async with s.begin(): await alerts.record_metric(s, "host_agent", res, "cpu_pct", 95.0) async with app.db_sessionmaker() as s: async with s.begin(): await s.execute(update(MaintenanceWindow).where(MaintenanceWindow.id == window_id) .values(end_at=datetime.now(timezone.utc))) async with app.db_sessionmaker() as s: async with s.begin(): await alerts.record_metric(s, "host_agent", res, "cpu_pct", 5.0) await alerts.notify_after_maintenance(app) # nothing left firing to release await _settle() assert sent == [] events = await _events(app, rule_id) assert [(frm, to) for frm, to, _, _ in events] == [("inactive", "firing"), ("firing", "resolved")] assert events[1][2] == alerts.HELD # A firing that was told (released, or never muted) still gets its end. async with app.db_sessionmaker() as s: async with s.begin(): await alerts.record_metric(s, "host_agent", res, "cpu_pct", 96.0) await alerts.record_metric(s, "host_agent", res, "cpu_pct", 4.0) await _settle() assert sent == [("firing", 96.0), ("resolved", 4.0)] with patch.object(alerts, "_dispatch_notification", fake_dispatch): asyncio.run(_go()) def test_an_alert_action_outside_its_source_is_refused(app, tmp_path): from steward.core import alerts from steward.models.ansible import AnsibleRun from sqlalchemy import func, select (tmp_path / "fix.yml").write_text("- hosts: all\n tasks: []\n") (tmp_path / "inventory.ini").write_text("localhost\n") app.config["ANSIBLE"] = {"sources": [{"name": "t", "type": "local", "path": str(tmp_path)}]} async def quiet(*args, **kwargs): pass async def _go(): tag = uuid.uuid4().hex[:8] rule_id, res, _ = await _rule( app, tag, ansible_action={"source": "t", "playbook": "../../etc/evil.yml", "inventory": "inventory.ini"}) async with app.db_sessionmaker() as s: before = (await s.execute(select(func.count()).select_from(AnsibleRun))).scalar_one() async with app.db_sessionmaker() as s: async with s.begin(): await alerts.record_metric(s, "host_agent", res, "cpu_pct", 95.0) loop = asyncio.get_running_loop() deadline = loop.time() + 30 while alerts._background: assert loop.time() < deadline, "alert tasks did not finish" await asyncio.wait(set(alerts._background), timeout=deadline - loop.time()) [(frm, to, note, run_id)] = await _events(app, rule_id) assert (frm, to, run_id) == ("inactive", "firing", None) assert note == "Ansible action not started: Source 't' has no playbook '../../etc/evil.yml'" async with app.db_sessionmaker() as s: after = (await s.execute(select(func.count()).select_from(AnsibleRun))).scalar_one() assert after == before with patch.object(alerts, "_dispatch_notification", quiet): asyncio.run(_go())