diff --git a/steward/core/alerts.py b/steward/core/alerts.py index 533369e..c05d752 100644 --- a/steward/core/alerts.py +++ b/steward/core/alerts.py @@ -121,6 +121,19 @@ async def _went_quiet_with_its_agent(session, rule_id: str) -> bool: return bool(note and AGENT_OFFLINE in note) +async def _firing_was_held(session, rule_id: str) -> bool: + """Whether the firing a resolution ends was muted by a maintenance window + and never released: nobody was told it fired, so its end is no news. A + host's agent going quiet for its own maintenance reboot resolved after + the job's window closed, and mailed "resolved" for an alert nobody had + heard of (#5769).""" + note = (await session.execute( + select(AlertEvent.note).where(AlertEvent.rule_id == rule_id, + AlertEvent.to_state == "firing") + .order_by(AlertEvent.transitioned_at.desc()).limit(1))).scalar_one_or_none() + return bool(note and MUTED in note and RELEASED not in note) + + def alert_extra_vars(rule: AlertRule, value: float) -> dict: """Ansible extra-vars describing the alert that fired. @@ -196,6 +209,7 @@ def _in_window(w: MaintenanceWindow, source_module: str, resource_name: str) -> MUTED = "notification muted: maintenance window" RELEASED = "notified when the window ended" +HELD = "not notified: its firing was held by a maintenance window" def _muted_by(windows, rule: AlertRule) -> str | None: @@ -379,6 +393,10 @@ async def record_metrics(session: AsyncSession, rows: list[PluginMetric], phases for notif_type, event_id, note in notifications: if notif_type == "data_resumed" and not rule.notify_no_data: continue + if notif_type == "resolved" and await _firing_was_held(session, rule.id): + event = await session.get(AlertEvent, event_id) + event.note = "; ".join(n for n in (event.note, HELD) if n) + continue # Back with its agent: the agent's return is the news (#5721). if notif_type == "data_resumed" and host_behind(rule) is not None \ and await _went_quiet_with_its_agent(session, rule.id): diff --git a/tests/integration/test_alert_dispatch.py b/tests/integration/test_alert_dispatch.py index 3678197..e8a3cb7 100644 --- a/tests/integration/test_alert_dispatch.py +++ b/tests/integration/test_alert_dispatch.py @@ -147,6 +147,60 @@ def test_maintenance_mutes_but_the_state_moves_and_notifies_when_it_ends(app): asyncio.run(_go()) +def test_an_alert_that_fired_and_ended_unheard_is_not_mailed_resolved(app): + """#5769: an agent offline for its own maintenance reboot fired inside the + job's window (muted) and resolved just after the window closed, before + the minute's release. Nobody heard it fire, so its end isn't mailed.""" + from sqlalchemy import update + from steward.core import alerts + from steward.models.alerts import MaintenanceWindow + + sent = [] + + async def fake_dispatch(app_, notif_type, rule, value, event_id, note=None): + sent.append((notif_type, value)) + + async def _go(): + tag = uuid.uuid4().hex[:8] + rule_id, res, user_id = await _rule(app, tag) + now = datetime.now(timezone.utc) + async with app.db_sessionmaker() as s: + async with s.begin(): + window = MaintenanceWindow( + name=f"reboot {tag}", start_at=now - timedelta(minutes=1), + end_at=now + timedelta(hours=1), scope_source="host_agent", + scope_resource=res, created_by=user_id) + s.add(window) + window_id = window.id + async with app.db_sessionmaker() as s: + async with s.begin(): + await alerts.record_metric(s, "host_agent", res, "cpu_pct", 95.0) + async with app.db_sessionmaker() as s: + async with s.begin(): + await s.execute(update(MaintenanceWindow).where(MaintenanceWindow.id == window_id) + .values(end_at=datetime.now(timezone.utc))) + async with app.db_sessionmaker() as s: + async with s.begin(): + await alerts.record_metric(s, "host_agent", res, "cpu_pct", 5.0) + await alerts.notify_after_maintenance(app) # nothing left firing to release + await _settle() + assert sent == [] + events = await _events(app, rule_id) + assert [(frm, to) for frm, to, _, _ in events] == [("inactive", "firing"), ("firing", "resolved")] + assert events[1][2] == alerts.HELD + + # A firing that was told (released, or never muted) still gets its end. + async with app.db_sessionmaker() as s: + async with s.begin(): + await alerts.record_metric(s, "host_agent", res, "cpu_pct", 96.0) + await alerts.record_metric(s, "host_agent", res, "cpu_pct", 4.0) + await _settle() + assert sent == [("firing", 96.0), ("resolved", 4.0)] + + with patch.object(alerts, "_dispatch_notification", fake_dispatch): + asyncio.run(_go()) + + def test_an_alert_action_outside_its_source_is_refused(app, tmp_path): from steward.core import alerts from steward.models.ansible import AnsibleRun