fix(alerts): an alert that fired and ended inside maintenance isn't mailed resolved (#6097)

During Docker02's maintenance reboot its Agent offline alert fired muted by
the job's window, then resolved just after the window closed: the resolution
was mailed for a firing nobody had been told of. A resolution whose firing
was held and never released is now recorded with a note, not sent.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/alerts.py

@@ -121,6 +121,19 @@ async def _went_quiet_with_its_agent(session, rule_id: str) -> bool:
     return bool(note and AGENT_OFFLINE in note)
 
 
+async def _firing_was_held(session, rule_id: str) -> bool:
+    """Whether the firing a resolution ends was muted by a maintenance window
+    and never released: nobody was told it fired, so its end is no news. A
+    host's agent going quiet for its own maintenance reboot resolved after
+    the job's window closed, and mailed "resolved" for an alert nobody had
+    heard of (#5769)."""
+    note = (await session.execute(
+        select(AlertEvent.note).where(AlertEvent.rule_id == rule_id,
+                                      AlertEvent.to_state == "firing")
+        .order_by(AlertEvent.transitioned_at.desc()).limit(1))).scalar_one_or_none()
+    return bool(note and MUTED in note and RELEASED not in note)
+
+
 def alert_extra_vars(rule: AlertRule, value: float) -> dict:
     """Ansible extra-vars describing the alert that fired.
 
@@ -196,6 +209,7 @@ def _in_window(w: MaintenanceWindow, source_module: str, resource_name: str) ->
 
 MUTED = "notification muted: maintenance window"
 RELEASED = "notified when the window ended"
+HELD = "not notified: its firing was held by a maintenance window"
 
 
 def _muted_by(windows, rule: AlertRule) -> str | None:
@@ -379,6 +393,10 @@ async def record_metrics(session: AsyncSession, rows: list[PluginMetric], phases
             for notif_type, event_id, note in notifications:
                 if notif_type == "data_resumed" and not rule.notify_no_data:
                     continue
+                if notif_type == "resolved" and await _firing_was_held(session, rule.id):
+                    event = await session.get(AlertEvent, event_id)
+                    event.note = "; ".join(n for n in (event.note, HELD) if n)
+                    continue
                 # Back with its agent: the agent's return is the news (#5721).
                 if notif_type == "data_resumed" and host_behind(rule) is not None \
                         and await _went_quiet_with_its_agent(session, rule.id):

tests/integration/test_alert_dispatch.py

@@ -147,6 +147,60 @@ def test_maintenance_mutes_but_the_state_moves_and_notifies_when_it_ends(app):
         asyncio.run(_go())
 
 
+def test_an_alert_that_fired_and_ended_unheard_is_not_mailed_resolved(app):
+    """#5769: an agent offline for its own maintenance reboot fired inside the
+    job's window (muted) and resolved just after the window closed, before
+    the minute's release. Nobody heard it fire, so its end isn't mailed."""
+    from sqlalchemy import update
+    from steward.core import alerts
+    from steward.models.alerts import MaintenanceWindow
+
+    sent = []
+
+    async def fake_dispatch(app_, notif_type, rule, value, event_id, note=None):
+        sent.append((notif_type, value))
+
+    async def _go():
+        tag = uuid.uuid4().hex[:8]
+        rule_id, res, user_id = await _rule(app, tag)
+        now = datetime.now(timezone.utc)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                window = MaintenanceWindow(
+                    name=f"reboot {tag}", start_at=now - timedelta(minutes=1),
+                    end_at=now + timedelta(hours=1), scope_source="host_agent",
+                    scope_resource=res, created_by=user_id)
+                s.add(window)
+            window_id = window.id
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await alerts.record_metric(s, "host_agent", res, "cpu_pct", 95.0)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await s.execute(update(MaintenanceWindow).where(MaintenanceWindow.id == window_id)
+                                .values(end_at=datetime.now(timezone.utc)))
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await alerts.record_metric(s, "host_agent", res, "cpu_pct", 5.0)
+        await alerts.notify_after_maintenance(app)        # nothing left firing to release
+        await _settle()
+        assert sent == []
+        events = await _events(app, rule_id)
+        assert [(frm, to) for frm, to, _, _ in events] == [("inactive", "firing"), ("firing", "resolved")]
+        assert events[1][2] == alerts.HELD
+
+        # A firing that was told (released, or never muted) still gets its end.
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await alerts.record_metric(s, "host_agent", res, "cpu_pct", 96.0)
+                await alerts.record_metric(s, "host_agent", res, "cpu_pct", 4.0)
+        await _settle()
+        assert sent == [("firing", 96.0), ("resolved", 4.0)]
+
+    with patch.object(alerts, "_dispatch_notification", fake_dispatch):
+        asyncio.run(_go())
+
+
 def test_an_alert_action_outside_its_source_is_refused(app, tmp_path):
     from steward.core import alerts
     from steward.models.ansible import AnsibleRun