fix(alerts): an alert that fired and ended inside maintenance isn't mailed resolved (#6097)
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 1m29s
CI / integration (push) Successful in 1m39s
CI / publish (push) Successful in 1m9s

During Docker02's maintenance reboot its Agent offline alert fired muted by
the job's window, then resolved just after the window closed: the resolution
was mailed for a firing nobody had been told of. A resolution whose firing
was held and never released is now recorded with a note, not sent.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-19 07:41:51 -04:00
co-authored by Claude Opus 5.5
parent ed9da56efa
commit da81010a75
2 changed files with 72 additions and 0 deletions
+18
View File
@@ -121,6 +121,19 @@ async def _went_quiet_with_its_agent(session, rule_id: str) -> bool:
return bool(note and AGENT_OFFLINE in note)
async def _firing_was_held(session, rule_id: str) -> bool:
"""Whether the firing a resolution ends was muted by a maintenance window
and never released: nobody was told it fired, so its end is no news. A
host's agent going quiet for its own maintenance reboot resolved after
the job's window closed, and mailed "resolved" for an alert nobody had
heard of (#5769)."""
note = (await session.execute(
select(AlertEvent.note).where(AlertEvent.rule_id == rule_id,
AlertEvent.to_state == "firing")
.order_by(AlertEvent.transitioned_at.desc()).limit(1))).scalar_one_or_none()
return bool(note and MUTED in note and RELEASED not in note)
def alert_extra_vars(rule: AlertRule, value: float) -> dict:
"""Ansible extra-vars describing the alert that fired.
@@ -196,6 +209,7 @@ def _in_window(w: MaintenanceWindow, source_module: str, resource_name: str) ->
MUTED = "notification muted: maintenance window"
RELEASED = "notified when the window ended"
HELD = "not notified: its firing was held by a maintenance window"
def _muted_by(windows, rule: AlertRule) -> str | None:
@@ -379,6 +393,10 @@ async def record_metrics(session: AsyncSession, rows: list[PluginMetric], phases
for notif_type, event_id, note in notifications:
if notif_type == "data_resumed" and not rule.notify_no_data:
continue
if notif_type == "resolved" and await _firing_was_held(session, rule.id):
event = await session.get(AlertEvent, event_id)
event.note = "; ".join(n for n in (event.note, HELD) if n)
continue
# Back with its agent: the agent's return is the news (#5721).
if notif_type == "data_resumed" and host_behind(rule) is not None \
and await _went_quiet_with_its_agent(session, rule.id):