fix(alerts): an alert that fired and ended inside maintenance isn't mailed resolved (#6097)
During Docker02's maintenance reboot its Agent offline alert fired muted by the job's window, then resolved just after the window closed: the resolution was mailed for a firing nobody had been told of. A resolution whose firing was held and never released is now recorded with a note, not sent. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -121,6 +121,19 @@ async def _went_quiet_with_its_agent(session, rule_id: str) -> bool:
|
||||
return bool(note and AGENT_OFFLINE in note)
|
||||
|
||||
|
||||
async def _firing_was_held(session, rule_id: str) -> bool:
|
||||
"""Whether the firing a resolution ends was muted by a maintenance window
|
||||
and never released: nobody was told it fired, so its end is no news. A
|
||||
host's agent going quiet for its own maintenance reboot resolved after
|
||||
the job's window closed, and mailed "resolved" for an alert nobody had
|
||||
heard of (#5769)."""
|
||||
note = (await session.execute(
|
||||
select(AlertEvent.note).where(AlertEvent.rule_id == rule_id,
|
||||
AlertEvent.to_state == "firing")
|
||||
.order_by(AlertEvent.transitioned_at.desc()).limit(1))).scalar_one_or_none()
|
||||
return bool(note and MUTED in note and RELEASED not in note)
|
||||
|
||||
|
||||
def alert_extra_vars(rule: AlertRule, value: float) -> dict:
|
||||
"""Ansible extra-vars describing the alert that fired.
|
||||
|
||||
@@ -196,6 +209,7 @@ def _in_window(w: MaintenanceWindow, source_module: str, resource_name: str) ->
|
||||
|
||||
MUTED = "notification muted: maintenance window"
|
||||
RELEASED = "notified when the window ended"
|
||||
HELD = "not notified: its firing was held by a maintenance window"
|
||||
|
||||
|
||||
def _muted_by(windows, rule: AlertRule) -> str | None:
|
||||
@@ -379,6 +393,10 @@ async def record_metrics(session: AsyncSession, rows: list[PluginMetric], phases
|
||||
for notif_type, event_id, note in notifications:
|
||||
if notif_type == "data_resumed" and not rule.notify_no_data:
|
||||
continue
|
||||
if notif_type == "resolved" and await _firing_was_held(session, rule.id):
|
||||
event = await session.get(AlertEvent, event_id)
|
||||
event.note = "; ".join(n for n in (event.note, HELD) if n)
|
||||
continue
|
||||
# Back with its agent: the agent's return is the news (#5721).
|
||||
if notif_type == "data_resumed" and host_behind(rule) is not None \
|
||||
and await _went_quiet_with_its_agent(session, rule.id):
|
||||
|
||||
Reference in New Issue
Block a user