bvandeusenandClaude Opus 5.5
997aa50939
#5465 "One email per burst of alerts". When a switch reboots, every host's agent-offline rule fires at once and each account got one email per rule. Alert email now waits 30 s from the first alert: each recipient then gets one message listing every alert in the window, oldest first, in their own zone ("[Steward] 3 alerts: 2 firing, 1 resolved"). One alert alone keeps its usual email. Webhooks still post per alert, at once. Every alert's event records the email's outcome. A burst held when Steward stops is not sent; the alerts stay on the record and on the Alerts page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
106 lines
3.9 KiB
Python
106 lines
3.9 KiB
Python
"""Alert emails gathered into one message per burst (#5465).
|
|
|
|
When the network drops or a switch reboots, every host's agent-offline rule
|
|
fires together, and one email per rule per account turns one incident into a
|
|
dozen messages. So alert email waits: the first alert opens a window of
|
|
BURST_WINDOW_S, and when it closes each recipient gets one email for every
|
|
alert that arrived in it. An alert on its own still reads as it always has.
|
|
|
|
Webhooks are not held here: each alert posts on its own, at once, since an
|
|
integration expects one alert per post (core.alerts._dispatch_notification).
|
|
Every alert's event still records the email's outcome.
|
|
|
|
A burst held when Steward stops is not sent; the alerts stay on the record
|
|
and on the Alerts page.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import select
|
|
|
|
if TYPE_CHECKING:
|
|
from quart import Quart
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Long enough to gather what one incident sets off (the agent-offline check
|
|
# runs each minute, agents report every 10-60 s), short enough that a lone
|
|
# alert is still news when it lands.
|
|
BURST_WINDOW_S = 30.0
|
|
|
|
_held: list[tuple[dict, str]] = [] # (alert data, event id), oldest first
|
|
_flusher: asyncio.Task | None = None
|
|
|
|
|
|
def hold(app: "Quart", alert: dict, event_id: str) -> None:
|
|
"""Add an alert to the email burst, opening a window if none is open."""
|
|
global _flusher
|
|
_held.append((alert, event_id))
|
|
if _flusher is None or _flusher.done():
|
|
_flusher = asyncio.create_task(_after_window(app))
|
|
|
|
|
|
async def _after_window(app: "Quart") -> None:
|
|
"""Wait out the window, then send what it gathered. Alerts that arrive
|
|
while the burst is being sent open the next window."""
|
|
global _flusher
|
|
try:
|
|
await asyncio.sleep(BURST_WINDOW_S)
|
|
await flush(app)
|
|
except Exception:
|
|
logger.exception("Alert email burst failed")
|
|
finally:
|
|
_flusher = None
|
|
if _held:
|
|
_flusher = asyncio.create_task(_after_window(app))
|
|
|
|
|
|
async def flush(app: "Quart") -> None:
|
|
"""Send every held alert now: one email per recipient, then record the
|
|
outcome on each alert's event."""
|
|
from steward.core.mail import recipients
|
|
items = _held[:]
|
|
_held.clear()
|
|
if not items:
|
|
return
|
|
smtp = app.config.get("SMTP", {})
|
|
if not smtp.get("host"):
|
|
return
|
|
async with app.db_sessionmaker() as db:
|
|
to = await recipients(db, "alerts", smtp)
|
|
result = await email_burst(smtp, [alert for alert, _ in items], to)
|
|
if result is not None:
|
|
await _record(app, [event_id for _, event_id in items], result)
|
|
|
|
|
|
async def email_burst(cfg: dict, alerts: list[dict], to: list) -> dict | None:
|
|
"""Email `alerts` to each recipient as one message, in their own zone:
|
|
the usual alert email for one alert, a list for several. Returns
|
|
{"sent", "error"}, or None when there is nobody to send to."""
|
|
from steward.core.mail import alert_email, burst_email, message, send
|
|
if not to or not alerts:
|
|
return None
|
|
try:
|
|
messages = []
|
|
for r in to:
|
|
parts = alert_email(cfg, alerts[0], r) if len(alerts) == 1 else burst_email(cfg, alerts, r)
|
|
messages.append(message(cfg, r.address, *parts))
|
|
await send(cfg, messages)
|
|
return {"sent": True, "error": None}
|
|
except Exception as exc:
|
|
logger.error("Alert email failed: %s", exc)
|
|
return {"sent": False, "error": str(exc)}
|
|
|
|
|
|
async def _record(app: "Quart", event_ids: list[str], result: dict) -> None:
|
|
from steward.models.alerts import AlertEvent
|
|
async with app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
for event in (await db.execute(
|
|
select(AlertEvent).where(AlertEvent.id.in_(event_ids)))).scalars():
|
|
event.email_sent = result["sent"]
|
|
event.email_error = result.get("error")
|