"""Alert emails gathered into one message per burst (#5465). When the network drops or a switch reboots, every host's agent-offline rule fires together, and one email per rule per account turns one incident into a dozen messages. So alert email waits: the first alert opens a window of BURST_WINDOW_S, and when it closes each recipient gets one email for every alert that arrived in it. An alert on its own still reads as it always has. Webhooks are not held here: each alert posts on its own, at once, since an integration expects one alert per post (core.alerts._dispatch_notification). Every alert's event still records the email's outcome. A burst held when Steward stops is not sent; the alerts stay on the record and on the Alerts page. """ from __future__ import annotations import asyncio import logging from typing import TYPE_CHECKING from sqlalchemy import select if TYPE_CHECKING: from quart import Quart logger = logging.getLogger(__name__) # Long enough to gather what one incident sets off (the agent-offline check # runs each minute, agents report every 10-60 s), short enough that a lone # alert is still news when it lands. BURST_WINDOW_S = 30.0 _held: list[tuple[dict, str]] = [] # (alert data, event id), oldest first _flusher: asyncio.Task | None = None def hold(app: "Quart", alert: dict, event_id: str) -> None: """Add an alert to the email burst, opening a window if none is open.""" global _flusher _held.append((alert, event_id)) if _flusher is None or _flusher.done(): _flusher = asyncio.create_task(_after_window(app)) async def _after_window(app: "Quart") -> None: """Wait out the window, then send what it gathered. Alerts that arrive while the burst is being sent open the next window.""" global _flusher try: await asyncio.sleep(BURST_WINDOW_S) await flush(app) except Exception: logger.exception("Alert email burst failed") finally: _flusher = None if _held: _flusher = asyncio.create_task(_after_window(app)) async def flush(app: "Quart") -> None: """Send every held alert now: one email per recipient, then record the outcome on each alert's event.""" from steward.core.mail import recipients items = _held[:] _held.clear() if not items: return smtp = app.config.get("SMTP", {}) if not smtp.get("host"): return async with app.db_sessionmaker() as db: to = await recipients(db, "alerts", smtp) result = await email_burst(smtp, [alert for alert, _ in items], to) if result is not None: await _record(app, [event_id for _, event_id in items], result) async def email_burst(cfg: dict, alerts: list[dict], to: list) -> dict | None: """Email `alerts` to each recipient as one message, in their own zone: the usual alert email for one alert, a list for several. Returns {"sent", "error"}, or None when there is nobody to send to.""" from steward.core.mail import alert_email, burst_email, message, send if not to or not alerts: return None try: messages = [] for r in to: parts = alert_email(cfg, alerts[0], r) if len(alerts) == 1 else burst_email(cfg, alerts, r) messages.append(message(cfg, r.address, *parts)) await send(cfg, messages) return {"sent": True, "error": None} except Exception as exc: logger.error("Alert email failed: %s", exc) return {"sent": False, "error": str(exc)} async def _record(app: "Quart", event_ids: list[str], result: dict) -> None: from steward.models.alerts import AlertEvent async with app.db_sessionmaker() as db: async with db.begin(): for event in (await db.execute( select(AlertEvent).where(AlertEvent.id.in_(event_ids)))).scalars(): event.email_sent = result["sent"] event.email_error = result.get("error")