feat(alerts): alerts that arrive together go out as one email (#5465)

#5465 "One email per burst of alerts". When a switch reboots, every host's
agent-offline rule fires at once and each account got one email per rule.
Alert email now waits 30 s from the first alert: each recipient then gets one
message listing every alert in the window, oldest first, in their own zone
("[Steward] 3 alerts: 2 firing, 1 resolved"). One alert alone keeps its
usual email. Webhooks still post per alert, at once. Every alert's event
records the email's outcome. A burst held when Steward stops is not sent;
the alerts stay on the record and on the Alerts page.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/alert_burst.py

@@ -0,0 +1,105 @@
+"""Alert emails gathered into one message per burst (#5465).
+
+When the network drops or a switch reboots, every host's agent-offline rule
+fires together, and one email per rule per account turns one incident into a
+dozen messages. So alert email waits: the first alert opens a window of
+BURST_WINDOW_S, and when it closes each recipient gets one email for every
+alert that arrived in it. An alert on its own still reads as it always has.
+
+Webhooks are not held here: each alert posts on its own, at once, since an
+integration expects one alert per post (core.alerts._dispatch_notification).
+Every alert's event still records the email's outcome.
+
+A burst held when Steward stops is not sent; the alerts stay on the record
+and on the Alerts page.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import TYPE_CHECKING
+
+from sqlalchemy import select
+
+if TYPE_CHECKING:
+    from quart import Quart
+
+logger = logging.getLogger(__name__)
+
+# Long enough to gather what one incident sets off (the agent-offline check
+# runs each minute, agents report every 10-60 s), short enough that a lone
+# alert is still news when it lands.
+BURST_WINDOW_S = 30.0
+
+_held: list[tuple[dict, str]] = []          # (alert data, event id), oldest first
+_flusher: asyncio.Task | None = None
+
+
+def hold(app: "Quart", alert: dict, event_id: str) -> None:
+    """Add an alert to the email burst, opening a window if none is open."""
+    global _flusher
+    _held.append((alert, event_id))
+    if _flusher is None or _flusher.done():
+        _flusher = asyncio.create_task(_after_window(app))
+
+
+async def _after_window(app: "Quart") -> None:
+    """Wait out the window, then send what it gathered. Alerts that arrive
+    while the burst is being sent open the next window."""
+    global _flusher
+    try:
+        await asyncio.sleep(BURST_WINDOW_S)
+        await flush(app)
+    except Exception:
+        logger.exception("Alert email burst failed")
+    finally:
+        _flusher = None
+        if _held:
+            _flusher = asyncio.create_task(_after_window(app))
+
+
+async def flush(app: "Quart") -> None:
+    """Send every held alert now: one email per recipient, then record the
+    outcome on each alert's event."""
+    from steward.core.mail import recipients
+    items = _held[:]
+    _held.clear()
+    if not items:
+        return
+    smtp = app.config.get("SMTP", {})
+    if not smtp.get("host"):
+        return
+    async with app.db_sessionmaker() as db:
+        to = await recipients(db, "alerts", smtp)
+    result = await email_burst(smtp, [alert for alert, _ in items], to)
+    if result is not None:
+        await _record(app, [event_id for _, event_id in items], result)
+
+
+async def email_burst(cfg: dict, alerts: list[dict], to: list) -> dict | None:
+    """Email `alerts` to each recipient as one message, in their own zone:
+    the usual alert email for one alert, a list for several. Returns
+    {"sent", "error"}, or None when there is nobody to send to."""
+    from steward.core.mail import alert_email, burst_email, message, send
+    if not to or not alerts:
+        return None
+    try:
+        messages = []
+        for r in to:
+            parts = alert_email(cfg, alerts[0], r) if len(alerts) == 1 else burst_email(cfg, alerts, r)
+            messages.append(message(cfg, r.address, *parts))
+        await send(cfg, messages)
+        return {"sent": True, "error": None}
+    except Exception as exc:
+        logger.error("Alert email failed: %s", exc)
+        return {"sent": False, "error": str(exc)}
+
+
+async def _record(app: "Quart", event_ids: list[str], result: dict) -> None:
+    from steward.models.alerts import AlertEvent
+    async with app.db_sessionmaker() as db:
+        async with db.begin():
+            for event in (await db.execute(
+                    select(AlertEvent).where(AlertEvent.id.in_(event_ids)))).scalars():
+                event.email_sent = result["sent"]
+                event.email_error = result.get("error")