feat(alerts): alerts that arrive together go out as one email (#5465)
CI / lint (push) Successful in 2s
CI / integration (push) Successful in 1m9s
CI / unit (push) Successful in 1m26s
CI / publish (push) Successful in 1m13s

#5465 "One email per burst of alerts". When a switch reboots, every host's
agent-offline rule fires at once and each account got one email per rule.
Alert email now waits 30 s from the first alert: each recipient then gets one
message listing every alert in the window, oldest first, in their own zone
("[Steward] 3 alerts: 2 firing, 1 resolved"). One alert alone keeps its
usual email. Webhooks still post per alert, at once. Every alert's event
records the email's outcome. A burst held when Steward stops is not sent;
the alerts stay on the record and on the Alerts page.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 08:29:52 -04:00
co-authored by Claude Opus 5.5
parent 48a0de5600
commit 997aa50939
6 changed files with 264 additions and 20 deletions
+16 -20
View File
@@ -495,7 +495,9 @@ async def _dispatch_notification(
event_id: str,
note: str | None = None,
) -> None:
"""Run after the transaction commits. Sends notifications and updates the event row."""
"""Run after the transaction commits. Holds the alert's email for its
burst (core.alert_burst), posts its webhook, and records the webhook's
outcome on the event; the burst records the email's."""
from steward.core.notifications import dispatch_notifications
alert_data = {
@@ -511,29 +513,23 @@ async def _dispatch_notification(
"timestamp": datetime.now(timezone.utc).isoformat(),
}
from steward.core.mail import recipients
smtp_cfg = app.config.get("SMTP", {})
webhook_cfg = app.config.get("WEBHOOK", {})
async with app.db_sessionmaker() as db:
to = await recipients(db, "alerts", smtp_cfg) if smtp_cfg.get("host") else []
results = await dispatch_notifications(smtp_cfg, webhook_cfg, alert_data, to)
# Email waits a moment so alerts that arrive together go out as one
# message per recipient (#5465); the webhook posts this alert now.
from steward.core import alert_burst
if app.config.get("SMTP", {}).get("host"):
alert_burst.hold(app, alert_data, event_id)
results = await dispatch_notifications({}, app.config.get("WEBHOOK", {}), alert_data)
# Update event row with delivery results in a new transaction
webhook_r = results.get("webhook")
if not webhook_r:
return
async with app.db_sessionmaker() as db:
async with db.begin():
event_result = await db.execute(
select(AlertEvent).where(AlertEvent.id == event_id)
)
event = event_result.scalar_one_or_none()
event = (await db.execute(
select(AlertEvent).where(AlertEvent.id == event_id))).scalar_one_or_none()
if event:
email_r = results.get("email")
if email_r:
event.email_sent = email_r["sent"]
event.email_error = email_r.get("error")
webhook_r = results.get("webhook")
if webhook_r:
event.webhook_sent = webhook_r["sent"]
event.webhook_error = webhook_r.get("error")
event.webhook_sent = webhook_r["sent"]
event.webhook_error = webhook_r.get("error")
async def _run_ansible_action(