feat(alerts): alerts that arrive together go out as one email (#5465)
#5465 "One email per burst of alerts". When a switch reboots, every host's agent-offline rule fires at once and each account got one email per rule. Alert email now waits 30 s from the first alert: each recipient then gets one message listing every alert in the window, oldest first, in their own zone ("[Steward] 3 alerts: 2 firing, 1 resolved"). One alert alone keeps its usual email. Webhooks still post per alert, at once. Every alert's event records the email's outcome. A burst held when Steward stops is not sent; the alerts stay on the record and on the Alerts page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+16
-20
@@ -495,7 +495,9 @@ async def _dispatch_notification(
|
||||
event_id: str,
|
||||
note: str | None = None,
|
||||
) -> None:
|
||||
"""Run after the transaction commits. Sends notifications and updates the event row."""
|
||||
"""Run after the transaction commits. Holds the alert's email for its
|
||||
burst (core.alert_burst), posts its webhook, and records the webhook's
|
||||
outcome on the event; the burst records the email's."""
|
||||
from steward.core.notifications import dispatch_notifications
|
||||
|
||||
alert_data = {
|
||||
@@ -511,29 +513,23 @@ async def _dispatch_notification(
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
from steward.core.mail import recipients
|
||||
smtp_cfg = app.config.get("SMTP", {})
|
||||
webhook_cfg = app.config.get("WEBHOOK", {})
|
||||
async with app.db_sessionmaker() as db:
|
||||
to = await recipients(db, "alerts", smtp_cfg) if smtp_cfg.get("host") else []
|
||||
results = await dispatch_notifications(smtp_cfg, webhook_cfg, alert_data, to)
|
||||
# Email waits a moment so alerts that arrive together go out as one
|
||||
# message per recipient (#5465); the webhook posts this alert now.
|
||||
from steward.core import alert_burst
|
||||
if app.config.get("SMTP", {}).get("host"):
|
||||
alert_burst.hold(app, alert_data, event_id)
|
||||
results = await dispatch_notifications({}, app.config.get("WEBHOOK", {}), alert_data)
|
||||
|
||||
# Update event row with delivery results in a new transaction
|
||||
webhook_r = results.get("webhook")
|
||||
if not webhook_r:
|
||||
return
|
||||
async with app.db_sessionmaker() as db:
|
||||
async with db.begin():
|
||||
event_result = await db.execute(
|
||||
select(AlertEvent).where(AlertEvent.id == event_id)
|
||||
)
|
||||
event = event_result.scalar_one_or_none()
|
||||
event = (await db.execute(
|
||||
select(AlertEvent).where(AlertEvent.id == event_id))).scalar_one_or_none()
|
||||
if event:
|
||||
email_r = results.get("email")
|
||||
if email_r:
|
||||
event.email_sent = email_r["sent"]
|
||||
event.email_error = email_r.get("error")
|
||||
webhook_r = results.get("webhook")
|
||||
if webhook_r:
|
||||
event.webhook_sent = webhook_r["sent"]
|
||||
event.webhook_error = webhook_r.get("error")
|
||||
event.webhook_sent = webhook_r["sent"]
|
||||
event.webhook_error = webhook_r.get("error")
|
||||
|
||||
|
||||
async def _run_ansible_action(
|
||||
|
||||
Reference in New Issue
Block a user