feat(alerts): alert email waits 2 minutes in the outbox and each alert records its outcome (#5507)
CI / lint (push) Successful in 3s
CI / integration (push) Failing after 1m13s
CI / unit (push) Successful in 1m25s
CI / publish (push) Skipped

#5507 "Alert emails through the outbox, gathered for 2 minutes"
(milestone 501). Replaces the in-memory core/alert_burst.py.

- outbox.hold_alert: an alert joins each recipient's held alert email, or
  opens one due 2 minutes later (System 156 ruling). A Postgres advisory
  lock keeps two alerts at once from opening two windows for one reader.
- email_outbox_events.alert_json (migration 0043) keeps the alert as it
  fired, so the email written when the window closes says what it said.
- write_held writes it then, in the reader's zone: the usual alert email
  for one alert, the list for several (outbox.alert_words).
- after_settle records on each AlertEvent, once every email carrying it
  has settled: email_sent, or email_error naming each reader it didn't
  reach and the server's reason.
- A held email survives a restart and goes out when its window closes.
- A refusal of a message's one recipient no longer repeats the address.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 15:59:17 -04:00
co-authored by Claude Opus 5.5
parent 4e77ad0e7f
commit 5237ffc663
12 changed files with 356 additions and 212 deletions
+28 -2
View File
@@ -46,7 +46,9 @@ def test_reason_is_what_the_server_said():
assert outbox.reason(smtplib.SMTPDataError(554, b"5.7.1 rejected\r\n as spam")) \
== "554 5.7.1 rejected as spam"
assert outbox.reason(smtplib.SMTPRecipientsRefused({"a@x": (550, b"no such user")})) \
== "a@x: 550 no such user"
== "550 no such user"
assert outbox.reason(smtplib.SMTPRecipientsRefused({"a@x": (550, b"no"), "b@x": (450, b"busy")})) \
== "a@x: 550 no; b@x: 450 busy"
assert outbox.reason(TimeoutError()) == "The mail server didn't answer in time"
@@ -61,7 +63,7 @@ def test_a_refusal_fails_at_once_with_the_servers_words():
row = _row()
outbox.settle(row, smtplib.SMTPRecipientsRefused({"ann@example.com": (550, b"no such user")}), NOW)
assert row.state == "failed" and row.next_attempt_at is None
assert row.last_error == "ann@example.com: 550 no such user"
assert row.last_error == "550 no such user"
def test_a_temporary_failure_waits_and_tries_again():
@@ -160,3 +162,27 @@ def test_no_connection_is_every_messages_outcome(monkeypatch):
monkeypatch.setattr(notifications, "smtp_open", _open)
assert outbox.deliver_sync({}, _msgs(2)) == [refused, refused]
CFG = {"host": "mx", "port": 587, "from": "alerts@example.com", "base_url": "https://steward.example/"}
ANN = mail.Recipient("ann@example.com", ZoneInfo("America/New_York"), False, "ann")
def _alert(host: str, state: str = "FIRING", value: float = 400.0) -> dict:
return {"rule_name": f"Agent offline: {host}", "state": state, "metric": "seconds_since_seen",
"value": value, "operator": ">", "threshold": 180.0, "resource": host,
"source_module": "host_agent", "note": None, "timestamp": "2026-10-09T12:00:00+00:00"}
def test_one_alert_alone_keeps_its_usual_email():
assert outbox.alert_words(CFG, [_alert("web01")], ANN) == mail.alert_email(CFG, _alert("web01"), ANN)
def test_several_alerts_are_one_email_listing_each_oldest_first():
burst = [_alert("web01"), _alert("web02"), _alert("db01", "RESOLVED", 12.0)]
subject, text, html = outbox.alert_words(CFG, burst, ANN)
assert subject == "[Steward] 3 alerts: 2 firing, 1 resolved"
for host in ("web01", "web02", "db01"):
assert f"Agent offline: {host}" in text and f"Agent offline: {host}" in html
assert text.index("web01") < text.index("web02") < text.index("db01")
assert "Fri 9 Oct 2026 08:00 EDT" in text # the reader's zone