feat(alerts): alert email waits 2 minutes in the outbox and each alert records its outcome (#5507)
#5507 "Alert emails through the outbox, gathered for 2 minutes" (milestone 501). Replaces the in-memory core/alert_burst.py. - outbox.hold_alert: an alert joins each recipient's held alert email, or opens one due 2 minutes later (System 156 ruling). A Postgres advisory lock keeps two alerts at once from opening two windows for one reader. - email_outbox_events.alert_json (migration 0043) keeps the alert as it fired, so the email written when the window closes says what it said. - write_held writes it then, in the reader's zone: the usual alert email for one alert, the list for several (outbox.alert_words). - after_settle records on each AlertEvent, once every email carrying it has settled: email_sent, or email_error naming each reader it didn't reach and the server's reason. - A held email survives a restart and goes out when its window closes. - A refusal of a message's one recipient no longer repeats the address. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -46,7 +46,9 @@ def test_reason_is_what_the_server_said():
|
||||
assert outbox.reason(smtplib.SMTPDataError(554, b"5.7.1 rejected\r\n as spam")) \
|
||||
== "554 5.7.1 rejected as spam"
|
||||
assert outbox.reason(smtplib.SMTPRecipientsRefused({"a@x": (550, b"no such user")})) \
|
||||
== "a@x: 550 no such user"
|
||||
== "550 no such user"
|
||||
assert outbox.reason(smtplib.SMTPRecipientsRefused({"a@x": (550, b"no"), "b@x": (450, b"busy")})) \
|
||||
== "a@x: 550 no; b@x: 450 busy"
|
||||
assert outbox.reason(TimeoutError()) == "The mail server didn't answer in time"
|
||||
|
||||
|
||||
@@ -61,7 +63,7 @@ def test_a_refusal_fails_at_once_with_the_servers_words():
|
||||
row = _row()
|
||||
outbox.settle(row, smtplib.SMTPRecipientsRefused({"ann@example.com": (550, b"no such user")}), NOW)
|
||||
assert row.state == "failed" and row.next_attempt_at is None
|
||||
assert row.last_error == "ann@example.com: 550 no such user"
|
||||
assert row.last_error == "550 no such user"
|
||||
|
||||
|
||||
def test_a_temporary_failure_waits_and_tries_again():
|
||||
@@ -160,3 +162,27 @@ def test_no_connection_is_every_messages_outcome(monkeypatch):
|
||||
|
||||
monkeypatch.setattr(notifications, "smtp_open", _open)
|
||||
assert outbox.deliver_sync({}, _msgs(2)) == [refused, refused]
|
||||
|
||||
|
||||
CFG = {"host": "mx", "port": 587, "from": "alerts@example.com", "base_url": "https://steward.example/"}
|
||||
ANN = mail.Recipient("ann@example.com", ZoneInfo("America/New_York"), False, "ann")
|
||||
|
||||
|
||||
def _alert(host: str, state: str = "FIRING", value: float = 400.0) -> dict:
|
||||
return {"rule_name": f"Agent offline: {host}", "state": state, "metric": "seconds_since_seen",
|
||||
"value": value, "operator": ">", "threshold": 180.0, "resource": host,
|
||||
"source_module": "host_agent", "note": None, "timestamp": "2026-10-09T12:00:00+00:00"}
|
||||
|
||||
|
||||
def test_one_alert_alone_keeps_its_usual_email():
|
||||
assert outbox.alert_words(CFG, [_alert("web01")], ANN) == mail.alert_email(CFG, _alert("web01"), ANN)
|
||||
|
||||
|
||||
def test_several_alerts_are_one_email_listing_each_oldest_first():
|
||||
burst = [_alert("web01"), _alert("web02"), _alert("db01", "RESOLVED", 12.0)]
|
||||
subject, text, html = outbox.alert_words(CFG, burst, ANN)
|
||||
assert subject == "[Steward] 3 alerts: 2 firing, 1 resolved"
|
||||
for host in ("web01", "web02", "db01"):
|
||||
assert f"Agent offline: {host}" in text and f"Agent offline: {host}" in html
|
||||
assert text.index("web01") < text.index("web02") < text.index("db01")
|
||||
assert "Fri 9 Oct 2026 08:00 EDT" in text # the reader's zone
|
||||
|
||||
Reference in New Issue
Block a user