feat(alerts): alert email waits 2 minutes in the outbox and each alert records its outcome (#5507)

#5507 "Alert emails through the outbox, gathered for 2 minutes"
(milestone 501). Replaces the in-memory core/alert_burst.py.

- outbox.hold_alert: an alert joins each recipient's held alert email, or
  opens one due 2 minutes later (System 156 ruling). A Postgres advisory
  lock keeps two alerts at once from opening two windows for one reader.
- email_outbox_events.alert_json (migration 0043) keeps the alert as it
  fired, so the email written when the window closes says what it said.
- write_held writes it then, in the reader's zone: the usual alert email
  for one alert, the list for several (outbox.alert_words).
- after_settle records on each AlertEvent, once every email carrying it
  has settled: email_sent, or email_error naming each reader it didn't
  reach and the server's reason.
- A held email survives a restart and goes out when its window closes.
- A refusal of a message's one recipient no longer repeats the address.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_alert_burst.py

@@ -1,79 +0,0 @@
-"""Alert emails gathered into one message per burst (#5465): alerts that
-arrive within the window go out as one email per recipient listing each of
-them; one alert alone keeps its usual email; a failed send says so for every
-alert in the burst; an alert arriving after a window closes opens the next."""
-import asyncio
-from email.message import EmailMessage
-from zoneinfo import ZoneInfo
-
-from steward.core import alert_burst, mail
-from steward.core import notifications as n
-
-NY = ZoneInfo("America/New_York")
-ANN = mail.Recipient("ann@example.com", NY, False, "ann")
-BOB = mail.Recipient("bob@example.com", NY, False, "bob")
-CFG = {"host": "mx", "port": 587, "from": "alerts@example.com", "base_url": "https://steward.example/"}
-
-
-def _alert(host: str, state: str = "FIRING", value: float = 400.0) -> dict:
-    return {"rule_name": f"Agent offline: {host}", "state": state, "metric": "seconds_since_seen",
-            "value": value, "operator": ">", "threshold": 180.0, "resource": host,
-            "source_module": "host_agent", "note": None, "timestamp": "2026-10-09T12:00:00+00:00"}
-
-
-def _capture(monkeypatch) -> list[EmailMessage]:
-    out: list[EmailMessage] = []
-    monkeypatch.setattr(n, "_smtp_send_sync", lambda c, msgs: out.extend(msgs))
-    return out
-
-
-async def test_one_alert_alone_keeps_its_usual_email(monkeypatch):
-    sent = _capture(monkeypatch)
-    assert await alert_burst.email_burst(CFG, [_alert("web01")], [ANN]) == {"sent": True, "error": None}
-    assert len(sent) == 1
-    assert sent[0]["Subject"] == mail.alert_email(CFG, _alert("web01"), ANN)[0]
-
-
-async def test_a_burst_is_one_email_per_recipient_listing_each_alert(monkeypatch):
-    sent = _capture(monkeypatch)
-    burst = [_alert("web01"), _alert("web02"), _alert("db01", "RESOLVED", 12.0)]
-    assert (await alert_burst.email_burst(CFG, burst, [ANN, BOB]))["sent"]
-    assert [m["To"] for m in sent] == ["ann@example.com", "bob@example.com"]
-    assert sent[0]["Subject"] == "[Steward] 3 alerts: 2 firing, 1 resolved"
-    text = sent[0].get_body(("plain",)).get_content()
-    html = sent[0].get_body(("html",)).get_content()
-    for host in ("web01", "web02", "db01"):
-        assert f"Agent offline: {host}" in text and f"Agent offline: {host}" in html
-    assert text.index("web01") < text.index("web02") < text.index("db01")   # oldest first
-    assert "Fri 9 Oct 2026 08:00 EDT" in text                                # the reader's zone
-
-
-async def test_a_failed_send_is_said_for_the_burst(monkeypatch):
-    def boom(cfg, msgs):
-        raise OSError("connection refused")
-    monkeypatch.setattr(n, "_smtp_send_sync", boom)
-    got = await alert_burst.email_burst(CFG, [_alert("a"), _alert("b")], [ANN])
-    assert got == {"sent": False, "error": "connection refused"}
-
-
-async def test_nobody_to_send_to_sends_nothing(monkeypatch):
-    sent = _capture(monkeypatch)
-    assert await alert_burst.email_burst(CFG, [_alert("a")], []) is None and sent == []
-
-
-async def test_alerts_within_the_window_are_flushed_together(monkeypatch):
-    flushed: list[list[str]] = []
-
-    async def fake_flush(app):
-        flushed.append([a["resource"] for a, _ in alert_burst._held])
-        alert_burst._held.clear()
-
-    monkeypatch.setattr(alert_burst, "BURST_WINDOW_S", 0.05)
-    monkeypatch.setattr(alert_burst, "flush", fake_flush)
-    alert_burst._held.clear()
-    for host in ("a", "b", "c"):
-        alert_burst.hold(None, _alert(host), f"ev-{host}")
-    await asyncio.sleep(0.15)
-    alert_burst.hold(None, _alert("d"), "ev-d")      # after the window: the next burst
-    await asyncio.sleep(0.15)
-    assert flushed == [["a", "b", "c"], ["d"]]

tests/core/test_outbox.py

@@ -46,7 +46,9 @@ def test_reason_is_what_the_server_said():
     assert outbox.reason(smtplib.SMTPDataError(554, b"5.7.1  rejected\r\n as spam")) \
         == "554 5.7.1 rejected as spam"
     assert outbox.reason(smtplib.SMTPRecipientsRefused({"a@x": (550, b"no such user")})) \
-        == "a@x: 550 no such user"
+        == "550 no such user"
+    assert outbox.reason(smtplib.SMTPRecipientsRefused({"a@x": (550, b"no"), "b@x": (450, b"busy")})) \
+        == "a@x: 550 no; b@x: 450 busy"
     assert outbox.reason(TimeoutError()) == "The mail server didn't answer in time"
 
 
@@ -61,7 +63,7 @@ def test_a_refusal_fails_at_once_with_the_servers_words():
     row = _row()
     outbox.settle(row, smtplib.SMTPRecipientsRefused({"ann@example.com": (550, b"no such user")}), NOW)
     assert row.state == "failed" and row.next_attempt_at is None
-    assert row.last_error == "ann@example.com: 550 no such user"
+    assert row.last_error == "550 no such user"
 
 
 def test_a_temporary_failure_waits_and_tries_again():
@@ -160,3 +162,27 @@ def test_no_connection_is_every_messages_outcome(monkeypatch):
 
     monkeypatch.setattr(notifications, "smtp_open", _open)
     assert outbox.deliver_sync({}, _msgs(2)) == [refused, refused]
+
+
+CFG = {"host": "mx", "port": 587, "from": "alerts@example.com", "base_url": "https://steward.example/"}
+ANN = mail.Recipient("ann@example.com", ZoneInfo("America/New_York"), False, "ann")
+
+
+def _alert(host: str, state: str = "FIRING", value: float = 400.0) -> dict:
+    return {"rule_name": f"Agent offline: {host}", "state": state, "metric": "seconds_since_seen",
+            "value": value, "operator": ">", "threshold": 180.0, "resource": host,
+            "source_module": "host_agent", "note": None, "timestamp": "2026-10-09T12:00:00+00:00"}
+
+
+def test_one_alert_alone_keeps_its_usual_email():
+    assert outbox.alert_words(CFG, [_alert("web01")], ANN) == mail.alert_email(CFG, _alert("web01"), ANN)
+
+
+def test_several_alerts_are_one_email_listing_each_oldest_first():
+    burst = [_alert("web01"), _alert("web02"), _alert("db01", "RESOLVED", 12.0)]
+    subject, text, html = outbox.alert_words(CFG, burst, ANN)
+    assert subject == "[Steward] 3 alerts: 2 firing, 1 resolved"
+    for host in ("web01", "web02", "db01"):
+        assert f"Agent offline: {host}" in text and f"Agent offline: {host}" in html
+    assert text.index("web01") < text.index("web02") < text.index("db01")
+    assert "Fri 9 Oct 2026 08:00 EDT" in text                                # the reader's zone