feat(email): every email is a row in email_outbox until the server accepts it (#5506)

#5506 "Email outbox: a table of outgoing email and a sender that retries
until accepted, for up to an hour" (milestone 501).

- email_outbox (migration 0042) holds each message per recipient, its
  words, its state (held / pending / sending / sent / failed), attempts and
  the server's last answer. email_outbox_events links alert events for
  #5507.
- core/outbox.py: enqueue; a sender task every 15 s claims what is due
  (SKIP LOCKED, stale claims released), sends the batch over one
  connection with no session held, and settles each row:
  - accepted -> sent;
  - a 5xx refusal -> failed at once with the server's words;
  - no connection, a timeout, a 4xx -> retried after 30 s, 1, 2, 5, 10,
    15 min, the last try at the hour, then failed;
  - a test (retry=False) settles on its first answer.
  It runs on startup, so a row left by a restart goes out at boot.
- notifications.smtp_open is the one place a connection is opened.
- mail.message takes the html part as optional.
- cleanup ages email_outbox rows on the data window.

Senders move onto it in #5507 and #5508.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_outbox.py

@@ -0,0 +1,162 @@
+"""The email outbox's decisions (#5506): which failures are worth another
+try, how long to wait, when to give up, and what one connection's batch
+reports per message. The table and the sender's ticks are in
+tests/integration/test_email_outbox.py."""
+import smtplib
+from datetime import datetime, timedelta, timezone
+from email.message import EmailMessage
+from zoneinfo import ZoneInfo
+
+import pytest
+
+from steward.core import mail, outbox
+from steward.models.outbox import EmailOutbox
+
+NOW = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc)
+
+
+def _row(**kw) -> EmailOutbox:
+    base = dict(kind="report", recipient="ann@example.com", retry=True, attempts=0,
+                state="sending", first_attempt_at=None, claimed_at=NOW)
+    return EmailOutbox(**{**base, **kw})
+
+
+@pytest.mark.parametrize("exc, kind", [
+    (smtplib.SMTPRecipientsRefused({"a@x": (550, b"no such user")}), "permanent"),
+    (smtplib.SMTPRecipientsRefused({"a@x": (550, b"no"), "b@x": (451, b"later")}), "temporary"),
+    (smtplib.SMTPDataError(554, b"rejected as spam"), "permanent"),
+    (smtplib.SMTPSenderRefused(553, b"not yours", "steward@x"), "permanent"),
+    (smtplib.SMTPAuthenticationError(535, b"bad credentials"), "permanent"),
+    (smtplib.SMTPDataError(451, b"greylisted, try later"), "temporary"),
+    (smtplib.SMTPServerDisconnected("gone"), "temporary"),
+    (ConnectionRefusedError(111, "refused"), "temporary"),
+    (TimeoutError(), "temporary"),
+    (smtplib.SMTPNotSupportedError("STARTTLS extension not supported"), "permanent"),
+    (ValueError("no words"), "permanent"),
+])
+def test_a_refusal_is_final_and_anything_that_may_pass_is_retried(exc, kind):
+    assert outbox.classify(exc) == kind
+
+
+def test_backoff_grows_then_holds_at_its_last_step():
+    assert [outbox.backoff(n) for n in range(1, 9)] == [30, 60, 120, 300, 600, 900, 900, 900]
+
+
+def test_reason_is_what_the_server_said():
+    assert outbox.reason(smtplib.SMTPDataError(554, b"5.7.1  rejected\r\n as spam")) \
+        == "554 5.7.1 rejected as spam"
+    assert outbox.reason(smtplib.SMTPRecipientsRefused({"a@x": (550, b"no such user")})) \
+        == "a@x: 550 no such user"
+    assert outbox.reason(TimeoutError()) == "The mail server didn't answer in time"
+
+
+def test_accepted_is_sent():
+    row = _row()
+    outbox.settle(row, None, NOW)
+    assert (row.state, row.sent_at, row.attempts, row.claimed_at) == ("sent", NOW, 1, None)
+    assert row.first_attempt_at == NOW and row.next_attempt_at is None
+
+
+def test_a_refusal_fails_at_once_with_the_servers_words():
+    row = _row()
+    outbox.settle(row, smtplib.SMTPRecipientsRefused({"ann@example.com": (550, b"no such user")}), NOW)
+    assert row.state == "failed" and row.next_attempt_at is None
+    assert row.last_error == "ann@example.com: 550 no such user"
+
+
+def test_a_temporary_failure_waits_and_tries_again():
+    row = _row()
+    outbox.settle(row, smtplib.SMTPDataError(451, b"try later"), NOW)
+    assert row.state == "pending" and row.next_attempt_at == NOW + timedelta(seconds=30)
+    later = NOW + timedelta(seconds=30)
+    outbox.settle(row, ConnectionRefusedError(), later)
+    assert row.attempts == 2 and row.next_attempt_at == later + timedelta(seconds=60)
+    assert row.first_attempt_at == NOW
+
+
+def test_the_last_try_lands_at_the_hour_and_then_it_fails():
+    row = _row(attempts=7, first_attempt_at=NOW)
+    near = NOW + timedelta(seconds=3300)
+    outbox.settle(row, TimeoutError(), near)
+    # 900 s would overshoot the hour: the try is pulled back to it.
+    assert row.state == "pending" and row.next_attempt_at == NOW + timedelta(seconds=3600)
+    outbox.settle(row, TimeoutError(), NOW + timedelta(seconds=3600))
+    assert row.state == "failed"
+    assert row.last_error == "The mail server didn't answer in time (gave up after an hour of retries)"
+
+
+def test_a_test_send_settles_on_its_first_answer():
+    row = _row(kind="test", retry=False)
+    outbox.settle(row, ConnectionRefusedError(), NOW)
+    assert row.state == "failed" and row.last_error == "The mail server refused the connection"
+
+
+def test_enqueue_keeps_the_readers_zone_and_holds_until_send_after():
+    added = []
+
+    class _Db:
+        add = added.append
+
+    ann = mail.Recipient("ann@example.com", ZoneInfo("America/New_York"), True, "ann")
+    now_row = outbox.enqueue(_Db, kind="report", to=ann, subject="s", text="t", now=NOW)
+    assert (now_row.state, now_row.next_attempt_at, now_row.tz, now_row.show_utc) == \
+        ("pending", NOW, "America/New_York", True)
+    held = outbox.enqueue(_Db, kind="alert", to="bob@example.com",
+                          send_after=NOW + timedelta(minutes=2), now=NOW)
+    assert (held.state, held.next_attempt_at, held.tz) == ("held", NOW + timedelta(minutes=2), "UTC")
+    assert added == [now_row, held]
+
+
+class _FakeSMTP:
+    def __init__(self, answers):
+        self.answers, self.sent = list(answers), []
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        return False
+
+    def send_message(self, msg):
+        answer = self.answers.pop(0)
+        if answer is not None:
+            raise answer
+        self.sent.append(msg)
+
+
+def _msgs(n):
+    out = []
+    for i in range(n):
+        m = EmailMessage()
+        m["Subject"] = f"m{i}"
+        out.append(m)
+    return out
+
+
+def test_one_connection_reports_each_message(monkeypatch):
+    from steward.core import notifications
+    smtp = _FakeSMTP([None, smtplib.SMTPRecipientsRefused({"b@x": (550, b"no")}), None])
+    monkeypatch.setattr(notifications, "smtp_open", lambda cfg: smtp)
+    out = outbox.deliver_sync({}, _msgs(3))
+    assert out[0] is None and isinstance(out[1], smtplib.SMTPRecipientsRefused) and out[2] is None
+    assert len(smtp.sent) == 2
+
+
+def test_a_dropped_line_is_every_unsent_messages_outcome(monkeypatch):
+    from steward.core import notifications
+    gone = smtplib.SMTPServerDisconnected("gone")
+    monkeypatch.setattr(notifications, "smtp_open", lambda cfg: _FakeSMTP([None, gone]))
+    out = outbox.deliver_sync({}, _msgs(4))
+    assert out[0] is None and out[1] is gone
+    assert all(isinstance(o, smtplib.SMTPServerDisconnected) for o in out[2:]) and len(out) == 4
+
+
+def test_no_connection_is_every_messages_outcome(monkeypatch):
+    from steward.core import notifications
+    refused = ConnectionRefusedError(111, "refused")
+
+    def _open(cfg):
+        raise refused
+
+    monkeypatch.setattr(notifications, "smtp_open", _open)
+    assert outbox.deliver_sync({}, _msgs(2)) == [refused, refused]