feat(email): every email is a row in email_outbox until the server accepts it (#5506)

#5506 "Email outbox: a table of outgoing email and a sender that retries
until accepted, for up to an hour" (milestone 501).

- email_outbox (migration 0042) holds each message per recipient, its
  words, its state (held / pending / sending / sent / failed), attempts and
  the server's last answer. email_outbox_events links alert events for
  #5507.
- core/outbox.py: enqueue; a sender task every 15 s claims what is due
  (SKIP LOCKED, stale claims released), sends the batch over one
  connection with no session held, and settles each row:
  - accepted -> sent;
  - a 5xx refusal -> failed at once with the server's words;
  - no connection, a timeout, a 4xx -> retried after 30 s, 1, 2, 5, 10,
    15 min, the last try at the hour, then failed;
  - a test (retry=False) settles on its first answer.
  It runs on startup, so a row left by a restart goes out at boot.
- notifications.smtp_open is the one place a connection is opened.
- mail.message takes the html part as optional.
- cleanup ages email_outbox rows on the data window.

Senders move onto it in #5507 and #5508.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_outbox.py

@@ -0,0 +1,162 @@
+"""The email outbox's decisions (#5506): which failures are worth another
+try, how long to wait, when to give up, and what one connection's batch
+reports per message. The table and the sender's ticks are in
+tests/integration/test_email_outbox.py."""
+import smtplib
+from datetime import datetime, timedelta, timezone
+from email.message import EmailMessage
+from zoneinfo import ZoneInfo
+
+import pytest
+
+from steward.core import mail, outbox
+from steward.models.outbox import EmailOutbox
+
+NOW = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc)
+
+
+def _row(**kw) -> EmailOutbox:
+    base = dict(kind="report", recipient="ann@example.com", retry=True, attempts=0,
+                state="sending", first_attempt_at=None, claimed_at=NOW)
+    return EmailOutbox(**{**base, **kw})
+
+
+@pytest.mark.parametrize("exc, kind", [
+    (smtplib.SMTPRecipientsRefused({"a@x": (550, b"no such user")}), "permanent"),
+    (smtplib.SMTPRecipientsRefused({"a@x": (550, b"no"), "b@x": (451, b"later")}), "temporary"),
+    (smtplib.SMTPDataError(554, b"rejected as spam"), "permanent"),
+    (smtplib.SMTPSenderRefused(553, b"not yours", "steward@x"), "permanent"),
+    (smtplib.SMTPAuthenticationError(535, b"bad credentials"), "permanent"),
+    (smtplib.SMTPDataError(451, b"greylisted, try later"), "temporary"),
+    (smtplib.SMTPServerDisconnected("gone"), "temporary"),
+    (ConnectionRefusedError(111, "refused"), "temporary"),
+    (TimeoutError(), "temporary"),
+    (smtplib.SMTPNotSupportedError("STARTTLS extension not supported"), "permanent"),
+    (ValueError("no words"), "permanent"),
+])
+def test_a_refusal_is_final_and_anything_that_may_pass_is_retried(exc, kind):
+    assert outbox.classify(exc) == kind
+
+
+def test_backoff_grows_then_holds_at_its_last_step():
+    assert [outbox.backoff(n) for n in range(1, 9)] == [30, 60, 120, 300, 600, 900, 900, 900]
+
+
+def test_reason_is_what_the_server_said():
+    assert outbox.reason(smtplib.SMTPDataError(554, b"5.7.1  rejected\r\n as spam")) \
+        == "554 5.7.1 rejected as spam"
+    assert outbox.reason(smtplib.SMTPRecipientsRefused({"a@x": (550, b"no such user")})) \
+        == "a@x: 550 no such user"
+    assert outbox.reason(TimeoutError()) == "The mail server didn't answer in time"
+
+
+def test_accepted_is_sent():
+    row = _row()
+    outbox.settle(row, None, NOW)
+    assert (row.state, row.sent_at, row.attempts, row.claimed_at) == ("sent", NOW, 1, None)
+    assert row.first_attempt_at == NOW and row.next_attempt_at is None
+
+
+def test_a_refusal_fails_at_once_with_the_servers_words():
+    row = _row()
+    outbox.settle(row, smtplib.SMTPRecipientsRefused({"ann@example.com": (550, b"no such user")}), NOW)
+    assert row.state == "failed" and row.next_attempt_at is None
+    assert row.last_error == "ann@example.com: 550 no such user"
+
+
+def test_a_temporary_failure_waits_and_tries_again():
+    row = _row()
+    outbox.settle(row, smtplib.SMTPDataError(451, b"try later"), NOW)
+    assert row.state == "pending" and row.next_attempt_at == NOW + timedelta(seconds=30)
+    later = NOW + timedelta(seconds=30)
+    outbox.settle(row, ConnectionRefusedError(), later)
+    assert row.attempts == 2 and row.next_attempt_at == later + timedelta(seconds=60)
+    assert row.first_attempt_at == NOW
+
+
+def test_the_last_try_lands_at_the_hour_and_then_it_fails():
+    row = _row(attempts=7, first_attempt_at=NOW)
+    near = NOW + timedelta(seconds=3300)
+    outbox.settle(row, TimeoutError(), near)
+    # 900 s would overshoot the hour: the try is pulled back to it.
+    assert row.state == "pending" and row.next_attempt_at == NOW + timedelta(seconds=3600)
+    outbox.settle(row, TimeoutError(), NOW + timedelta(seconds=3600))
+    assert row.state == "failed"
+    assert row.last_error == "The mail server didn't answer in time (gave up after an hour of retries)"
+
+
+def test_a_test_send_settles_on_its_first_answer():
+    row = _row(kind="test", retry=False)
+    outbox.settle(row, ConnectionRefusedError(), NOW)
+    assert row.state == "failed" and row.last_error == "The mail server refused the connection"
+
+
+def test_enqueue_keeps_the_readers_zone_and_holds_until_send_after():
+    added = []
+
+    class _Db:
+        add = added.append
+
+    ann = mail.Recipient("ann@example.com", ZoneInfo("America/New_York"), True, "ann")
+    now_row = outbox.enqueue(_Db, kind="report", to=ann, subject="s", text="t", now=NOW)
+    assert (now_row.state, now_row.next_attempt_at, now_row.tz, now_row.show_utc) == \
+        ("pending", NOW, "America/New_York", True)
+    held = outbox.enqueue(_Db, kind="alert", to="bob@example.com",
+                          send_after=NOW + timedelta(minutes=2), now=NOW)
+    assert (held.state, held.next_attempt_at, held.tz) == ("held", NOW + timedelta(minutes=2), "UTC")
+    assert added == [now_row, held]
+
+
+class _FakeSMTP:
+    def __init__(self, answers):
+        self.answers, self.sent = list(answers), []
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        return False
+
+    def send_message(self, msg):
+        answer = self.answers.pop(0)
+        if answer is not None:
+            raise answer
+        self.sent.append(msg)
+
+
+def _msgs(n):
+    out = []
+    for i in range(n):
+        m = EmailMessage()
+        m["Subject"] = f"m{i}"
+        out.append(m)
+    return out
+
+
+def test_one_connection_reports_each_message(monkeypatch):
+    from steward.core import notifications
+    smtp = _FakeSMTP([None, smtplib.SMTPRecipientsRefused({"b@x": (550, b"no")}), None])
+    monkeypatch.setattr(notifications, "smtp_open", lambda cfg: smtp)
+    out = outbox.deliver_sync({}, _msgs(3))
+    assert out[0] is None and isinstance(out[1], smtplib.SMTPRecipientsRefused) and out[2] is None
+    assert len(smtp.sent) == 2
+
+
+def test_a_dropped_line_is_every_unsent_messages_outcome(monkeypatch):
+    from steward.core import notifications
+    gone = smtplib.SMTPServerDisconnected("gone")
+    monkeypatch.setattr(notifications, "smtp_open", lambda cfg: _FakeSMTP([None, gone]))
+    out = outbox.deliver_sync({}, _msgs(4))
+    assert out[0] is None and out[1] is gone
+    assert all(isinstance(o, smtplib.SMTPServerDisconnected) for o in out[2:]) and len(out) == 4
+
+
+def test_no_connection_is_every_messages_outcome(monkeypatch):
+    from steward.core import notifications
+    refused = ConnectionRefusedError(111, "refused")
+
+    def _open(cfg):
+        raise refused
+
+    monkeypatch.setattr(notifications, "smtp_open", _open)
+    assert outbox.deliver_sync({}, _msgs(2)) == [refused, refused]

tests/integration/test_email_outbox.py

@@ -0,0 +1,108 @@
+"""Integration: the email outbox's sender against Postgres (#5506).
+
+A due row is sent on the sender's first tick, a refusal fails it, a
+temporary failure leaves it waiting for its retry, a held row waits for its
+window, and a claim left by a run that died goes back to the queue. Only the
+SMTP conversation itself is replaced.
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import smtplib
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    app = create_app(testing=False)
+    app.config["SMTP"] = {"host": "mx.test", "port": 587, "from": "steward@test"}
+    return app
+
+
+def test_the_sender_settles_each_row_on_what_the_server_said(app, monkeypatch):
+    from sqlalchemy import select
+    from steward.core import outbox
+    from steward.models.outbox import EmailOutbox
+
+    tag = uuid.uuid4().hex[:8]
+    now = datetime.now(timezone.utc)
+    to = {k: f"{k}-{tag}@test" for k in ("ok", "refused", "later", "held", "stale")}
+    tried: list[str] = []
+
+    def fake_deliver(cfg, messages):
+        out = []
+        for m in messages:
+            tried.append(m["To"])
+            if m["To"] == to["refused"]:
+                out.append(smtplib.SMTPRecipientsRefused({m["To"]: (550, b"no such user")}))
+            elif m["To"] == to["later"]:
+                out.append(smtplib.SMTPDataError(451, b"greylisted"))
+            else:
+                out.append(None)
+        return out
+
+    monkeypatch.setattr(outbox, "deliver_sync", fake_deliver)
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                for k in ("ok", "refused", "later"):
+                    outbox.enqueue(s, kind="report", to=to[k], subject=f"s {k}", text="t", html="<p>t</p>")
+                outbox.enqueue(s, kind="report", to=to["held"], subject="s", text="t",
+                               send_after=now + timedelta(minutes=2))
+                # Claimed by a sender that died ten minutes ago.
+                dead = outbox.enqueue(s, kind="report", to=to["stale"], subject="s", text="t")
+                dead.state, dead.claimed_at = "sending", now - timedelta(minutes=10)
+        await outbox.send_due(app)
+        async with app.db_sessionmaker() as s:
+            rows = (await s.execute(select(EmailOutbox)
+                                    .where(EmailOutbox.recipient.in_(to.values())))).scalars().all()
+            return {r.recipient: r for r in rows}
+
+    rows = asyncio.run(_go())
+    by = {k: rows[to[k]] for k in to}
+    assert by["ok"].state == "sent" and by["ok"].sent_at is not None and by["ok"].attempts == 1
+    assert by["refused"].state == "failed"
+    assert by["refused"].last_error == f"{to['refused']}: 550 no such user"
+    later = by["later"]
+    assert later.state == "pending" and later.attempts == 1 and later.last_error == "451 greylisted"
+    assert later.next_attempt_at > later.first_attempt_at
+    # The held row is not due for two minutes; nothing went to it.
+    assert by["held"].state == "held" and by["held"].attempts == 0 and to["held"] not in tried
+    # The dead run's claim was released and the row sent in the same tick.
+    assert by["stale"].state == "sent" and by["stale"].claimed_at is None
+    assert sorted(t for t in tried if t.endswith(f"{tag}@test")) == sorted(
+        to[k] for k in ("ok", "refused", "later", "stale"))
+
+
+def test_with_no_mail_server_a_row_fails_and_says_where_to_set_one(app, monkeypatch):
+    from sqlalchemy import select
+    from steward.core import outbox
+    from steward.models.outbox import EmailOutbox
+
+    app.config["SMTP"] = {}
+    monkeypatch.setattr(outbox, "deliver_sync", lambda cfg, msgs: pytest.fail("nothing to send to"))
+    addr = f"nomx-{uuid.uuid4().hex[:8]}@test"
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                row = outbox.enqueue(s, kind="report", to=addr, subject="s", text="t")
+                await s.flush()
+                rid = row.id
+        await outbox.attempt(app, [rid])
+        async with app.db_sessionmaker() as s:
+            return (await s.execute(select(EmailOutbox).where(EmailOutbox.recipient == addr))).scalar_one()
+
+    row = asyncio.run(_go())
+    assert row.state == "failed"
+    assert row.last_error == "No mail server is set (Settings → Notifications)"