feat(email): the report, invites, resets and test sends go through the outbox; only it talks SMTP (#5508)

#5508 "Every other email through the outbox: weekly report, invites,
password resets, and the test buttons" (milestone 501).

- outbox.send_each queues one email per reader (claimed, so the sender
  doesn't take it too) and tries them at once, for a page that answers
  with what happened; outbox.said words the result. attempt takes a cfg
  for a test of the form's unsaved settings and returns the settled rows.
- Weekly report (scheduled and Send now) and invites: queued, tried at
  once; one the server couldn't take yet is retried for an hour. An invite
  that didn't go shows its link with the reason, as before.
- Password reset: queued in the reset link's own transaction; the sender
  delivers it on its next tick.
- Settings -> Notifications test email and a rule's Send test: queued and
  recorded with retry=False, tried once (System 156 ruling).
- Removed mail.send, notifications._send_email and _smtp_send_sync;
  dispatch_notifications is webhook only. A guard asserts only the outbox
  (and smtp_open itself) opens an SMTP connection, with a self-test.

Also fixes run 8941 (#5507's integration test): its Also-send-to
addresses were "@test", which is_real_address rightly drops; they are
example.com now.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_mail.py

@@ -6,7 +6,6 @@ from datetime import datetime, timezone
 from zoneinfo import ZoneInfo
 
 from steward.core import mail
-from steward.core import notifications as n
 from steward.core import reports
 
 ALERT = {"rule_name": 'Disk <b>"root"</b> on web01', "state": "FIRING", "metric": "disk_used_pct",
@@ -46,9 +45,8 @@ def test_every_colour_is_a_theme_value():
 
 
 async def _sent(monkeypatch, cfg, alert, to=(EXTRA_NY,)):
-    out = []
-    monkeypatch.setattr(n, "_smtp_send_sync", lambda c, msgs: out.extend(msgs))
-    assert (await n._send_email(cfg, alert, list(to)))["sent"]
+    """The message each of `to` is handed for `alert`, as the outbox writes it."""
+    out = [mail.message(cfg, r.address, *mail.alert_email(cfg, alert, r)) for r in to]
     return out[0] if len(to) == 1 else out
 
 
@@ -140,12 +138,21 @@ async def test_the_report_is_sent_from_the_configured_address(monkeypatch):
         assert kind == "report"
         return [PERSON_NY, EXTRA_NY]
 
+    from types import SimpleNamespace
+    from steward.core import outbox
     out = []
+
+    async def _send_each(app, kind, to, words, **kw):
+        # Queued then tried at once (#5508); here every one is accepted.
+        assert kind == "report" and kw.get("retry", True)
+        out.extend(mail.message(app.config["SMTP"], r.address, *words(r)) for r in to)
+        return [SimpleNamespace(recipient=r.address, state="sent", last_error=None) for r in to]
+
     monkeypatch.setattr(reports, "build_report_data", _data)
     monkeypatch.setattr(reports, "recipients", _recipients)
-    monkeypatch.setattr(n, "_smtp_send_sync", lambda c, msgs: out.extend(msgs))
+    monkeypatch.setattr(outbox, "send_each", _send_each)
     ok, said = await reports.send_report(_App())
-    assert ok and said == "Report sent to ann@example.com, ops@example.com."
+    assert ok and said == "Sent to ann@example.com, ops@example.com."
     assert [(m["From"], m["To"]) for m in out] == [("alerts@example.com", "ann@example.com"),
                                                    ("alerts@example.com", "ops@example.com")]
     assert all(m.get_content_type() == "multipart/alternative" for m in out)

tests/core/test_notifications.py

@@ -49,7 +49,7 @@ def test_a_bad_template_raises_valueerror_saying_why(tpl, says):
         n.render_webhook(tpl, ALERT)
 
 
-# ── smtp_mode / _smtp_send_sync ───────────────────────────────────────────────
+# ── smtp_mode / smtp_open (via outbox.deliver_sync) ───────────────────────────────────────────────
 
 @pytest.mark.parametrize("port, tls, mode", [
     (465, True, "ssl"), (465, False, "ssl"), ("465", False, "ssl"),
@@ -98,27 +98,30 @@ class SMTP_SSL(_FakeSMTP):
     (25, False, [("connect", "SMTP", "mx", 25, n.SMTP_TIMEOUT_S)]),
 ])
 def test_smtp_send_connects_by_mode_with_a_timeout(monkeypatch, port, tls, expected):
+    from email.message import EmailMessage
+    from steward.core import outbox
     monkeypatch.setattr(n.smtplib, "SMTP", SMTP)
     monkeypatch.setattr(n.smtplib, "SMTP_SSL", SMTP_SSL)
     _FakeSMTP.made = []
-    msgs = [n.EmailMessage(), n.EmailMessage()]
+    msgs = [EmailMessage(), EmailMessage()]
     msgs[0]["From"], msgs[1]["From"] = "steward@example.com", "other@example.com"
-    n._smtp_send_sync({"host": "mx", "port": port, "tls": tls, "username": "u", "password": "p"}, msgs)
+    cfg = {"host": "mx", "port": port, "tls": tls, "username": "u", "password": "p"}
+    assert outbox.deliver_sync(cfg, msgs) == [None, None]
     (smtp,) = _FakeSMTP.made  # every message over one connection
     assert smtp.calls == expected + [("login", "u", "p"), ("send", "steward@example.com"),
                                      ("send", "other@example.com")]
 
 
-async def test_email_from_falls_back_and_a_line_break_in_the_name_still_sends(monkeypatch):
+def test_email_from_falls_back_and_a_line_break_in_the_name_still_sends():
+    from steward.core.mail import alert_email, message
     sent = []
-    monkeypatch.setattr(n, "_smtp_send_sync",
-                        lambda cfg, msgs: sent.extend((m["From"], m["Subject"]) for m in msgs))
-    to = [Recipient("a@example.com", timezone.utc, False)]
+    to = Recipient("a@example.com", timezone.utc, False)
     base = {"host": "mx", "port": 587}
     for cfg in ({**base, "from": "alerts@example.com", "username": "u@example.com"},
                 {**base, "from": "", "username": "u@example.com"},
                 {**base, "from": "", "username": ""}):
-        assert (await n._send_email(cfg, ALERT, to))["sent"]
+        m = message(cfg, to.address, *alert_email(cfg, ALERT, to))
+        sent.append((m["From"], m["Subject"]))
     assert [f for f, _ in sent] == ["alerts@example.com", "u@example.com", "steward@localhost"]
     # The rule name has a line break; the subject folds it to a space rather than failing.
     assert sent[0][1] == '[Steward] Firing: Disk "root" on web01 \\ backslash — web01 94%'

tests/core/test_outbox.py

@@ -186,3 +186,40 @@ def test_several_alerts_are_one_email_listing_each_oldest_first():
         assert f"Agent offline: {host}" in text and f"Agent offline: {host}" in html
     assert text.index("web01") < text.index("web02") < text.index("db01")
     assert "Fri 9 Oct 2026 08:00 EDT" in text                                # the reader's zone
+
+
+def test_said_names_who_got_it_who_refused_and_who_is_still_being_tried():
+    rows = [_row(recipient="a@x", state="sent"),
+            _row(recipient="b@x", state="failed", last_error="550 no such user"),
+            _row(recipient="c@x", state="pending", last_error="451 greylisted")]
+    ok, said = outbox.said(rows)
+    assert not ok
+    assert said == ("Sent to a@x. Not sent to b@x: 550 no such user. Not sent yet to c@x "
+                    "(451 greylisted); Steward will try again for up to an hour.")
+    assert outbox.said(rows[:1]) == (True, "Sent to a@x.")
+    assert outbox.said([])[0] is False
+
+
+# The guard: every email goes through the queue (System 156 ruling), so the
+# only code that opens a connection to the mail server is the outbox's.
+def _smtp_openers(sources: dict[str, str]) -> set[str]:
+    import re
+    opens = re.compile(r"\bsmtp_open\(|\bsmtplib\.SMTP(_SSL)?\(")
+    return {path for path, src in sources.items()
+            if any(opens.search(line) and not line.lstrip().startswith(("#", "def "))
+                   for line in src.splitlines())}
+
+
+def test_only_the_outbox_talks_to_the_mail_server():
+    from pathlib import Path
+    root = Path(__file__).resolve().parents[2]          # the checkout
+    sources = {str(p.relative_to(root)): p.read_text() for p in (root / "steward").rglob("*.py")}
+    sources.update({str(p.relative_to(root)): p.read_text() for p in (root / "plugins").rglob("*.py")}
+                   if (root / "plugins").is_dir() else {})
+    assert _smtp_openers(sources) == {"steward/core/outbox.py", "steward/core/notifications.py"}
+
+
+def test_the_smtp_guard_can_fail():
+    assert _smtp_openers({"x.py": "    with smtplib.SMTP(host) as s:"}) == {"x.py"}
+    assert _smtp_openers({"y.py": "    smtp = notifications.smtp_open(cfg)"}) == {"y.py"}
+    assert _smtp_openers({"z.py": "# smtp_open(cfg) is the outbox's\ndef smtp_open(cfg):"}) == set()

tests/integration/_mail.py

@@ -0,0 +1,29 @@
+"""Email in integration tests: every email is a queued row the outbox sends
+(#5506), so a test catches what the mail server would have been handed by
+replacing the one SMTP conversation, outbox.deliver_sync."""
+from __future__ import annotations
+
+from email.message import EmailMessage
+
+
+def capture(app, monkeypatch, answer=None) -> list[EmailMessage]:
+    """Set a mail server and collect every message the outbox delivers.
+    `answer(msg)` is the server's reply to each (None: accepted)."""
+    from steward.core import outbox
+    sent: list[EmailMessage] = []
+
+    def deliver(cfg, messages):
+        sent.extend(messages)
+        return [answer(m) if answer else None for m in messages]
+
+    app.config["SMTP"] = {**app.config.get("SMTP", {}), "host": "mx.example", "port": 587,
+                          "from": "steward@example.com"}
+    monkeypatch.setattr(outbox, "deliver_sync", deliver)
+    return sent
+
+
+def to(sent: list[EmailMessage], *addresses: str) -> list[EmailMessage]:
+    """The messages among `sent` addressed to any of `addresses` (other
+    tests' queued email can go out in the same tick)."""
+    want = {a.lower() for a in addresses}
+    return [m for m in sent if m["To"].lower() in want]

tests/integration/test_alert_email_outbox.py

@@ -81,7 +81,7 @@ def test_alerts_in_one_window_are_one_email_and_each_event_records_it(app, monke
     from steward.core import outbox
 
     tag = uuid.uuid4().hex[:8]
-    extra = f"oncall-{tag}@test"
+    extra = f"oncall-{tag}@example.com"
     app.config["SMTP"] = {"host": "mx.test", "port": 587, "from": "steward@test",
                           "recipients": [extra], "tz": "America/New_York"}
     t0 = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc)
@@ -142,7 +142,7 @@ def test_an_email_whose_alerts_were_deleted_fails_and_says_so(app, monkeypatch):
     from steward.models.alerts import AlertEvent
 
     tag = uuid.uuid4().hex[:8]
-    extra = f"gone-{tag}@test"
+    extra = f"gone-{tag}@example.com"
     app.config["SMTP"] = {"host": "mx.test", "port": 587, "from": "steward@test", "recipients": [extra]}
     monkeypatch.setattr(outbox, "deliver_sync", lambda cfg, msgs: [None] * len(msgs))
     t0 = datetime.now(timezone.utc)
@@ -155,8 +155,8 @@ def test_an_email_whose_alerts_were_deleted_fails_and_says_so(app, monkeypatch):
         async with app.db_sessionmaker() as s:
             async with s.begin():
                 await s.execute(delete(AlertEvent).where(AlertEvent.id == ev))
-        states = await outbox.attempt(app, [row.id])
-        return states[row.id], row.id
+        done = await outbox.attempt(app, [row.id])
+        return done[row.id].state, row.id
 
     state, rid = asyncio.run(_go())
     assert state == "failed"

tests/integration/test_alert_send_test.py

@@ -39,11 +39,8 @@ def test_send_test_reaches_each_channel_and_leaves_the_rule_alone(app, monkeypat
     from steward.models.audit import AuditEvent
     from steward.models.users import User, UserRole
 
-    emails, hooks = [], []
-
-    async def fake_email(cfg, alert, to):
-        emails.append((alert, [r.address for r in to]))
-        return {"sent": True, "error": None}
+    from tests.integration._mail import capture
+    hooks = []
 
     async def fake_hook(cfg, alert):
         hooks.append(alert)
@@ -52,10 +49,9 @@ def test_send_test_reaches_each_channel_and_leaves_the_rule_alone(app, monkeypat
     async def fake_recipients(db, kind, smtp):
         return [mail.Recipient("ann@example.com", ZoneInfo("UTC"), False, "ann")]
 
-    monkeypatch.setattr(notifications, "_send_email", fake_email)
+    emails = capture(app, monkeypatch)
     monkeypatch.setattr(notifications, "_send_webhook", fake_hook)
     monkeypatch.setattr(mail, "recipients", fake_recipients)
-    app.config["SMTP"] = {"host": "mx.example", "port": 587, "from": "alerts@example.com"}
     app.config["WEBHOOK"] = {"url": "https://hook.example/api/1"}
 
     async def _go():
@@ -83,12 +79,20 @@ def test_send_test_reaches_each_channel_and_leaves_the_rule_alone(app, monkeypat
         assert resp.status_code == 200
         page = await resp.get_data(as_text=True)
         assert "alert-error" in page
-        assert "Email sent to ann@example.com." in page
+        assert "Email: Sent to ann@example.com." in page
         assert "Webhook to hook.example failed: connection refused" in page
 
-        # What went out is this rule, marked TEST, at its latest reading.
-        ((alert, to),) = emails
-        assert to == ["ann@example.com"] and hooks == [alert]
+        # What went out is this rule, marked TEST, at its latest reading: one
+        # queued test email (recorded, not retried) and the webhook.
+        (msg,) = emails
+        (alert,) = hooks
+        assert msg["To"] == "ann@example.com" and f"Hot {tag}" in msg["Subject"]
+        from steward.models.outbox import EmailOutbox
+        async with app.db_sessionmaker() as s:
+            row = (await s.execute(select(EmailOutbox).where(
+                EmailOutbox.recipient == "ann@example.com", EmailOutbox.kind == "test",
+                EmailOutbox.subject == msg["Subject"]))).scalars().one()
+        assert (row.state, row.retry, row.attempts) == ("sent", False, 1)
         assert (alert["rule_name"], alert["state"], alert["resource"], alert["value"]) == (
             f"Hot {tag}", "TEST", host, 42.0)
 

tests/integration/test_invites.py

@@ -62,16 +62,9 @@ async def _audits(app, uid):
     return [(r.action, json.loads(r.detail_json or "{}")) for r in rows]
 
 
-def _mail(app, monkeypatch):
-    from steward.core import mail
-    sent = []
-
-    async def fake_send(cfg, messages):
-        sent.extend(messages)
-
-    app.config["SMTP"] = {"host": "mx.example", "port": 587, "from": "steward@example.com"}
-    monkeypatch.setattr(mail, "send", fake_send)
-    return sent
+def _mail(app, monkeypatch, answer=None):
+    from tests.integration._mail import capture
+    return capture(app, monkeypatch, answer)
 
 
 def test_an_invite_is_emailed_and_the_invitee_makes_their_own_account_once(app, monkeypatch):
@@ -193,3 +186,27 @@ def test_an_address_with_an_account_is_not_invited_and_only_an_admin_invites(app
                                     form={"email": "y@example.com"})).status_code == 403
 
     asyncio.run(_go())
+
+
+def test_an_invite_the_server_refuses_or_defers_shows_the_link_and_why(app, monkeypatch):
+    # Sent through the queue (#5508) and tried at once: a refusal says so; a
+    # "try later" says Steward keeps trying; either way the link is shown.
+    import smtplib
+    answers = {}
+    _mail(app, monkeypatch, lambda m: answers.get(m["To"]))
+
+    async def _go():
+        admin, aid, _ = await _admin(app)
+        refused, later = f"no-{aid[:8]}@example.com", f"later-{aid[:8]}@example.com"
+        answers[refused] = smtplib.SMTPRecipientsRefused({refused: (550, b"no such user")})
+        answers[later] = smtplib.SMTPDataError(451, b"greylisted")
+        resp = await admin.post("/settings/users/invite", headers=CSRF,
+                                form={"email": refused, "role": "viewer", "days": "7"})
+        page = await resp.get_data(as_text=True)
+        assert f"Sending to {refused} failed (550 no such user)." in page and "/invite/" in page
+        resp = await admin.post("/settings/users/invite", headers=CSRF,
+                                form={"email": later, "role": "viewer", "days": "7"})
+        page = await resp.get_data(as_text=True)
+        assert "451 greylisted" in page and "keep trying for up to an hour" in page
+
+    asyncio.run(_go())

tests/integration/test_notification_test_send.py

@@ -90,14 +90,14 @@ def test_test_webhook_sends_the_form_template_and_shows_the_body(app, monkeypatc
 
 
 def test_test_email_uses_the_form_settings_and_names_the_tls_mode(app, monkeypatch):
-    from steward.core import notifications
+    from steward.core import outbox
     sent = []
 
-    async def fake_send(cfg, alert, to):
-        sent.append((cfg, [r.address for r in to]))
-        return {"sent": True, "error": None}
+    def fake_deliver(cfg, messages):
+        sent.append((cfg, [m["To"] for m in messages]))
+        return [None] * len(messages)
 
-    monkeypatch.setattr(notifications, "_send_email", fake_send)
+    monkeypatch.setattr(outbox, "deliver_sync", fake_deliver)
 
     async def _go():
         client, uid = await _admin_client(app)
@@ -110,10 +110,18 @@ def test_test_email_uses_the_form_settings_and_names_the_tls_mode(app, monkeypat
         assert "alert-success" in page
         assert "Sent to a@example.com, b@example.com via mx.example:465 (implicit TLS)." in page
         assert "Rule: Test alert" in page
-        # The admin's own address (uuid@test) isn't one mail can reach, so only the typed ones.
+        # The admin's own address (uuid@test) isn't one mail can reach, so only the typed ones,
+        # sent through the queue with the form's unsaved values, recorded, and not retried.
         assert sent == [({"host": "mx.example", "port": 465, "tls": False, "username": "u",
                           "password": "pw", "from": "alerts@example.com", "base_url": ""},
                          ["a@example.com", "b@example.com"])]
+        from sqlalchemy import select
+        from steward.models.outbox import EmailOutbox
+        async with app.db_sessionmaker() as s:
+            rows = (await s.execute(select(EmailOutbox).where(
+                EmailOutbox.kind == "test", EmailOutbox.recipient.in_(["a@example.com", "b@example.com"]))
+                .order_by(EmailOutbox.created_at.desc()).limit(2))).scalars().all()
+        assert {(r.state, r.retry) for r in rows} == {("sent", False)}
 
         resp = await client.post("/settings/notifications/test-email", form={
             "smtp.host": "", "smtp.port": "587", "smtp.recipients": ""}, headers=csrf)

tests/integration/test_passwords.py

@@ -123,15 +123,14 @@ _KEY = re.compile(r"/reset/([A-Za-z0-9_-]{20,})")
 
 
 def _mail(app, monkeypatch):
-    from steward.core import mail
-    sent = []
+    from tests.integration._mail import capture
+    return capture(app, monkeypatch)
 
-    async def fake_send(cfg, messages):
-        sent.extend(messages)
 
-    app.config["SMTP"] = {"host": "mx.example", "port": 587, "from": "steward@example.com"}
-    monkeypatch.setattr(mail, "send", fake_send)
-    return sent
+async def _deliver(app):
+    """The outbox sender's next tick: a reset email is queued (#5508)."""
+    from steward.core import outbox
+    await outbox.send_due(app)
 
 
 async def _stranger(app):
@@ -158,8 +157,11 @@ def test_a_forgotten_password_is_reset_from_the_emailed_link(app, monkeypatch):
             resp = await guest.post("/forgot", headers=CSRF, form={"name": who})
             answers.append((resp.status_code, "on its way" in await resp.get_data(as_text=True)))
         assert answers == [(200, True)] * 3
-        assert [m["To"] for m in sent] == [f"{name}@example.com"]
-        text = sent[0].get_body(("plain",)).get_content()
+        await _deliver(app)
+        from tests.integration._mail import to
+        mine = to(sent, f"{name}@example.com", f"{sso_name}@example.com", f"nobody-{uid[:8]}@example.com")
+        assert [m["To"] for m in mine] == [f"{name}@example.com"]
+        text = mine[0].get_body(("plain",)).get_content()
         assert name in text and "BST" in text
         key = _KEY.search(text).group(1)
 
@@ -197,7 +199,10 @@ def test_asking_again_kills_the_first_link_and_an_old_link_expires(app, monkeypa
         guest = await _stranger(app)
         for _ in range(2):
             await guest.post("/forgot", headers=CSRF, form={"name": name})
-        first, second = (_KEY.search(m.get_body(("plain",)).get_content()).group(1) for m in sent)
+        await _deliver(app)
+        from tests.integration._mail import to
+        first, second = (_KEY.search(m.get_body(("plain",)).get_content()).group(1)
+                         for m in to(sent, f"{name}@example.com"))
         assert (await guest.get(f"/reset/{first}")).status_code == 404
         assert (await guest.get(f"/reset/{second}")).status_code == 200