feat(email): the report, invites, resets and test sends go through the outbox; only it talks SMTP (#5508)

#5508 "Every other email through the outbox: weekly report, invites,
password resets, and the test buttons" (milestone 501).

- outbox.send_each queues one email per reader (claimed, so the sender
  doesn't take it too) and tries them at once, for a page that answers
  with what happened; outbox.said words the result. attempt takes a cfg
  for a test of the form's unsaved settings and returns the settled rows.
- Weekly report (scheduled and Send now) and invites: queued, tried at
  once; one the server couldn't take yet is retried for an hour. An invite
  that didn't go shows its link with the reason, as before.
- Password reset: queued in the reset link's own transaction; the sender
  delivers it on its next tick.
- Settings -> Notifications test email and a rule's Send test: queued and
  recorded with retry=False, tried once (System 156 ruling).
- Removed mail.send, notifications._send_email and _smtp_send_sync;
  dispatch_notifications is webhook only. A guard asserts only the outbox
  (and smtp_open itself) opens an SMTP connection, with a self-test.

Also fixes run 8941 (#5507's integration test): its Also-send-to
addresses were "@test", which is_real_address rightly drops; they are
example.com now.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/_mail.py

@@ -0,0 +1,29 @@
+"""Email in integration tests: every email is a queued row the outbox sends
+(#5506), so a test catches what the mail server would have been handed by
+replacing the one SMTP conversation, outbox.deliver_sync."""
+from __future__ import annotations
+
+from email.message import EmailMessage
+
+
+def capture(app, monkeypatch, answer=None) -> list[EmailMessage]:
+    """Set a mail server and collect every message the outbox delivers.
+    `answer(msg)` is the server's reply to each (None: accepted)."""
+    from steward.core import outbox
+    sent: list[EmailMessage] = []
+
+    def deliver(cfg, messages):
+        sent.extend(messages)
+        return [answer(m) if answer else None for m in messages]
+
+    app.config["SMTP"] = {**app.config.get("SMTP", {}), "host": "mx.example", "port": 587,
+                          "from": "steward@example.com"}
+    monkeypatch.setattr(outbox, "deliver_sync", deliver)
+    return sent
+
+
+def to(sent: list[EmailMessage], *addresses: str) -> list[EmailMessage]:
+    """The messages among `sent` addressed to any of `addresses` (other
+    tests' queued email can go out in the same tick)."""
+    want = {a.lower() for a in addresses}
+    return [m for m in sent if m["To"].lower() in want]

tests/integration/test_alert_email_outbox.py

@@ -81,7 +81,7 @@ def test_alerts_in_one_window_are_one_email_and_each_event_records_it(app, monke
     from steward.core import outbox
 
     tag = uuid.uuid4().hex[:8]
-    extra = f"oncall-{tag}@test"
+    extra = f"oncall-{tag}@example.com"
     app.config["SMTP"] = {"host": "mx.test", "port": 587, "from": "steward@test",
                           "recipients": [extra], "tz": "America/New_York"}
     t0 = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc)
@@ -142,7 +142,7 @@ def test_an_email_whose_alerts_were_deleted_fails_and_says_so(app, monkeypatch):
     from steward.models.alerts import AlertEvent
 
     tag = uuid.uuid4().hex[:8]
-    extra = f"gone-{tag}@test"
+    extra = f"gone-{tag}@example.com"
     app.config["SMTP"] = {"host": "mx.test", "port": 587, "from": "steward@test", "recipients": [extra]}
     monkeypatch.setattr(outbox, "deliver_sync", lambda cfg, msgs: [None] * len(msgs))
     t0 = datetime.now(timezone.utc)
@@ -155,8 +155,8 @@ def test_an_email_whose_alerts_were_deleted_fails_and_says_so(app, monkeypatch):
         async with app.db_sessionmaker() as s:
             async with s.begin():
                 await s.execute(delete(AlertEvent).where(AlertEvent.id == ev))
-        states = await outbox.attempt(app, [row.id])
-        return states[row.id], row.id
+        done = await outbox.attempt(app, [row.id])
+        return done[row.id].state, row.id
 
     state, rid = asyncio.run(_go())
     assert state == "failed"

tests/integration/test_alert_send_test.py

@@ -39,11 +39,8 @@ def test_send_test_reaches_each_channel_and_leaves_the_rule_alone(app, monkeypat
     from steward.models.audit import AuditEvent
     from steward.models.users import User, UserRole
 
-    emails, hooks = [], []
-
-    async def fake_email(cfg, alert, to):
-        emails.append((alert, [r.address for r in to]))
-        return {"sent": True, "error": None}
+    from tests.integration._mail import capture
+    hooks = []
 
     async def fake_hook(cfg, alert):
         hooks.append(alert)
@@ -52,10 +49,9 @@ def test_send_test_reaches_each_channel_and_leaves_the_rule_alone(app, monkeypat
     async def fake_recipients(db, kind, smtp):
         return [mail.Recipient("ann@example.com", ZoneInfo("UTC"), False, "ann")]
 
-    monkeypatch.setattr(notifications, "_send_email", fake_email)
+    emails = capture(app, monkeypatch)
     monkeypatch.setattr(notifications, "_send_webhook", fake_hook)
     monkeypatch.setattr(mail, "recipients", fake_recipients)
-    app.config["SMTP"] = {"host": "mx.example", "port": 587, "from": "alerts@example.com"}
     app.config["WEBHOOK"] = {"url": "https://hook.example/api/1"}
 
     async def _go():
@@ -83,12 +79,20 @@ def test_send_test_reaches_each_channel_and_leaves_the_rule_alone(app, monkeypat
         assert resp.status_code == 200
         page = await resp.get_data(as_text=True)
         assert "alert-error" in page
-        assert "Email sent to ann@example.com." in page
+        assert "Email: Sent to ann@example.com." in page
         assert "Webhook to hook.example failed: connection refused" in page
 
-        # What went out is this rule, marked TEST, at its latest reading.
-        ((alert, to),) = emails
-        assert to == ["ann@example.com"] and hooks == [alert]
+        # What went out is this rule, marked TEST, at its latest reading: one
+        # queued test email (recorded, not retried) and the webhook.
+        (msg,) = emails
+        (alert,) = hooks
+        assert msg["To"] == "ann@example.com" and f"Hot {tag}" in msg["Subject"]
+        from steward.models.outbox import EmailOutbox
+        async with app.db_sessionmaker() as s:
+            row = (await s.execute(select(EmailOutbox).where(
+                EmailOutbox.recipient == "ann@example.com", EmailOutbox.kind == "test",
+                EmailOutbox.subject == msg["Subject"]))).scalars().one()
+        assert (row.state, row.retry, row.attempts) == ("sent", False, 1)
         assert (alert["rule_name"], alert["state"], alert["resource"], alert["value"]) == (
             f"Hot {tag}", "TEST", host, 42.0)
 

tests/integration/test_invites.py

@@ -62,16 +62,9 @@ async def _audits(app, uid):
     return [(r.action, json.loads(r.detail_json or "{}")) for r in rows]
 
 
-def _mail(app, monkeypatch):
-    from steward.core import mail
-    sent = []
-
-    async def fake_send(cfg, messages):
-        sent.extend(messages)
-
-    app.config["SMTP"] = {"host": "mx.example", "port": 587, "from": "steward@example.com"}
-    monkeypatch.setattr(mail, "send", fake_send)
-    return sent
+def _mail(app, monkeypatch, answer=None):
+    from tests.integration._mail import capture
+    return capture(app, monkeypatch, answer)
 
 
 def test_an_invite_is_emailed_and_the_invitee_makes_their_own_account_once(app, monkeypatch):
@@ -193,3 +186,27 @@ def test_an_address_with_an_account_is_not_invited_and_only_an_admin_invites(app
                                     form={"email": "y@example.com"})).status_code == 403
 
     asyncio.run(_go())
+
+
+def test_an_invite_the_server_refuses_or_defers_shows_the_link_and_why(app, monkeypatch):
+    # Sent through the queue (#5508) and tried at once: a refusal says so; a
+    # "try later" says Steward keeps trying; either way the link is shown.
+    import smtplib
+    answers = {}
+    _mail(app, monkeypatch, lambda m: answers.get(m["To"]))
+
+    async def _go():
+        admin, aid, _ = await _admin(app)
+        refused, later = f"no-{aid[:8]}@example.com", f"later-{aid[:8]}@example.com"
+        answers[refused] = smtplib.SMTPRecipientsRefused({refused: (550, b"no such user")})
+        answers[later] = smtplib.SMTPDataError(451, b"greylisted")
+        resp = await admin.post("/settings/users/invite", headers=CSRF,
+                                form={"email": refused, "role": "viewer", "days": "7"})
+        page = await resp.get_data(as_text=True)
+        assert f"Sending to {refused} failed (550 no such user)." in page and "/invite/" in page
+        resp = await admin.post("/settings/users/invite", headers=CSRF,
+                                form={"email": later, "role": "viewer", "days": "7"})
+        page = await resp.get_data(as_text=True)
+        assert "451 greylisted" in page and "keep trying for up to an hour" in page
+
+    asyncio.run(_go())

tests/integration/test_notification_test_send.py

@@ -90,14 +90,14 @@ def test_test_webhook_sends_the_form_template_and_shows_the_body(app, monkeypatc
 
 
 def test_test_email_uses_the_form_settings_and_names_the_tls_mode(app, monkeypatch):
-    from steward.core import notifications
+    from steward.core import outbox
     sent = []
 
-    async def fake_send(cfg, alert, to):
-        sent.append((cfg, [r.address for r in to]))
-        return {"sent": True, "error": None}
+    def fake_deliver(cfg, messages):
+        sent.append((cfg, [m["To"] for m in messages]))
+        return [None] * len(messages)
 
-    monkeypatch.setattr(notifications, "_send_email", fake_send)
+    monkeypatch.setattr(outbox, "deliver_sync", fake_deliver)
 
     async def _go():
         client, uid = await _admin_client(app)
@@ -110,10 +110,18 @@ def test_test_email_uses_the_form_settings_and_names_the_tls_mode(app, monkeypat
         assert "alert-success" in page
         assert "Sent to a@example.com, b@example.com via mx.example:465 (implicit TLS)." in page
         assert "Rule: Test alert" in page
-        # The admin's own address (uuid@test) isn't one mail can reach, so only the typed ones.
+        # The admin's own address (uuid@test) isn't one mail can reach, so only the typed ones,
+        # sent through the queue with the form's unsaved values, recorded, and not retried.
         assert sent == [({"host": "mx.example", "port": 465, "tls": False, "username": "u",
                           "password": "pw", "from": "alerts@example.com", "base_url": ""},
                          ["a@example.com", "b@example.com"])]
+        from sqlalchemy import select
+        from steward.models.outbox import EmailOutbox
+        async with app.db_sessionmaker() as s:
+            rows = (await s.execute(select(EmailOutbox).where(
+                EmailOutbox.kind == "test", EmailOutbox.recipient.in_(["a@example.com", "b@example.com"]))
+                .order_by(EmailOutbox.created_at.desc()).limit(2))).scalars().all()
+        assert {(r.state, r.retry) for r in rows} == {("sent", False)}
 
         resp = await client.post("/settings/notifications/test-email", form={
             "smtp.host": "", "smtp.port": "587", "smtp.recipients": ""}, headers=csrf)

tests/integration/test_passwords.py

@@ -123,15 +123,14 @@ _KEY = re.compile(r"/reset/([A-Za-z0-9_-]{20,})")
 
 
 def _mail(app, monkeypatch):
-    from steward.core import mail
-    sent = []
+    from tests.integration._mail import capture
+    return capture(app, monkeypatch)
 
-    async def fake_send(cfg, messages):
-        sent.extend(messages)
 
-    app.config["SMTP"] = {"host": "mx.example", "port": 587, "from": "steward@example.com"}
-    monkeypatch.setattr(mail, "send", fake_send)
-    return sent
+async def _deliver(app):
+    """The outbox sender's next tick: a reset email is queued (#5508)."""
+    from steward.core import outbox
+    await outbox.send_due(app)
 
 
 async def _stranger(app):
@@ -158,8 +157,11 @@ def test_a_forgotten_password_is_reset_from_the_emailed_link(app, monkeypatch):
             resp = await guest.post("/forgot", headers=CSRF, form={"name": who})
             answers.append((resp.status_code, "on its way" in await resp.get_data(as_text=True)))
         assert answers == [(200, True)] * 3
-        assert [m["To"] for m in sent] == [f"{name}@example.com"]
-        text = sent[0].get_body(("plain",)).get_content()
+        await _deliver(app)
+        from tests.integration._mail import to
+        mine = to(sent, f"{name}@example.com", f"{sso_name}@example.com", f"nobody-{uid[:8]}@example.com")
+        assert [m["To"] for m in mine] == [f"{name}@example.com"]
+        text = mine[0].get_body(("plain",)).get_content()
         assert name in text and "BST" in text
         key = _KEY.search(text).group(1)
 
@@ -197,7 +199,10 @@ def test_asking_again_kills_the_first_link_and_an_old_link_expires(app, monkeypa
         guest = await _stranger(app)
         for _ in range(2):
             await guest.post("/forgot", headers=CSRF, form={"name": name})
-        first, second = (_KEY.search(m.get_body(("plain",)).get_content()).group(1) for m in sent)
+        await _deliver(app)
+        from tests.integration._mail import to
+        first, second = (_KEY.search(m.get_body(("plain",)).get_content()).group(1)
+                         for m in to(sent, f"{name}@example.com"))
         assert (await guest.get(f"/reset/{first}")).status_code == 404
         assert (await guest.get(f"/reset/{second}")).status_code == 200