feat(email): delivery you can see: each alert's email in history, outgoing email on Settings -> System (#5509)

#5509 "Delivery you can see: each alert's email state in history, and
outgoing email on Settings -> System" (milestone 501).

- Alerts -> History: an alert's email reads from the outbox (#5506):
  "gathering" with when it sends, "sending", "retrying" with the attempt,
  next try and the server's last answer, "sent", or "failed" naming each
  reader it didn't reach and why (outbox.delivery / email_states). Events
  from before the outbox keep what they recorded. The Notified column
  shows once an alert has a queued email.
- Settings -> System gets "Outgoing email": every email of the last 7
  days not yet sent, failures first, then the longest retrying, with
  recipient, kind, state and last error, and a count of what was sent.
  Failures and anything still retrying after 5 minutes lead the page's
  verdict (outbox.outgoing).
- get_alert_history (MCP) adds `email`: the same delivery state.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_alert_episodes.py

@@ -74,14 +74,18 @@ def test_an_episode_split_by_paging_says_so():
     assert not newer[0]["began_here"] and newer[0]["end"] == NOW - timedelta(minutes=10)
 
 
+def _d(label, state="sent", error=None):
+    return {"label": label, "state": state, "at": None, "attempts": None, "error": error}
+
+
 def test_who_was_told_is_gathered_once_per_episode():
     r = _rule()
     eps = episodes(_newest_first(
-        _t(r, 30, "inactive", "firing", 200.0, deliveries=[("email", False, "timed out"), ("webhook", True, None)],
+        _t(r, 30, "inactive", "firing", 200.0, deliveries=[_d("email", "failed", "timed out"), _d("webhook")],
            run="run-1"),
-        _t(r, 20, "firing", "resolved", 5.0, deliveries=[("webhook", True, None)], muted_by="patch")),
+        _t(r, 20, "firing", "resolved", 5.0, deliveries=[_d("webhook")], muted_by="patch")),
         newest_page=True, now=NOW, since=SINCE)
-    assert eps[0]["deliveries"] == [("email", False, "timed out"), ("webhook", True, None)]
+    assert eps[0]["deliveries"] == [_d("email", "failed", "timed out"), _d("webhook")]
     assert eps[0]["muted_by"] == "patch" and eps[0]["runs"] == ["run-1"]
 
 

tests/core/test_alert_history_row.py

@@ -14,7 +14,7 @@ RULE = SimpleNamespace(metric_name="cpu_pct", operator=AlertOperator(">"), thres
 
 
 def _event(**kw):
-    base = dict(transitioned_at=datetime(2026, 10, 6, tzinfo=timezone.utc), from_state="inactive",
+    base = dict(id="ev-1", transitioned_at=datetime(2026, 10, 6, tzinfo=timezone.utc), from_state="inactive",
                 to_state="firing", metric_value=97.5, note=None, email_sent=None, email_error=None,
                 webhook_sent=None, webhook_error=None, ansible_run_id=None, actor_id=None)
     return SimpleNamespace(**{**base, **kw})
@@ -44,12 +44,25 @@ def test_value_and_condition_read_in_words():
 
 def test_only_attempted_deliveries_are_listed():
     row = _history_row(_event(email_sent=False, email_error="timed out", webhook_sent=None), RULE)
-    assert row["deliveries"] == [("email", False, "timed out")]
+    assert [(d["label"], d["state"], d["error"]) for d in row["deliveries"]] == [("email", "failed", "timed out")]
     row = _history_row(_event(email_sent=True, webhook_sent=True), RULE)
-    assert [(label, sent) for label, sent, _ in row["deliveries"]] == [("email", True), ("webhook", True)]
+    assert [(d["label"], d["state"]) for d in row["deliveries"]] == [("email", "sent"), ("webhook", "sent")]
     assert _history_row(_event(), RULE)["deliveries"] == []
 
 
+def test_an_email_through_the_outbox_shows_its_live_state():
+    """#5509: the outbox's state for the event outranks what the event holds
+    (nothing yet, while it gathers or retries)."""
+    from datetime import datetime, timezone
+    at = datetime(2026, 10, 9, 19, 50, tzinfo=timezone.utc)
+    ev = _event(email_sent=None, webhook_sent=True)
+    live = {ev.id: {"state": "retrying", "at": at, "attempts": 3, "error": "451 greylisted"}}
+    row = _history_row(ev, RULE, emails=live)
+    assert row["deliveries"][0] == {"label": "email", "state": "retrying", "at": at, "attempts": 3,
+                                    "error": "451 greylisted"}
+    assert row["deliveries"][1]["label"] == "webhook"
+
+
 def test_a_transition_a_person_made_says_who():
     """#5463: an acknowledgement, and a rule change that reset the alert,
     name the person; one whose account is gone still reads as a person's."""

tests/core/test_outbox.py

@@ -223,3 +223,23 @@ def test_the_smtp_guard_can_fail():
     assert _smtp_openers({"x.py": "    with smtplib.SMTP(host) as s:"}) == {"x.py"}
     assert _smtp_openers({"y.py": "    smtp = notifications.smtp_open(cfg)"}) == {"y.py"}
     assert _smtp_openers({"z.py": "# smtp_open(cfg) is the outbox's\ndef smtp_open(cfg):"}) == set()
+
+
+# ── How an alert's email reads in history (#5509) ────────────────────────────
+
+def test_an_alerts_email_reads_by_its_least_finished_copy():
+    later, sent_at = NOW + timedelta(minutes=2), NOW + timedelta(seconds=5)
+    held = _row(state="held", send_after=later, created_at=NOW)
+    retrying = _row(recipient="b@x", state="pending", attempts=3, next_attempt_at=later,
+                    last_error="451 greylisted", created_at=NOW)
+    sent = _row(recipient="c@x", state="sent", attempts=1, sent_at=sent_at, created_at=NOW)
+    failed = _row(recipient="d@x", state="failed", attempts=1, first_attempt_at=NOW,
+                  last_error="550 no such user", created_at=NOW)
+    first = _row(recipient="e@x", state="pending", attempts=0, next_attempt_at=NOW, created_at=NOW)
+    assert outbox.delivery([sent, held, failed]) == {"state": "held", "at": later, "attempts": 0, "error": None}
+    assert outbox.delivery([sent, retrying]) == {"state": "retrying", "at": later, "attempts": 3,
+                                                 "error": "451 greylisted"}
+    assert outbox.delivery([first])["state"] == "sending"
+    assert outbox.delivery([sent, failed]) == {"state": "failed", "at": NOW, "attempts": 1,
+                                               "error": "d@x: 550 no such user"}
+    assert outbox.delivery([sent]) == {"state": "sent", "at": sent_at, "attempts": 1, "error": None}

tests/core/test_steward_health.py

@@ -13,10 +13,15 @@ STORAGE = {"total": 3 * 2**30, "rest": {"count": 0, "bytes": 0},
                       {"name": "docker_logs", "bytes": 512 * 2**20, "rows": None}]}
 
 
-async def _page(app, ctx, storage=STORAGE, cleanup=None):
+NO_EMAIL = {"rows": [], "more": 0, "sent": 0, "problems": []}
+
+
+async def _page(app, ctx, storage=STORAGE, cleanup=None, email=NO_EMAIL):
+    from steward.core.outbox import KIND_LABELS
     async with app.test_request_context("/settings/system/"):
         session["user_role"] = "admin"
-        page = await render_template("settings/system.html", **ctx, storage=storage, cleanup=cleanup)
+        page = await render_template("settings/system.html", **ctx, storage=storage, cleanup=cleanup,
+                                     email=email, kind_labels=KIND_LABELS)
     return " ".join(page.split())
 
 
@@ -109,3 +114,46 @@ async def test_the_last_cleanup_keeps_only_what_went(monkeypatch):
     assert got["removed"] == [("docker_logs: past the age window", 7)]
     stored = {}
     assert await health.last_cleanup(None) is None
+
+
+# ── Outgoing email (#5509) ────────────────────────────────────────────────────
+
+def _mail(**kw):
+    from steward.models.outbox import EmailOutbox
+    base = dict(kind="alert", recipient="ann@example.com", state="pending", attempts=0,
+                created_at=AT, send_after=None, next_attempt_at=None, last_error=None)
+    return EmailOutbox(**{**base, **kw})
+
+
+async def test_outgoing_email_lists_what_has_not_gone_failures_first(app, previews):
+    previews()
+    got = await health.process_health(None)
+    for r in got["rows"]:
+        r["spark"] = ""
+    email = {"sent": 1234, "more": 3, "problems": ["1 email failed in the last 7 days"], "rows": [
+        _mail(state="failed", kind="invite", recipient="bob@example.com", attempts=1,
+              last_error="550 no such user"),
+        _mail(state="pending", attempts=3, next_attempt_at=AT, last_error="451 greylisted"),
+        _mail(state="held", kind="alert", send_after=AT)]}
+    got["problems"] = email["problems"] + got["problems"]
+    page = await _page(app, got, email=email)
+    # The verdict leads with it; the section says each one's state and why.
+    assert '<p class="page-verdict"><span class="tone-warn">1 email failed in the last 7 days.' in page
+    assert "1,234 sent in the last 7 days." in page and "Nothing waiting" not in page
+    assert page.index("bob@example.com") < page.index("451 greylisted")
+    assert "Invite" in page and "550 no such user" in page
+    assert "retrying</span> <span class=\"tone-dim\">attempt 3, next" in page
+    assert "gathering</span> <span class=\"tone-dim\">sends" in page
+    assert "+3 more not yet sent" in page
+    assert 'title="<' not in page
+    assert page.index("Steward itself") < page.index("Outgoing email") < page.index("Storage")
+
+
+async def test_nothing_waiting_says_so(app, previews):
+    previews()
+    got = await health.process_health(None)
+    for r in got["rows"]:
+        r["spark"] = ""
+    page = await _page(app, got, email={**NO_EMAIL, "sent": 2})
+    assert "Nothing waiting to send. 2 sent in the last 7 days." in page
+    assert "Steward is running normally." in page

tests/integration/test_email_delivery_pages.py

@@ -0,0 +1,99 @@
+"""Integration: email delivery you can see (#5509), against Postgres.
+
+An alert's email gathering for its window reads so in the alert's history
+and on Settings → System; once refused it reads "failed" with the server's
+words in both places, and the System page's verdict names it.
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import smtplib
+import time
+import uuid
+from datetime import datetime, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+async def _admin(app, tag):
+    from steward.models.users import User, UserRole
+    uid = str(uuid.uuid4())
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            s.add(User(id=uid, username=f"dl-{tag}", email=f"dl-{tag}@test", password_hash="x",
+                       role=UserRole.admin, is_active=True))
+    client = app.test_client()
+    async with client.session_transaction() as sess:
+        sess.update(user_id=uid, user_role="admin", username=f"dl-{tag}", _csrf="test-token",
+                    login_at=time.time())
+    return client, uid
+
+
+def test_an_alerts_email_reads_gathering_then_failed_in_history_and_on_system(app, monkeypatch):
+    from steward.alerts import service as alert_svc
+    from steward.core import outbox
+    from steward.core.alerts import record_metrics
+    from steward.models.alerts import AlertEvent
+    from steward.models.metrics import PluginMetric
+    from tests.integration._mail import capture
+
+    tag = uuid.uuid4().hex[:8]
+    extra = f"pager-{tag}@example.com"
+    capture(app, monkeypatch, lambda m: smtplib.SMTPRecipientsRefused({m["To"]: (550, b"mailbox gone")})
+            if m["To"] == extra else None)
+    app.config["SMTP"]["recipients"] = [extra]
+
+    async def _go():
+        client, uid = await _admin(app, tag)
+        now = datetime.now(timezone.utc)
+        res = f"box-{tag}"
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await record_metrics(s, [PluginMetric(source_module="host_agent", resource_name=res,
+                                                      metric_name="cpu_pct", value=1.0)])
+                await s.flush()
+                fields = await alert_svc.rule_fields(
+                    s, name=f"hot {tag}", source_module="host_agent", resource_name=res,
+                    metric_name="cpu_pct", operator=">", threshold=90)
+                rule = await alert_svc.add_rule(s, fields, created_by=uid)
+                event = AlertEvent(rule_id=rule.id, from_state="inactive", to_state="firing",
+                                   metric_value=95.0, transitioned_at=now)
+                s.add(event)
+            rule_id, event_id = rule.id, event.id
+        await outbox.hold_alert(app, {"rule_name": f"hot {tag}", "state": "FIRING", "metric": "cpu_pct",
+                                      "value": 95.0, "operator": ">", "threshold": 90.0, "resource": res,
+                                      "source_module": "host_agent", "note": None,
+                                      "timestamp": now.isoformat()}, event_id, now=now)
+
+        history = await (await client.get(f"/alerts/history?rule={rule_id}")).get_data(as_text=True)
+        assert "email gathering" in history and "waiting for other alerts; sends" in history
+        system = await (await client.get("/settings/system/")).get_data(as_text=True)
+        assert extra in system and "gathering" in system
+
+        # The window closes; the server refuses the pager address.
+        from sqlalchemy import select
+        from steward.models.outbox import email_outbox_events as link
+        async with app.db_sessionmaker() as s:
+            ids = list((await s.execute(select(link.c.outbox_id).where(
+                link.c.event_id == event_id))).scalars())
+        await outbox.attempt(app, ids)
+
+        history = await (await client.get(f"/alerts/history?rule={rule_id}")).get_data(as_text=True)
+        assert "email failed" in history and f"{extra}: 550 mailbox gone" in history
+        system = " ".join((await (await client.get("/settings/system/")).get_data(as_text=True)).split())
+        verdict = system[system.index("page-verdict"):system.index("</p>", system.index("page-verdict"))]
+        assert "failed in the last 7 days" in verdict
+        assert extra in system and "550 mailbox gone" in system
+
+    asyncio.run(_go())