feat(email): delivery you can see: each alert's email in history, outgoing email on Settings -> System (#5509)

#5509 "Delivery you can see: each alert's email state in history, and
outgoing email on Settings -> System" (milestone 501).

- Alerts -> History: an alert's email reads from the outbox (#5506):
  "gathering" with when it sends, "sending", "retrying" with the attempt,
  next try and the server's last answer, "sent", or "failed" naming each
  reader it didn't reach and why (outbox.delivery / email_states). Events
  from before the outbox keep what they recorded. The Notified column
  shows once an alert has a queued email.
- Settings -> System gets "Outgoing email": every email of the last 7
  days not yet sent, failures first, then the longest retrying, with
  recipient, kind, state and last error, and a count of what was sent.
  Failures and anything still retrying after 5 minutes lead the page's
  verdict (outbox.outgoing).
- get_alert_history (MCP) adds `email`: the same delivery state.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/core/alerting.md

@@ -184,6 +184,14 @@ seconds and settles each email on the server's answer:
 Once every email carrying an alert has settled, that alert's event records
 whether the email went, and to whom it didn't.
 
+Where to see it: each alert's row in **Alerts → History** says what its email
+is doing (gathering, with when it sends; retrying, with the attempt, the next
+try and the server's last answer; sent; failed, with who wasn't reached and
+why), and `get_alert_history` over MCP reports the same. **Settings → System →
+Outgoing email** lists every email of the last 7 days not yet sent, failures
+first, and the page's verdict names failures and any email still retrying
+after 5 minutes.
+
 An email a person is waiting on is tried at once instead of on the next tick:
 the weekly report's **Send now**, an invite, and the test buttons, so the page
 can say what the server answered. A report or invite the server couldn't take

steward/alerts/episodes.py

@@ -87,8 +87,8 @@ def episodes(rows: list[dict], *, newest_page: bool, now: datetime, since: datet
         seen, deliveries = set(), []
         for s in steps:
             for d in s["deliveries"]:
-                if (d[0], d[1]) not in seen:
-                    seen.add((d[0], d[1]))
+                if (d["label"], d["state"]) not in seen:
+                    seen.add((d["label"], d["state"]))
                     deliveries.append(d)
         ep["deliveries"] = deliveries
         ep["muted_by"] = next((s["muted_by"] for s in steps if s["muted_by"]), None)

steward/alerts/routes.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 from datetime import datetime, timezone
 from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
 from steward.core.audit import log_audit
+from steward.core import outbox
 from sqlalchemy import func, select
 from steward.auth.middleware import require_role
 from steward.alerts import service as alert_svc
@@ -177,9 +178,20 @@ async def active_fragment():
         rules_off=sum(1 for r in rules if not r.enabled))
 
 
-def _history_row(event, rule, names: dict | None = None) -> dict:
+def _delivery(label: str, sent: bool | None, error: str | None) -> dict | None:
+    """A channel's outcome as recorded on the event; None when not tried."""
+    if sent is None:
+        return None
+    return {"label": label, "state": "sent" if sent else "failed", "at": None, "attempts": None,
+            "error": None if sent else error}
+
+
+def _history_row(event, rule, names: dict | None = None, emails: dict | None = None) -> dict:
     """One transition as the history page shows it. `names` maps user ids to
-    usernames: a transition a person made says who (#5463)."""
+    usernames: a transition a person made says who (#5463). `emails` is
+    outbox.email_states for the page: the email's live state (gathering,
+    retrying, sent, failed) where it went through the outbox (#5509), else
+    what the event recorded."""
     from steward.core.alerts import MUTED, RELEASED
     from steward.core.formatting import metric_label, metric_value
     notes, muted_by, released = [], None, False
@@ -208,9 +220,10 @@ def _history_row(event, rule, names: dict | None = None) -> dict:
         "from_state": event.from_state, "to_state": event.to_state,
         "value": value, "raw": event.metric_value, "condition": alert_svc.rule_condition(rule),
         "notes": notes, "muted_by": muted_by, "released": released,
-        "deliveries": [(label, sent, error) for label, sent, error in (
-            ("email", event.email_sent, event.email_error),
-            ("webhook", event.webhook_sent, event.webhook_error)) if sent is not None],
+        "deliveries": [d for d in (
+            {"label": "email", **(emails or {})[event.id]} if event.id in (emails or {})
+            else _delivery("email", event.email_sent, event.email_error),
+            _delivery("webhook", event.webhook_sent, event.webhook_error)) if d],
         "ansible_run_id": event.ansible_run_id,
     }
 
@@ -235,13 +248,14 @@ async def alert_history():
         rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before)
         show_notified, show_action = await alert_svc.history_columns(db, since, rule_id)
         names = await alert_svc.usernames(db, (e.actor_id for e, _ in rows))
+        emails = await outbox.email_states(db, [e.id for e, _ in rows])
         verdict = None
         if chosen and not before:
             every = await alert_svc.history(db, since, rule_id, HISTORY_SUMMARY_ROWS)
             verdict = summary(episodes([_history_row(e, r) for e, r in every], newest_page=True,
                                        now=now, since=since))
     rows, after = next_page(rows, lambda row: (row[0].transitioned_at, row[0].id))
-    ctx = {"episodes": episodes([_history_row(e, r, names) for e, r in rows],
+    ctx = {"episodes": episodes([_history_row(e, r, names, emails) for e, r in rows],
                                 newest_page=not before, now=now, since=since),
            "after": after, "rules": rules, "chosen": chosen, "current_range": range_key,
            "show_notified": show_notified, "show_action": show_action, "verdict": verdict}

steward/alerts/service.py

@@ -322,9 +322,12 @@ async def history_columns(db, since: datetime, rule_id: str | None = None) -> tu
     base = select(AlertEvent.id).where(AlertEvent.transitioned_at >= since)
     if rule_id:
         base = base.where(AlertEvent.rule_id == rule_id)
+    from steward.models.outbox import email_outbox_events
+    queued = select(email_outbox_events.c.event_id).where(
+        email_outbox_events.c.event_id == AlertEvent.id).exists()
     notified = await db.scalar(select(base.where(or_(
         AlertEvent.email_sent.is_not(None), AlertEvent.webhook_sent.is_not(None),
-        AlertEvent.note.like(f"%{MUTED}%"))).exists()))
+        AlertEvent.note.like(f"%{MUTED}%"), queued)).exists()))
     acted = await db.scalar(select(base.where(AlertEvent.ansible_run_id.is_not(None)).exists()))
     return bool(notified), bool(acted)
 

steward/core/outbox.py

@@ -356,3 +356,81 @@ def said(rows: list[EmailOutbox]) -> tuple[bool, str]:
         ([f"Not sent yet to {'; '.join(later)}; Steward will try again for up to an hour."]
          if later else [])
     return bool(rows) and not failed and not later, " ".join(parts)
+
+
+# ── What the pages show (#5509) ───────────────────────────────────────────────
+
+KIND_LABELS = {"alert": "Alert", "report": "Weekly report", "invite": "Invite",
+               "reset": "Password reset", "test": "Test"}
+# Retrying longer than this is worth a line in Settings → System's verdict.
+SLOW_S = 300
+SHOWN_DAYS = 7
+
+
+def delivery(rows: list[EmailOutbox]) -> dict:
+    """One alert event's email, over every reader's copy, as history says
+    it: {"state", "at", "attempts", "error"}. Gathering while any copy is
+    held (at: when it sends); sending/retrying while any is still due (at:
+    the next try); failed when any reader wasn't reached (error names each);
+    else sent (at: the last acceptance)."""
+    if held := [r for r in rows if r.state == "held"]:
+        return {"state": "held", "at": min(r.send_after for r in held), "attempts": 0, "error": None}
+    if due := [r for r in rows if r.state in ("pending", "sending")]:
+        worst = max(due, key=lambda r: r.attempts)
+        return {"state": "retrying" if worst.attempts else "sending",
+                "at": min((r.next_attempt_at for r in due if r.next_attempt_at), default=None),
+                "attempts": worst.attempts, "error": worst.last_error}
+    if failed := [r for r in rows if r.state == "failed"]:
+        return {"state": "failed", "at": max(r.first_attempt_at or r.created_at for r in failed),
+                "attempts": max(r.attempts for r in failed),
+                "error": "; ".join(f"{r.recipient}: {r.last_error}" for r in failed)}
+    return {"state": "sent", "at": max((r.sent_at for r in rows if r.sent_at), default=None),
+            "attempts": max(r.attempts for r in rows), "error": None}
+
+
+async def email_states(db, event_ids: list[str]) -> dict[str, dict]:
+    """{event id: delivery(...)} for the alert events whose email went
+    through the outbox; an event with none (no reader, or from before it)
+    is absent."""
+    if not event_ids:
+        return {}
+    link = email_outbox_events.c
+    by: dict[str, list] = {}
+    for event_id, row in (await db.execute(
+            select(link.event_id, EmailOutbox).join(EmailOutbox, EmailOutbox.id == link.outbox_id)
+            .where(link.event_id.in_(event_ids)))).all():
+        by.setdefault(event_id, []).append(row)
+    return {eid: delivery(rows) for eid, rows in by.items()}
+
+
+async def outgoing(db, now: datetime | None = None, limit: int = 20) -> dict:
+    """Settings → System's "Outgoing email": every email of the last
+    SHOWN_DAYS not yet sent (failed first, then retrying longest, then
+    sending and gathering), how many were sent, and the problems for the
+    page's verdict. {"rows": [EmailOutbox], "more": n, "sent": n,
+    "problems": [str]}."""
+    from sqlalchemy import case, func
+    now = now or datetime.now(timezone.utc)
+    since = now - timedelta(days=SHOWN_DAYS)
+    open_states = ("failed", "pending", "sending", "held")
+    order = case({"failed": 0, "pending": 1, "sending": 2, "held": 3}, value=EmailOutbox.state)
+    base = select(EmailOutbox).where(EmailOutbox.created_at >= since,
+                                     EmailOutbox.state.in_(open_states))
+    rows = list((await db.execute(
+        base.order_by(order, EmailOutbox.first_attempt_at.asc().nulls_last(),
+                      EmailOutbox.created_at.desc()).limit(limit))).scalars())
+    counts = dict((await db.execute(
+        select(EmailOutbox.state, func.count()).where(EmailOutbox.created_at >= since)
+        .group_by(EmailOutbox.state))).all())
+    slow = (await db.execute(select(func.count()).select_from(EmailOutbox).where(
+        EmailOutbox.state.in_(("pending", "sending")),
+        EmailOutbox.first_attempt_at < now - timedelta(seconds=SLOW_S)))).scalar_one()
+    problems = []
+    if failed := counts.get("failed", 0):
+        problems.append(f"{failed} email{'s' if failed != 1 else ''} failed in the last {SHOWN_DAYS} days")
+    if slow:
+        problems.append(f"{slow} email{'s' if slow != 1 else ''} still not accepted by the mail "
+                        f"server after {SLOW_S // 60} minutes of trying")
+    shown = sum(counts.get(s, 0) for s in open_states)
+    return {"rows": rows, "more": max(shown - len(rows), 0), "sent": counts.get("sent", 0),
+            "problems": problems}

steward/mcp/tools/alerts.py

@@ -6,6 +6,7 @@ from datetime import datetime, timedelta, timezone
 from sqlalchemy import select
 
 from steward.alerts import service as alert_svc
+from steward.core import outbox
 from steward.mcp._context import current_caller, mcp_app
 from steward.models.alerts import AlertOperator, AlertRule, AlertState, MaintenanceWindow
 from steward.models.users import UserRole
@@ -82,7 +83,11 @@ async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50)
 
     Each transition carries the metric value that caused it and whether the
     email/webhook notification went out (with the error if not), plus the id
-    of any Ansible run the rule's action triggered. `by` names the person who
+    of any Ansible run the rule's action triggered. `email` is the email's
+    delivery: state "held" (gathering other alerts; `at` is when it sends),
+    "sending", "retrying" (`attempts`, next try `at`, last `error`), "sent"
+    or "failed" (`error` names each reader not reached); null when no email
+    was queued for it. `by` names the person who
     made it happen (an acknowledgement, or a rule edit that reset the alert);
     null when a reading or Steward itself did.
     """
@@ -93,6 +98,7 @@ async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50)
         chosen = pick(rule, rules, kind="alert rule", list_tool="list_alert_rules") if rule else None
         rows = await alert_svc.history(db, since, chosen.id if chosen else None, limit)
         names = await alert_svc.usernames(db, (e.actor_id for e, _ in rows))
+        emails = await outbox.email_states(db, [e.id for e, _ in rows])
     events = [
         {
             "rule_id": r.id,
@@ -106,6 +112,7 @@ async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50)
             "age_s": age_s(e.transitioned_at),
             "email_sent": e.email_sent,
             "email_error": e.email_error,
+            "email": ({**emails[e.id], "at": iso(emails[e.id]["at"])} if e.id in emails else None),
             "webhook_sent": e.webhook_sent,
             "webhook_error": e.webhook_error,
             "ansible_run_id": e.ansible_run_id,

steward/settings/routes.py

@@ -630,7 +630,10 @@ async def _reports_page(settings: dict, flash):
 async def system():
     """Steward's own health (#5467): its process metrics now and over the day,
     each one an alert rule can watch (source "Steward itself"); then its
-    tables' sizes and what the last hourly cleanup removed (#5469)."""
+    tables' sizes and what the last hourly cleanup removed (#5469); and
+    the email not yet sent (#5509), whose failures and slow retries join the
+    page's verdict."""
+    from steward.core import outbox
     from steward.core.health import last_cleanup, process_health, table_sizes
     from steward.core.status import sparkline_svg
     async with current_app.db_sessionmaker() as db:
@@ -638,11 +641,14 @@ async def system():
         # Table sizes and the last cleanup (#5469): Postgres's catalog and
         # what the cleanup kept, never a scan of the tables themselves.
         storage, cleanup = await table_sizes(db), await last_cleanup(db)
+        email = await outbox.outgoing(db)
+    health["problems"] = email["problems"] + health["problems"]
     for row in health["rows"]:
         # From zero to the row's own peak, with a floor (UI & Design ruling 8).
         row["spark"] = sparkline_svg(row["series"], width=120, height=20,
                                      stroke="currentColor", floor=1.0) if len(row["series"]) > 1 else ""
-    return await render_template("settings/system.html", **health, storage=storage, cleanup=cleanup)
+    return await render_template("settings/system.html", **health, storage=storage, cleanup=cleanup,
+                                 email=email, kind_labels=outbox.KIND_LABELS)
 
 
 @settings_bp.get("/reports/")

steward/templates/alerts/_history_page.html

@@ -48,11 +48,20 @@
           <span class="badge badge-yellow" title="A maintenance window muted this notification">muted · {{ ep.muted_by }}</span>
           {% if ep.released %}<div class="cell-note">notified when the window ended</div>{% endif %}
           {% endif %}
-          {% for label, sent, error in ep.deliveries %}
+          {# Each channel's outcome; an email through the outbox (#5509) also
+             says when it sends, or how its retries stand, or why it failed. #}
+          {% for d in ep.deliveries %}
           <div>
-            {% if sent %}<span class="badge badge-green">{{ label }} sent</span>
-            {% else %}<span class="badge badge-red">{{ label }} failed</span>
-            {% if error %}<div class="cell-note" style="max-width:28rem;overflow-wrap:anywhere;">{{ error }}</div>{% endif %}
+            {% if d.state == "sent" %}<span class="badge badge-green">{{ d.label }} sent</span>
+            {%- if d.at %} <span class="tone-dim">{{ d.at|when }}</span>{% endif %}
+            {% elif d.state == "held" %}<span class="badge badge-yellow">{{ d.label }} gathering</span>
+            <div class="cell-note">waiting for other alerts; sends {{ d.at|when }}</div>
+            {% elif d.state == "sending" %}<span class="badge badge-yellow">{{ d.label }} sending</span>
+            {% elif d.state == "retrying" %}<span class="badge badge-yellow">{{ d.label }} retrying</span>
+            <div class="cell-note" style="max-width:28rem;overflow-wrap:anywhere;">attempt {{ d.attempts }}
+              {%- if d.at %}, next {{ d.at|when }}{% endif %}{% if d.error %}: {{ d.error }}{% endif %}</div>
+            {% else %}<span class="badge badge-red">{{ d.label }} failed</span>
+            {% if d.error %}<div class="cell-note" style="max-width:28rem;overflow-wrap:anywhere;">{{ d.error }}</div>{% endif %}
             {% endif %}
           </div>
           {% endfor %}

steward/templates/settings/system.html

@@ -9,8 +9,8 @@
 {% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("System", "")]) }}{% endblock %}
 {% block content %}
 <p class="page-verdict">
-  {%- if not seen %}<span class="tone-dim">No readings yet: the first arrive within a minute of Steward starting.</span>
-  {%- elif problems %}{% set said = problems|join("; ") %}<span class="tone-warn">{{ said[:1]|upper }}{{ said[1:] }}.</span>
+  {%- if problems %}{% set said = problems|join("; ") %}<span class="tone-warn">{{ said[:1]|upper }}{{ said[1:] }}.</span>
+  {%- elif not seen %}<span class="tone-dim">No readings yet: the first arrive within a minute of Steward starting.</span>
   {%- else %}<span class="tone-ok">Steward is running normally.</span>{% endif -%}
 </p>
 
@@ -39,6 +39,46 @@
   Recorded each minute (the cleanup each hour). Any of these can be watched by an
   <a href="/alerts/rules/new">alert rule</a>: pick the source "Steward itself".</p>
 
+{# Outgoing email (#5509): every email waits in the outbox until the mail
+   server takes it. What hasn't gone yet, failures first, then the longest
+   retrying; sent mail is only counted. #}
+<h2 class="section-title" style="margin:2rem 0 0.5rem;">Outgoing email</h2>
+<p style="margin:0 0 1rem;">
+  {%- if not email.rows %}Nothing waiting to send. {% endif -%}
+  {{ "{:,}".format(email.sent) }} sent in the last 7 days.</p>
+{% if email.rows %}
+<div class="card-flush">
+  <table class="table">
+    <thead><tr><th>To</th><th>What</th><th>State</th><th>Queued</th></tr></thead>
+    <tbody>
+      {% for r in email.rows %}
+      <tr>
+        <td class="cell-name"><span title="{{ r.recipient }}">{{ r.recipient }}</span></td>
+        <td class="nowrap">{{ kind_labels.get(r.kind, r.kind) }}</td>
+        <td>
+          {%- if r.state == "failed" %}<span class="badge badge-red">failed</span>
+          {%- elif r.state == "pending" and r.attempts %}<span class="badge badge-yellow">retrying</span>
+            <span class="tone-dim">attempt {{ r.attempts }}{% if r.next_attempt_at %}, next {{ r.next_attempt_at|when }}{% endif %}</span>
+          {%- elif r.state == "held" %}<span class="badge badge-yellow">gathering</span>
+            <span class="tone-dim">sends {{ r.send_after|when }}</span>
+          {%- else %}<span class="badge badge-yellow">sending</span>{% endif %}
+          {%- if r.last_error %}<div class="cell-note" style="max-width:32rem;overflow-wrap:anywhere;">{{ r.last_error }}</div>{% endif -%}
+        </td>
+        <td class="nowrap">{{ r.created_at|ago }}</td>
+      </tr>
+      {% endfor %}
+      {% if email.more %}
+      <tr><td colspan="4" class="tone-dim">+{{ email.more }} more not yet sent</td></tr>
+      {% endif %}
+    </tbody>
+  </table>
+</div>
+{% endif %}
+<p class="field-hint" style="margin-top:0.75rem;">
+  An email the server refuses is failed at once; one it can't take yet is tried
+  again for up to an hour. The mail server is set under
+  <a href="/settings/notifications/">Notifications</a>.</p>
+
 {# Storage (#5469): Postgres's own figures for each table, then what the
    hourly cleanup last removed. Read from the catalog, never by counting. #}
 <h2 class="section-title" style="margin:2rem 0 0.5rem;">Storage</h2>

tests/core/test_alert_episodes.py

@@ -74,14 +74,18 @@ def test_an_episode_split_by_paging_says_so():
     assert not newer[0]["began_here"] and newer[0]["end"] == NOW - timedelta(minutes=10)
 
 
+def _d(label, state="sent", error=None):
+    return {"label": label, "state": state, "at": None, "attempts": None, "error": error}
+
+
 def test_who_was_told_is_gathered_once_per_episode():
     r = _rule()
     eps = episodes(_newest_first(
-        _t(r, 30, "inactive", "firing", 200.0, deliveries=[("email", False, "timed out"), ("webhook", True, None)],
+        _t(r, 30, "inactive", "firing", 200.0, deliveries=[_d("email", "failed", "timed out"), _d("webhook")],
            run="run-1"),
-        _t(r, 20, "firing", "resolved", 5.0, deliveries=[("webhook", True, None)], muted_by="patch")),
+        _t(r, 20, "firing", "resolved", 5.0, deliveries=[_d("webhook")], muted_by="patch")),
         newest_page=True, now=NOW, since=SINCE)
-    assert eps[0]["deliveries"] == [("email", False, "timed out"), ("webhook", True, None)]
+    assert eps[0]["deliveries"] == [_d("email", "failed", "timed out"), _d("webhook")]
     assert eps[0]["muted_by"] == "patch" and eps[0]["runs"] == ["run-1"]
 
 

tests/core/test_alert_history_row.py

@@ -14,7 +14,7 @@ RULE = SimpleNamespace(metric_name="cpu_pct", operator=AlertOperator(">"), thres
 
 
 def _event(**kw):
-    base = dict(transitioned_at=datetime(2026, 10, 6, tzinfo=timezone.utc), from_state="inactive",
+    base = dict(id="ev-1", transitioned_at=datetime(2026, 10, 6, tzinfo=timezone.utc), from_state="inactive",
                 to_state="firing", metric_value=97.5, note=None, email_sent=None, email_error=None,
                 webhook_sent=None, webhook_error=None, ansible_run_id=None, actor_id=None)
     return SimpleNamespace(**{**base, **kw})
@@ -44,12 +44,25 @@ def test_value_and_condition_read_in_words():
 
 def test_only_attempted_deliveries_are_listed():
     row = _history_row(_event(email_sent=False, email_error="timed out", webhook_sent=None), RULE)
-    assert row["deliveries"] == [("email", False, "timed out")]
+    assert [(d["label"], d["state"], d["error"]) for d in row["deliveries"]] == [("email", "failed", "timed out")]
     row = _history_row(_event(email_sent=True, webhook_sent=True), RULE)
-    assert [(label, sent) for label, sent, _ in row["deliveries"]] == [("email", True), ("webhook", True)]
+    assert [(d["label"], d["state"]) for d in row["deliveries"]] == [("email", "sent"), ("webhook", "sent")]
     assert _history_row(_event(), RULE)["deliveries"] == []
 
 
+def test_an_email_through_the_outbox_shows_its_live_state():
+    """#5509: the outbox's state for the event outranks what the event holds
+    (nothing yet, while it gathers or retries)."""
+    from datetime import datetime, timezone
+    at = datetime(2026, 10, 9, 19, 50, tzinfo=timezone.utc)
+    ev = _event(email_sent=None, webhook_sent=True)
+    live = {ev.id: {"state": "retrying", "at": at, "attempts": 3, "error": "451 greylisted"}}
+    row = _history_row(ev, RULE, emails=live)
+    assert row["deliveries"][0] == {"label": "email", "state": "retrying", "at": at, "attempts": 3,
+                                    "error": "451 greylisted"}
+    assert row["deliveries"][1]["label"] == "webhook"
+
+
 def test_a_transition_a_person_made_says_who():
     """#5463: an acknowledgement, and a rule change that reset the alert,
     name the person; one whose account is gone still reads as a person's."""

tests/core/test_outbox.py

@@ -223,3 +223,23 @@ def test_the_smtp_guard_can_fail():
     assert _smtp_openers({"x.py": "    with smtplib.SMTP(host) as s:"}) == {"x.py"}
     assert _smtp_openers({"y.py": "    smtp = notifications.smtp_open(cfg)"}) == {"y.py"}
     assert _smtp_openers({"z.py": "# smtp_open(cfg) is the outbox's\ndef smtp_open(cfg):"}) == set()
+
+
+# ── How an alert's email reads in history (#5509) ────────────────────────────
+
+def test_an_alerts_email_reads_by_its_least_finished_copy():
+    later, sent_at = NOW + timedelta(minutes=2), NOW + timedelta(seconds=5)
+    held = _row(state="held", send_after=later, created_at=NOW)
+    retrying = _row(recipient="b@x", state="pending", attempts=3, next_attempt_at=later,
+                    last_error="451 greylisted", created_at=NOW)
+    sent = _row(recipient="c@x", state="sent", attempts=1, sent_at=sent_at, created_at=NOW)
+    failed = _row(recipient="d@x", state="failed", attempts=1, first_attempt_at=NOW,
+                  last_error="550 no such user", created_at=NOW)
+    first = _row(recipient="e@x", state="pending", attempts=0, next_attempt_at=NOW, created_at=NOW)
+    assert outbox.delivery([sent, held, failed]) == {"state": "held", "at": later, "attempts": 0, "error": None}
+    assert outbox.delivery([sent, retrying]) == {"state": "retrying", "at": later, "attempts": 3,
+                                                 "error": "451 greylisted"}
+    assert outbox.delivery([first])["state"] == "sending"
+    assert outbox.delivery([sent, failed]) == {"state": "failed", "at": NOW, "attempts": 1,
+                                               "error": "d@x: 550 no such user"}
+    assert outbox.delivery([sent]) == {"state": "sent", "at": sent_at, "attempts": 1, "error": None}

tests/core/test_steward_health.py

@@ -13,10 +13,15 @@ STORAGE = {"total": 3 * 2**30, "rest": {"count": 0, "bytes": 0},
                       {"name": "docker_logs", "bytes": 512 * 2**20, "rows": None}]}
 
 
-async def _page(app, ctx, storage=STORAGE, cleanup=None):
+NO_EMAIL = {"rows": [], "more": 0, "sent": 0, "problems": []}
+
+
+async def _page(app, ctx, storage=STORAGE, cleanup=None, email=NO_EMAIL):
+    from steward.core.outbox import KIND_LABELS
     async with app.test_request_context("/settings/system/"):
         session["user_role"] = "admin"
-        page = await render_template("settings/system.html", **ctx, storage=storage, cleanup=cleanup)
+        page = await render_template("settings/system.html", **ctx, storage=storage, cleanup=cleanup,
+                                     email=email, kind_labels=KIND_LABELS)
     return " ".join(page.split())
 
 
@@ -109,3 +114,46 @@ async def test_the_last_cleanup_keeps_only_what_went(monkeypatch):
     assert got["removed"] == [("docker_logs: past the age window", 7)]
     stored = {}
     assert await health.last_cleanup(None) is None
+
+
+# ── Outgoing email (#5509) ────────────────────────────────────────────────────
+
+def _mail(**kw):
+    from steward.models.outbox import EmailOutbox
+    base = dict(kind="alert", recipient="ann@example.com", state="pending", attempts=0,
+                created_at=AT, send_after=None, next_attempt_at=None, last_error=None)
+    return EmailOutbox(**{**base, **kw})
+
+
+async def test_outgoing_email_lists_what_has_not_gone_failures_first(app, previews):
+    previews()
+    got = await health.process_health(None)
+    for r in got["rows"]:
+        r["spark"] = ""
+    email = {"sent": 1234, "more": 3, "problems": ["1 email failed in the last 7 days"], "rows": [
+        _mail(state="failed", kind="invite", recipient="bob@example.com", attempts=1,
+              last_error="550 no such user"),
+        _mail(state="pending", attempts=3, next_attempt_at=AT, last_error="451 greylisted"),
+        _mail(state="held", kind="alert", send_after=AT)]}
+    got["problems"] = email["problems"] + got["problems"]
+    page = await _page(app, got, email=email)
+    # The verdict leads with it; the section says each one's state and why.
+    assert '<p class="page-verdict"><span class="tone-warn">1 email failed in the last 7 days.' in page
+    assert "1,234 sent in the last 7 days." in page and "Nothing waiting" not in page
+    assert page.index("bob@example.com") < page.index("451 greylisted")
+    assert "Invite" in page and "550 no such user" in page
+    assert "retrying</span> <span class=\"tone-dim\">attempt 3, next" in page
+    assert "gathering</span> <span class=\"tone-dim\">sends" in page
+    assert "+3 more not yet sent" in page
+    assert 'title="<' not in page
+    assert page.index("Steward itself") < page.index("Outgoing email") < page.index("Storage")
+
+
+async def test_nothing_waiting_says_so(app, previews):
+    previews()
+    got = await health.process_health(None)
+    for r in got["rows"]:
+        r["spark"] = ""
+    page = await _page(app, got, email={**NO_EMAIL, "sent": 2})
+    assert "Nothing waiting to send. 2 sent in the last 7 days." in page
+    assert "Steward is running normally." in page

tests/integration/test_email_delivery_pages.py

@@ -0,0 +1,99 @@
+"""Integration: email delivery you can see (#5509), against Postgres.
+
+An alert's email gathering for its window reads so in the alert's history
+and on Settings → System; once refused it reads "failed" with the server's
+words in both places, and the System page's verdict names it.
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import smtplib
+import time
+import uuid
+from datetime import datetime, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+async def _admin(app, tag):
+    from steward.models.users import User, UserRole
+    uid = str(uuid.uuid4())
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            s.add(User(id=uid, username=f"dl-{tag}", email=f"dl-{tag}@test", password_hash="x",
+                       role=UserRole.admin, is_active=True))
+    client = app.test_client()
+    async with client.session_transaction() as sess:
+        sess.update(user_id=uid, user_role="admin", username=f"dl-{tag}", _csrf="test-token",
+                    login_at=time.time())
+    return client, uid
+
+
+def test_an_alerts_email_reads_gathering_then_failed_in_history_and_on_system(app, monkeypatch):
+    from steward.alerts import service as alert_svc
+    from steward.core import outbox
+    from steward.core.alerts import record_metrics
+    from steward.models.alerts import AlertEvent
+    from steward.models.metrics import PluginMetric
+    from tests.integration._mail import capture
+
+    tag = uuid.uuid4().hex[:8]
+    extra = f"pager-{tag}@example.com"
+    capture(app, monkeypatch, lambda m: smtplib.SMTPRecipientsRefused({m["To"]: (550, b"mailbox gone")})
+            if m["To"] == extra else None)
+    app.config["SMTP"]["recipients"] = [extra]
+
+    async def _go():
+        client, uid = await _admin(app, tag)
+        now = datetime.now(timezone.utc)
+        res = f"box-{tag}"
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await record_metrics(s, [PluginMetric(source_module="host_agent", resource_name=res,
+                                                      metric_name="cpu_pct", value=1.0)])
+                await s.flush()
+                fields = await alert_svc.rule_fields(
+                    s, name=f"hot {tag}", source_module="host_agent", resource_name=res,
+                    metric_name="cpu_pct", operator=">", threshold=90)
+                rule = await alert_svc.add_rule(s, fields, created_by=uid)
+                event = AlertEvent(rule_id=rule.id, from_state="inactive", to_state="firing",
+                                   metric_value=95.0, transitioned_at=now)
+                s.add(event)
+            rule_id, event_id = rule.id, event.id
+        await outbox.hold_alert(app, {"rule_name": f"hot {tag}", "state": "FIRING", "metric": "cpu_pct",
+                                      "value": 95.0, "operator": ">", "threshold": 90.0, "resource": res,
+                                      "source_module": "host_agent", "note": None,
+                                      "timestamp": now.isoformat()}, event_id, now=now)
+
+        history = await (await client.get(f"/alerts/history?rule={rule_id}")).get_data(as_text=True)
+        assert "email gathering" in history and "waiting for other alerts; sends" in history
+        system = await (await client.get("/settings/system/")).get_data(as_text=True)
+        assert extra in system and "gathering" in system
+
+        # The window closes; the server refuses the pager address.
+        from sqlalchemy import select
+        from steward.models.outbox import email_outbox_events as link
+        async with app.db_sessionmaker() as s:
+            ids = list((await s.execute(select(link.c.outbox_id).where(
+                link.c.event_id == event_id))).scalars())
+        await outbox.attempt(app, ids)
+
+        history = await (await client.get(f"/alerts/history?rule={rule_id}")).get_data(as_text=True)
+        assert "email failed" in history and f"{extra}: 550 mailbox gone" in history
+        system = " ".join((await (await client.get("/settings/system/")).get_data(as_text=True)).split())
+        verdict = system[system.index("page-verdict"):system.index("</p>", system.index("page-verdict"))]
+        assert "failed in the last 7 days" in verdict
+        assert extra in system and "550 mailbox gone" in system
+
+    asyncio.run(_go())