feat(email): delivery you can see: each alert's email in history, outgoing email on Settings -> System (#5509)

#5509 "Delivery you can see: each alert's email state in history, and
outgoing email on Settings -> System" (milestone 501).

- Alerts -> History: an alert's email reads from the outbox (#5506):
  "gathering" with when it sends, "sending", "retrying" with the attempt,
  next try and the server's last answer, "sent", or "failed" naming each
  reader it didn't reach and why (outbox.delivery / email_states). Events
  from before the outbox keep what they recorded. The Notified column
  shows once an alert has a queued email.
- Settings -> System gets "Outgoing email": every email of the last 7
  days not yet sent, failures first, then the longest retrying, with
  recipient, kind, state and last error, and a count of what was sent.
  Failures and anything still retrying after 5 minutes lead the page's
  verdict (outbox.outgoing).
- get_alert_history (MCP) adds `email`: the same delivery state.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/episodes.py

@@ -87,8 +87,8 @@ def episodes(rows: list[dict], *, newest_page: bool, now: datetime, since: datet
         seen, deliveries = set(), []
         for s in steps:
             for d in s["deliveries"]:
-                if (d[0], d[1]) not in seen:
-                    seen.add((d[0], d[1]))
+                if (d["label"], d["state"]) not in seen:
+                    seen.add((d["label"], d["state"]))
                     deliveries.append(d)
         ep["deliveries"] = deliveries
         ep["muted_by"] = next((s["muted_by"] for s in steps if s["muted_by"]), None)

steward/alerts/routes.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 from datetime import datetime, timezone
 from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
 from steward.core.audit import log_audit
+from steward.core import outbox
 from sqlalchemy import func, select
 from steward.auth.middleware import require_role
 from steward.alerts import service as alert_svc
@@ -177,9 +178,20 @@ async def active_fragment():
         rules_off=sum(1 for r in rules if not r.enabled))
 
 
-def _history_row(event, rule, names: dict | None = None) -> dict:
+def _delivery(label: str, sent: bool | None, error: str | None) -> dict | None:
+    """A channel's outcome as recorded on the event; None when not tried."""
+    if sent is None:
+        return None
+    return {"label": label, "state": "sent" if sent else "failed", "at": None, "attempts": None,
+            "error": None if sent else error}
+
+
+def _history_row(event, rule, names: dict | None = None, emails: dict | None = None) -> dict:
     """One transition as the history page shows it. `names` maps user ids to
-    usernames: a transition a person made says who (#5463)."""
+    usernames: a transition a person made says who (#5463). `emails` is
+    outbox.email_states for the page: the email's live state (gathering,
+    retrying, sent, failed) where it went through the outbox (#5509), else
+    what the event recorded."""
     from steward.core.alerts import MUTED, RELEASED
     from steward.core.formatting import metric_label, metric_value
     notes, muted_by, released = [], None, False
@@ -208,9 +220,10 @@ def _history_row(event, rule, names: dict | None = None) -> dict:
         "from_state": event.from_state, "to_state": event.to_state,
         "value": value, "raw": event.metric_value, "condition": alert_svc.rule_condition(rule),
         "notes": notes, "muted_by": muted_by, "released": released,
-        "deliveries": [(label, sent, error) for label, sent, error in (
-            ("email", event.email_sent, event.email_error),
-            ("webhook", event.webhook_sent, event.webhook_error)) if sent is not None],
+        "deliveries": [d for d in (
+            {"label": "email", **(emails or {})[event.id]} if event.id in (emails or {})
+            else _delivery("email", event.email_sent, event.email_error),
+            _delivery("webhook", event.webhook_sent, event.webhook_error)) if d],
         "ansible_run_id": event.ansible_run_id,
     }
 
@@ -235,13 +248,14 @@ async def alert_history():
         rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before)
         show_notified, show_action = await alert_svc.history_columns(db, since, rule_id)
         names = await alert_svc.usernames(db, (e.actor_id for e, _ in rows))
+        emails = await outbox.email_states(db, [e.id for e, _ in rows])
         verdict = None
         if chosen and not before:
             every = await alert_svc.history(db, since, rule_id, HISTORY_SUMMARY_ROWS)
             verdict = summary(episodes([_history_row(e, r) for e, r in every], newest_page=True,
                                        now=now, since=since))
     rows, after = next_page(rows, lambda row: (row[0].transitioned_at, row[0].id))
-    ctx = {"episodes": episodes([_history_row(e, r, names) for e, r in rows],
+    ctx = {"episodes": episodes([_history_row(e, r, names, emails) for e, r in rows],
                                 newest_page=not before, now=now, since=since),
            "after": after, "rules": rules, "chosen": chosen, "current_range": range_key,
            "show_notified": show_notified, "show_action": show_action, "verdict": verdict}

steward/alerts/service.py

@@ -322,9 +322,12 @@ async def history_columns(db, since: datetime, rule_id: str | None = None) -> tu
     base = select(AlertEvent.id).where(AlertEvent.transitioned_at >= since)
     if rule_id:
         base = base.where(AlertEvent.rule_id == rule_id)
+    from steward.models.outbox import email_outbox_events
+    queued = select(email_outbox_events.c.event_id).where(
+        email_outbox_events.c.event_id == AlertEvent.id).exists()
     notified = await db.scalar(select(base.where(or_(
         AlertEvent.email_sent.is_not(None), AlertEvent.webhook_sent.is_not(None),
-        AlertEvent.note.like(f"%{MUTED}%"))).exists()))
+        AlertEvent.note.like(f"%{MUTED}%"), queued)).exists()))
     acted = await db.scalar(select(base.where(AlertEvent.ansible_run_id.is_not(None)).exists()))
     return bool(notified), bool(acted)
 

steward/core/outbox.py

@@ -356,3 +356,81 @@ def said(rows: list[EmailOutbox]) -> tuple[bool, str]:
         ([f"Not sent yet to {'; '.join(later)}; Steward will try again for up to an hour."]
          if later else [])
     return bool(rows) and not failed and not later, " ".join(parts)
+
+
+# ── What the pages show (#5509) ───────────────────────────────────────────────
+
+KIND_LABELS = {"alert": "Alert", "report": "Weekly report", "invite": "Invite",
+               "reset": "Password reset", "test": "Test"}
+# Retrying longer than this is worth a line in Settings → System's verdict.
+SLOW_S = 300
+SHOWN_DAYS = 7
+
+
+def delivery(rows: list[EmailOutbox]) -> dict:
+    """One alert event's email, over every reader's copy, as history says
+    it: {"state", "at", "attempts", "error"}. Gathering while any copy is
+    held (at: when it sends); sending/retrying while any is still due (at:
+    the next try); failed when any reader wasn't reached (error names each);
+    else sent (at: the last acceptance)."""
+    if held := [r for r in rows if r.state == "held"]:
+        return {"state": "held", "at": min(r.send_after for r in held), "attempts": 0, "error": None}
+    if due := [r for r in rows if r.state in ("pending", "sending")]:
+        worst = max(due, key=lambda r: r.attempts)
+        return {"state": "retrying" if worst.attempts else "sending",
+                "at": min((r.next_attempt_at for r in due if r.next_attempt_at), default=None),
+                "attempts": worst.attempts, "error": worst.last_error}
+    if failed := [r for r in rows if r.state == "failed"]:
+        return {"state": "failed", "at": max(r.first_attempt_at or r.created_at for r in failed),
+                "attempts": max(r.attempts for r in failed),
+                "error": "; ".join(f"{r.recipient}: {r.last_error}" for r in failed)}
+    return {"state": "sent", "at": max((r.sent_at for r in rows if r.sent_at), default=None),
+            "attempts": max(r.attempts for r in rows), "error": None}
+
+
+async def email_states(db, event_ids: list[str]) -> dict[str, dict]:
+    """{event id: delivery(...)} for the alert events whose email went
+    through the outbox; an event with none (no reader, or from before it)
+    is absent."""
+    if not event_ids:
+        return {}
+    link = email_outbox_events.c
+    by: dict[str, list] = {}
+    for event_id, row in (await db.execute(
+            select(link.event_id, EmailOutbox).join(EmailOutbox, EmailOutbox.id == link.outbox_id)
+            .where(link.event_id.in_(event_ids)))).all():
+        by.setdefault(event_id, []).append(row)
+    return {eid: delivery(rows) for eid, rows in by.items()}
+
+
+async def outgoing(db, now: datetime | None = None, limit: int = 20) -> dict:
+    """Settings → System's "Outgoing email": every email of the last
+    SHOWN_DAYS not yet sent (failed first, then retrying longest, then
+    sending and gathering), how many were sent, and the problems for the
+    page's verdict. {"rows": [EmailOutbox], "more": n, "sent": n,
+    "problems": [str]}."""
+    from sqlalchemy import case, func
+    now = now or datetime.now(timezone.utc)
+    since = now - timedelta(days=SHOWN_DAYS)
+    open_states = ("failed", "pending", "sending", "held")
+    order = case({"failed": 0, "pending": 1, "sending": 2, "held": 3}, value=EmailOutbox.state)
+    base = select(EmailOutbox).where(EmailOutbox.created_at >= since,
+                                     EmailOutbox.state.in_(open_states))
+    rows = list((await db.execute(
+        base.order_by(order, EmailOutbox.first_attempt_at.asc().nulls_last(),
+                      EmailOutbox.created_at.desc()).limit(limit))).scalars())
+    counts = dict((await db.execute(
+        select(EmailOutbox.state, func.count()).where(EmailOutbox.created_at >= since)
+        .group_by(EmailOutbox.state))).all())
+    slow = (await db.execute(select(func.count()).select_from(EmailOutbox).where(
+        EmailOutbox.state.in_(("pending", "sending")),
+        EmailOutbox.first_attempt_at < now - timedelta(seconds=SLOW_S)))).scalar_one()
+    problems = []
+    if failed := counts.get("failed", 0):
+        problems.append(f"{failed} email{'s' if failed != 1 else ''} failed in the last {SHOWN_DAYS} days")
+    if slow:
+        problems.append(f"{slow} email{'s' if slow != 1 else ''} still not accepted by the mail "
+                        f"server after {SLOW_S // 60} minutes of trying")
+    shown = sum(counts.get(s, 0) for s in open_states)
+    return {"rows": rows, "more": max(shown - len(rows), 0), "sent": counts.get("sent", 0),
+            "problems": problems}

steward/mcp/tools/alerts.py

@@ -6,6 +6,7 @@ from datetime import datetime, timedelta, timezone
 from sqlalchemy import select
 
 from steward.alerts import service as alert_svc
+from steward.core import outbox
 from steward.mcp._context import current_caller, mcp_app
 from steward.models.alerts import AlertOperator, AlertRule, AlertState, MaintenanceWindow
 from steward.models.users import UserRole
@@ -82,7 +83,11 @@ async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50)
 
     Each transition carries the metric value that caused it and whether the
     email/webhook notification went out (with the error if not), plus the id
-    of any Ansible run the rule's action triggered. `by` names the person who
+    of any Ansible run the rule's action triggered. `email` is the email's
+    delivery: state "held" (gathering other alerts; `at` is when it sends),
+    "sending", "retrying" (`attempts`, next try `at`, last `error`), "sent"
+    or "failed" (`error` names each reader not reached); null when no email
+    was queued for it. `by` names the person who
     made it happen (an acknowledgement, or a rule edit that reset the alert);
     null when a reading or Steward itself did.
     """
@@ -93,6 +98,7 @@ async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50)
         chosen = pick(rule, rules, kind="alert rule", list_tool="list_alert_rules") if rule else None
         rows = await alert_svc.history(db, since, chosen.id if chosen else None, limit)
         names = await alert_svc.usernames(db, (e.actor_id for e, _ in rows))
+        emails = await outbox.email_states(db, [e.id for e, _ in rows])
     events = [
         {
             "rule_id": r.id,
@@ -106,6 +112,7 @@ async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50)
             "age_s": age_s(e.transitioned_at),
             "email_sent": e.email_sent,
             "email_error": e.email_error,
+            "email": ({**emails[e.id], "at": iso(emails[e.id]["at"])} if e.id in emails else None),
             "webhook_sent": e.webhook_sent,
             "webhook_error": e.webhook_error,
             "ansible_run_id": e.ansible_run_id,

steward/settings/routes.py

@@ -630,7 +630,10 @@ async def _reports_page(settings: dict, flash):
 async def system():
     """Steward's own health (#5467): its process metrics now and over the day,
     each one an alert rule can watch (source "Steward itself"); then its
-    tables' sizes and what the last hourly cleanup removed (#5469)."""
+    tables' sizes and what the last hourly cleanup removed (#5469); and
+    the email not yet sent (#5509), whose failures and slow retries join the
+    page's verdict."""
+    from steward.core import outbox
     from steward.core.health import last_cleanup, process_health, table_sizes
     from steward.core.status import sparkline_svg
     async with current_app.db_sessionmaker() as db:
@@ -638,11 +641,14 @@ async def system():
         # Table sizes and the last cleanup (#5469): Postgres's catalog and
         # what the cleanup kept, never a scan of the tables themselves.
         storage, cleanup = await table_sizes(db), await last_cleanup(db)
+        email = await outbox.outgoing(db)
+    health["problems"] = email["problems"] + health["problems"]
     for row in health["rows"]:
         # From zero to the row's own peak, with a floor (UI & Design ruling 8).
         row["spark"] = sparkline_svg(row["series"], width=120, height=20,
                                      stroke="currentColor", floor=1.0) if len(row["series"]) > 1 else ""
-    return await render_template("settings/system.html", **health, storage=storage, cleanup=cleanup)
+    return await render_template("settings/system.html", **health, storage=storage, cleanup=cleanup,
+                                 email=email, kind_labels=outbox.KIND_LABELS)
 
 
 @settings_bp.get("/reports/")

steward/templates/alerts/_history_page.html

@@ -48,11 +48,20 @@
           <span class="badge badge-yellow" title="A maintenance window muted this notification">muted · {{ ep.muted_by }}</span>
           {% if ep.released %}<div class="cell-note">notified when the window ended</div>{% endif %}
           {% endif %}
-          {% for label, sent, error in ep.deliveries %}
+          {# Each channel's outcome; an email through the outbox (#5509) also
+             says when it sends, or how its retries stand, or why it failed. #}
+          {% for d in ep.deliveries %}
           <div>
-            {% if sent %}<span class="badge badge-green">{{ label }} sent</span>
-            {% else %}<span class="badge badge-red">{{ label }} failed</span>
-            {% if error %}<div class="cell-note" style="max-width:28rem;overflow-wrap:anywhere;">{{ error }}</div>{% endif %}
+            {% if d.state == "sent" %}<span class="badge badge-green">{{ d.label }} sent</span>
+            {%- if d.at %} <span class="tone-dim">{{ d.at|when }}</span>{% endif %}
+            {% elif d.state == "held" %}<span class="badge badge-yellow">{{ d.label }} gathering</span>
+            <div class="cell-note">waiting for other alerts; sends {{ d.at|when }}</div>
+            {% elif d.state == "sending" %}<span class="badge badge-yellow">{{ d.label }} sending</span>
+            {% elif d.state == "retrying" %}<span class="badge badge-yellow">{{ d.label }} retrying</span>
+            <div class="cell-note" style="max-width:28rem;overflow-wrap:anywhere;">attempt {{ d.attempts }}
+              {%- if d.at %}, next {{ d.at|when }}{% endif %}{% if d.error %}: {{ d.error }}{% endif %}</div>
+            {% else %}<span class="badge badge-red">{{ d.label }} failed</span>
+            {% if d.error %}<div class="cell-note" style="max-width:28rem;overflow-wrap:anywhere;">{{ d.error }}</div>{% endif %}
             {% endif %}
           </div>
           {% endfor %}

steward/templates/settings/system.html

@@ -9,8 +9,8 @@
 {% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("System", "")]) }}{% endblock %}
 {% block content %}
 <p class="page-verdict">
-  {%- if not seen %}<span class="tone-dim">No readings yet: the first arrive within a minute of Steward starting.</span>
-  {%- elif problems %}{% set said = problems|join("; ") %}<span class="tone-warn">{{ said[:1]|upper }}{{ said[1:] }}.</span>
+  {%- if problems %}{% set said = problems|join("; ") %}<span class="tone-warn">{{ said[:1]|upper }}{{ said[1:] }}.</span>
+  {%- elif not seen %}<span class="tone-dim">No readings yet: the first arrive within a minute of Steward starting.</span>
   {%- else %}<span class="tone-ok">Steward is running normally.</span>{% endif -%}
 </p>
 
@@ -39,6 +39,46 @@
   Recorded each minute (the cleanup each hour). Any of these can be watched by an
   <a href="/alerts/rules/new">alert rule</a>: pick the source "Steward itself".</p>
 
+{# Outgoing email (#5509): every email waits in the outbox until the mail
+   server takes it. What hasn't gone yet, failures first, then the longest
+   retrying; sent mail is only counted. #}
+<h2 class="section-title" style="margin:2rem 0 0.5rem;">Outgoing email</h2>
+<p style="margin:0 0 1rem;">
+  {%- if not email.rows %}Nothing waiting to send. {% endif -%}
+  {{ "{:,}".format(email.sent) }} sent in the last 7 days.</p>
+{% if email.rows %}
+<div class="card-flush">
+  <table class="table">
+    <thead><tr><th>To</th><th>What</th><th>State</th><th>Queued</th></tr></thead>
+    <tbody>
+      {% for r in email.rows %}
+      <tr>
+        <td class="cell-name"><span title="{{ r.recipient }}">{{ r.recipient }}</span></td>
+        <td class="nowrap">{{ kind_labels.get(r.kind, r.kind) }}</td>
+        <td>
+          {%- if r.state == "failed" %}<span class="badge badge-red">failed</span>
+          {%- elif r.state == "pending" and r.attempts %}<span class="badge badge-yellow">retrying</span>
+            <span class="tone-dim">attempt {{ r.attempts }}{% if r.next_attempt_at %}, next {{ r.next_attempt_at|when }}{% endif %}</span>
+          {%- elif r.state == "held" %}<span class="badge badge-yellow">gathering</span>
+            <span class="tone-dim">sends {{ r.send_after|when }}</span>
+          {%- else %}<span class="badge badge-yellow">sending</span>{% endif %}
+          {%- if r.last_error %}<div class="cell-note" style="max-width:32rem;overflow-wrap:anywhere;">{{ r.last_error }}</div>{% endif -%}
+        </td>
+        <td class="nowrap">{{ r.created_at|ago }}</td>
+      </tr>
+      {% endfor %}
+      {% if email.more %}
+      <tr><td colspan="4" class="tone-dim">+{{ email.more }} more not yet sent</td></tr>
+      {% endif %}
+    </tbody>
+  </table>
+</div>
+{% endif %}
+<p class="field-hint" style="margin-top:0.75rem;">
+  An email the server refuses is failed at once; one it can't take yet is tried
+  again for up to an hour. The mail server is set under
+  <a href="/settings/notifications/">Notifications</a>.</p>
+
 {# Storage (#5469): Postgres's own figures for each table, then what the
    hourly cleanup last removed. Read from the catalog, never by counting. #}
 <h2 class="section-title" style="margin:2rem 0 0.5rem;">Storage</h2>