feat(email): delivery you can see: each alert's email in history, outgoing email on Settings -> System (#5509)

#5509 "Delivery you can see: each alert's email state in history, and
outgoing email on Settings -> System" (milestone 501).

- Alerts -> History: an alert's email reads from the outbox (#5506):
  "gathering" with when it sends, "sending", "retrying" with the attempt,
  next try and the server's last answer, "sent", or "failed" naming each
  reader it didn't reach and why (outbox.delivery / email_states). Events
  from before the outbox keep what they recorded. The Notified column
  shows once an alert has a queued email.
- Settings -> System gets "Outgoing email": every email of the last 7
  days not yet sent, failures first, then the longest retrying, with
  recipient, kind, state and last error, and a count of what was sent.
  Failures and anything still retrying after 5 minutes lead the page's
  verdict (outbox.outgoing).
- get_alert_history (MCP) adds `email`: the same delivery state.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/episodes.py

@@ -87,8 +87,8 @@ def episodes(rows: list[dict], *, newest_page: bool, now: datetime, since: datet
         seen, deliveries = set(), []
         for s in steps:
             for d in s["deliveries"]:
-                if (d[0], d[1]) not in seen:
-                    seen.add((d[0], d[1]))
+                if (d["label"], d["state"]) not in seen:
+                    seen.add((d["label"], d["state"]))
                     deliveries.append(d)
         ep["deliveries"] = deliveries
         ep["muted_by"] = next((s["muted_by"] for s in steps if s["muted_by"]), None)

steward/alerts/routes.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 from datetime import datetime, timezone
 from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
 from steward.core.audit import log_audit
+from steward.core import outbox
 from sqlalchemy import func, select
 from steward.auth.middleware import require_role
 from steward.alerts import service as alert_svc
@@ -177,9 +178,20 @@ async def active_fragment():
         rules_off=sum(1 for r in rules if not r.enabled))
 
 
-def _history_row(event, rule, names: dict | None = None) -> dict:
+def _delivery(label: str, sent: bool | None, error: str | None) -> dict | None:
+    """A channel's outcome as recorded on the event; None when not tried."""
+    if sent is None:
+        return None
+    return {"label": label, "state": "sent" if sent else "failed", "at": None, "attempts": None,
+            "error": None if sent else error}
+
+
+def _history_row(event, rule, names: dict | None = None, emails: dict | None = None) -> dict:
     """One transition as the history page shows it. `names` maps user ids to
-    usernames: a transition a person made says who (#5463)."""
+    usernames: a transition a person made says who (#5463). `emails` is
+    outbox.email_states for the page: the email's live state (gathering,
+    retrying, sent, failed) where it went through the outbox (#5509), else
+    what the event recorded."""
     from steward.core.alerts import MUTED, RELEASED
     from steward.core.formatting import metric_label, metric_value
     notes, muted_by, released = [], None, False
@@ -208,9 +220,10 @@ def _history_row(event, rule, names: dict | None = None) -> dict:
         "from_state": event.from_state, "to_state": event.to_state,
         "value": value, "raw": event.metric_value, "condition": alert_svc.rule_condition(rule),
         "notes": notes, "muted_by": muted_by, "released": released,
-        "deliveries": [(label, sent, error) for label, sent, error in (
-            ("email", event.email_sent, event.email_error),
-            ("webhook", event.webhook_sent, event.webhook_error)) if sent is not None],
+        "deliveries": [d for d in (
+            {"label": "email", **(emails or {})[event.id]} if event.id in (emails or {})
+            else _delivery("email", event.email_sent, event.email_error),
+            _delivery("webhook", event.webhook_sent, event.webhook_error)) if d],
         "ansible_run_id": event.ansible_run_id,
     }
 
@@ -235,13 +248,14 @@ async def alert_history():
         rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before)
         show_notified, show_action = await alert_svc.history_columns(db, since, rule_id)
         names = await alert_svc.usernames(db, (e.actor_id for e, _ in rows))
+        emails = await outbox.email_states(db, [e.id for e, _ in rows])
         verdict = None
         if chosen and not before:
             every = await alert_svc.history(db, since, rule_id, HISTORY_SUMMARY_ROWS)
             verdict = summary(episodes([_history_row(e, r) for e, r in every], newest_page=True,
                                        now=now, since=since))
     rows, after = next_page(rows, lambda row: (row[0].transitioned_at, row[0].id))
-    ctx = {"episodes": episodes([_history_row(e, r, names) for e, r in rows],
+    ctx = {"episodes": episodes([_history_row(e, r, names, emails) for e, r in rows],
                                 newest_page=not before, now=now, since=since),
            "after": after, "rules": rules, "chosen": chosen, "current_range": range_key,
            "show_notified": show_notified, "show_action": show_action, "verdict": verdict}

steward/alerts/service.py

@@ -322,9 +322,12 @@ async def history_columns(db, since: datetime, rule_id: str | None = None) -> tu
     base = select(AlertEvent.id).where(AlertEvent.transitioned_at >= since)
     if rule_id:
         base = base.where(AlertEvent.rule_id == rule_id)
+    from steward.models.outbox import email_outbox_events
+    queued = select(email_outbox_events.c.event_id).where(
+        email_outbox_events.c.event_id == AlertEvent.id).exists()
     notified = await db.scalar(select(base.where(or_(
         AlertEvent.email_sent.is_not(None), AlertEvent.webhook_sent.is_not(None),
-        AlertEvent.note.like(f"%{MUTED}%"))).exists()))
+        AlertEvent.note.like(f"%{MUTED}%"), queued)).exists()))
     acted = await db.scalar(select(base.where(AlertEvent.ansible_run_id.is_not(None)).exists()))
     return bool(notified), bool(acted)