From defad2e98b0bf0c658ea7049dd8d54f6d07a7a0e Mon Sep 17 00:00:00 2001
From: Bryan Van Deusen
Date: Fri, 9 Oct 2026 16:12:40 -0400
Subject: [PATCH] feat(email): delivery you can see: each alert's email in
history, outgoing email on Settings -> System (#5509)
#5509 "Delivery you can see: each alert's email state in history, and
outgoing email on Settings -> System" (milestone 501).
- Alerts -> History: an alert's email reads from the outbox (#5506):
"gathering" with when it sends, "sending", "retrying" with the attempt,
next try and the server's last answer, "sent", or "failed" naming each
reader it didn't reach and why (outbox.delivery / email_states). Events
from before the outbox keep what they recorded. The Notified column
shows once an alert has a queued email.
- Settings -> System gets "Outgoing email": every email of the last 7
days not yet sent, failures first, then the longest retrying, with
recipient, kind, state and last error, and a count of what was sent.
Failures and anything still retrying after 5 minutes lead the page's
verdict (outbox.outgoing).
- get_alert_history (MCP) adds `email`: the same delivery state.
Co-Authored-By: Claude Opus 5.5
---
docs/core/alerting.md | 8 ++
steward/alerts/episodes.py | 4 +-
steward/alerts/routes.py | 26 +++--
steward/alerts/service.py | 5 +-
steward/core/outbox.py | 78 +++++++++++++++
steward/mcp/tools/alerts.py | 9 +-
steward/settings/routes.py | 10 +-
steward/templates/alerts/_history_page.html | 17 +++-
steward/templates/settings/system.html | 44 ++++++++-
tests/core/test_alert_episodes.py | 10 +-
tests/core/test_alert_history_row.py | 19 +++-
tests/core/test_outbox.py | 20 ++++
tests/core/test_steward_health.py | 52 +++++++++-
.../integration/test_email_delivery_pages.py | 99 +++++++++++++++++++
14 files changed, 375 insertions(+), 26 deletions(-)
create mode 100644 tests/integration/test_email_delivery_pages.py
diff --git a/docs/core/alerting.md b/docs/core/alerting.md
index dbfc94d..77a718c 100644
--- a/docs/core/alerting.md
+++ b/docs/core/alerting.md
@@ -184,6 +184,14 @@ seconds and settles each email on the server's answer:
Once every email carrying an alert has settled, that alert's event records
whether the email went, and to whom it didn't.
+Where to see it: each alert's row in **Alerts → History** says what its email
+is doing (gathering, with when it sends; retrying, with the attempt, the next
+try and the server's last answer; sent; failed, with who wasn't reached and
+why), and `get_alert_history` over MCP reports the same. **Settings → System →
+Outgoing email** lists every email of the last 7 days not yet sent, failures
+first, and the page's verdict names failures and any email still retrying
+after 5 minutes.
+
An email a person is waiting on is tried at once instead of on the next tick:
the weekly report's **Send now**, an invite, and the test buttons, so the page
can say what the server answered. A report or invite the server couldn't take
diff --git a/steward/alerts/episodes.py b/steward/alerts/episodes.py
index 502544e..3d63522 100644
--- a/steward/alerts/episodes.py
+++ b/steward/alerts/episodes.py
@@ -87,8 +87,8 @@ def episodes(rows: list[dict], *, newest_page: bool, now: datetime, since: datet
seen, deliveries = set(), []
for s in steps:
for d in s["deliveries"]:
- if (d[0], d[1]) not in seen:
- seen.add((d[0], d[1]))
+ if (d["label"], d["state"]) not in seen:
+ seen.add((d["label"], d["state"]))
deliveries.append(d)
ep["deliveries"] = deliveries
ep["muted_by"] = next((s["muted_by"] for s in steps if s["muted_by"]), None)
diff --git a/steward/alerts/routes.py b/steward/alerts/routes.py
index 734ad14..760a009 100644
--- a/steward/alerts/routes.py
+++ b/steward/alerts/routes.py
@@ -2,6 +2,7 @@ from __future__ import annotations
from datetime import datetime, timezone
from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
from steward.core.audit import log_audit
+from steward.core import outbox
from sqlalchemy import func, select
from steward.auth.middleware import require_role
from steward.alerts import service as alert_svc
@@ -177,9 +178,20 @@ async def active_fragment():
rules_off=sum(1 for r in rules if not r.enabled))
-def _history_row(event, rule, names: dict | None = None) -> dict:
+def _delivery(label: str, sent: bool | None, error: str | None) -> dict | None:
+ """A channel's outcome as recorded on the event; None when not tried."""
+ if sent is None:
+ return None
+ return {"label": label, "state": "sent" if sent else "failed", "at": None, "attempts": None,
+ "error": None if sent else error}
+
+
+def _history_row(event, rule, names: dict | None = None, emails: dict | None = None) -> dict:
"""One transition as the history page shows it. `names` maps user ids to
- usernames: a transition a person made says who (#5463)."""
+ usernames: a transition a person made says who (#5463). `emails` is
+ outbox.email_states for the page: the email's live state (gathering,
+ retrying, sent, failed) where it went through the outbox (#5509), else
+ what the event recorded."""
from steward.core.alerts import MUTED, RELEASED
from steward.core.formatting import metric_label, metric_value
notes, muted_by, released = [], None, False
@@ -208,9 +220,10 @@ def _history_row(event, rule, names: dict | None = None) -> dict:
"from_state": event.from_state, "to_state": event.to_state,
"value": value, "raw": event.metric_value, "condition": alert_svc.rule_condition(rule),
"notes": notes, "muted_by": muted_by, "released": released,
- "deliveries": [(label, sent, error) for label, sent, error in (
- ("email", event.email_sent, event.email_error),
- ("webhook", event.webhook_sent, event.webhook_error)) if sent is not None],
+ "deliveries": [d for d in (
+ {"label": "email", **(emails or {})[event.id]} if event.id in (emails or {})
+ else _delivery("email", event.email_sent, event.email_error),
+ _delivery("webhook", event.webhook_sent, event.webhook_error)) if d],
"ansible_run_id": event.ansible_run_id,
}
@@ -235,13 +248,14 @@ async def alert_history():
rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before)
show_notified, show_action = await alert_svc.history_columns(db, since, rule_id)
names = await alert_svc.usernames(db, (e.actor_id for e, _ in rows))
+ emails = await outbox.email_states(db, [e.id for e, _ in rows])
verdict = None
if chosen and not before:
every = await alert_svc.history(db, since, rule_id, HISTORY_SUMMARY_ROWS)
verdict = summary(episodes([_history_row(e, r) for e, r in every], newest_page=True,
now=now, since=since))
rows, after = next_page(rows, lambda row: (row[0].transitioned_at, row[0].id))
- ctx = {"episodes": episodes([_history_row(e, r, names) for e, r in rows],
+ ctx = {"episodes": episodes([_history_row(e, r, names, emails) for e, r in rows],
newest_page=not before, now=now, since=since),
"after": after, "rules": rules, "chosen": chosen, "current_range": range_key,
"show_notified": show_notified, "show_action": show_action, "verdict": verdict}
diff --git a/steward/alerts/service.py b/steward/alerts/service.py
index a5b8b43..70f46ae 100644
--- a/steward/alerts/service.py
+++ b/steward/alerts/service.py
@@ -322,9 +322,12 @@ async def history_columns(db, since: datetime, rule_id: str | None = None) -> tu
base = select(AlertEvent.id).where(AlertEvent.transitioned_at >= since)
if rule_id:
base = base.where(AlertEvent.rule_id == rule_id)
+ from steward.models.outbox import email_outbox_events
+ queued = select(email_outbox_events.c.event_id).where(
+ email_outbox_events.c.event_id == AlertEvent.id).exists()
notified = await db.scalar(select(base.where(or_(
AlertEvent.email_sent.is_not(None), AlertEvent.webhook_sent.is_not(None),
- AlertEvent.note.like(f"%{MUTED}%"))).exists()))
+ AlertEvent.note.like(f"%{MUTED}%"), queued)).exists()))
acted = await db.scalar(select(base.where(AlertEvent.ansible_run_id.is_not(None)).exists()))
return bool(notified), bool(acted)
diff --git a/steward/core/outbox.py b/steward/core/outbox.py
index 3d2a427..44b2646 100644
--- a/steward/core/outbox.py
+++ b/steward/core/outbox.py
@@ -356,3 +356,81 @@ def said(rows: list[EmailOutbox]) -> tuple[bool, str]:
([f"Not sent yet to {'; '.join(later)}; Steward will try again for up to an hour."]
if later else [])
return bool(rows) and not failed and not later, " ".join(parts)
+
+
+# ── What the pages show (#5509) ───────────────────────────────────────────────
+
+KIND_LABELS = {"alert": "Alert", "report": "Weekly report", "invite": "Invite",
+ "reset": "Password reset", "test": "Test"}
+# Retrying longer than this is worth a line in Settings → System's verdict.
+SLOW_S = 300
+SHOWN_DAYS = 7
+
+
+def delivery(rows: list[EmailOutbox]) -> dict:
+ """One alert event's email, over every reader's copy, as history says
+ it: {"state", "at", "attempts", "error"}. Gathering while any copy is
+ held (at: when it sends); sending/retrying while any is still due (at:
+ the next try); failed when any reader wasn't reached (error names each);
+ else sent (at: the last acceptance)."""
+ if held := [r for r in rows if r.state == "held"]:
+ return {"state": "held", "at": min(r.send_after for r in held), "attempts": 0, "error": None}
+ if due := [r for r in rows if r.state in ("pending", "sending")]:
+ worst = max(due, key=lambda r: r.attempts)
+ return {"state": "retrying" if worst.attempts else "sending",
+ "at": min((r.next_attempt_at for r in due if r.next_attempt_at), default=None),
+ "attempts": worst.attempts, "error": worst.last_error}
+ if failed := [r for r in rows if r.state == "failed"]:
+ return {"state": "failed", "at": max(r.first_attempt_at or r.created_at for r in failed),
+ "attempts": max(r.attempts for r in failed),
+ "error": "; ".join(f"{r.recipient}: {r.last_error}" for r in failed)}
+ return {"state": "sent", "at": max((r.sent_at for r in rows if r.sent_at), default=None),
+ "attempts": max(r.attempts for r in rows), "error": None}
+
+
+async def email_states(db, event_ids: list[str]) -> dict[str, dict]:
+ """{event id: delivery(...)} for the alert events whose email went
+ through the outbox; an event with none (no reader, or from before it)
+ is absent."""
+ if not event_ids:
+ return {}
+ link = email_outbox_events.c
+ by: dict[str, list] = {}
+ for event_id, row in (await db.execute(
+ select(link.event_id, EmailOutbox).join(EmailOutbox, EmailOutbox.id == link.outbox_id)
+ .where(link.event_id.in_(event_ids)))).all():
+ by.setdefault(event_id, []).append(row)
+ return {eid: delivery(rows) for eid, rows in by.items()}
+
+
+async def outgoing(db, now: datetime | None = None, limit: int = 20) -> dict:
+ """Settings → System's "Outgoing email": every email of the last
+ SHOWN_DAYS not yet sent (failed first, then retrying longest, then
+ sending and gathering), how many were sent, and the problems for the
+ page's verdict. {"rows": [EmailOutbox], "more": n, "sent": n,
+ "problems": [str]}."""
+ from sqlalchemy import case, func
+ now = now or datetime.now(timezone.utc)
+ since = now - timedelta(days=SHOWN_DAYS)
+ open_states = ("failed", "pending", "sending", "held")
+ order = case({"failed": 0, "pending": 1, "sending": 2, "held": 3}, value=EmailOutbox.state)
+ base = select(EmailOutbox).where(EmailOutbox.created_at >= since,
+ EmailOutbox.state.in_(open_states))
+ rows = list((await db.execute(
+ base.order_by(order, EmailOutbox.first_attempt_at.asc().nulls_last(),
+ EmailOutbox.created_at.desc()).limit(limit))).scalars())
+ counts = dict((await db.execute(
+ select(EmailOutbox.state, func.count()).where(EmailOutbox.created_at >= since)
+ .group_by(EmailOutbox.state))).all())
+ slow = (await db.execute(select(func.count()).select_from(EmailOutbox).where(
+ EmailOutbox.state.in_(("pending", "sending")),
+ EmailOutbox.first_attempt_at < now - timedelta(seconds=SLOW_S)))).scalar_one()
+ problems = []
+ if failed := counts.get("failed", 0):
+ problems.append(f"{failed} email{'s' if failed != 1 else ''} failed in the last {SHOWN_DAYS} days")
+ if slow:
+ problems.append(f"{slow} email{'s' if slow != 1 else ''} still not accepted by the mail "
+ f"server after {SLOW_S // 60} minutes of trying")
+ shown = sum(counts.get(s, 0) for s in open_states)
+ return {"rows": rows, "more": max(shown - len(rows), 0), "sent": counts.get("sent", 0),
+ "problems": problems}
diff --git a/steward/mcp/tools/alerts.py b/steward/mcp/tools/alerts.py
index e386e8d..6772021 100644
--- a/steward/mcp/tools/alerts.py
+++ b/steward/mcp/tools/alerts.py
@@ -6,6 +6,7 @@ from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from steward.alerts import service as alert_svc
+from steward.core import outbox
from steward.mcp._context import current_caller, mcp_app
from steward.models.alerts import AlertOperator, AlertRule, AlertState, MaintenanceWindow
from steward.models.users import UserRole
@@ -82,7 +83,11 @@ async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50)
Each transition carries the metric value that caused it and whether the
email/webhook notification went out (with the error if not), plus the id
- of any Ansible run the rule's action triggered. `by` names the person who
+ of any Ansible run the rule's action triggered. `email` is the email's
+ delivery: state "held" (gathering other alerts; `at` is when it sends),
+ "sending", "retrying" (`attempts`, next try `at`, last `error`), "sent"
+ or "failed" (`error` names each reader not reached); null when no email
+ was queued for it. `by` names the person who
made it happen (an acknowledgement, or a rule edit that reset the alert);
null when a reading or Steward itself did.
"""
@@ -93,6 +98,7 @@ async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50)
chosen = pick(rule, rules, kind="alert rule", list_tool="list_alert_rules") if rule else None
rows = await alert_svc.history(db, since, chosen.id if chosen else None, limit)
names = await alert_svc.usernames(db, (e.actor_id for e, _ in rows))
+ emails = await outbox.email_states(db, [e.id for e, _ in rows])
events = [
{
"rule_id": r.id,
@@ -106,6 +112,7 @@ async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50)
"age_s": age_s(e.transitioned_at),
"email_sent": e.email_sent,
"email_error": e.email_error,
+ "email": ({**emails[e.id], "at": iso(emails[e.id]["at"])} if e.id in emails else None),
"webhook_sent": e.webhook_sent,
"webhook_error": e.webhook_error,
"ansible_run_id": e.ansible_run_id,
diff --git a/steward/settings/routes.py b/steward/settings/routes.py
index 3247dce..1862397 100644
--- a/steward/settings/routes.py
+++ b/steward/settings/routes.py
@@ -630,7 +630,10 @@ async def _reports_page(settings: dict, flash):
async def system():
"""Steward's own health (#5467): its process metrics now and over the day,
each one an alert rule can watch (source "Steward itself"); then its
- tables' sizes and what the last hourly cleanup removed (#5469)."""
+ tables' sizes and what the last hourly cleanup removed (#5469); and
+ the email not yet sent (#5509), whose failures and slow retries join the
+ page's verdict."""
+ from steward.core import outbox
from steward.core.health import last_cleanup, process_health, table_sizes
from steward.core.status import sparkline_svg
async with current_app.db_sessionmaker() as db:
@@ -638,11 +641,14 @@ async def system():
# Table sizes and the last cleanup (#5469): Postgres's catalog and
# what the cleanup kept, never a scan of the tables themselves.
storage, cleanup = await table_sizes(db), await last_cleanup(db)
+ email = await outbox.outgoing(db)
+ health["problems"] = email["problems"] + health["problems"]
for row in health["rows"]:
# From zero to the row's own peak, with a floor (UI & Design ruling 8).
row["spark"] = sparkline_svg(row["series"], width=120, height=20,
stroke="currentColor", floor=1.0) if len(row["series"]) > 1 else ""
- return await render_template("settings/system.html", **health, storage=storage, cleanup=cleanup)
+ return await render_template("settings/system.html", **health, storage=storage, cleanup=cleanup,
+ email=email, kind_labels=outbox.KIND_LABELS)
@settings_bp.get("/reports/")
diff --git a/steward/templates/alerts/_history_page.html b/steward/templates/alerts/_history_page.html
index fa9f8aa..256c1f0 100644
--- a/steward/templates/alerts/_history_page.html
+++ b/steward/templates/alerts/_history_page.html
@@ -48,11 +48,20 @@
muted · {{ ep.muted_by }}
{% if ep.released %}notified when the window ended
{% endif %}
{% endif %}
- {% for label, sent, error in ep.deliveries %}
+ {# Each channel's outcome; an email through the outbox (#5509) also
+ says when it sends, or how its retries stand, or why it failed. #}
+ {% for d in ep.deliveries %}
- {% if sent %}
{{ label }} sent
- {% else %}
{{ label }} failed
- {% if error %}
{{ error }}
{% endif %}
+ {% if d.state == "sent" %}
{{ d.label }} sent
+ {%- if d.at %}
{{ d.at|when }}{% endif %}
+ {% elif d.state == "held" %}
{{ d.label }} gathering
+
waiting for other alerts; sends {{ d.at|when }}
+ {% elif d.state == "sending" %}
{{ d.label }} sending
+ {% elif d.state == "retrying" %}
{{ d.label }} retrying
+
attempt {{ d.attempts }}
+ {%- if d.at %}, next {{ d.at|when }}{% endif %}{% if d.error %}: {{ d.error }}{% endif %}
+ {% else %}
{{ d.label }} failed
+ {% if d.error %}
{{ d.error }}
{% endif %}
{% endif %}
{% endfor %}
diff --git a/steward/templates/settings/system.html b/steward/templates/settings/system.html
index d7db32c..3fba74c 100644
--- a/steward/templates/settings/system.html
+++ b/steward/templates/settings/system.html
@@ -9,8 +9,8 @@
{% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("System", "")]) }}{% endblock %}
{% block content %}
- {%- if not seen %}No readings yet: the first arrive within a minute of Steward starting.
- {%- elif problems %}{% set said = problems|join("; ") %}{{ said[:1]|upper }}{{ said[1:] }}.
+ {%- if problems %}{% set said = problems|join("; ") %}{{ said[:1]|upper }}{{ said[1:] }}.
+ {%- elif not seen %}No readings yet: the first arrive within a minute of Steward starting.
{%- else %}Steward is running normally.{% endif -%}
@@ -39,6 +39,46 @@
Recorded each minute (the cleanup each hour). Any of these can be watched by an
alert rule: pick the source "Steward itself".
+{# Outgoing email (#5509): every email waits in the outbox until the mail
+ server takes it. What hasn't gone yet, failures first, then the longest
+ retrying; sent mail is only counted. #}
+Outgoing email
+
+ {%- if not email.rows %}Nothing waiting to send. {% endif -%}
+ {{ "{:,}".format(email.sent) }} sent in the last 7 days.
+{% if email.rows %}
+
+
+ | To | What | State | Queued |
+
+ {% for r in email.rows %}
+
+ | {{ r.recipient }} |
+ {{ kind_labels.get(r.kind, r.kind) }} |
+
+ {%- if r.state == "failed" %}failed
+ {%- elif r.state == "pending" and r.attempts %}retrying
+ attempt {{ r.attempts }}{% if r.next_attempt_at %}, next {{ r.next_attempt_at|when }}{% endif %}
+ {%- elif r.state == "held" %}gathering
+ sends {{ r.send_after|when }}
+ {%- else %}sending{% endif %}
+ {%- if r.last_error %} {{ r.last_error }} {% endif -%}
+ |
+ {{ r.created_at|ago }} |
+
+ {% endfor %}
+ {% if email.more %}
+ | +{{ email.more }} more not yet sent |
+ {% endif %}
+
+
+
+{% endif %}
+
+ An email the server refuses is failed at once; one it can't take yet is tried
+ again for up to an hour. The mail server is set under
+ Notifications.
+
{# Storage (#5469): Postgres's own figures for each table, then what the
hourly cleanup last removed. Read from the catalog, never by counting. #}
Storage
diff --git a/tests/core/test_alert_episodes.py b/tests/core/test_alert_episodes.py
index de18f1d..1d08632 100644
--- a/tests/core/test_alert_episodes.py
+++ b/tests/core/test_alert_episodes.py
@@ -74,14 +74,18 @@ def test_an_episode_split_by_paging_says_so():
assert not newer[0]["began_here"] and newer[0]["end"] == NOW - timedelta(minutes=10)
+def _d(label, state="sent", error=None):
+ return {"label": label, "state": state, "at": None, "attempts": None, "error": error}
+
+
def test_who_was_told_is_gathered_once_per_episode():
r = _rule()
eps = episodes(_newest_first(
- _t(r, 30, "inactive", "firing", 200.0, deliveries=[("email", False, "timed out"), ("webhook", True, None)],
+ _t(r, 30, "inactive", "firing", 200.0, deliveries=[_d("email", "failed", "timed out"), _d("webhook")],
run="run-1"),
- _t(r, 20, "firing", "resolved", 5.0, deliveries=[("webhook", True, None)], muted_by="patch")),
+ _t(r, 20, "firing", "resolved", 5.0, deliveries=[_d("webhook")], muted_by="patch")),
newest_page=True, now=NOW, since=SINCE)
- assert eps[0]["deliveries"] == [("email", False, "timed out"), ("webhook", True, None)]
+ assert eps[0]["deliveries"] == [_d("email", "failed", "timed out"), _d("webhook")]
assert eps[0]["muted_by"] == "patch" and eps[0]["runs"] == ["run-1"]
diff --git a/tests/core/test_alert_history_row.py b/tests/core/test_alert_history_row.py
index 4bbd54f..9b7d158 100644
--- a/tests/core/test_alert_history_row.py
+++ b/tests/core/test_alert_history_row.py
@@ -14,7 +14,7 @@ RULE = SimpleNamespace(metric_name="cpu_pct", operator=AlertOperator(">"), thres
def _event(**kw):
- base = dict(transitioned_at=datetime(2026, 10, 6, tzinfo=timezone.utc), from_state="inactive",
+ base = dict(id="ev-1", transitioned_at=datetime(2026, 10, 6, tzinfo=timezone.utc), from_state="inactive",
to_state="firing", metric_value=97.5, note=None, email_sent=None, email_error=None,
webhook_sent=None, webhook_error=None, ansible_run_id=None, actor_id=None)
return SimpleNamespace(**{**base, **kw})
@@ -44,12 +44,25 @@ def test_value_and_condition_read_in_words():
def test_only_attempted_deliveries_are_listed():
row = _history_row(_event(email_sent=False, email_error="timed out", webhook_sent=None), RULE)
- assert row["deliveries"] == [("email", False, "timed out")]
+ assert [(d["label"], d["state"], d["error"]) for d in row["deliveries"]] == [("email", "failed", "timed out")]
row = _history_row(_event(email_sent=True, webhook_sent=True), RULE)
- assert [(label, sent) for label, sent, _ in row["deliveries"]] == [("email", True), ("webhook", True)]
+ assert [(d["label"], d["state"]) for d in row["deliveries"]] == [("email", "sent"), ("webhook", "sent")]
assert _history_row(_event(), RULE)["deliveries"] == []
+def test_an_email_through_the_outbox_shows_its_live_state():
+ """#5509: the outbox's state for the event outranks what the event holds
+ (nothing yet, while it gathers or retries)."""
+ from datetime import datetime, timezone
+ at = datetime(2026, 10, 9, 19, 50, tzinfo=timezone.utc)
+ ev = _event(email_sent=None, webhook_sent=True)
+ live = {ev.id: {"state": "retrying", "at": at, "attempts": 3, "error": "451 greylisted"}}
+ row = _history_row(ev, RULE, emails=live)
+ assert row["deliveries"][0] == {"label": "email", "state": "retrying", "at": at, "attempts": 3,
+ "error": "451 greylisted"}
+ assert row["deliveries"][1]["label"] == "webhook"
+
+
def test_a_transition_a_person_made_says_who():
"""#5463: an acknowledgement, and a rule change that reset the alert,
name the person; one whose account is gone still reads as a person's."""
diff --git a/tests/core/test_outbox.py b/tests/core/test_outbox.py
index b1a0cfd..08039df 100644
--- a/tests/core/test_outbox.py
+++ b/tests/core/test_outbox.py
@@ -223,3 +223,23 @@ def test_the_smtp_guard_can_fail():
assert _smtp_openers({"x.py": " with smtplib.SMTP(host) as s:"}) == {"x.py"}
assert _smtp_openers({"y.py": " smtp = notifications.smtp_open(cfg)"}) == {"y.py"}
assert _smtp_openers({"z.py": "# smtp_open(cfg) is the outbox's\ndef smtp_open(cfg):"}) == set()
+
+
+# ── How an alert's email reads in history (#5509) ────────────────────────────
+
+def test_an_alerts_email_reads_by_its_least_finished_copy():
+ later, sent_at = NOW + timedelta(minutes=2), NOW + timedelta(seconds=5)
+ held = _row(state="held", send_after=later, created_at=NOW)
+ retrying = _row(recipient="b@x", state="pending", attempts=3, next_attempt_at=later,
+ last_error="451 greylisted", created_at=NOW)
+ sent = _row(recipient="c@x", state="sent", attempts=1, sent_at=sent_at, created_at=NOW)
+ failed = _row(recipient="d@x", state="failed", attempts=1, first_attempt_at=NOW,
+ last_error="550 no such user", created_at=NOW)
+ first = _row(recipient="e@x", state="pending", attempts=0, next_attempt_at=NOW, created_at=NOW)
+ assert outbox.delivery([sent, held, failed]) == {"state": "held", "at": later, "attempts": 0, "error": None}
+ assert outbox.delivery([sent, retrying]) == {"state": "retrying", "at": later, "attempts": 3,
+ "error": "451 greylisted"}
+ assert outbox.delivery([first])["state"] == "sending"
+ assert outbox.delivery([sent, failed]) == {"state": "failed", "at": NOW, "attempts": 1,
+ "error": "d@x: 550 no such user"}
+ assert outbox.delivery([sent]) == {"state": "sent", "at": sent_at, "attempts": 1, "error": None}
diff --git a/tests/core/test_steward_health.py b/tests/core/test_steward_health.py
index 5ed9673..4f0f404 100644
--- a/tests/core/test_steward_health.py
+++ b/tests/core/test_steward_health.py
@@ -13,10 +13,15 @@ STORAGE = {"total": 3 * 2**30, "rest": {"count": 0, "bytes": 0},
{"name": "docker_logs", "bytes": 512 * 2**20, "rows": None}]}
-async def _page(app, ctx, storage=STORAGE, cleanup=None):
+NO_EMAIL = {"rows": [], "more": 0, "sent": 0, "problems": []}
+
+
+async def _page(app, ctx, storage=STORAGE, cleanup=None, email=NO_EMAIL):
+ from steward.core.outbox import KIND_LABELS
async with app.test_request_context("/settings/system/"):
session["user_role"] = "admin"
- page = await render_template("settings/system.html", **ctx, storage=storage, cleanup=cleanup)
+ page = await render_template("settings/system.html", **ctx, storage=storage, cleanup=cleanup,
+ email=email, kind_labels=KIND_LABELS)
return " ".join(page.split())
@@ -109,3 +114,46 @@ async def test_the_last_cleanup_keeps_only_what_went(monkeypatch):
assert got["removed"] == [("docker_logs: past the age window", 7)]
stored = {}
assert await health.last_cleanup(None) is None
+
+
+# ── Outgoing email (#5509) ────────────────────────────────────────────────────
+
+def _mail(**kw):
+ from steward.models.outbox import EmailOutbox
+ base = dict(kind="alert", recipient="ann@example.com", state="pending", attempts=0,
+ created_at=AT, send_after=None, next_attempt_at=None, last_error=None)
+ return EmailOutbox(**{**base, **kw})
+
+
+async def test_outgoing_email_lists_what_has_not_gone_failures_first(app, previews):
+ previews()
+ got = await health.process_health(None)
+ for r in got["rows"]:
+ r["spark"] = ""
+ email = {"sent": 1234, "more": 3, "problems": ["1 email failed in the last 7 days"], "rows": [
+ _mail(state="failed", kind="invite", recipient="bob@example.com", attempts=1,
+ last_error="550 no such user"),
+ _mail(state="pending", attempts=3, next_attempt_at=AT, last_error="451 greylisted"),
+ _mail(state="held", kind="alert", send_after=AT)]}
+ got["problems"] = email["problems"] + got["problems"]
+ page = await _page(app, got, email=email)
+ # The verdict leads with it; the section says each one's state and why.
+ assert '1 email failed in the last 7 days.' in page
+ assert "1,234 sent in the last 7 days." in page and "Nothing waiting" not in page
+ assert page.index("bob@example.com") < page.index("451 greylisted")
+ assert "Invite" in page and "550 no such user" in page
+ assert "retrying attempt 3, next" in page
+ assert "gathering sends" in page
+ assert "+3 more not yet sent" in page
+ assert 'title="<' not in page
+ assert page.index("Steward itself") < page.index("Outgoing email") < page.index("Storage")
+
+
+async def test_nothing_waiting_says_so(app, previews):
+ previews()
+ got = await health.process_health(None)
+ for r in got["rows"]:
+ r["spark"] = ""
+ page = await _page(app, got, email={**NO_EMAIL, "sent": 2})
+ assert "Nothing waiting to send. 2 sent in the last 7 days." in page
+ assert "Steward is running normally." in page
diff --git a/tests/integration/test_email_delivery_pages.py b/tests/integration/test_email_delivery_pages.py
new file mode 100644
index 0000000..42bb7ca
--- /dev/null
+++ b/tests/integration/test_email_delivery_pages.py
@@ -0,0 +1,99 @@
+"""Integration: email delivery you can see (#5509), against Postgres.
+
+An alert's email gathering for its window reads so in the alert's history
+and on Settings → System; once refused it reads "failed" with the server's
+words in both places, and the System page's verdict names it.
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import smtplib
+import time
+import uuid
+from datetime import datetime, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+ if not os.environ.get("STEWARD_DATABASE_URL"):
+ pytest.skip("needs Postgres")
+ from steward.app import create_app
+ return create_app(testing=False)
+
+
+async def _admin(app, tag):
+ from steward.models.users import User, UserRole
+ uid = str(uuid.uuid4())
+ async with app.db_sessionmaker() as s:
+ async with s.begin():
+ s.add(User(id=uid, username=f"dl-{tag}", email=f"dl-{tag}@test", password_hash="x",
+ role=UserRole.admin, is_active=True))
+ client = app.test_client()
+ async with client.session_transaction() as sess:
+ sess.update(user_id=uid, user_role="admin", username=f"dl-{tag}", _csrf="test-token",
+ login_at=time.time())
+ return client, uid
+
+
+def test_an_alerts_email_reads_gathering_then_failed_in_history_and_on_system(app, monkeypatch):
+ from steward.alerts import service as alert_svc
+ from steward.core import outbox
+ from steward.core.alerts import record_metrics
+ from steward.models.alerts import AlertEvent
+ from steward.models.metrics import PluginMetric
+ from tests.integration._mail import capture
+
+ tag = uuid.uuid4().hex[:8]
+ extra = f"pager-{tag}@example.com"
+ capture(app, monkeypatch, lambda m: smtplib.SMTPRecipientsRefused({m["To"]: (550, b"mailbox gone")})
+ if m["To"] == extra else None)
+ app.config["SMTP"]["recipients"] = [extra]
+
+ async def _go():
+ client, uid = await _admin(app, tag)
+ now = datetime.now(timezone.utc)
+ res = f"box-{tag}"
+ async with app.db_sessionmaker() as s:
+ async with s.begin():
+ await record_metrics(s, [PluginMetric(source_module="host_agent", resource_name=res,
+ metric_name="cpu_pct", value=1.0)])
+ await s.flush()
+ fields = await alert_svc.rule_fields(
+ s, name=f"hot {tag}", source_module="host_agent", resource_name=res,
+ metric_name="cpu_pct", operator=">", threshold=90)
+ rule = await alert_svc.add_rule(s, fields, created_by=uid)
+ event = AlertEvent(rule_id=rule.id, from_state="inactive", to_state="firing",
+ metric_value=95.0, transitioned_at=now)
+ s.add(event)
+ rule_id, event_id = rule.id, event.id
+ await outbox.hold_alert(app, {"rule_name": f"hot {tag}", "state": "FIRING", "metric": "cpu_pct",
+ "value": 95.0, "operator": ">", "threshold": 90.0, "resource": res,
+ "source_module": "host_agent", "note": None,
+ "timestamp": now.isoformat()}, event_id, now=now)
+
+ history = await (await client.get(f"/alerts/history?rule={rule_id}")).get_data(as_text=True)
+ assert "email gathering" in history and "waiting for other alerts; sends" in history
+ system = await (await client.get("/settings/system/")).get_data(as_text=True)
+ assert extra in system and "gathering" in system
+
+ # The window closes; the server refuses the pager address.
+ from sqlalchemy import select
+ from steward.models.outbox import email_outbox_events as link
+ async with app.db_sessionmaker() as s:
+ ids = list((await s.execute(select(link.c.outbox_id).where(
+ link.c.event_id == event_id))).scalars())
+ await outbox.attempt(app, ids)
+
+ history = await (await client.get(f"/alerts/history?rule={rule_id}")).get_data(as_text=True)
+ assert "email failed" in history and f"{extra}: 550 mailbox gone" in history
+ system = " ".join((await (await client.get("/settings/system/")).get_data(as_text=True)).split())
+ verdict = system[system.index("page-verdict"):system.index("
", system.index("page-verdict"))]
+ assert "failed in the last 7 days" in verdict
+ assert extra in system and "550 mailbox gone" in system
+
+ asyncio.run(_go())