feat(email): every email is a row in email_outbox until the server accepts it (#5506)

#5506 "Email outbox: a table of outgoing email and a sender that retries
until accepted, for up to an hour" (milestone 501).

- email_outbox (migration 0042) holds each message per recipient, its
  words, its state (held / pending / sending / sent / failed), attempts and
  the server's last answer. email_outbox_events links alert events for
  #5507.
- core/outbox.py: enqueue; a sender task every 15 s claims what is due
  (SKIP LOCKED, stale claims released), sends the batch over one
  connection with no session held, and settles each row:
  - accepted -> sent;
  - a 5xx refusal -> failed at once with the server's words;
  - no connection, a timeout, a 4xx -> retried after 30 s, 1, 2, 5, 10,
    15 min, the last try at the hour, then failed;
  - a test (retry=False) settles on its first answer.
  It runs on startup, so a row left by a restart goes out at boot.
- notifications.smtp_open is the one place a connection is opened.
- mail.message takes the html part as optional.
- cleanup ages email_outbox rows on the data window.

Senders move onto it in #5507 and #5508.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/app.py

@@ -438,7 +438,21 @@ def _register_core_tasks(app: Quart) -> None:
         from .core.self_monitor import record_self_metrics
         await record_self_metrics(app)
 
+    async def run_email_outbox():
+        from .core.outbox import send_due
+        await send_due(app)
+
+    from .core.outbox import SENDER_TICK_S
     app._task_registry.extend([
+        # Every email is a row until the mail server accepts it (#5506); this
+        # sends what is due and retries what failed. On startup too, so what a
+        # restart left unsent goes out at once.
+        ScheduledTask(
+            name="email_outbox",
+            coro_factory=run_email_outbox,
+            interval_seconds=SENDER_TICK_S,
+            run_on_startup=True,
+        ),
         ScheduledTask(
             name="weekly_report_check",
             coro_factory=run_report_check,

steward/core/cleanup.py

@@ -12,6 +12,7 @@ from steward.models.alerts import AlertEvent
 from steward.models.ansible import AnsibleRun
 from steward.models.audit import AuditEvent
 from steward.models.monitors import MonitorResult
+from steward.models.outbox import EmailOutbox
 
 if TYPE_CHECKING:
     from quart import Quart
@@ -23,7 +24,8 @@ logger = logging.getLogger(__name__)
 # own with prune_by_age() in setup(), so the core never imports plugin models.
 # Tables kept another way are not listed: plugin_metrics and the docker tables
 # are rolled up (below), and the audit log has its own, longer window.
-_AGED: list = [MonitorResult.checked_at, AnsibleRun.started_at, AlertEvent.transitioned_at]
+_AGED: list = [MonitorResult.checked_at, AnsibleRun.started_at, AlertEvent.transitioned_at,
+               EmailOutbox.created_at]
 
 
 def prune_by_age(column) -> None:

steward/core/mail.py

@@ -180,14 +180,15 @@ def sender(cfg: dict) -> str:
     return cfg.get("from") or cfg.get("username") or "steward@localhost"
 
 
-def message(cfg: dict, to: str, subject: str, text: str, html: str) -> EmailMessage:
+def message(cfg: dict, to: str, subject: str, text: str, html: str | None = None) -> EmailMessage:
     msg = EmailMessage()
     # A header may not hold a line break; a rule name can (MCP takes any string).
     msg["Subject"] = " ".join(subject.split())
     msg["From"] = sender(cfg)
     msg["To"] = to
     msg.set_content(text)
-    msg.add_alternative(html, subtype="html")
+    if html:
+        msg.add_alternative(html, subtype="html")
     return msg
 
 

steward/core/notifications.py

@@ -55,7 +55,9 @@ async def _send_email(cfg: dict, alert: dict, recipients: list) -> dict:
         return {"sent": False, "error": str(exc)}
 
 
-def _smtp_send_sync(cfg: dict, messages: list[EmailMessage]) -> None:
+def smtp_open(cfg: dict) -> smtplib.SMTP:
+    """A connection to the mail server, TLS and login done, every step
+    bounded by SMTP_TIMEOUT_S. The caller closes it (use it in `with`)."""
     port = int(cfg.get("port") or 587)
     mode = smtp_mode(port, cfg.get("tls", True))
     ctx = ssl.create_default_context()
@@ -63,11 +65,19 @@ def _smtp_send_sync(cfg: dict, messages: list[EmailMessage]) -> None:
         smtp = smtplib.SMTP_SSL(cfg["host"], port, timeout=SMTP_TIMEOUT_S, context=ctx)
     else:
         smtp = smtplib.SMTP(cfg["host"], port, timeout=SMTP_TIMEOUT_S)
-    with smtp:
+    try:
         if mode == "starttls":
             smtp.starttls(context=ctx)
         if cfg.get("username"):
             smtp.login(cfg["username"], cfg.get("password", ""))
+    except BaseException:
+        smtp.close()
+        raise
+    return smtp
+
+
+def _smtp_send_sync(cfg: dict, messages: list[EmailMessage]) -> None:
+    with smtp_open(cfg) as smtp:
         for msg in messages:
             smtp.send_message(msg)
 

steward/core/outbox.py

@@ -0,0 +1,233 @@
+"""Every email Steward sends goes out through here (#5506).
+
+An email is a row in email_outbox until the mail server accepts it (System
+156 ruling, "Every email goes through a queue on the record"). The sender,
+send_due, runs every SENDER_TICK_S. Each tick it:
+- claims what is due;
+- sends the batch over one connection;
+- settles each row on what the server said.
+
+How a row settles:
+- **accepted:** sent.
+- **refused outright** (a 5xx reply, e.g. "no such mailbox"): failed at once,
+  with the server's reason. Retrying would only repeat the refusal.
+- **temporary** (no connection, a timeout, a 4xx reply): tried again after
+  BACKOFF_S, and once more at the hour. If the server still hasn't taken it
+  by then, it is failed with the last error.
+- **a test** (retry=False): settles on its first answer either way.
+
+Nothing is kept in memory. A row left behind by a restart is still pending
+(or held, or claimed by a run that died), and the next tick picks it up.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import smtplib
+from datetime import datetime, timedelta, timezone
+from email.message import EmailMessage
+from typing import TYPE_CHECKING
+
+from sqlalchemy import or_, select, update
+
+from steward.models.outbox import EmailOutbox
+
+if TYPE_CHECKING:
+    from quart import Quart
+    from steward.core.mail import Recipient
+
+logger = logging.getLogger(__name__)
+
+SENDER_TICK_S = 15
+# Waits after the 1st, 2nd, … temporary failure; every later wait is the last.
+BACKOFF_S = (30, 60, 120, 300, 600, 900)
+# An email the server hasn't accepted this long after its first attempt is
+# failed (operator ruling: "retry fails at an hour").
+GIVE_UP_S = 3600
+# A claim older than this belongs to a run that died mid-send: back to pending.
+STALE_CLAIM_S = 300
+BATCH = 50
+
+
+def backoff(attempts: int) -> int:
+    """Seconds to wait after the `attempts`-th failed try."""
+    return BACKOFF_S[min(max(attempts, 1), len(BACKOFF_S)) - 1]
+
+
+def classify(exc: BaseException) -> str:
+    """"permanent" when the mail server refused it (5xx), or the message
+    can't be sent as written; "temporary" when trying later may work: no
+    connection, a timeout, a dropped line, or a 4xx "try again"."""
+    if isinstance(exc, smtplib.SMTPRecipientsRefused):
+        codes = [code for code, _ in exc.recipients.values()]
+        return "permanent" if codes and all(500 <= c < 600 for c in codes) else "temporary"
+    if isinstance(exc, smtplib.SMTPResponseException):
+        return "permanent" if 500 <= exc.smtp_code < 600 else "temporary"
+    if isinstance(exc, smtplib.SMTPNotSupportedError):
+        return "permanent"            # e.g. STARTTLS asked of a server without it
+    if isinstance(exc, (OSError, TimeoutError)):
+        return "temporary"            # SMTPServerDisconnected, refused, unreachable, TLS
+    return "permanent"                # a message we couldn't build: trying again repeats it
+
+
+def reason(exc: BaseException) -> str:
+    """What went wrong, as the server said it where it said anything."""
+    def said(code, msg) -> str:
+        text = msg.decode(errors="replace") if isinstance(msg, bytes) else str(msg)
+        return f"{code} {' '.join(text.split())}"
+    if isinstance(exc, smtplib.SMTPRecipientsRefused):
+        return "; ".join(f"{to}: {said(c, m)}" for to, (c, m) in exc.recipients.items())
+    if isinstance(exc, smtplib.SMTPResponseException):
+        return said(exc.smtp_code, exc.smtp_error)
+    if isinstance(exc, TimeoutError):
+        return "The mail server didn't answer in time"
+    if isinstance(exc, ConnectionRefusedError):
+        return "The mail server refused the connection"
+    if isinstance(exc, smtplib.SMTPServerDisconnected):
+        return f"The mail server closed the connection{': ' + str(exc) if str(exc) else ''}"
+    return str(exc) or type(exc).__name__
+
+
+def enqueue(db, *, kind: str, to: "Recipient | str", subject: str | None = None,
+            text: str | None = None, html: str | None = None,
+            send_after: datetime | None = None, retry: bool = True,
+            now: datetime | None = None) -> EmailOutbox:
+    """Add one email to the queue in `db`'s transaction (the caller commits).
+    `send_after` holds it until then (an alert burst, #5507); otherwise it is
+    due at once. `to` is a mail.Recipient, or a bare address read in UTC."""
+    now = now or datetime.now(timezone.utc)
+    address, tz, show_utc = (to, "UTC", False) if isinstance(to, str) else \
+        (to.address, getattr(to.tz, "key", None) or "UTC", to.show_utc)
+    row = EmailOutbox(kind=kind, recipient=address, tz=tz, show_utc=show_utc,
+                      subject=subject, text=text, html=html, retry=retry, created_at=now,
+                      state="held" if send_after else "pending", send_after=send_after,
+                      next_attempt_at=send_after or now)
+    db.add(row)
+    return row
+
+
+def settle(row: EmailOutbox, outcome: BaseException | None, now: datetime) -> None:
+    """Record one attempt's outcome on `row`: None is the server's acceptance."""
+    row.attempts += 1
+    row.first_attempt_at = row.first_attempt_at or now
+    row.claimed_at = None
+    if outcome is None:
+        row.state, row.sent_at, row.last_error, row.next_attempt_at = "sent", now, None, None
+        return
+    row.last_error = reason(outcome)
+    deadline = row.first_attempt_at + timedelta(seconds=GIVE_UP_S)
+    if classify(outcome) == "permanent" or not row.retry or now >= deadline:
+        row.state, row.next_attempt_at = "failed", None
+        if now >= deadline and classify(outcome) == "temporary" and row.retry:
+            row.last_error += " (gave up after an hour of retries)"
+        return
+    # Retry; the last try lands at the hour, not after it.
+    row.state = "pending"
+    row.next_attempt_at = min(now + timedelta(seconds=backoff(row.attempts)), deadline)
+
+
+def deliver_sync(cfg: dict, messages: list[EmailMessage]) -> list[BaseException | None]:
+    """Send each message over one connection; per message, None when the
+    server accepted it, else what went wrong. A failure to connect, or a
+    line dropped mid-batch, is every unsent message's outcome."""
+    from steward.core.notifications import smtp_open
+    outcomes: list[BaseException | None] = []
+    try:
+        with smtp_open(cfg) as smtp:
+            for msg in messages:
+                try:
+                    smtp.send_message(msg)
+                    outcomes.append(None)
+                except smtplib.SMTPServerDisconnected as exc:
+                    outcomes.append(exc)
+                    break
+                except Exception as exc:
+                    outcomes.append(exc)
+    except Exception as exc:
+        outcomes += [exc] * (len(messages) - len(outcomes))
+    return outcomes + [smtplib.SMTPServerDisconnected("connection lost")] * (len(messages) - len(outcomes))
+
+
+async def write_held(app: "Quart", row: EmailOutbox) -> None:
+    """Fill in a row queued without its words: an alert burst, written when
+    its window closes (#5507). Raises when it can't be written."""
+    raise ValueError(f"no words to send for a {row.kind} email")
+
+
+async def attempt(app: "Quart", ids: list[str], now: datetime | None = None) -> dict[str, str]:
+    """Try to send the claimed rows `ids` once, and settle each. The words
+    are written and kept first, so a retry sends the same email; no session
+    is held while the mail server is talked to. Returns {id: state after}."""
+    from steward.core.mail import message
+    if not ids:
+        return {}
+    cfg = app.config.get("SMTP", {})
+    outcomes: dict[str, BaseException | None] = {}
+    sendable: list[tuple[str, EmailMessage]] = []
+    async with app.db_sessionmaker() as db:
+        for row in (await db.execute(select(EmailOutbox).where(EmailOutbox.id.in_(ids)))).scalars():
+            if not cfg.get("host"):
+                outcomes[row.id] = smtplib.SMTPNotSupportedError(
+                    "No mail server is set (Settings → Notifications)")
+                continue
+            try:
+                if row.subject is None:
+                    await write_held(app, row)
+                sendable.append((row.id, message(cfg, row.recipient, row.subject, row.text, row.html)))
+            except Exception as exc:
+                logger.exception("email %s: could not be written", row.id)
+                outcomes[row.id] = exc
+        await db.commit()
+    if sendable:
+        sent = await asyncio.get_running_loop().run_in_executor(
+            None, deliver_sync, cfg, [m for _, m in sendable])
+        outcomes.update({rid: out for (rid, _), out in zip(sendable, sent)})
+    now = now or datetime.now(timezone.utc)
+    async with app.db_sessionmaker() as db:
+        async with db.begin():
+            rows = list((await db.execute(
+                select(EmailOutbox).where(EmailOutbox.id.in_(ids)))).scalars())
+            for row in rows:
+                settle(row, outcomes.get(row.id, RuntimeError("not tried")), now)
+                if row.state == "failed":
+                    logger.warning("email %s to %s failed: %s", row.kind, row.recipient, row.last_error)
+            await after_settle(db, rows)
+        return {row.id: row.state for row in rows}
+
+
+async def after_settle(db, rows: list[EmailOutbox]) -> None:
+    """What else follows from these rows' new states, in the same
+    transaction: an alert event's email outcome (#5507)."""
+
+
+async def claim_due(app: "Quart", now: datetime, limit: int = BATCH) -> list[str]:
+    """Mark what is due as being sent, and return its ids: pending rows whose
+    time has come, held rows whose window has closed, and rows a dead run
+    had claimed. SKIP LOCKED, so two senders never claim one row."""
+    stale = now - timedelta(seconds=STALE_CLAIM_S)
+    async with app.db_sessionmaker() as db:
+        async with db.begin():
+            await db.execute(update(EmailOutbox)
+                             .where(EmailOutbox.state == "sending", EmailOutbox.claimed_at < stale)
+                             .values(state="pending", claimed_at=None, next_attempt_at=now))
+            ids = list((await db.execute(
+                select(EmailOutbox.id)
+                .where(or_(EmailOutbox.state == "pending", EmailOutbox.state == "held"),
+                       EmailOutbox.next_attempt_at <= now)
+                .order_by(EmailOutbox.next_attempt_at)
+                .limit(limit)
+                .with_for_update(skip_locked=True))).scalars())
+            if ids:
+                await db.execute(update(EmailOutbox).where(EmailOutbox.id.in_(ids))
+                                 .values(state="sending", claimed_at=now))
+    return ids
+
+
+async def send_due(app: "Quart") -> None:
+    """The sender's tick: claim what is due and try it once."""
+    now = datetime.now(timezone.utc)
+    ids = await claim_due(app, now)
+    if ids:
+        states = await attempt(app, ids, now)
+        logger.info("email outbox: %d tried, %d sent", len(states),
+                    sum(1 for s in states.values() if s == "sent"))

steward/migrations/versions/0042_email_outbox.py

@@ -0,0 +1,57 @@
+"""Outgoing email as rows until the mail server accepts them (#5506), and
+which alert events each one carries (#5507).
+
+Revision ID: 0042_email_outbox
+Revises: 0041_alert_rule_covers
+Create Date: 2026-10-09
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "0042_email_outbox"
+down_revision: Union[str, None] = "0041_alert_rule_covers"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "email_outbox",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("kind", sa.String(16), nullable=False),
+        sa.Column("recipient", sa.String(320), nullable=False),
+        sa.Column("tz", sa.String(64), nullable=False, server_default="UTC"),
+        sa.Column("show_utc", sa.Boolean, nullable=False, server_default=sa.false()),
+        sa.Column("subject", sa.Text, nullable=True),
+        sa.Column("text", sa.Text, nullable=True),
+        sa.Column("html", sa.Text, nullable=True),
+        sa.Column("state", sa.String(10), nullable=False, server_default="pending"),
+        sa.Column("retry", sa.Boolean, nullable=False, server_default=sa.true()),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("send_after", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("first_attempt_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("attempts", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("claimed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_error", sa.Text, nullable=True),
+        sa.Column("sent_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.create_index("ix_email_outbox_state_due", "email_outbox", ["state", "next_attempt_at"])
+    op.create_index("ix_email_outbox_created", "email_outbox", ["created_at"])
+    op.create_table(
+        "email_outbox_events",
+        sa.Column("outbox_id", sa.String(36),
+                  sa.ForeignKey("email_outbox.id", ondelete="CASCADE"), primary_key=True),
+        sa.Column("event_id", sa.String(36),
+                  sa.ForeignKey("alert_events.id", ondelete="CASCADE"), primary_key=True),
+    )
+    op.create_index("ix_email_outbox_events_event_id", "email_outbox_events", ["event_id"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_email_outbox_events_event_id", table_name="email_outbox_events")
+    op.drop_table("email_outbox_events")
+    op.drop_index("ix_email_outbox_created", table_name="email_outbox")
+    op.drop_index("ix_email_outbox_state_due", table_name="email_outbox")
+    op.drop_table("email_outbox")

steward/models/__init__.py

@@ -12,6 +12,7 @@ from .ansible_inventory import AnsibleTarget, AnsibleGroup, ansible_target_group
 from .ansible_schedule import AnsibleSchedule
 from .settings import AppSetting
 from .dashboard import Dashboard, DashboardWidget, DashboardShareToken
+from .outbox import EmailOutbox, email_outbox_events
 
 __all__ = [
     "Base", "User", "ApiToken", "UserInvite", "PasswordReset",
@@ -24,4 +25,5 @@ __all__ = [
     "AnsibleSchedule",
     "AppSetting",
     "Dashboard", "DashboardWidget", "DashboardShareToken",
+    "EmailOutbox", "email_outbox_events",
 ]

steward/models/outbox.py

@@ -0,0 +1,68 @@
+"""Outgoing email, one row per message per recipient, until the mail server
+accepts it (#5506). Sent by steward/core/outbox.py; the System 156 ruling
+"Every email goes through a queue on the record" is why it is a table and
+not something held in memory: a restart picks up whatever is unsent.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from sqlalchemy import (Boolean, Column, DateTime, ForeignKey, Index, Integer, String, Table,
+                        Text)
+from sqlalchemy.orm import Mapped, mapped_column
+
+from steward.core.ids import time_ordered_id
+
+from .base import Base
+
+# held:    an alert email gathering its burst until send_after (#5507)
+# pending: due at next_attempt_at, first try or a retry after a temporary failure
+# sending: claimed by a sender; back to pending if the claim goes stale
+# sent:    the mail server accepted it
+# failed:  refused outright, a test that didn't go, or an hour of retries
+STATES = ("held", "pending", "sending", "sent", "failed")
+
+
+class EmailOutbox(Base):
+    __tablename__ = "email_outbox"
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
+    kind: Mapped[str] = mapped_column(String(16), nullable=False)   # alert | report | invite | reset | test
+    recipient: Mapped[str] = mapped_column(String(320), nullable=False)
+    # How the recipient reads times, kept so an alert email rendered when its
+    # burst closes is still written in their zone.
+    tz: Mapped[str] = mapped_column(String(64), nullable=False, default="UTC")
+    show_utc: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    # Rendered at enqueue, or (an alert burst) when the window closes; a retry
+    # sends the same words.
+    subject: Mapped[str | None] = mapped_column(Text, nullable=True)
+    text: Mapped[str | None] = mapped_column(Text, nullable=True)
+    html: Mapped[str | None] = mapped_column(Text, nullable=True)
+    state: Mapped[str] = mapped_column(String(10), nullable=False, default="pending")
+    retry: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
+    send_after: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    first_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    claimed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    sent_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+
+    __table_args__ = (
+        # The sender's question each tick: what is due, in each open state.
+        Index("ix_email_outbox_state_due", "state", "next_attempt_at"),
+        Index("ix_email_outbox_created", "created_at"),
+    )
+
+
+# Which alert events an email carries (#5507): a burst carries several, and
+# each event is carried by one email per recipient.
+email_outbox_events = Table(
+    "email_outbox_events", Base.metadata,
+    Column("outbox_id", String(36), ForeignKey("email_outbox.id", ondelete="CASCADE"),
+           primary_key=True),
+    Column("event_id", String(36), ForeignKey("alert_events.id", ondelete="CASCADE"),
+           primary_key=True, index=True),
+)