feat(email): every email is a row in email_outbox until the server accepts it (#5506)

#5506 "Email outbox: a table of outgoing email and a sender that retries
until accepted, for up to an hour" (milestone 501).

- email_outbox (migration 0042) holds each message per recipient, its
  words, its state (held / pending / sending / sent / failed), attempts and
  the server's last answer. email_outbox_events links alert events for
  #5507.
- core/outbox.py: enqueue; a sender task every 15 s claims what is due
  (SKIP LOCKED, stale claims released), sends the batch over one
  connection with no session held, and settles each row:
  - accepted -> sent;
  - a 5xx refusal -> failed at once with the server's words;
  - no connection, a timeout, a 4xx -> retried after 30 s, 1, 2, 5, 10,
    15 min, the last try at the hour, then failed;
  - a test (retry=False) settles on its first answer.
  It runs on startup, so a row left by a restart goes out at boot.
- notifications.smtp_open is the one place a connection is opened.
- mail.message takes the html part as optional.
- cleanup ages email_outbox rows on the data window.

Senders move onto it in #5507 and #5508.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/models/__init__.py

@@ -12,6 +12,7 @@ from .ansible_inventory import AnsibleTarget, AnsibleGroup, ansible_target_group
 from .ansible_schedule import AnsibleSchedule
 from .settings import AppSetting
 from .dashboard import Dashboard, DashboardWidget, DashboardShareToken
+from .outbox import EmailOutbox, email_outbox_events
 
 __all__ = [
     "Base", "User", "ApiToken", "UserInvite", "PasswordReset",
@@ -24,4 +25,5 @@ __all__ = [
     "AnsibleSchedule",
     "AppSetting",
     "Dashboard", "DashboardWidget", "DashboardShareToken",
+    "EmailOutbox", "email_outbox_events",
 ]

steward/models/outbox.py

@@ -0,0 +1,68 @@
+"""Outgoing email, one row per message per recipient, until the mail server
+accepts it (#5506). Sent by steward/core/outbox.py; the System 156 ruling
+"Every email goes through a queue on the record" is why it is a table and
+not something held in memory: a restart picks up whatever is unsent.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from sqlalchemy import (Boolean, Column, DateTime, ForeignKey, Index, Integer, String, Table,
+                        Text)
+from sqlalchemy.orm import Mapped, mapped_column
+
+from steward.core.ids import time_ordered_id
+
+from .base import Base
+
+# held:    an alert email gathering its burst until send_after (#5507)
+# pending: due at next_attempt_at, first try or a retry after a temporary failure
+# sending: claimed by a sender; back to pending if the claim goes stale
+# sent:    the mail server accepted it
+# failed:  refused outright, a test that didn't go, or an hour of retries
+STATES = ("held", "pending", "sending", "sent", "failed")
+
+
+class EmailOutbox(Base):
+    __tablename__ = "email_outbox"
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
+    kind: Mapped[str] = mapped_column(String(16), nullable=False)   # alert | report | invite | reset | test
+    recipient: Mapped[str] = mapped_column(String(320), nullable=False)
+    # How the recipient reads times, kept so an alert email rendered when its
+    # burst closes is still written in their zone.
+    tz: Mapped[str] = mapped_column(String(64), nullable=False, default="UTC")
+    show_utc: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    # Rendered at enqueue, or (an alert burst) when the window closes; a retry
+    # sends the same words.
+    subject: Mapped[str | None] = mapped_column(Text, nullable=True)
+    text: Mapped[str | None] = mapped_column(Text, nullable=True)
+    html: Mapped[str | None] = mapped_column(Text, nullable=True)
+    state: Mapped[str] = mapped_column(String(10), nullable=False, default="pending")
+    retry: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
+    send_after: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    first_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    claimed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    sent_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+
+    __table_args__ = (
+        # The sender's question each tick: what is due, in each open state.
+        Index("ix_email_outbox_state_due", "state", "next_attempt_at"),
+        Index("ix_email_outbox_created", "created_at"),
+    )
+
+
+# Which alert events an email carries (#5507): a burst carries several, and
+# each event is carried by one email per recipient.
+email_outbox_events = Table(
+    "email_outbox_events", Base.metadata,
+    Column("outbox_id", String(36), ForeignKey("email_outbox.id", ondelete="CASCADE"),
+           primary_key=True),
+    Column("event_id", String(36), ForeignKey("alert_events.id", ondelete="CASCADE"),
+           primary_key=True, index=True),
+)