Files
FabledSteward/steward/core/notifications.py
T
bvandeusenandClaude Opus 5.5 4e77ad0e7f
CI / lint (push) Successful in 2s
CI / integration (push) Successful in 1m15s
CI / unit (push) Successful in 1m29s
CI / publish (push) Successful in 1m30s
feat(email): every email is a row in email_outbox until the server accepts it (#5506)
#5506 "Email outbox: a table of outgoing email and a sender that retries
until accepted, for up to an hour" (milestone 501).

- email_outbox (migration 0042) holds each message per recipient, its
  words, its state (held / pending / sending / sent / failed), attempts and
  the server's last answer. email_outbox_events links alert events for
  #5507.
- core/outbox.py: enqueue; a sender task every 15 s claims what is due
  (SKIP LOCKED, stale claims released), sends the batch over one
  connection with no session held, and settles each row:
  - accepted -> sent;
  - a 5xx refusal -> failed at once with the server's words;
  - no connection, a timeout, a 4xx -> retried after 30 s, 1, 2, 5, 10,
    15 min, the last try at the hour, then failed;
  - a test (retry=False) settles on its first answer.
  It runs on startup, so a row left by a restart goes out at boot.
- notifications.smtp_open is the one place a connection is opened.
- mail.message takes the html part as optional.
- cleanup ages email_outbox rows on the data window.

Senders move onto it in #5507 and #5508.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 15:53:53 -04:00

139 lines
4.8 KiB
Python

from __future__ import annotations
import json
import logging
import smtplib
import ssl
import types
from email.message import EmailMessage
import httpx
from jinja2.sandbox import SandboxedEnvironment
logger = logging.getLogger(__name__)
async def dispatch_notifications(
smtp_cfg: dict,
webhook_cfg: dict,
alert_data: dict,
recipients: list = (),
) -> dict:
"""Dispatch all configured notification channels: email to each of
`recipients` (mail.Recipient, from mail.recipients) when a server is set.
Returns results dict."""
results = {}
if smtp_cfg.get("host") and recipients:
results["email"] = await _send_email(smtp_cfg, alert_data, recipients)
if webhook_cfg.get("url"):
results["webhook"] = await _send_webhook(webhook_cfg, alert_data)
return results
SMTP_TIMEOUT_S = 15
def smtp_mode(port, tls) -> str:
"""How to talk to the SMTP server: "ssl" (implicit TLS, port 465),
"starttls" (upgrade a plain connection, when TLS is ticked) or "plain"."""
if int(port or 0) == 465:
return "ssl"
return "starttls" if tls else "plain"
async def _send_email(cfg: dict, alert: dict, recipients: list) -> dict:
"""Each recipient their own copy, in their own zone, over one connection."""
from steward.core.mail import alert_email, message, send
try:
messages = [message(cfg, to.address, *alert_email(cfg, alert, to)) for to in recipients]
await send(cfg, messages)
return {"sent": True, "error": None}
except Exception as exc:
logger.error(f"Email notification failed: {exc}")
return {"sent": False, "error": str(exc)}
def smtp_open(cfg: dict) -> smtplib.SMTP:
"""A connection to the mail server, TLS and login done, every step
bounded by SMTP_TIMEOUT_S. The caller closes it (use it in `with`)."""
port = int(cfg.get("port") or 587)
mode = smtp_mode(port, cfg.get("tls", True))
ctx = ssl.create_default_context()
if mode == "ssl":
smtp = smtplib.SMTP_SSL(cfg["host"], port, timeout=SMTP_TIMEOUT_S, context=ctx)
else:
smtp = smtplib.SMTP(cfg["host"], port, timeout=SMTP_TIMEOUT_S)
try:
if mode == "starttls":
smtp.starttls(context=ctx)
if cfg.get("username"):
smtp.login(cfg["username"], cfg.get("password", ""))
except BaseException:
smtp.close()
raise
return smtp
def _smtp_send_sync(cfg: dict, messages: list[EmailMessage]) -> None:
with smtp_open(cfg) as smtp:
for msg in messages:
smtp.send_message(msg)
def _json_str(value) -> str:
"""A value as it may sit between the quotes of a JSON string."""
return json.dumps("" if value is None else str(value))[1:-1]
def render_webhook(template_str: str, alert: dict) -> str:
"""The webhook body for an alert, checked to be JSON.
In the template, `alert.<field>` is escaped to sit inside a JSON string
("{{ alert.rule_name }}" stays valid whatever the name holds), with a
missing value reading "no data"; `raw.<field>` is the plain value for use
with |tojson ({{ raw.value | tojson }} gives a number or null). The
template runs sandboxed. ValueError says why a body could not be made.
"""
shown = dict(alert)
if shown.get("value") is None:
shown["value"] = "no data"
env = SandboxedEnvironment()
try:
rendered = env.from_string(template_str).render(
alert=types.SimpleNamespace(**{k: _json_str(v) for k, v in shown.items()}),
raw=types.SimpleNamespace(**alert))
except Exception as exc:
raise ValueError(f"Template render failed: {exc}") from None
try:
json.loads(rendered)
except json.JSONDecodeError as exc:
raise ValueError(f"Rendered webhook template is not valid JSON: {exc}") from None
return rendered
async def _send_webhook(cfg: dict, alert: dict) -> dict:
template_str = cfg.get("template") or '{"content": "{{ alert.state }} — {{ alert.rule_name }}"}'
try:
rendered = render_webhook(template_str, alert)
except ValueError as exc:
logger.error(str(exc))
return {"sent": False, "error": str(exc)}
try:
async with httpx.AsyncClient(timeout=10) as client:
response = await client.post(
cfg["url"],
content=rendered,
headers={"Content-Type": "application/json"},
)
response.raise_for_status()
return {"sent": True, "error": None}
except Exception as exc:
# The URL is a secret (it usually carries its own token) and httpx
# puts it in its error messages.
error = str(exc).replace(cfg["url"], "<webhook URL>")
logger.error(f"Webhook notification failed: {error}")
return {"sent": False, "error": error}