diff --git a/docs/core/alerting.md b/docs/core/alerting.md index dbfc94d..77a718c 100644 --- a/docs/core/alerting.md +++ b/docs/core/alerting.md @@ -184,6 +184,14 @@ seconds and settles each email on the server's answer: Once every email carrying an alert has settled, that alert's event records whether the email went, and to whom it didn't. +Where to see it: each alert's row in **Alerts → History** says what its email +is doing (gathering, with when it sends; retrying, with the attempt, the next +try and the server's last answer; sent; failed, with who wasn't reached and +why), and `get_alert_history` over MCP reports the same. **Settings → System → +Outgoing email** lists every email of the last 7 days not yet sent, failures +first, and the page's verdict names failures and any email still retrying +after 5 minutes. + An email a person is waiting on is tried at once instead of on the next tick: the weekly report's **Send now**, an invite, and the test buttons, so the page can say what the server answered. A report or invite the server couldn't take diff --git a/steward/alerts/episodes.py b/steward/alerts/episodes.py index 502544e..3d63522 100644 --- a/steward/alerts/episodes.py +++ b/steward/alerts/episodes.py @@ -87,8 +87,8 @@ def episodes(rows: list[dict], *, newest_page: bool, now: datetime, since: datet seen, deliveries = set(), [] for s in steps: for d in s["deliveries"]: - if (d[0], d[1]) not in seen: - seen.add((d[0], d[1])) + if (d["label"], d["state"]) not in seen: + seen.add((d["label"], d["state"])) deliveries.append(d) ep["deliveries"] = deliveries ep["muted_by"] = next((s["muted_by"] for s in steps if s["muted_by"]), None) diff --git a/steward/alerts/routes.py b/steward/alerts/routes.py index 734ad14..760a009 100644 --- a/steward/alerts/routes.py +++ b/steward/alerts/routes.py @@ -2,6 +2,7 @@ from __future__ import annotations from datetime import datetime, timezone from quart import Blueprint, render_template, request, redirect, url_for, current_app, session from steward.core.audit import log_audit +from steward.core import outbox from sqlalchemy import func, select from steward.auth.middleware import require_role from steward.alerts import service as alert_svc @@ -177,9 +178,20 @@ async def active_fragment(): rules_off=sum(1 for r in rules if not r.enabled)) -def _history_row(event, rule, names: dict | None = None) -> dict: +def _delivery(label: str, sent: bool | None, error: str | None) -> dict | None: + """A channel's outcome as recorded on the event; None when not tried.""" + if sent is None: + return None + return {"label": label, "state": "sent" if sent else "failed", "at": None, "attempts": None, + "error": None if sent else error} + + +def _history_row(event, rule, names: dict | None = None, emails: dict | None = None) -> dict: """One transition as the history page shows it. `names` maps user ids to - usernames: a transition a person made says who (#5463).""" + usernames: a transition a person made says who (#5463). `emails` is + outbox.email_states for the page: the email's live state (gathering, + retrying, sent, failed) where it went through the outbox (#5509), else + what the event recorded.""" from steward.core.alerts import MUTED, RELEASED from steward.core.formatting import metric_label, metric_value notes, muted_by, released = [], None, False @@ -208,9 +220,10 @@ def _history_row(event, rule, names: dict | None = None) -> dict: "from_state": event.from_state, "to_state": event.to_state, "value": value, "raw": event.metric_value, "condition": alert_svc.rule_condition(rule), "notes": notes, "muted_by": muted_by, "released": released, - "deliveries": [(label, sent, error) for label, sent, error in ( - ("email", event.email_sent, event.email_error), - ("webhook", event.webhook_sent, event.webhook_error)) if sent is not None], + "deliveries": [d for d in ( + {"label": "email", **(emails or {})[event.id]} if event.id in (emails or {}) + else _delivery("email", event.email_sent, event.email_error), + _delivery("webhook", event.webhook_sent, event.webhook_error)) if d], "ansible_run_id": event.ansible_run_id, } @@ -235,13 +248,14 @@ async def alert_history(): rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before) show_notified, show_action = await alert_svc.history_columns(db, since, rule_id) names = await alert_svc.usernames(db, (e.actor_id for e, _ in rows)) + emails = await outbox.email_states(db, [e.id for e, _ in rows]) verdict = None if chosen and not before: every = await alert_svc.history(db, since, rule_id, HISTORY_SUMMARY_ROWS) verdict = summary(episodes([_history_row(e, r) for e, r in every], newest_page=True, now=now, since=since)) rows, after = next_page(rows, lambda row: (row[0].transitioned_at, row[0].id)) - ctx = {"episodes": episodes([_history_row(e, r, names) for e, r in rows], + ctx = {"episodes": episodes([_history_row(e, r, names, emails) for e, r in rows], newest_page=not before, now=now, since=since), "after": after, "rules": rules, "chosen": chosen, "current_range": range_key, "show_notified": show_notified, "show_action": show_action, "verdict": verdict} diff --git a/steward/alerts/service.py b/steward/alerts/service.py index a5b8b43..70f46ae 100644 --- a/steward/alerts/service.py +++ b/steward/alerts/service.py @@ -322,9 +322,12 @@ async def history_columns(db, since: datetime, rule_id: str | None = None) -> tu base = select(AlertEvent.id).where(AlertEvent.transitioned_at >= since) if rule_id: base = base.where(AlertEvent.rule_id == rule_id) + from steward.models.outbox import email_outbox_events + queued = select(email_outbox_events.c.event_id).where( + email_outbox_events.c.event_id == AlertEvent.id).exists() notified = await db.scalar(select(base.where(or_( AlertEvent.email_sent.is_not(None), AlertEvent.webhook_sent.is_not(None), - AlertEvent.note.like(f"%{MUTED}%"))).exists())) + AlertEvent.note.like(f"%{MUTED}%"), queued)).exists())) acted = await db.scalar(select(base.where(AlertEvent.ansible_run_id.is_not(None)).exists())) return bool(notified), bool(acted) diff --git a/steward/core/outbox.py b/steward/core/outbox.py index 3d2a427..44b2646 100644 --- a/steward/core/outbox.py +++ b/steward/core/outbox.py @@ -356,3 +356,81 @@ def said(rows: list[EmailOutbox]) -> tuple[bool, str]: ([f"Not sent yet to {'; '.join(later)}; Steward will try again for up to an hour."] if later else []) return bool(rows) and not failed and not later, " ".join(parts) + + +# ── What the pages show (#5509) ─────────────────────────────────────────────── + +KIND_LABELS = {"alert": "Alert", "report": "Weekly report", "invite": "Invite", + "reset": "Password reset", "test": "Test"} +# Retrying longer than this is worth a line in Settings → System's verdict. +SLOW_S = 300 +SHOWN_DAYS = 7 + + +def delivery(rows: list[EmailOutbox]) -> dict: + """One alert event's email, over every reader's copy, as history says + it: {"state", "at", "attempts", "error"}. Gathering while any copy is + held (at: when it sends); sending/retrying while any is still due (at: + the next try); failed when any reader wasn't reached (error names each); + else sent (at: the last acceptance).""" + if held := [r for r in rows if r.state == "held"]: + return {"state": "held", "at": min(r.send_after for r in held), "attempts": 0, "error": None} + if due := [r for r in rows if r.state in ("pending", "sending")]: + worst = max(due, key=lambda r: r.attempts) + return {"state": "retrying" if worst.attempts else "sending", + "at": min((r.next_attempt_at for r in due if r.next_attempt_at), default=None), + "attempts": worst.attempts, "error": worst.last_error} + if failed := [r for r in rows if r.state == "failed"]: + return {"state": "failed", "at": max(r.first_attempt_at or r.created_at for r in failed), + "attempts": max(r.attempts for r in failed), + "error": "; ".join(f"{r.recipient}: {r.last_error}" for r in failed)} + return {"state": "sent", "at": max((r.sent_at for r in rows if r.sent_at), default=None), + "attempts": max(r.attempts for r in rows), "error": None} + + +async def email_states(db, event_ids: list[str]) -> dict[str, dict]: + """{event id: delivery(...)} for the alert events whose email went + through the outbox; an event with none (no reader, or from before it) + is absent.""" + if not event_ids: + return {} + link = email_outbox_events.c + by: dict[str, list] = {} + for event_id, row in (await db.execute( + select(link.event_id, EmailOutbox).join(EmailOutbox, EmailOutbox.id == link.outbox_id) + .where(link.event_id.in_(event_ids)))).all(): + by.setdefault(event_id, []).append(row) + return {eid: delivery(rows) for eid, rows in by.items()} + + +async def outgoing(db, now: datetime | None = None, limit: int = 20) -> dict: + """Settings → System's "Outgoing email": every email of the last + SHOWN_DAYS not yet sent (failed first, then retrying longest, then + sending and gathering), how many were sent, and the problems for the + page's verdict. {"rows": [EmailOutbox], "more": n, "sent": n, + "problems": [str]}.""" + from sqlalchemy import case, func + now = now or datetime.now(timezone.utc) + since = now - timedelta(days=SHOWN_DAYS) + open_states = ("failed", "pending", "sending", "held") + order = case({"failed": 0, "pending": 1, "sending": 2, "held": 3}, value=EmailOutbox.state) + base = select(EmailOutbox).where(EmailOutbox.created_at >= since, + EmailOutbox.state.in_(open_states)) + rows = list((await db.execute( + base.order_by(order, EmailOutbox.first_attempt_at.asc().nulls_last(), + EmailOutbox.created_at.desc()).limit(limit))).scalars()) + counts = dict((await db.execute( + select(EmailOutbox.state, func.count()).where(EmailOutbox.created_at >= since) + .group_by(EmailOutbox.state))).all()) + slow = (await db.execute(select(func.count()).select_from(EmailOutbox).where( + EmailOutbox.state.in_(("pending", "sending")), + EmailOutbox.first_attempt_at < now - timedelta(seconds=SLOW_S)))).scalar_one() + problems = [] + if failed := counts.get("failed", 0): + problems.append(f"{failed} email{'s' if failed != 1 else ''} failed in the last {SHOWN_DAYS} days") + if slow: + problems.append(f"{slow} email{'s' if slow != 1 else ''} still not accepted by the mail " + f"server after {SLOW_S // 60} minutes of trying") + shown = sum(counts.get(s, 0) for s in open_states) + return {"rows": rows, "more": max(shown - len(rows), 0), "sent": counts.get("sent", 0), + "problems": problems} diff --git a/steward/mcp/tools/alerts.py b/steward/mcp/tools/alerts.py index e386e8d..6772021 100644 --- a/steward/mcp/tools/alerts.py +++ b/steward/mcp/tools/alerts.py @@ -6,6 +6,7 @@ from datetime import datetime, timedelta, timezone from sqlalchemy import select from steward.alerts import service as alert_svc +from steward.core import outbox from steward.mcp._context import current_caller, mcp_app from steward.models.alerts import AlertOperator, AlertRule, AlertState, MaintenanceWindow from steward.models.users import UserRole @@ -82,7 +83,11 @@ async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50) Each transition carries the metric value that caused it and whether the email/webhook notification went out (with the error if not), plus the id - of any Ansible run the rule's action triggered. `by` names the person who + of any Ansible run the rule's action triggered. `email` is the email's + delivery: state "held" (gathering other alerts; `at` is when it sends), + "sending", "retrying" (`attempts`, next try `at`, last `error`), "sent" + or "failed" (`error` names each reader not reached); null when no email + was queued for it. `by` names the person who made it happen (an acknowledgement, or a rule edit that reset the alert); null when a reading or Steward itself did. """ @@ -93,6 +98,7 @@ async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50) chosen = pick(rule, rules, kind="alert rule", list_tool="list_alert_rules") if rule else None rows = await alert_svc.history(db, since, chosen.id if chosen else None, limit) names = await alert_svc.usernames(db, (e.actor_id for e, _ in rows)) + emails = await outbox.email_states(db, [e.id for e, _ in rows]) events = [ { "rule_id": r.id, @@ -106,6 +112,7 @@ async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50) "age_s": age_s(e.transitioned_at), "email_sent": e.email_sent, "email_error": e.email_error, + "email": ({**emails[e.id], "at": iso(emails[e.id]["at"])} if e.id in emails else None), "webhook_sent": e.webhook_sent, "webhook_error": e.webhook_error, "ansible_run_id": e.ansible_run_id, diff --git a/steward/settings/routes.py b/steward/settings/routes.py index 3247dce..1862397 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -630,7 +630,10 @@ async def _reports_page(settings: dict, flash): async def system(): """Steward's own health (#5467): its process metrics now and over the day, each one an alert rule can watch (source "Steward itself"); then its - tables' sizes and what the last hourly cleanup removed (#5469).""" + tables' sizes and what the last hourly cleanup removed (#5469); and + the email not yet sent (#5509), whose failures and slow retries join the + page's verdict.""" + from steward.core import outbox from steward.core.health import last_cleanup, process_health, table_sizes from steward.core.status import sparkline_svg async with current_app.db_sessionmaker() as db: @@ -638,11 +641,14 @@ async def system(): # Table sizes and the last cleanup (#5469): Postgres's catalog and # what the cleanup kept, never a scan of the tables themselves. storage, cleanup = await table_sizes(db), await last_cleanup(db) + email = await outbox.outgoing(db) + health["problems"] = email["problems"] + health["problems"] for row in health["rows"]: # From zero to the row's own peak, with a floor (UI & Design ruling 8). row["spark"] = sparkline_svg(row["series"], width=120, height=20, stroke="currentColor", floor=1.0) if len(row["series"]) > 1 else "" - return await render_template("settings/system.html", **health, storage=storage, cleanup=cleanup) + return await render_template("settings/system.html", **health, storage=storage, cleanup=cleanup, + email=email, kind_labels=outbox.KIND_LABELS) @settings_bp.get("/reports/") diff --git a/steward/templates/alerts/_history_page.html b/steward/templates/alerts/_history_page.html index fa9f8aa..256c1f0 100644 --- a/steward/templates/alerts/_history_page.html +++ b/steward/templates/alerts/_history_page.html @@ -48,11 +48,20 @@ muted · {{ ep.muted_by }} {% if ep.released %}
notified when the window ended
{% endif %} {% endif %} - {% for label, sent, error in ep.deliveries %} + {# Each channel's outcome; an email through the outbox (#5509) also + says when it sends, or how its retries stand, or why it failed. #} + {% for d in ep.deliveries %}
- {% if sent %}{{ label }} sent - {% else %}{{ label }} failed - {% if error %}
{{ error }}
{% endif %} + {% if d.state == "sent" %}{{ d.label }} sent + {%- if d.at %} {{ d.at|when }}{% endif %} + {% elif d.state == "held" %}{{ d.label }} gathering +
waiting for other alerts; sends {{ d.at|when }}
+ {% elif d.state == "sending" %}{{ d.label }} sending + {% elif d.state == "retrying" %}{{ d.label }} retrying +
attempt {{ d.attempts }} + {%- if d.at %}, next {{ d.at|when }}{% endif %}{% if d.error %}: {{ d.error }}{% endif %}
+ {% else %}{{ d.label }} failed + {% if d.error %}
{{ d.error }}
{% endif %} {% endif %}
{% endfor %} diff --git a/steward/templates/settings/system.html b/steward/templates/settings/system.html index d7db32c..3fba74c 100644 --- a/steward/templates/settings/system.html +++ b/steward/templates/settings/system.html @@ -9,8 +9,8 @@ {% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("System", "")]) }}{% endblock %} {% block content %}

- {%- if not seen %}No readings yet: the first arrive within a minute of Steward starting. - {%- elif problems %}{% set said = problems|join("; ") %}{{ said[:1]|upper }}{{ said[1:] }}. + {%- if problems %}{% set said = problems|join("; ") %}{{ said[:1]|upper }}{{ said[1:] }}. + {%- elif not seen %}No readings yet: the first arrive within a minute of Steward starting. {%- else %}Steward is running normally.{% endif -%}

@@ -39,6 +39,46 @@ Recorded each minute (the cleanup each hour). Any of these can be watched by an alert rule: pick the source "Steward itself".

+{# Outgoing email (#5509): every email waits in the outbox until the mail + server takes it. What hasn't gone yet, failures first, then the longest + retrying; sent mail is only counted. #} +

Outgoing email

+

+ {%- if not email.rows %}Nothing waiting to send. {% endif -%} + {{ "{:,}".format(email.sent) }} sent in the last 7 days.

+{% if email.rows %} +
+ + + + {% for r in email.rows %} + + + + + + + {% endfor %} + {% if email.more %} + + {% endif %} + +
ToWhatStateQueued
{{ r.recipient }}{{ kind_labels.get(r.kind, r.kind) }} + {%- if r.state == "failed" %}failed + {%- elif r.state == "pending" and r.attempts %}retrying + attempt {{ r.attempts }}{% if r.next_attempt_at %}, next {{ r.next_attempt_at|when }}{% endif %} + {%- elif r.state == "held" %}gathering + sends {{ r.send_after|when }} + {%- else %}sending{% endif %} + {%- if r.last_error %}
{{ r.last_error }}
{% endif -%} + 		{{ r.created_at|ago }}
+{{ email.more }} more not yet sent
+
+{% endif %} +

+ An email the server refuses is failed at once; one it can't take yet is tried + again for up to an hour. The mail server is set under + Notifications.

+ {# Storage (#5469): Postgres's own figures for each table, then what the hourly cleanup last removed. Read from the catalog, never by counting. #}

Storage

diff --git a/tests/core/test_alert_episodes.py b/tests/core/test_alert_episodes.py index de18f1d..1d08632 100644 --- a/tests/core/test_alert_episodes.py +++ b/tests/core/test_alert_episodes.py @@ -74,14 +74,18 @@ def test_an_episode_split_by_paging_says_so(): assert not newer[0]["began_here"] and newer[0]["end"] == NOW - timedelta(minutes=10) +def _d(label, state="sent", error=None): + return {"label": label, "state": state, "at": None, "attempts": None, "error": error} + + def test_who_was_told_is_gathered_once_per_episode(): r = _rule() eps = episodes(_newest_first( - _t(r, 30, "inactive", "firing", 200.0, deliveries=[("email", False, "timed out"), ("webhook", True, None)], + _t(r, 30, "inactive", "firing", 200.0, deliveries=[_d("email", "failed", "timed out"), _d("webhook")], run="run-1"), - _t(r, 20, "firing", "resolved", 5.0, deliveries=[("webhook", True, None)], muted_by="patch")), + _t(r, 20, "firing", "resolved", 5.0, deliveries=[_d("webhook")], muted_by="patch")), newest_page=True, now=NOW, since=SINCE) - assert eps[0]["deliveries"] == [("email", False, "timed out"), ("webhook", True, None)] + assert eps[0]["deliveries"] == [_d("email", "failed", "timed out"), _d("webhook")] assert eps[0]["muted_by"] == "patch" and eps[0]["runs"] == ["run-1"] diff --git a/tests/core/test_alert_history_row.py b/tests/core/test_alert_history_row.py index 4bbd54f..9b7d158 100644 --- a/tests/core/test_alert_history_row.py +++ b/tests/core/test_alert_history_row.py @@ -14,7 +14,7 @@ RULE = SimpleNamespace(metric_name="cpu_pct", operator=AlertOperator(">"), thres def _event(**kw): - base = dict(transitioned_at=datetime(2026, 10, 6, tzinfo=timezone.utc), from_state="inactive", + base = dict(id="ev-1", transitioned_at=datetime(2026, 10, 6, tzinfo=timezone.utc), from_state="inactive", to_state="firing", metric_value=97.5, note=None, email_sent=None, email_error=None, webhook_sent=None, webhook_error=None, ansible_run_id=None, actor_id=None) return SimpleNamespace(**{**base, **kw}) @@ -44,12 +44,25 @@ def test_value_and_condition_read_in_words(): def test_only_attempted_deliveries_are_listed(): row = _history_row(_event(email_sent=False, email_error="timed out", webhook_sent=None), RULE) - assert row["deliveries"] == [("email", False, "timed out")] + assert [(d["label"], d["state"], d["error"]) for d in row["deliveries"]] == [("email", "failed", "timed out")] row = _history_row(_event(email_sent=True, webhook_sent=True), RULE) - assert [(label, sent) for label, sent, _ in row["deliveries"]] == [("email", True), ("webhook", True)] + assert [(d["label"], d["state"]) for d in row["deliveries"]] == [("email", "sent"), ("webhook", "sent")] assert _history_row(_event(), RULE)["deliveries"] == [] +def test_an_email_through_the_outbox_shows_its_live_state(): + """#5509: the outbox's state for the event outranks what the event holds + (nothing yet, while it gathers or retries).""" + from datetime import datetime, timezone + at = datetime(2026, 10, 9, 19, 50, tzinfo=timezone.utc) + ev = _event(email_sent=None, webhook_sent=True) + live = {ev.id: {"state": "retrying", "at": at, "attempts": 3, "error": "451 greylisted"}} + row = _history_row(ev, RULE, emails=live) + assert row["deliveries"][0] == {"label": "email", "state": "retrying", "at": at, "attempts": 3, + "error": "451 greylisted"} + assert row["deliveries"][1]["label"] == "webhook" + + def test_a_transition_a_person_made_says_who(): """#5463: an acknowledgement, and a rule change that reset the alert, name the person; one whose account is gone still reads as a person's.""" diff --git a/tests/core/test_outbox.py b/tests/core/test_outbox.py index b1a0cfd..08039df 100644 --- a/tests/core/test_outbox.py +++ b/tests/core/test_outbox.py @@ -223,3 +223,23 @@ def test_the_smtp_guard_can_fail(): assert _smtp_openers({"x.py": " with smtplib.SMTP(host) as s:"}) == {"x.py"} assert _smtp_openers({"y.py": " smtp = notifications.smtp_open(cfg)"}) == {"y.py"} assert _smtp_openers({"z.py": "# smtp_open(cfg) is the outbox's\ndef smtp_open(cfg):"}) == set() + + +# ── How an alert's email reads in history (#5509) ──────────────────────────── + +def test_an_alerts_email_reads_by_its_least_finished_copy(): + later, sent_at = NOW + timedelta(minutes=2), NOW + timedelta(seconds=5) + held = _row(state="held", send_after=later, created_at=NOW) + retrying = _row(recipient="b@x", state="pending", attempts=3, next_attempt_at=later, + last_error="451 greylisted", created_at=NOW) + sent = _row(recipient="c@x", state="sent", attempts=1, sent_at=sent_at, created_at=NOW) + failed = _row(recipient="d@x", state="failed", attempts=1, first_attempt_at=NOW, + last_error="550 no such user", created_at=NOW) + first = _row(recipient="e@x", state="pending", attempts=0, next_attempt_at=NOW, created_at=NOW) + assert outbox.delivery([sent, held, failed]) == {"state": "held", "at": later, "attempts": 0, "error": None} + assert outbox.delivery([sent, retrying]) == {"state": "retrying", "at": later, "attempts": 3, + "error": "451 greylisted"} + assert outbox.delivery([first])["state"] == "sending" + assert outbox.delivery([sent, failed]) == {"state": "failed", "at": NOW, "attempts": 1, + "error": "d@x: 550 no such user"} + assert outbox.delivery([sent]) == {"state": "sent", "at": sent_at, "attempts": 1, "error": None} diff --git a/tests/core/test_steward_health.py b/tests/core/test_steward_health.py index 5ed9673..4f0f404 100644 --- a/tests/core/test_steward_health.py +++ b/tests/core/test_steward_health.py @@ -13,10 +13,15 @@ STORAGE = {"total": 3 * 2**30, "rest": {"count": 0, "bytes": 0}, {"name": "docker_logs", "bytes": 512 * 2**20, "rows": None}]} -async def _page(app, ctx, storage=STORAGE, cleanup=None): +NO_EMAIL = {"rows": [], "more": 0, "sent": 0, "problems": []} + + +async def _page(app, ctx, storage=STORAGE, cleanup=None, email=NO_EMAIL): + from steward.core.outbox import KIND_LABELS async with app.test_request_context("/settings/system/"): session["user_role"] = "admin" - page = await render_template("settings/system.html", **ctx, storage=storage, cleanup=cleanup) + page = await render_template("settings/system.html", **ctx, storage=storage, cleanup=cleanup, + email=email, kind_labels=KIND_LABELS) return " ".join(page.split()) @@ -109,3 +114,46 @@ async def test_the_last_cleanup_keeps_only_what_went(monkeypatch): assert got["removed"] == [("docker_logs: past the age window", 7)] stored = {} assert await health.last_cleanup(None) is None + + +# ── Outgoing email (#5509) ──────────────────────────────────────────────────── + +def _mail(**kw): + from steward.models.outbox import EmailOutbox + base = dict(kind="alert", recipient="ann@example.com", state="pending", attempts=0, + created_at=AT, send_after=None, next_attempt_at=None, last_error=None) + return EmailOutbox(**{**base, **kw}) + + +async def test_outgoing_email_lists_what_has_not_gone_failures_first(app, previews): + previews() + got = await health.process_health(None) + for r in got["rows"]: + r["spark"] = "" + email = {"sent": 1234, "more": 3, "problems": ["1 email failed in the last 7 days"], "rows": [ + _mail(state="failed", kind="invite", recipient="bob@example.com", attempts=1, + last_error="550 no such user"), + _mail(state="pending", attempts=3, next_attempt_at=AT, last_error="451 greylisted"), + _mail(state="held", kind="alert", send_after=AT)]} + got["problems"] = email["problems"] + got["problems"] + page = await _page(app, got, email=email) + # The verdict leads with it; the section says each one's state and why. + assert '

1 email failed in the last 7 days.' in page + assert "1,234 sent in the last 7 days." in page and "Nothing waiting" not in page + assert page.index("bob@example.com") < page.index("451 greylisted") + assert "Invite" in page and "550 no such user" in page + assert "retrying attempt 3, next" in page + assert "gathering sends" in page + assert "+3 more not yet sent" in page + assert 'title="<' not in page + assert page.index("Steward itself") < page.index("Outgoing email") < page.index("Storage") + + +async def test_nothing_waiting_says_so(app, previews): + previews() + got = await health.process_health(None) + for r in got["rows"]: + r["spark"] = "" + page = await _page(app, got, email={**NO_EMAIL, "sent": 2}) + assert "Nothing waiting to send. 2 sent in the last 7 days." in page + assert "Steward is running normally." in page diff --git a/tests/integration/test_email_delivery_pages.py b/tests/integration/test_email_delivery_pages.py new file mode 100644 index 0000000..42bb7ca --- /dev/null +++ b/tests/integration/test_email_delivery_pages.py @@ -0,0 +1,99 @@ +"""Integration: email delivery you can see (#5509), against Postgres. + +An alert's email gathering for its window reads so in the alert's history +and on Settings → System; once refused it reads "failed" with the server's +words in both places, and the System page's verdict names it. +""" +from __future__ import annotations + +import asyncio +import os +import smtplib +import time +import uuid +from datetime import datetime, timezone + +import pytest + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +async def _admin(app, tag): + from steward.models.users import User, UserRole + uid = str(uuid.uuid4()) + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add(User(id=uid, username=f"dl-{tag}", email=f"dl-{tag}@test", password_hash="x", + role=UserRole.admin, is_active=True)) + client = app.test_client() + async with client.session_transaction() as sess: + sess.update(user_id=uid, user_role="admin", username=f"dl-{tag}", _csrf="test-token", + login_at=time.time()) + return client, uid + + +def test_an_alerts_email_reads_gathering_then_failed_in_history_and_on_system(app, monkeypatch): + from steward.alerts import service as alert_svc + from steward.core import outbox + from steward.core.alerts import record_metrics + from steward.models.alerts import AlertEvent + from steward.models.metrics import PluginMetric + from tests.integration._mail import capture + + tag = uuid.uuid4().hex[:8] + extra = f"pager-{tag}@example.com" + capture(app, monkeypatch, lambda m: smtplib.SMTPRecipientsRefused({m["To"]: (550, b"mailbox gone")}) + if m["To"] == extra else None) + app.config["SMTP"]["recipients"] = [extra] + + async def _go(): + client, uid = await _admin(app, tag) + now = datetime.now(timezone.utc) + res = f"box-{tag}" + async with app.db_sessionmaker() as s: + async with s.begin(): + await record_metrics(s, [PluginMetric(source_module="host_agent", resource_name=res, + metric_name="cpu_pct", value=1.0)]) + await s.flush() + fields = await alert_svc.rule_fields( + s, name=f"hot {tag}", source_module="host_agent", resource_name=res, + metric_name="cpu_pct", operator=">", threshold=90) + rule = await alert_svc.add_rule(s, fields, created_by=uid) + event = AlertEvent(rule_id=rule.id, from_state="inactive", to_state="firing", + metric_value=95.0, transitioned_at=now) + s.add(event) + rule_id, event_id = rule.id, event.id + await outbox.hold_alert(app, {"rule_name": f"hot {tag}", "state": "FIRING", "metric": "cpu_pct", + "value": 95.0, "operator": ">", "threshold": 90.0, "resource": res, + "source_module": "host_agent", "note": None, + "timestamp": now.isoformat()}, event_id, now=now) + + history = await (await client.get(f"/alerts/history?rule={rule_id}")).get_data(as_text=True) + assert "email gathering" in history and "waiting for other alerts; sends" in history + system = await (await client.get("/settings/system/")).get_data(as_text=True) + assert extra in system and "gathering" in system + + # The window closes; the server refuses the pager address. + from sqlalchemy import select + from steward.models.outbox import email_outbox_events as link + async with app.db_sessionmaker() as s: + ids = list((await s.execute(select(link.c.outbox_id).where( + link.c.event_id == event_id))).scalars()) + await outbox.attempt(app, ids) + + history = await (await client.get(f"/alerts/history?rule={rule_id}")).get_data(as_text=True) + assert "email failed" in history and f"{extra}: 550 mailbox gone" in history + system = " ".join((await (await client.get("/settings/system/")).get_data(as_text=True)).split()) + verdict = system[system.index("page-verdict"):system.index("

", system.index("page-verdict"))] + assert "failed in the last 7 days" in verdict + assert extra in system and "550 mailbox gone" in system + + asyncio.run(_go())