bvandeusenandClaude Opus 5.5
defad2e98b
#5509 "Delivery you can see: each alert's email state in history, and outgoing email on Settings -> System" (milestone 501). - Alerts -> History: an alert's email reads from the outbox (#5506): "gathering" with when it sends, "sending", "retrying" with the attempt, next try and the server's last answer, "sent", or "failed" naming each reader it didn't reach and why (outbox.delivery / email_states). Events from before the outbox keep what they recorded. The Notified column shows once an alert has a queued email. - Settings -> System gets "Outgoing email": every email of the last 7 days not yet sent, failures first, then the longest retrying, with recipient, kind, state and last error, and a count of what was sent. Failures and anything still retrying after 5 minutes lead the page's verdict (outbox.outgoing). - get_alert_history (MCP) adds `email`: the same delivery state. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
100 lines
4.4 KiB
Python
100 lines
4.4 KiB
Python
"""Integration: email delivery you can see (#5509), against Postgres.
|
|
|
|
An alert's email gathering for its window reads so in the alert's history
|
|
and on Settings → System; once refused it reads "failed" with the server's
|
|
words in both places, and the System page's verdict names it.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import smtplib
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
async def _admin(app, tag):
|
|
from steward.models.users import User, UserRole
|
|
uid = str(uuid.uuid4())
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
s.add(User(id=uid, username=f"dl-{tag}", email=f"dl-{tag}@test", password_hash="x",
|
|
role=UserRole.admin, is_active=True))
|
|
client = app.test_client()
|
|
async with client.session_transaction() as sess:
|
|
sess.update(user_id=uid, user_role="admin", username=f"dl-{tag}", _csrf="test-token",
|
|
login_at=time.time())
|
|
return client, uid
|
|
|
|
|
|
def test_an_alerts_email_reads_gathering_then_failed_in_history_and_on_system(app, monkeypatch):
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core import outbox
|
|
from steward.core.alerts import record_metrics
|
|
from steward.models.alerts import AlertEvent
|
|
from steward.models.metrics import PluginMetric
|
|
from tests.integration._mail import capture
|
|
|
|
tag = uuid.uuid4().hex[:8]
|
|
extra = f"pager-{tag}@example.com"
|
|
capture(app, monkeypatch, lambda m: smtplib.SMTPRecipientsRefused({m["To"]: (550, b"mailbox gone")})
|
|
if m["To"] == extra else None)
|
|
app.config["SMTP"]["recipients"] = [extra]
|
|
|
|
async def _go():
|
|
client, uid = await _admin(app, tag)
|
|
now = datetime.now(timezone.utc)
|
|
res = f"box-{tag}"
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await record_metrics(s, [PluginMetric(source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", value=1.0)])
|
|
await s.flush()
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=f"hot {tag}", source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", operator=">", threshold=90)
|
|
rule = await alert_svc.add_rule(s, fields, created_by=uid)
|
|
event = AlertEvent(rule_id=rule.id, from_state="inactive", to_state="firing",
|
|
metric_value=95.0, transitioned_at=now)
|
|
s.add(event)
|
|
rule_id, event_id = rule.id, event.id
|
|
await outbox.hold_alert(app, {"rule_name": f"hot {tag}", "state": "FIRING", "metric": "cpu_pct",
|
|
"value": 95.0, "operator": ">", "threshold": 90.0, "resource": res,
|
|
"source_module": "host_agent", "note": None,
|
|
"timestamp": now.isoformat()}, event_id, now=now)
|
|
|
|
history = await (await client.get(f"/alerts/history?rule={rule_id}")).get_data(as_text=True)
|
|
assert "email gathering" in history and "waiting for other alerts; sends" in history
|
|
system = await (await client.get("/settings/system/")).get_data(as_text=True)
|
|
assert extra in system and "gathering" in system
|
|
|
|
# The window closes; the server refuses the pager address.
|
|
from sqlalchemy import select
|
|
from steward.models.outbox import email_outbox_events as link
|
|
async with app.db_sessionmaker() as s:
|
|
ids = list((await s.execute(select(link.c.outbox_id).where(
|
|
link.c.event_id == event_id))).scalars())
|
|
await outbox.attempt(app, ids)
|
|
|
|
history = await (await client.get(f"/alerts/history?rule={rule_id}")).get_data(as_text=True)
|
|
assert "email failed" in history and f"{extra}: 550 mailbox gone" in history
|
|
system = " ".join((await (await client.get("/settings/system/")).get_data(as_text=True)).split())
|
|
verdict = system[system.index("page-verdict"):system.index("</p>", system.index("page-verdict"))]
|
|
assert "failed in the last 7 days" in verdict
|
|
assert extra in system and "550 mailbox gone" in system
|
|
|
|
asyncio.run(_go())
|