feat(alerts): Send test on a rule checks that its alerts reach someone (#5466)

#5466 "Send test: check a rule's delivery from the rule". A rule's page gets
a Send test button (operator role), and MCP gets send_test_alert. Each sends
a TEST alert for that rule now, by email to everyone who gets alerts and by
webhook. The alert carries the rule's name and condition, at the latest
reading of what it watches; a rule covering hosts uses its first host. The
result for each channel shows under the button, or "Nothing is set up to
send to". Nothing fires, no event is written, and the test is audited as
alert_rule.tested. An alert with no reading now says "has no reading yet"
instead of "is None".

Docs: alerting.md also gains the burst email from #5465 "One email per burst
of alerts".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/core/alerting.md

@@ -134,6 +134,14 @@ Configured at `/settings/` under SMTP. Settings:
 
 Email is skipped if `smtp.host` is empty.
 
+Alerts that arrive together go out as one email (`core/alert_burst.py`). The
+first alert opens a 30-second window. When it closes, each recipient gets one
+message listing every alert in the window, oldest first, in their own zone,
+with a subject like "[Steward] 3 alerts: 2 firing, 1 resolved". One alert on
+its own keeps its usual email. Every alert's event records whether the email
+went. A burst still waiting when Steward stops is not sent; its alerts stay on
+the Alerts page and in the history.
+
 ### Webhook
 
 Configured at `/settings/` under Webhook. The body is a Jinja2 template rendered to JSON. Content-Type is always `application/json`. If the rendered template is not valid JSON, the delivery is logged as failed and no request is sent.
@@ -154,7 +162,18 @@ Default template (Discord-compatible):
 {"content": "**{{ alert.state }}** — {{ alert.resource }} — {{ alert.rule_name }} ({{ alert.metric }} = {{ alert.value }})"}
 ```
 
-Webhook is skipped if `webhook.url` is empty.
+Webhook is skipped if `webhook.url` is empty. Webhooks are not gathered into
+bursts: each alert posts on its own, at once.
+
+### Send test
+
+**Send test** on a rule's page (operator role), or `send_test_alert` over MCP,
+sends a `TEST` alert for that rule now, by email and webhook, the same way a
+real alert would go. It uses the rule's name and condition and the latest
+reading of what the rule watches. A rule that covers hosts uses its first
+host. Each channel's result shows under the button. Nothing fires, the rule's
+history doesn't change, and the test is recorded in the audit log as
+`alert_rule.tested`.
 
 ---
 

docs/core/mcp.md

@@ -86,7 +86,7 @@ Each role includes the tools of the roles above it in this list.
 |---|---|
 | Automation | `run_playbook`, `cancel_ansible_run`, `create_schedule`, `update_schedule`, `delete_schedule` |
 | Monitors | `create_monitor`, `update_monitor`, `set_monitor_enabled` |
-| Alerts | `create_alert_rule`, `update_alert_rule`, `acknowledge_alert`, `create_maintenance_window`, `end_maintenance_window` |
+| Alerts | `create_alert_rule`, `update_alert_rule`, `acknowledge_alert`, `send_test_alert`, `create_maintenance_window`, `end_maintenance_window` |
 
 ### Admin: destructive and remediation
 

steward/alerts/routes.py

@@ -420,6 +420,21 @@ async def toggle_rule(rule_id: str):
     return redirect(url_for("alerts.list_alerts"))
 
 
+@alerts_bp.post("/rules/<rule_id>/test")
+@require_role(UserRole.operator)
+async def test_rule(rule_id: str):
+    """Send test (#5466): a TEST notification for this rule, now, by every
+    channel its alerts use; the result shows under the button."""
+    try:
+        sent = await alert_svc.send_test(current_app, rule_id)
+    except LookupError:
+        return "Not found", 404
+    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
+                    "alert_rule.tested", entity_type="alert_rule", entity_id=sent["rule"],
+                    detail={"rule_id": rule_id, "ok": sent["ok"], "lines": sent["lines"]})
+    return await render_template("alerts/_test_result.html", ok=sent["ok"], lines=sent["lines"])
+
+
 @alerts_bp.post("/rules/<rule_id>/delete")
 @require_role(UserRole.admin)
 async def delete_rule(rule_id: str):

steward/alerts/service.py

@@ -602,6 +602,59 @@ def part_of(rule: AlertRule, rules) -> AlertRule | None:
     return next((r for r in rules if r.id == rule.parent_id), None) if rule.parent_id else None
 
 
+async def test_alert(db, rule: AlertRule) -> dict:
+    """What a Send test carries for `rule` (#5466): a TEST alert with the
+    rule's name and condition, on its resource at the latest reading. A rule
+    covering hosts tests on its first host."""
+    target = rule
+    if rule.covers != "resource":
+        target = (await db.execute(
+            select(AlertRule).where(AlertRule.parent_id == rule.id)
+            .order_by(AlertRule.resource_name).limit(1))).scalar_one_or_none() or rule
+    return {"rule_name": rule.name, "state": "TEST", "metric": rule.metric_name,
+            "value": await latest_value(db, target) if target.resource_name else None,
+            "operator": rule.operator.value, "threshold": rule.threshold,
+            "resource": target.resource_name or covers_text(rule),
+            "source_module": rule.source_module,
+            "note": "A test of this rule's notifications. Nothing has fired.",
+            "timestamp": datetime.now(timezone.utc).isoformat()}
+
+
+async def send_test(app, rule_id: str) -> dict:
+    """Send a TEST notification for a rule now, by every channel a real alert
+    of it would use: email to whoever gets alerts, each in their own zone, and
+    the webhook. Not held for a burst, and not recorded in the rule's history.
+
+    Returns {"rule", "ok", "lines"}: ok when something was sent and nothing
+    failed; lines say what happened to each channel. LookupError for no rule.
+    """
+    from urllib.parse import urlsplit
+    from steward.core.mail import recipients
+    from steward.core.notifications import dispatch_notifications
+    smtp, webhook = app.config.get("SMTP", {}), app.config.get("WEBHOOK", {})
+    async with app.db_sessionmaker() as db:
+        rule = await db.get(AlertRule, rule_id)
+        if rule is None:
+            raise LookupError(rule_id)
+        alert = await test_alert(db, rule)
+        to = await recipients(db, "alerts", smtp) if smtp.get("host") else []
+    results = await dispatch_notifications(smtp, webhook, alert, to)
+    lines = []
+    if email := results.get("email"):
+        who = ", ".join(r.address for r in to)
+        lines.append(f"Email sent to {who}." if email["sent"]
+                     else f"Email to {who} failed: {email['error']}")
+    if hook := results.get("webhook"):
+        host = urlsplit(webhook["url"]).hostname or webhook["url"]
+        lines.append(f"Webhook to {host} delivered." if hook["sent"]
+                     else f"Webhook to {host} failed: {hook['error']}")
+    if not lines:
+        lines.append("Nothing is set up to send to: no email server with someone to send to, "
+                     "and no webhook.")
+    return {"rule": rule.name, "ok": bool(results) and all(r["sent"] for r in results.values()),
+            "lines": lines, "email": results.get("email"), "webhook": results.get("webhook")}
+
+
 class PartOfCoveringRule(ValueError):
     """A child of a covering rule is changed through its parent, never alone."""
 

steward/audit/service.py

@@ -149,6 +149,7 @@ ACTIONS = {
     "alert_rule.disabled": "turned off alert rule",
     "alert_rule.deleted": "deleted alert rule",
     "alert_rule.acknowledged": "acknowledged the alert from",
+    "alert_rule.tested": "sent a test alert for rule",
     "alert_rule.action_set": "set the playbook run by alert rule",
     "alert_rule.action_cleared": "removed the playbook from alert rule",
     "maintenance_window.created": "scheduled maintenance",

steward/core/mail.py

@@ -222,6 +222,8 @@ def alert_facts(alert: dict, tz: tzinfo, show_utc: bool = True) -> dict:
     elif state == "resolved":
         headline = f"{metric_label(metric)} on {resource} is back within its limit" + (
             f", at {reading}." if reading else ".")
+    elif reading is None:
+        headline = f"{metric_label(metric)} on {resource} has no reading yet."
     else:
         headline = f"{metric_label(metric)} on {resource} is {reading}."
     at = datetime.fromisoformat(alert["timestamp"])

steward/mcp/tools/alerts.py

@@ -6,7 +6,7 @@ from datetime import datetime, timedelta, timezone
 from sqlalchemy import select
 
 from steward.alerts import service as alert_svc
-from steward.mcp._context import current_caller
+from steward.mcp._context import current_caller, mcp_app
 from steward.models.alerts import AlertOperator, AlertRule, AlertState, MaintenanceWindow
 from steward.models.users import UserRole
 
@@ -391,6 +391,25 @@ async def acknowledge_alert(rule: str) -> dict:
             "acknowledged_at": iso(state.acknowledged_at)}
 
 
+async def send_test_alert(rule: str) -> dict:
+    """Send a TEST notification for a rule now, to check it reaches someone:
+    email to everyone who gets alerts (each in their own zone) and the
+    webhook, the same channels a real alert of this rule would use. Nothing
+    fires and the rule's history is unchanged.
+
+    rule: the rule's name or id (list_alert_rules). Returns `ok` (something
+    was sent and nothing failed) and one line per channel saying what
+    happened. Recorded in the audit log.
+    """
+    async with db_session() as db:
+        r = await _rule(db, rule)
+    sent = await alert_svc.send_test(mcp_app(), r.id)
+    await audit_mcp("alert_rule.tested", "alert_rule", r.name,
+                    {"rule_id": r.id, "ok": sent["ok"], "lines": sent["lines"]})
+    return {"rule_id": r.id, "rule_name": r.name, "ok": sent["ok"], "result": sent["lines"],
+            "email": sent["email"], "webhook": sent["webhook"]}
+
+
 def _when(value: str, what: str) -> datetime:
     try:
         dt = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
@@ -467,7 +486,7 @@ def register(register_tool) -> None:
     for fn in (list_alerts, get_alert_history, list_alert_rules, list_maintenance_windows,
                get_alert_rule_options):
         register_tool(fn, role=UserRole.viewer, read_only=True)
-    for fn in (create_alert_rule, update_alert_rule, acknowledge_alert,
+    for fn in (create_alert_rule, update_alert_rule, acknowledge_alert, send_test_alert,
                create_maintenance_window, end_maintenance_window):
         register_tool(fn, role=UserRole.operator, read_only=False)
     register_tool(delete_alert_rule, role=UserRole.admin, read_only=False)

steward/templates/alerts/_test_result.html

@@ -0,0 +1,4 @@
+{# alerts/_test_result.html — what a rule's Send test did (#5466), one line per channel #}
+<div class="alert {% if ok %}alert-success{% else %}alert-error{% endif %}" style="margin:0.75rem 0 0;">
+  {% for line in lines %}<div>{{ line }}</div>{% endfor %}
+</div>

steward/templates/alerts/rules_form.html

@@ -141,6 +141,15 @@
         {%- else %}<span class="tone-warn">It notifies nobody yet:</span> no email server with recipients and no webhook is set
         {%- if session.user_role == 'admin' %} (<a href="/settings/notifications/">Settings → Notifications</a>){% endif %}.{% endif -%}
       </p>
+      {% if rule %}
+      <div class="form-group">
+        <button type="button" class="btn btn-sm btn-ghost" hx-post="/alerts/rules/{{ rule.id }}/test"
+                hx-target="#rule-test-result" hx-swap="innerHTML">Send test</button>
+        <p class="field-hint">Sends a TEST alert for this rule as saved, now, to everywhere above.
+          Nothing fires, and the rule's history doesn't change.</p>
+        <div id="rule-test-result" aria-live="polite"></div>
+      </div>
+      {% endif %}
       <div class="form-group">
         <label style="display:flex;align-items:center;gap:0.5rem;font-weight:normal;">
           <input type="checkbox" name="notify_no_data"

tests/core/test_alert_test_alert.py

@@ -0,0 +1,52 @@
+"""What a rule's Send test carries (#5466): the rule's own name and
+condition, marked TEST, on its resource at the latest reading; a rule
+covering hosts tests on its first host."""
+from types import SimpleNamespace
+
+from steward.alerts import service as alert_svc
+from steward.models.alerts import AlertOperator
+
+
+def _rule(**kw):
+    base = dict(id="r1", name="Hot", source_module="host_agent", resource_name="web01",
+                metric_name="cpu_pct", operator=AlertOperator.gt, threshold=90.0,
+                covers="resource", cover_group=None, parent_id=None)
+    return SimpleNamespace(**{**base, **kw})
+
+
+class _Db:
+    """Answers the one query test_alert makes: a covering rule's first host's rule."""
+
+    def __init__(self, child):
+        self.child = child
+
+    async def execute(self, stmt):
+        return SimpleNamespace(scalar_one_or_none=lambda: self.child)
+
+
+def _readings(monkeypatch, values):
+    async def latest(db, rule):
+        return values.get(rule.resource_name)
+    monkeypatch.setattr(alert_svc, "latest_value", latest)
+
+
+async def test_a_rule_tests_on_its_own_resource_at_its_latest_reading(monkeypatch):
+    _readings(monkeypatch, {"web01": 42.0})
+    got = await alert_svc.test_alert(_Db(None), _rule())
+    assert {k: got[k] for k in ("rule_name", "state", "resource", "value", "operator", "threshold")} == {
+        "rule_name": "Hot", "state": "TEST", "resource": "web01", "value": 42.0,
+        "operator": ">", "threshold": 90.0}
+
+
+async def test_a_covering_rule_tests_on_its_first_host(monkeypatch):
+    _readings(monkeypatch, {"db01": 12.0})
+    parent = _rule(resource_name="", covers="group", cover_group="web")
+    got = await alert_svc.test_alert(_Db(_rule(id="c1", name="Hot: db01", resource_name="db01",
+                                               parent_id="r1")), parent)
+    assert (got["rule_name"], got["resource"], got["value"]) == ("Hot", "db01", 12.0)
+
+
+async def test_a_covering_rule_with_no_hosts_yet_says_what_it_covers(monkeypatch):
+    _readings(monkeypatch, {})
+    got = await alert_svc.test_alert(_Db(None), _rule(resource_name="", covers="all_hosts"))
+    assert (got["resource"], got["value"]) == ("every host", None)

tests/core/test_mail.py

@@ -96,11 +96,13 @@ def test_without_steward_s_address_there_is_no_link_and_utc_is_said_once():
 
 def test_each_state_reads_as_a_sentence():
     head = {s: mail.alert_facts({**ALERT, "state": s, "value": v}, timezone.utc)["headline"]
-            for s, v in (("FIRING", 93.5), ("RESOLVED", 71.0), ("NO_DATA", None), ("DATA_RESUMED", 80.0))}
+            for s, v in (("FIRING", 93.5), ("RESOLVED", 71.0), ("NO_DATA", None), ("DATA_RESUMED", 80.0),
+                         ("TEST", None))}
     assert head == {"FIRING": "Disk used on web01 is 94%.",
                     "RESOLVED": "Disk used on web01 is back within its limit, at 71%.",
                     "NO_DATA": "Steward has had no disk used reading from web01.",
-                    "DATA_RESUMED": "web01 is reporting disk used again."}
+                    "DATA_RESUMED": "web01 is reporting disk used again.",
+                    "TEST": "Disk used on web01 has no reading yet."}
     down = mail.alert_facts({**ALERT, "metric": "is_up", "operator": "<", "threshold": 0.5, "value": 0.0},
                             timezone.utc)
     assert (down["headline"], down["condition"]) == ("web01 is down.", "is down")

tests/integration/test_alert_send_test.py

@@ -0,0 +1,116 @@
+"""Integration: Send test on an alert rule (#5466).
+
+The button on a rule's page, and send_test_alert over MCP, send a TEST alert
+for that rule now, by every channel a real alert of it would use, and say
+what happened to each. Nothing fires and the rule's history is unchanged;
+the audit log records the test. Only the delivery itself is replaced.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import time
+import uuid
+from zoneinfo import ZoneInfo
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def test_send_test_reaches_each_channel_and_leaves_the_rule_alone(app, monkeypatch):
+    from sqlalchemy import func, select
+    from steward.alerts import service as alert_svc
+    from steward.core import mail, notifications
+    from steward.core.alerts import record_metric
+    from steward.core.api_tokens import Caller
+    from steward.mcp import _context
+    from steward.mcp.tools import alerts as mcp_alerts
+    from steward.models.alerts import AlertEvent, AlertState, AlertStateEnum
+    from steward.models.audit import AuditEvent
+    from steward.models.users import User, UserRole
+
+    emails, hooks = [], []
+
+    async def fake_email(cfg, alert, to):
+        emails.append((alert, [r.address for r in to]))
+        return {"sent": True, "error": None}
+
+    async def fake_hook(cfg, alert):
+        hooks.append(alert)
+        return {"sent": False, "error": "connection refused"}
+
+    async def fake_recipients(db, kind, smtp):
+        return [mail.Recipient("ann@example.com", ZoneInfo("UTC"), False, "ann")]
+
+    monkeypatch.setattr(notifications, "_send_email", fake_email)
+    monkeypatch.setattr(notifications, "_send_webhook", fake_hook)
+    monkeypatch.setattr(mail, "recipients", fake_recipients)
+    app.config["SMTP"] = {"host": "mx.example", "port": 587, "from": "alerts@example.com"}
+    app.config["WEBHOOK"] = {"url": "https://hook.example/api/1"}
+
+    async def _go():
+        tag = uuid.uuid4().hex[:8]
+        host = f"st-{tag}"
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                user = User(username=f"st-{tag}", email=f"st-{tag}@test", password_hash="x",
+                            role=UserRole.operator, is_active=True)
+                s.add(user)
+                await s.flush()
+                await record_metric(s, "host_agent", host, "cpu_pct", 42.0)
+                fields = await alert_svc.rule_fields(
+                    s, name=f"Hot {tag}", source_module="host_agent", resource_name=host,
+                    metric_name="cpu_pct", operator=">", threshold=90)
+                rule = await alert_svc.add_rule(s, fields, created_by=user.id)
+            uid, rule_id = user.id, rule.id
+
+        client = app.test_client()
+        async with client.session_transaction() as sess:
+            sess.update(user_id=uid, user_role="operator", username=f"st-{tag}",
+                        _csrf="test-token", login_at=time.time())
+        resp = await client.post(f"/alerts/rules/{rule_id}/test",
+                                 headers={"X-CSRF-Token": "test-token"})
+        assert resp.status_code == 200
+        page = await resp.get_data(as_text=True)
+        assert "alert-error" in page
+        assert "Email sent to ann@example.com." in page
+        assert "Webhook to hook.example failed: connection refused" in page
+
+        # What went out is this rule, marked TEST, at its latest reading.
+        ((alert, to),) = emails
+        assert to == ["ann@example.com"] and hooks == [alert]
+        assert (alert["rule_name"], alert["state"], alert["resource"], alert["value"]) == (
+            f"Hot {tag}", "TEST", host, 42.0)
+
+        # Nothing fired: no event, the alert still inactive. The audit log has it.
+        async with app.db_sessionmaker() as s:
+            events = (await s.execute(select(func.count()).select_from(AlertEvent)
+                                      .where(AlertEvent.rule_id == rule_id))).scalar_one()
+            state = (await s.execute(select(AlertState).where(
+                AlertState.rule_id == rule_id))).scalar_one()
+            audits = (await s.execute(select(AuditEvent).where(
+                AuditEvent.user_id == uid))).scalars().all()
+        assert events == 0 and state.state == AlertStateEnum.inactive
+        assert [a.action for a in audits] == ["alert_rule.tested"]
+        assert json.loads(audits[0].detail_json)["ok"] is False
+
+        # Over MCP, with nothing set up to send to, it says so.
+        app.config["SMTP"], app.config["WEBHOOK"] = {}, {}
+        _context._app_ctx.set(app)
+        _context._caller_ctx.set(Caller(user_id=uid, username=f"st-{tag}",
+                                        role=UserRole.operator, token_id=7))
+        got = await mcp_alerts.send_test_alert(f"Hot {tag}")
+        assert got["ok"] is False and got["result"][0].startswith("Nothing is set up to send to")
+        assert len(emails) == 1 and len(hooks) == 1
+
+    asyncio.run(_go())

tests/mcp/test_monitoring_writes.py

@@ -19,7 +19,7 @@ from steward.models.users import UserRole
 NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc)
 
 OPERATOR_TOOLS = {"create_monitor", "update_monitor", "set_monitor_enabled", "create_alert_rule",
-                  "update_alert_rule", "acknowledge_alert", "create_maintenance_window",
+                  "update_alert_rule", "acknowledge_alert", "send_test_alert", "create_maintenance_window",
                   "end_maintenance_window"}
 ADMIN_TOOLS = {"delete_monitor", "delete_alert_rule"}