feat(alerts): Send test on a rule checks that its alerts reach someone (#5466)

#5466 "Send test: check a rule's delivery from the rule". A rule's page gets
a Send test button (operator role), and MCP gets send_test_alert. Each sends
a TEST alert for that rule now, by email to everyone who gets alerts and by
webhook. The alert carries the rule's name and condition, at the latest
reading of what it watches; a rule covering hosts uses its first host. The
result for each channel shows under the button, or "Nothing is set up to
send to". Nothing fires, no event is written, and the test is audited as
alert_rule.tested. An alert with no reading now says "has no reading yet"
instead of "is None".

Docs: alerting.md also gains the burst email from #5465 "One email per burst
of alerts".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/routes.py

@@ -420,6 +420,21 @@ async def toggle_rule(rule_id: str):
     return redirect(url_for("alerts.list_alerts"))
 
 
+@alerts_bp.post("/rules/<rule_id>/test")
+@require_role(UserRole.operator)
+async def test_rule(rule_id: str):
+    """Send test (#5466): a TEST notification for this rule, now, by every
+    channel its alerts use; the result shows under the button."""
+    try:
+        sent = await alert_svc.send_test(current_app, rule_id)
+    except LookupError:
+        return "Not found", 404
+    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
+                    "alert_rule.tested", entity_type="alert_rule", entity_id=sent["rule"],
+                    detail={"rule_id": rule_id, "ok": sent["ok"], "lines": sent["lines"]})
+    return await render_template("alerts/_test_result.html", ok=sent["ok"], lines=sent["lines"])
+
+
 @alerts_bp.post("/rules/<rule_id>/delete")
 @require_role(UserRole.admin)
 async def delete_rule(rule_id: str):

steward/alerts/service.py

@@ -602,6 +602,59 @@ def part_of(rule: AlertRule, rules) -> AlertRule | None:
     return next((r for r in rules if r.id == rule.parent_id), None) if rule.parent_id else None
 
 
+async def test_alert(db, rule: AlertRule) -> dict:
+    """What a Send test carries for `rule` (#5466): a TEST alert with the
+    rule's name and condition, on its resource at the latest reading. A rule
+    covering hosts tests on its first host."""
+    target = rule
+    if rule.covers != "resource":
+        target = (await db.execute(
+            select(AlertRule).where(AlertRule.parent_id == rule.id)
+            .order_by(AlertRule.resource_name).limit(1))).scalar_one_or_none() or rule
+    return {"rule_name": rule.name, "state": "TEST", "metric": rule.metric_name,
+            "value": await latest_value(db, target) if target.resource_name else None,
+            "operator": rule.operator.value, "threshold": rule.threshold,
+            "resource": target.resource_name or covers_text(rule),
+            "source_module": rule.source_module,
+            "note": "A test of this rule's notifications. Nothing has fired.",
+            "timestamp": datetime.now(timezone.utc).isoformat()}
+
+
+async def send_test(app, rule_id: str) -> dict:
+    """Send a TEST notification for a rule now, by every channel a real alert
+    of it would use: email to whoever gets alerts, each in their own zone, and
+    the webhook. Not held for a burst, and not recorded in the rule's history.
+
+    Returns {"rule", "ok", "lines"}: ok when something was sent and nothing
+    failed; lines say what happened to each channel. LookupError for no rule.
+    """
+    from urllib.parse import urlsplit
+    from steward.core.mail import recipients
+    from steward.core.notifications import dispatch_notifications
+    smtp, webhook = app.config.get("SMTP", {}), app.config.get("WEBHOOK", {})
+    async with app.db_sessionmaker() as db:
+        rule = await db.get(AlertRule, rule_id)
+        if rule is None:
+            raise LookupError(rule_id)
+        alert = await test_alert(db, rule)
+        to = await recipients(db, "alerts", smtp) if smtp.get("host") else []
+    results = await dispatch_notifications(smtp, webhook, alert, to)
+    lines = []
+    if email := results.get("email"):
+        who = ", ".join(r.address for r in to)
+        lines.append(f"Email sent to {who}." if email["sent"]
+                     else f"Email to {who} failed: {email['error']}")
+    if hook := results.get("webhook"):
+        host = urlsplit(webhook["url"]).hostname or webhook["url"]
+        lines.append(f"Webhook to {host} delivered." if hook["sent"]
+                     else f"Webhook to {host} failed: {hook['error']}")
+    if not lines:
+        lines.append("Nothing is set up to send to: no email server with someone to send to, "
+                     "and no webhook.")
+    return {"rule": rule.name, "ok": bool(results) and all(r["sent"] for r in results.values()),
+            "lines": lines, "email": results.get("email"), "webhook": results.get("webhook")}
+
+
 class PartOfCoveringRule(ValueError):
     """A child of a covering rule is changed through its parent, never alone."""