diff --git a/docs/core/alerting.md b/docs/core/alerting.md index 3b7f504..cf461f3 100644 --- a/docs/core/alerting.md +++ b/docs/core/alerting.md @@ -134,6 +134,14 @@ Configured at `/settings/` under SMTP. Settings: Email is skipped if `smtp.host` is empty. +Alerts that arrive together go out as one email (`core/alert_burst.py`). The +first alert opens a 30-second window. When it closes, each recipient gets one +message listing every alert in the window, oldest first, in their own zone, +with a subject like "[Steward] 3 alerts: 2 firing, 1 resolved". One alert on +its own keeps its usual email. Every alert's event records whether the email +went. A burst still waiting when Steward stops is not sent; its alerts stay on +the Alerts page and in the history. + ### Webhook Configured at `/settings/` under Webhook. The body is a Jinja2 template rendered to JSON. Content-Type is always `application/json`. If the rendered template is not valid JSON, the delivery is logged as failed and no request is sent. @@ -154,7 +162,18 @@ Default template (Discord-compatible): {"content": "**{{ alert.state }}** — {{ alert.resource }} — {{ alert.rule_name }} ({{ alert.metric }} = {{ alert.value }})"} ``` -Webhook is skipped if `webhook.url` is empty. +Webhook is skipped if `webhook.url` is empty. Webhooks are not gathered into +bursts: each alert posts on its own, at once. + +### Send test + +**Send test** on a rule's page (operator role), or `send_test_alert` over MCP, +sends a `TEST` alert for that rule now, by email and webhook, the same way a +real alert would go. It uses the rule's name and condition and the latest +reading of what the rule watches. A rule that covers hosts uses its first +host. Each channel's result shows under the button. Nothing fires, the rule's +history doesn't change, and the test is recorded in the audit log as +`alert_rule.tested`. --- diff --git a/docs/core/mcp.md b/docs/core/mcp.md index 5148d1f..271576e 100644 --- a/docs/core/mcp.md +++ b/docs/core/mcp.md @@ -86,7 +86,7 @@ Each role includes the tools of the roles above it in this list. |---|---| | Automation | `run_playbook`, `cancel_ansible_run`, `create_schedule`, `update_schedule`, `delete_schedule` | | Monitors | `create_monitor`, `update_monitor`, `set_monitor_enabled` | -| Alerts | `create_alert_rule`, `update_alert_rule`, `acknowledge_alert`, `create_maintenance_window`, `end_maintenance_window` | +| Alerts | `create_alert_rule`, `update_alert_rule`, `acknowledge_alert`, `send_test_alert`, `create_maintenance_window`, `end_maintenance_window` | ### Admin: destructive and remediation diff --git a/steward/alerts/routes.py b/steward/alerts/routes.py index c49e49c..8084621 100644 --- a/steward/alerts/routes.py +++ b/steward/alerts/routes.py @@ -420,6 +420,21 @@ async def toggle_rule(rule_id: str): return redirect(url_for("alerts.list_alerts")) +@alerts_bp.post("/rules//test") +@require_role(UserRole.operator) +async def test_rule(rule_id: str): + """Send test (#5466): a TEST notification for this rule, now, by every + channel its alerts use; the result shows under the button.""" + try: + sent = await alert_svc.send_test(current_app, rule_id) + except LookupError: + return "Not found", 404 + await log_audit(current_app, session.get("user_id"), session.get("username", ""), + "alert_rule.tested", entity_type="alert_rule", entity_id=sent["rule"], + detail={"rule_id": rule_id, "ok": sent["ok"], "lines": sent["lines"]}) + return await render_template("alerts/_test_result.html", ok=sent["ok"], lines=sent["lines"]) + + @alerts_bp.post("/rules//delete") @require_role(UserRole.admin) async def delete_rule(rule_id: str): diff --git a/steward/alerts/service.py b/steward/alerts/service.py index f591c70..77124d8 100644 --- a/steward/alerts/service.py +++ b/steward/alerts/service.py @@ -602,6 +602,59 @@ def part_of(rule: AlertRule, rules) -> AlertRule | None: return next((r for r in rules if r.id == rule.parent_id), None) if rule.parent_id else None +async def test_alert(db, rule: AlertRule) -> dict: + """What a Send test carries for `rule` (#5466): a TEST alert with the + rule's name and condition, on its resource at the latest reading. A rule + covering hosts tests on its first host.""" + target = rule + if rule.covers != "resource": + target = (await db.execute( + select(AlertRule).where(AlertRule.parent_id == rule.id) + .order_by(AlertRule.resource_name).limit(1))).scalar_one_or_none() or rule + return {"rule_name": rule.name, "state": "TEST", "metric": rule.metric_name, + "value": await latest_value(db, target) if target.resource_name else None, + "operator": rule.operator.value, "threshold": rule.threshold, + "resource": target.resource_name or covers_text(rule), + "source_module": rule.source_module, + "note": "A test of this rule's notifications. Nothing has fired.", + "timestamp": datetime.now(timezone.utc).isoformat()} + + +async def send_test(app, rule_id: str) -> dict: + """Send a TEST notification for a rule now, by every channel a real alert + of it would use: email to whoever gets alerts, each in their own zone, and + the webhook. Not held for a burst, and not recorded in the rule's history. + + Returns {"rule", "ok", "lines"}: ok when something was sent and nothing + failed; lines say what happened to each channel. LookupError for no rule. + """ + from urllib.parse import urlsplit + from steward.core.mail import recipients + from steward.core.notifications import dispatch_notifications + smtp, webhook = app.config.get("SMTP", {}), app.config.get("WEBHOOK", {}) + async with app.db_sessionmaker() as db: + rule = await db.get(AlertRule, rule_id) + if rule is None: + raise LookupError(rule_id) + alert = await test_alert(db, rule) + to = await recipients(db, "alerts", smtp) if smtp.get("host") else [] + results = await dispatch_notifications(smtp, webhook, alert, to) + lines = [] + if email := results.get("email"): + who = ", ".join(r.address for r in to) + lines.append(f"Email sent to {who}." if email["sent"] + else f"Email to {who} failed: {email['error']}") + if hook := results.get("webhook"): + host = urlsplit(webhook["url"]).hostname or webhook["url"] + lines.append(f"Webhook to {host} delivered." if hook["sent"] + else f"Webhook to {host} failed: {hook['error']}") + if not lines: + lines.append("Nothing is set up to send to: no email server with someone to send to, " + "and no webhook.") + return {"rule": rule.name, "ok": bool(results) and all(r["sent"] for r in results.values()), + "lines": lines, "email": results.get("email"), "webhook": results.get("webhook")} + + class PartOfCoveringRule(ValueError): """A child of a covering rule is changed through its parent, never alone.""" diff --git a/steward/audit/service.py b/steward/audit/service.py index 67c4f72..7366fc2 100644 --- a/steward/audit/service.py +++ b/steward/audit/service.py @@ -149,6 +149,7 @@ ACTIONS = { "alert_rule.disabled": "turned off alert rule", "alert_rule.deleted": "deleted alert rule", "alert_rule.acknowledged": "acknowledged the alert from", + "alert_rule.tested": "sent a test alert for rule", "alert_rule.action_set": "set the playbook run by alert rule", "alert_rule.action_cleared": "removed the playbook from alert rule", "maintenance_window.created": "scheduled maintenance", diff --git a/steward/core/mail.py b/steward/core/mail.py index 0a022e3..87ae983 100644 --- a/steward/core/mail.py +++ b/steward/core/mail.py @@ -222,6 +222,8 @@ def alert_facts(alert: dict, tz: tzinfo, show_utc: bool = True) -> dict: elif state == "resolved": headline = f"{metric_label(metric)} on {resource} is back within its limit" + ( f", at {reading}." if reading else ".") + elif reading is None: + headline = f"{metric_label(metric)} on {resource} has no reading yet." else: headline = f"{metric_label(metric)} on {resource} is {reading}." at = datetime.fromisoformat(alert["timestamp"]) diff --git a/steward/mcp/tools/alerts.py b/steward/mcp/tools/alerts.py index 4718ef8..e386e8d 100644 --- a/steward/mcp/tools/alerts.py +++ b/steward/mcp/tools/alerts.py @@ -6,7 +6,7 @@ from datetime import datetime, timedelta, timezone from sqlalchemy import select from steward.alerts import service as alert_svc -from steward.mcp._context import current_caller +from steward.mcp._context import current_caller, mcp_app from steward.models.alerts import AlertOperator, AlertRule, AlertState, MaintenanceWindow from steward.models.users import UserRole @@ -391,6 +391,25 @@ async def acknowledge_alert(rule: str) -> dict: "acknowledged_at": iso(state.acknowledged_at)} +async def send_test_alert(rule: str) -> dict: + """Send a TEST notification for a rule now, to check it reaches someone: + email to everyone who gets alerts (each in their own zone) and the + webhook, the same channels a real alert of this rule would use. Nothing + fires and the rule's history is unchanged. + + rule: the rule's name or id (list_alert_rules). Returns `ok` (something + was sent and nothing failed) and one line per channel saying what + happened. Recorded in the audit log. + """ + async with db_session() as db: + r = await _rule(db, rule) + sent = await alert_svc.send_test(mcp_app(), r.id) + await audit_mcp("alert_rule.tested", "alert_rule", r.name, + {"rule_id": r.id, "ok": sent["ok"], "lines": sent["lines"]}) + return {"rule_id": r.id, "rule_name": r.name, "ok": sent["ok"], "result": sent["lines"], + "email": sent["email"], "webhook": sent["webhook"]} + + def _when(value: str, what: str) -> datetime: try: dt = datetime.fromisoformat(value.strip().replace("Z", "+00:00")) @@ -467,7 +486,7 @@ def register(register_tool) -> None: for fn in (list_alerts, get_alert_history, list_alert_rules, list_maintenance_windows, get_alert_rule_options): register_tool(fn, role=UserRole.viewer, read_only=True) - for fn in (create_alert_rule, update_alert_rule, acknowledge_alert, + for fn in (create_alert_rule, update_alert_rule, acknowledge_alert, send_test_alert, create_maintenance_window, end_maintenance_window): register_tool(fn, role=UserRole.operator, read_only=False) register_tool(delete_alert_rule, role=UserRole.admin, read_only=False) diff --git a/steward/templates/alerts/_test_result.html b/steward/templates/alerts/_test_result.html new file mode 100644 index 0000000..303e129 --- /dev/null +++ b/steward/templates/alerts/_test_result.html @@ -0,0 +1,4 @@ +{# alerts/_test_result.html — what a rule's Send test did (#5466), one line per channel #} +
+ {% for line in lines %}
{{ line }}
{% endfor %} +
diff --git a/steward/templates/alerts/rules_form.html b/steward/templates/alerts/rules_form.html index 584f655..82bf25d 100644 --- a/steward/templates/alerts/rules_form.html +++ b/steward/templates/alerts/rules_form.html @@ -141,6 +141,15 @@ {%- else %}It notifies nobody yet: no email server with recipients and no webhook is set {%- if session.user_role == 'admin' %} (Settings → Notifications){% endif %}.{% endif -%}

+ {% if rule %} +
+ +

Sends a TEST alert for this rule as saved, now, to everywhere above. + Nothing fires, and the rule's history doesn't change.

+
+
+ {% endif %}