Files
FabledSteward/steward/alerts/routes.py
T
bvandeusenandClaude Opus 5.5 4fbe93bdf7
CI / lint (push) Successful in 3s
CI / integration (push) Successful in 45s
CI / unit (push) Failing after 1m7s
CI / publish (push) Skipped
feat(alerts): a sentence per alert, a link to what it watches, a count in the sidebar (#5124)
The Alerts page's active list now says each alert in one line around the
resource it watches, e.g. "cpu_pct on web-1 is 94% (> 90%), firing for 12m",
with web-1 linking to its host, monitor or container page (or the plugin's
page for Traefik, UniFi and SNMP). Values take their unit from the metric
name through formatting.metric_value. A silent series reads "No cpu_pct from
web-1 for 20m"; an up/down metric says the word.

Actions follow the routes' roles: acknowledge, edit, toggle and New rule for
operators and admins, Delete for admins only, nothing for viewers.

The sidebar shows how many alerts need attention beside Alerts, from an
async context processor that skips htmx fragments, logged-out pages and the
database-down page, and renders without the count if the query fails.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 18:12:50 -04:00

375 lines
15 KiB
Python

from __future__ import annotations
from datetime import datetime, timezone
from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
from steward.core.audit import log_audit
from sqlalchemy import select
from steward.auth.middleware import require_role
from steward.alerts import service as alert_svc
from steward.core.time_range import parse_range
from steward.ansible import sources as src_module
from steward.models.alerts import AlertRule, AlertOperator, MaintenanceWindow
from steward.models.users import UserRole
alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")
_OPERATORS = [(op.value, op.value) for op in AlertOperator]
def _rule_form(form) -> dict:
"""The rule form's fields, as alert_svc.rule_fields takes them."""
return {
"name": form.get("name", ""),
"source_module": form.get("source_module", ""),
"resource_name": form.get("resource_name", ""),
"metric_name": form.get("metric_name", ""),
"operator": form.get("operator", ""),
"threshold": form.get("threshold"),
"consecutive_failures_required": form.get("consecutive_failures_required"),
"consecutive_ok_required": form.get("consecutive_ok_required"),
"notify_no_data": "notify_no_data" in form,
}
def _form_time(value) -> datetime:
"""A datetime-local field, read as UTC."""
try:
return datetime.fromisoformat((value or "").strip()).replace(tzinfo=timezone.utc)
except ValueError:
raise ValueError(f"{value!r} is not a date and time.") from None
def _is_admin() -> bool:
return session.get("user_role") == UserRole.admin.value
def _ansible_source_names() -> list[str]:
try:
return [s["name"] for s in src_module.get_sources(current_app.config.get("ANSIBLE", {}))]
except Exception:
return []
def _parse_ansible_action(form) -> tuple[dict | None, str | None]:
"""Parse + validate the admin-only Ansible action from a rule form.
Returns (action_or_None, error_or_None). action is None when the action
checkbox is unchecked (i.e. the action is being cleared).
"""
if "ansible_enabled" not in form:
return None, None
source_name = (form.get("ansible_source", "") or "").strip()
playbook = (form.get("ansible_playbook", "") or "").strip()
inventory = (form.get("ansible_inventory", "") or "").strip()
if not (source_name and playbook and inventory):
return None, "Ansible action requires source, playbook, and inventory"
try:
srcs = src_module.get_sources(current_app.config.get("ANSIBLE", {}))
except Exception:
return None, "Ansible sources are misconfigured"
source = next((s for s in srcs if s["name"] == source_name), None)
if source is None:
return None, f"Ansible source '{source_name}' not found"
if playbook not in src_module.discover_playbooks(source["path"]):
return None, f"Playbook '{playbook}' not found in source '{source_name}'"
extra_vars: list[str] = []
for line in (form.get("ansible_extra_vars", "") or "").splitlines():
line = line.strip()
if not line:
continue
if "=" not in line:
return None, f"Invalid extra var (expected key=value): {line!r}"
extra_vars.append(line)
action: dict = {"source": source_name, "playbook": playbook, "inventory": inventory}
if extra_vars:
action["extra_vars"] = extra_vars
limit = (form.get("ansible_limit", "") or "").strip()
tags = (form.get("ansible_tags", "") or "").strip()
if limit:
action["limit"] = limit
if tags:
action["tags"] = tags
if "ansible_check" in form:
action["check"] = True
return action, None
@alerts_bp.get("/")
@require_role(UserRole.viewer)
async def list_alerts():
async with current_app.db_sessionmaker() as db:
active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
rules = await alert_svc.list_rules(db)
return await render_template("alerts/list.html", active=active, rules=rules, operators=_OPERATORS)
HISTORY_LIMIT = 200
def _history_row(event, rule) -> dict:
"""One transition as the history page shows it."""
from steward.core.alerts import MUTED, RELEASED
notes, muted_by, released = [], None, False
for part in (event.note or "").split("; "):
if part.startswith(MUTED):
muted_by = part[len(MUTED):].strip() or "a maintenance window"
elif part == RELEASED:
released = True
elif part:
notes.append(part)
value = "no data" if event.metric_value is None else f"{event.metric_value:g}"
return {
"at": event.transitioned_at, "rule": rule,
"from_state": event.from_state, "to_state": event.to_state,
"value": value, "condition": f"{rule.operator.value} {rule.threshold:g}",
"notes": notes, "muted_by": muted_by, "released": released,
"deliveries": [(label, sent, error) for label, sent, error in (
("email", event.email_sent, event.email_error),
("webhook", event.webhook_sent, event.webhook_error)) if sent is not None],
"ansible_run_id": event.ansible_run_id,
}
@alerts_bp.get("/history")
@require_role(UserRole.viewer)
async def alert_history():
"""Every alert transition in a range, newest first; one rule's with ?rule=."""
since, range_key = parse_range(request.args.get("range") or "7d")
async with current_app.db_sessionmaker() as db:
rules = await alert_svc.list_rules(db)
chosen = next((r for r in rules if r.id == request.args.get("rule")), None)
rows = await alert_svc.history(db, since, chosen.id if chosen else None, HISTORY_LIMIT)
ctx = {"events": [_history_row(e, r) for e, r in rows[:HISTORY_LIMIT]],
"truncated": len(rows) > HISTORY_LIMIT, "limit": HISTORY_LIMIT,
"rules": rules, "chosen": chosen, "current_range": range_key}
if request.headers.get("HX-Request"):
return await render_template("alerts/_history_rows.html", **ctx)
return await render_template("alerts/history.html", **ctx)
@alerts_bp.get("/api/rule_fields")
@require_role(UserRole.viewer)
async def rule_fields_api():
source = request.args.get("source_module", "")
current_resource = request.args.get("current_resource", "")
current_metric = request.args.get("current_metric", "")
async with current_app.db_sessionmaker() as db:
resources = await alert_svc.resources_for(db, source) if source else []
async with current_app.db_sessionmaker() as db:
metrics = await alert_svc.metrics_for(db, source)
return await render_template(
"alerts/_rule_fields.html",
resources=resources,
metrics=metrics,
current_resource=current_resource,
current_metric=current_metric,
)
@alerts_bp.get("/rules/new")
@require_role(UserRole.operator)
async def new_rule():
first_source = alert_svc.source_modules()[0]
async with current_app.db_sessionmaker() as db:
resources = await alert_svc.resources_for(db, first_source)
return await render_template(
"alerts/rules_form.html",
rule=None,
operators=_OPERATORS,
source_modules=alert_svc.source_modules(),
initial_source=first_source,
resources=resources,
metrics=alert_svc.metric_catalog().get(first_source, []),
ansible_sources=_ansible_source_names(),
)
@alerts_bp.get("/rules/<rule_id>/edit")
@require_role(UserRole.operator)
async def edit_rule(rule_id: str):
async with current_app.db_sessionmaker() as db:
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if rule is None:
return "Not found", 404
resources = await alert_svc.resources_for(db, rule.source_module)
return await render_template(
"alerts/rules_form.html",
rule=rule,
operators=_OPERATORS,
source_modules=alert_svc.source_modules(),
initial_source=rule.source_module,
resources=resources,
metrics=alert_svc.metric_catalog().get(rule.source_module, []),
ansible_sources=_ansible_source_names(),
)
@alerts_bp.post("/rules")
@require_role(UserRole.operator)
async def create_rule():
form = await request.form
user_id = session.get("user_id")
# Ansible action is admin-only; non-admins simply can't set one.
ansible_action = None
if _is_admin():
ansible_action, err = _parse_ansible_action(form)
if err:
return err, 400
async with current_app.db_sessionmaker() as db:
async with db.begin():
try:
fields = await alert_svc.rule_fields(db, **_rule_form(form))
except ValueError as exc:
return str(exc), 400
rule = await alert_svc.add_rule(db, fields, created_by=user_id,
ansible_action=ansible_action)
await log_audit(current_app, user_id, session.get("username", ""), "alert_rule.created",
entity_type="alert_rule", entity_id=rule.name,
detail={"source": rule.source_module, "resource": rule.resource_name,
"metric": rule.metric_name})
return redirect(url_for("alerts.list_alerts"))
@alerts_bp.post("/rules/<rule_id>")
@require_role(UserRole.operator)
async def update_rule(rule_id: str):
form = await request.form
# Validate the admin-only action before opening the transaction. Non-admins
# leave the existing action untouched (they can still edit everything else).
apply_action = _is_admin()
if apply_action:
ansible_action, err = _parse_ansible_action(form)
if err:
return err, 400
async with current_app.db_sessionmaker() as db:
async with db.begin():
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if rule is None:
return "Not found", 404
try:
fields = await alert_svc.rule_fields(db, **_rule_form(form))
except ValueError as exc:
return str(exc), 400
await alert_svc.apply_rule_change(db, rule, fields)
if apply_action:
rule.ansible_action = ansible_action
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
"alert_rule.updated", entity_type="alert_rule", entity_id=rule.name)
return redirect(url_for("alerts.list_alerts"))
@alerts_bp.post("/rules/<rule_id>/toggle")
@require_role(UserRole.operator)
async def toggle_rule(rule_id: str):
async with current_app.db_sessionmaker() as db:
async with db.begin():
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if rule is None:
return "Not found", 404
await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled)
action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
action, entity_type="alert_rule", entity_id=rule.name)
return redirect(url_for("alerts.list_alerts"))
@alerts_bp.post("/rules/<rule_id>/delete")
@require_role(UserRole.admin)
async def delete_rule(rule_id: str):
rule_name = None
async with current_app.db_sessionmaker() as db:
async with db.begin():
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if rule:
rule_name = rule.name
await db.delete(rule)
if rule_name:
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
"alert_rule.deleted", entity_type="alert_rule", entity_id=rule_name)
return redirect(url_for("alerts.list_alerts"))
@alerts_bp.get("/maintenance")
@require_role(UserRole.viewer)
async def list_maintenance():
now = datetime.now(timezone.utc)
async with current_app.db_sessionmaker() as db:
windows = await alert_svc.list_maintenance_windows(db)
return await render_template(
"alerts/maintenance.html",
windows=windows,
now=now,
source_modules=alert_svc.source_modules(),
)
@alerts_bp.get("/maintenance/new")
@require_role(UserRole.operator)
async def new_maintenance():
return await render_template(
"alerts/maintenance_form.html",
window=None,
source_modules=alert_svc.source_modules(),
)
@alerts_bp.post("/maintenance")
@require_role(UserRole.operator)
async def create_maintenance():
form = await request.form
user_id = session.get("user_id")
try:
fields = alert_svc.window_fields(
name=form.get("name", ""),
start_at=_form_time(form.get("start_at")), end_at=_form_time(form.get("end_at")),
scope_source=form.get("scope_source"), scope_resource=form.get("scope_resource"))
except ValueError as exc:
return str(exc), 400
window = MaintenanceWindow(created_by=user_id, **fields)
async with current_app.db_sessionmaker() as db:
async with db.begin():
db.add(window)
scope_desc = alert_svc.window_scope(window)
await log_audit(current_app, user_id, session.get("username", ""),
"maintenance_window.created", entity_type="maintenance_window",
entity_id=window.name, detail={"scope": scope_desc})
return redirect(url_for("alerts.list_maintenance"))
@alerts_bp.post("/maintenance/<window_id>/delete")
@require_role(UserRole.operator)
async def delete_maintenance(window_id: str):
window_name = None
async with current_app.db_sessionmaker() as db:
async with db.begin():
result = await db.execute(
select(MaintenanceWindow).where(MaintenanceWindow.id == window_id)
)
window = result.scalar_one_or_none()
if window:
window_name = window.name
await db.delete(window)
if window_name:
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
"maintenance_window.deleted", entity_type="maintenance_window",
entity_id=window_name)
return redirect(url_for("alerts.list_maintenance"))
@alerts_bp.post("/<rule_id>/acknowledge")
@require_role(UserRole.operator)
async def acknowledge(rule_id: str):
user_id = session.get("user_id")
async with current_app.db_sessionmaker() as db:
async with db.begin():
try:
await alert_svc.acknowledge(db, rule_id, user_id)
except LookupError:
return "Not found", 404
except ValueError as exc:
return f"Conflict: {exc}", 409
return redirect(url_for("alerts.list_alerts"))