bvandeusenandClaude Opus 5.5
dc22a35e9d
#5066 "Alert history page". /alerts/history lists every alert transition in a range (1h–30d, the shared range picker), newest first, all rules or one (?rule=). Each row: from → to, the value against the rule's condition ("no data" for a silent one), email and webhook sent or failed with the error, "muted · <window>" (and "notified when the window ended") taken out of the event note, other notes, and a link to the Ansible run an action started. Range and rule changes swap only the rows (HTMX). Viewer role. The Alerts page links each rule's history and the whole history. The query moves to alert_svc.history, which MCP get_alert_history now uses too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
375 lines
15 KiB
Python
375 lines
15 KiB
Python
from __future__ import annotations
|
|
from datetime import datetime, timezone
|
|
from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
|
|
from steward.core.audit import log_audit
|
|
from sqlalchemy import select
|
|
from steward.auth.middleware import require_role
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.time_range import parse_range
|
|
from steward.ansible import sources as src_module
|
|
from steward.models.alerts import AlertRule, AlertOperator, MaintenanceWindow
|
|
from steward.models.users import UserRole
|
|
|
|
alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")
|
|
|
|
_OPERATORS = [(op.value, op.value) for op in AlertOperator]
|
|
|
|
def _rule_form(form) -> dict:
|
|
"""The rule form's fields, as alert_svc.rule_fields takes them."""
|
|
return {
|
|
"name": form.get("name", ""),
|
|
"source_module": form.get("source_module", ""),
|
|
"resource_name": form.get("resource_name", ""),
|
|
"metric_name": form.get("metric_name", ""),
|
|
"operator": form.get("operator", ""),
|
|
"threshold": form.get("threshold"),
|
|
"consecutive_failures_required": form.get("consecutive_failures_required"),
|
|
"consecutive_ok_required": form.get("consecutive_ok_required"),
|
|
"notify_no_data": "notify_no_data" in form,
|
|
}
|
|
|
|
|
|
def _form_time(value) -> datetime:
|
|
"""A datetime-local field, read as UTC."""
|
|
try:
|
|
return datetime.fromisoformat((value or "").strip()).replace(tzinfo=timezone.utc)
|
|
except ValueError:
|
|
raise ValueError(f"{value!r} is not a date and time.") from None
|
|
|
|
|
|
def _is_admin() -> bool:
|
|
return session.get("user_role") == UserRole.admin.value
|
|
|
|
|
|
def _ansible_source_names() -> list[str]:
|
|
try:
|
|
return [s["name"] for s in src_module.get_sources(current_app.config.get("ANSIBLE", {}))]
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def _parse_ansible_action(form) -> tuple[dict | None, str | None]:
|
|
"""Parse + validate the admin-only Ansible action from a rule form.
|
|
|
|
Returns (action_or_None, error_or_None). action is None when the action
|
|
checkbox is unchecked (i.e. the action is being cleared).
|
|
"""
|
|
if "ansible_enabled" not in form:
|
|
return None, None
|
|
source_name = (form.get("ansible_source", "") or "").strip()
|
|
playbook = (form.get("ansible_playbook", "") or "").strip()
|
|
inventory = (form.get("ansible_inventory", "") or "").strip()
|
|
if not (source_name and playbook and inventory):
|
|
return None, "Ansible action requires source, playbook, and inventory"
|
|
try:
|
|
srcs = src_module.get_sources(current_app.config.get("ANSIBLE", {}))
|
|
except Exception:
|
|
return None, "Ansible sources are misconfigured"
|
|
source = next((s for s in srcs if s["name"] == source_name), None)
|
|
if source is None:
|
|
return None, f"Ansible source '{source_name}' not found"
|
|
if playbook not in src_module.discover_playbooks(source["path"]):
|
|
return None, f"Playbook '{playbook}' not found in source '{source_name}'"
|
|
extra_vars: list[str] = []
|
|
for line in (form.get("ansible_extra_vars", "") or "").splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
if "=" not in line:
|
|
return None, f"Invalid extra var (expected key=value): {line!r}"
|
|
extra_vars.append(line)
|
|
action: dict = {"source": source_name, "playbook": playbook, "inventory": inventory}
|
|
if extra_vars:
|
|
action["extra_vars"] = extra_vars
|
|
limit = (form.get("ansible_limit", "") or "").strip()
|
|
tags = (form.get("ansible_tags", "") or "").strip()
|
|
if limit:
|
|
action["limit"] = limit
|
|
if tags:
|
|
action["tags"] = tags
|
|
if "ansible_check" in form:
|
|
action["check"] = True
|
|
return action, None
|
|
|
|
|
|
@alerts_bp.get("/")
|
|
@require_role(UserRole.viewer)
|
|
async def list_alerts():
|
|
async with current_app.db_sessionmaker() as db:
|
|
active = await alert_svc.active_alerts(db)
|
|
rules = await alert_svc.list_rules(db)
|
|
|
|
return await render_template("alerts/list.html", active=active, rules=rules, operators=_OPERATORS)
|
|
|
|
|
|
HISTORY_LIMIT = 200
|
|
|
|
|
|
def _history_row(event, rule) -> dict:
|
|
"""One transition as the history page shows it."""
|
|
from steward.core.alerts import MUTED, RELEASED
|
|
notes, muted_by, released = [], None, False
|
|
for part in (event.note or "").split("; "):
|
|
if part.startswith(MUTED):
|
|
muted_by = part[len(MUTED):].strip() or "a maintenance window"
|
|
elif part == RELEASED:
|
|
released = True
|
|
elif part:
|
|
notes.append(part)
|
|
value = "no data" if event.metric_value is None else f"{event.metric_value:g}"
|
|
return {
|
|
"at": event.transitioned_at, "rule": rule,
|
|
"from_state": event.from_state, "to_state": event.to_state,
|
|
"value": value, "condition": f"{rule.operator.value} {rule.threshold:g}",
|
|
"notes": notes, "muted_by": muted_by, "released": released,
|
|
"deliveries": [(label, sent, error) for label, sent, error in (
|
|
("email", event.email_sent, event.email_error),
|
|
("webhook", event.webhook_sent, event.webhook_error)) if sent is not None],
|
|
"ansible_run_id": event.ansible_run_id,
|
|
}
|
|
|
|
|
|
@alerts_bp.get("/history")
|
|
@require_role(UserRole.viewer)
|
|
async def alert_history():
|
|
"""Every alert transition in a range, newest first; one rule's with ?rule=."""
|
|
since, range_key = parse_range(request.args.get("range") or "7d")
|
|
async with current_app.db_sessionmaker() as db:
|
|
rules = await alert_svc.list_rules(db)
|
|
chosen = next((r for r in rules if r.id == request.args.get("rule")), None)
|
|
rows = await alert_svc.history(db, since, chosen.id if chosen else None, HISTORY_LIMIT)
|
|
ctx = {"events": [_history_row(e, r) for e, r in rows[:HISTORY_LIMIT]],
|
|
"truncated": len(rows) > HISTORY_LIMIT, "limit": HISTORY_LIMIT,
|
|
"rules": rules, "chosen": chosen, "current_range": range_key}
|
|
if request.headers.get("HX-Request"):
|
|
return await render_template("alerts/_history_rows.html", **ctx)
|
|
return await render_template("alerts/history.html", **ctx)
|
|
|
|
|
|
@alerts_bp.get("/api/rule_fields")
|
|
@require_role(UserRole.viewer)
|
|
async def rule_fields_api():
|
|
source = request.args.get("source_module", "")
|
|
current_resource = request.args.get("current_resource", "")
|
|
current_metric = request.args.get("current_metric", "")
|
|
async with current_app.db_sessionmaker() as db:
|
|
resources = await alert_svc.resources_for(db, source) if source else []
|
|
async with current_app.db_sessionmaker() as db:
|
|
metrics = await alert_svc.metrics_for(db, source)
|
|
return await render_template(
|
|
"alerts/_rule_fields.html",
|
|
resources=resources,
|
|
metrics=metrics,
|
|
current_resource=current_resource,
|
|
current_metric=current_metric,
|
|
)
|
|
|
|
|
|
@alerts_bp.get("/rules/new")
|
|
@require_role(UserRole.operator)
|
|
async def new_rule():
|
|
first_source = alert_svc.source_modules()[0]
|
|
async with current_app.db_sessionmaker() as db:
|
|
resources = await alert_svc.resources_for(db, first_source)
|
|
return await render_template(
|
|
"alerts/rules_form.html",
|
|
rule=None,
|
|
operators=_OPERATORS,
|
|
source_modules=alert_svc.source_modules(),
|
|
initial_source=first_source,
|
|
resources=resources,
|
|
metrics=alert_svc.metric_catalog().get(first_source, []),
|
|
ansible_sources=_ansible_source_names(),
|
|
)
|
|
|
|
|
|
@alerts_bp.get("/rules/<rule_id>/edit")
|
|
@require_role(UserRole.operator)
|
|
async def edit_rule(rule_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule is None:
|
|
return "Not found", 404
|
|
resources = await alert_svc.resources_for(db, rule.source_module)
|
|
return await render_template(
|
|
"alerts/rules_form.html",
|
|
rule=rule,
|
|
operators=_OPERATORS,
|
|
source_modules=alert_svc.source_modules(),
|
|
initial_source=rule.source_module,
|
|
resources=resources,
|
|
metrics=alert_svc.metric_catalog().get(rule.source_module, []),
|
|
ansible_sources=_ansible_source_names(),
|
|
)
|
|
|
|
|
|
@alerts_bp.post("/rules")
|
|
@require_role(UserRole.operator)
|
|
async def create_rule():
|
|
form = await request.form
|
|
user_id = session.get("user_id")
|
|
# Ansible action is admin-only; non-admins simply can't set one.
|
|
ansible_action = None
|
|
if _is_admin():
|
|
ansible_action, err = _parse_ansible_action(form)
|
|
if err:
|
|
return err, 400
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
try:
|
|
fields = await alert_svc.rule_fields(db, **_rule_form(form))
|
|
except ValueError as exc:
|
|
return str(exc), 400
|
|
rule = await alert_svc.add_rule(db, fields, created_by=user_id,
|
|
ansible_action=ansible_action)
|
|
await log_audit(current_app, user_id, session.get("username", ""), "alert_rule.created",
|
|
entity_type="alert_rule", entity_id=rule.name,
|
|
detail={"source": rule.source_module, "resource": rule.resource_name,
|
|
"metric": rule.metric_name})
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>")
|
|
@require_role(UserRole.operator)
|
|
async def update_rule(rule_id: str):
|
|
form = await request.form
|
|
# Validate the admin-only action before opening the transaction. Non-admins
|
|
# leave the existing action untouched (they can still edit everything else).
|
|
apply_action = _is_admin()
|
|
if apply_action:
|
|
ansible_action, err = _parse_ansible_action(form)
|
|
if err:
|
|
return err, 400
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule is None:
|
|
return "Not found", 404
|
|
try:
|
|
fields = await alert_svc.rule_fields(db, **_rule_form(form))
|
|
except ValueError as exc:
|
|
return str(exc), 400
|
|
await alert_svc.apply_rule_change(db, rule, fields)
|
|
if apply_action:
|
|
rule.ansible_action = ansible_action
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"alert_rule.updated", entity_type="alert_rule", entity_id=rule.name)
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>/toggle")
|
|
@require_role(UserRole.operator)
|
|
async def toggle_rule(rule_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule is None:
|
|
return "Not found", 404
|
|
await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled)
|
|
action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
action, entity_type="alert_rule", entity_id=rule.name)
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>/delete")
|
|
@require_role(UserRole.admin)
|
|
async def delete_rule(rule_id: str):
|
|
rule_name = None
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule:
|
|
rule_name = rule.name
|
|
await db.delete(rule)
|
|
if rule_name:
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"alert_rule.deleted", entity_type="alert_rule", entity_id=rule_name)
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.get("/maintenance")
|
|
@require_role(UserRole.viewer)
|
|
async def list_maintenance():
|
|
now = datetime.now(timezone.utc)
|
|
async with current_app.db_sessionmaker() as db:
|
|
windows = await alert_svc.list_maintenance_windows(db)
|
|
return await render_template(
|
|
"alerts/maintenance.html",
|
|
windows=windows,
|
|
now=now,
|
|
source_modules=alert_svc.source_modules(),
|
|
)
|
|
|
|
|
|
@alerts_bp.get("/maintenance/new")
|
|
@require_role(UserRole.operator)
|
|
async def new_maintenance():
|
|
return await render_template(
|
|
"alerts/maintenance_form.html",
|
|
window=None,
|
|
source_modules=alert_svc.source_modules(),
|
|
)
|
|
|
|
|
|
@alerts_bp.post("/maintenance")
|
|
@require_role(UserRole.operator)
|
|
async def create_maintenance():
|
|
form = await request.form
|
|
user_id = session.get("user_id")
|
|
try:
|
|
fields = alert_svc.window_fields(
|
|
name=form.get("name", ""),
|
|
start_at=_form_time(form.get("start_at")), end_at=_form_time(form.get("end_at")),
|
|
scope_source=form.get("scope_source"), scope_resource=form.get("scope_resource"))
|
|
except ValueError as exc:
|
|
return str(exc), 400
|
|
window = MaintenanceWindow(created_by=user_id, **fields)
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
db.add(window)
|
|
scope_desc = alert_svc.window_scope(window)
|
|
await log_audit(current_app, user_id, session.get("username", ""),
|
|
"maintenance_window.created", entity_type="maintenance_window",
|
|
entity_id=window.name, detail={"scope": scope_desc})
|
|
return redirect(url_for("alerts.list_maintenance"))
|
|
|
|
|
|
@alerts_bp.post("/maintenance/<window_id>/delete")
|
|
@require_role(UserRole.operator)
|
|
async def delete_maintenance(window_id: str):
|
|
window_name = None
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(
|
|
select(MaintenanceWindow).where(MaintenanceWindow.id == window_id)
|
|
)
|
|
window = result.scalar_one_or_none()
|
|
if window:
|
|
window_name = window.name
|
|
await db.delete(window)
|
|
if window_name:
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"maintenance_window.deleted", entity_type="maintenance_window",
|
|
entity_id=window_name)
|
|
return redirect(url_for("alerts.list_maintenance"))
|
|
|
|
|
|
@alerts_bp.post("/<rule_id>/acknowledge")
|
|
@require_role(UserRole.operator)
|
|
async def acknowledge(rule_id: str):
|
|
user_id = session.get("user_id")
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
try:
|
|
await alert_svc.acknowledge(db, rule_id, user_id)
|
|
except LookupError:
|
|
return "Not found", 404
|
|
except ValueError as exc:
|
|
return f"Conflict: {exc}", 409
|
|
return redirect(url_for("alerts.list_alerts"))
|