The Alerts page's active list now says each alert in one line around the resource it watches, e.g. "cpu_pct on web-1 is 94% (> 90%), firing for 12m", with web-1 linking to its host, monitor or container page (or the plugin's page for Traefik, UniFi and SNMP). Values take their unit from the metric name through formatting.metric_value. A silent series reads "No cpu_pct from web-1 for 20m"; an up/down metric says the word. Actions follow the routes' roles: acknowledge, edit, toggle and New rule for operators and admins, Delete for admins only, nothing for viewers. The sidebar shows how many alerts need attention beside Alerts, from an async context processor that skips htmx fragments, logged-out pages and the database-down page, and renders without the count if the query fails. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
375 lines
15 KiB
Python
375 lines
15 KiB
Python
from __future__ import annotations
|
|
from datetime import datetime, timezone
|
|
from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
|
|
from steward.core.audit import log_audit
|
|
from sqlalchemy import select
|
|
from steward.auth.middleware import require_role
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.time_range import parse_range
|
|
from steward.ansible import sources as src_module
|
|
from steward.models.alerts import AlertRule, AlertOperator, MaintenanceWindow
|
|
from steward.models.users import UserRole
|
|
|
|
alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")
|
|
|
|
_OPERATORS = [(op.value, op.value) for op in AlertOperator]
|
|
|
|
def _rule_form(form) -> dict:
|
|
"""The rule form's fields, as alert_svc.rule_fields takes them."""
|
|
return {
|
|
"name": form.get("name", ""),
|
|
"source_module": form.get("source_module", ""),
|
|
"resource_name": form.get("resource_name", ""),
|
|
"metric_name": form.get("metric_name", ""),
|
|
"operator": form.get("operator", ""),
|
|
"threshold": form.get("threshold"),
|
|
"consecutive_failures_required": form.get("consecutive_failures_required"),
|
|
"consecutive_ok_required": form.get("consecutive_ok_required"),
|
|
"notify_no_data": "notify_no_data" in form,
|
|
}
|
|
|
|
|
|
def _form_time(value) -> datetime:
|
|
"""A datetime-local field, read as UTC."""
|
|
try:
|
|
return datetime.fromisoformat((value or "").strip()).replace(tzinfo=timezone.utc)
|
|
except ValueError:
|
|
raise ValueError(f"{value!r} is not a date and time.") from None
|
|
|
|
|
|
def _is_admin() -> bool:
|
|
return session.get("user_role") == UserRole.admin.value
|
|
|
|
|
|
def _ansible_source_names() -> list[str]:
|
|
try:
|
|
return [s["name"] for s in src_module.get_sources(current_app.config.get("ANSIBLE", {}))]
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def _parse_ansible_action(form) -> tuple[dict | None, str | None]:
|
|
"""Parse + validate the admin-only Ansible action from a rule form.
|
|
|
|
Returns (action_or_None, error_or_None). action is None when the action
|
|
checkbox is unchecked (i.e. the action is being cleared).
|
|
"""
|
|
if "ansible_enabled" not in form:
|
|
return None, None
|
|
source_name = (form.get("ansible_source", "") or "").strip()
|
|
playbook = (form.get("ansible_playbook", "") or "").strip()
|
|
inventory = (form.get("ansible_inventory", "") or "").strip()
|
|
if not (source_name and playbook and inventory):
|
|
return None, "Ansible action requires source, playbook, and inventory"
|
|
try:
|
|
srcs = src_module.get_sources(current_app.config.get("ANSIBLE", {}))
|
|
except Exception:
|
|
return None, "Ansible sources are misconfigured"
|
|
source = next((s for s in srcs if s["name"] == source_name), None)
|
|
if source is None:
|
|
return None, f"Ansible source '{source_name}' not found"
|
|
if playbook not in src_module.discover_playbooks(source["path"]):
|
|
return None, f"Playbook '{playbook}' not found in source '{source_name}'"
|
|
extra_vars: list[str] = []
|
|
for line in (form.get("ansible_extra_vars", "") or "").splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
if "=" not in line:
|
|
return None, f"Invalid extra var (expected key=value): {line!r}"
|
|
extra_vars.append(line)
|
|
action: dict = {"source": source_name, "playbook": playbook, "inventory": inventory}
|
|
if extra_vars:
|
|
action["extra_vars"] = extra_vars
|
|
limit = (form.get("ansible_limit", "") or "").strip()
|
|
tags = (form.get("ansible_tags", "") or "").strip()
|
|
if limit:
|
|
action["limit"] = limit
|
|
if tags:
|
|
action["tags"] = tags
|
|
if "ansible_check" in form:
|
|
action["check"] = True
|
|
return action, None
|
|
|
|
|
|
@alerts_bp.get("/")
|
|
@require_role(UserRole.viewer)
|
|
async def list_alerts():
|
|
async with current_app.db_sessionmaker() as db:
|
|
active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
|
|
rules = await alert_svc.list_rules(db)
|
|
|
|
return await render_template("alerts/list.html", active=active, rules=rules, operators=_OPERATORS)
|
|
|
|
|
|
HISTORY_LIMIT = 200
|
|
|
|
|
|
def _history_row(event, rule) -> dict:
|
|
"""One transition as the history page shows it."""
|
|
from steward.core.alerts import MUTED, RELEASED
|
|
notes, muted_by, released = [], None, False
|
|
for part in (event.note or "").split("; "):
|
|
if part.startswith(MUTED):
|
|
muted_by = part[len(MUTED):].strip() or "a maintenance window"
|
|
elif part == RELEASED:
|
|
released = True
|
|
elif part:
|
|
notes.append(part)
|
|
value = "no data" if event.metric_value is None else f"{event.metric_value:g}"
|
|
return {
|
|
"at": event.transitioned_at, "rule": rule,
|
|
"from_state": event.from_state, "to_state": event.to_state,
|
|
"value": value, "condition": f"{rule.operator.value} {rule.threshold:g}",
|
|
"notes": notes, "muted_by": muted_by, "released": released,
|
|
"deliveries": [(label, sent, error) for label, sent, error in (
|
|
("email", event.email_sent, event.email_error),
|
|
("webhook", event.webhook_sent, event.webhook_error)) if sent is not None],
|
|
"ansible_run_id": event.ansible_run_id,
|
|
}
|
|
|
|
|
|
@alerts_bp.get("/history")
|
|
@require_role(UserRole.viewer)
|
|
async def alert_history():
|
|
"""Every alert transition in a range, newest first; one rule's with ?rule=."""
|
|
since, range_key = parse_range(request.args.get("range") or "7d")
|
|
async with current_app.db_sessionmaker() as db:
|
|
rules = await alert_svc.list_rules(db)
|
|
chosen = next((r for r in rules if r.id == request.args.get("rule")), None)
|
|
rows = await alert_svc.history(db, since, chosen.id if chosen else None, HISTORY_LIMIT)
|
|
ctx = {"events": [_history_row(e, r) for e, r in rows[:HISTORY_LIMIT]],
|
|
"truncated": len(rows) > HISTORY_LIMIT, "limit": HISTORY_LIMIT,
|
|
"rules": rules, "chosen": chosen, "current_range": range_key}
|
|
if request.headers.get("HX-Request"):
|
|
return await render_template("alerts/_history_rows.html", **ctx)
|
|
return await render_template("alerts/history.html", **ctx)
|
|
|
|
|
|
@alerts_bp.get("/api/rule_fields")
|
|
@require_role(UserRole.viewer)
|
|
async def rule_fields_api():
|
|
source = request.args.get("source_module", "")
|
|
current_resource = request.args.get("current_resource", "")
|
|
current_metric = request.args.get("current_metric", "")
|
|
async with current_app.db_sessionmaker() as db:
|
|
resources = await alert_svc.resources_for(db, source) if source else []
|
|
async with current_app.db_sessionmaker() as db:
|
|
metrics = await alert_svc.metrics_for(db, source)
|
|
return await render_template(
|
|
"alerts/_rule_fields.html",
|
|
resources=resources,
|
|
metrics=metrics,
|
|
current_resource=current_resource,
|
|
current_metric=current_metric,
|
|
)
|
|
|
|
|
|
@alerts_bp.get("/rules/new")
|
|
@require_role(UserRole.operator)
|
|
async def new_rule():
|
|
first_source = alert_svc.source_modules()[0]
|
|
async with current_app.db_sessionmaker() as db:
|
|
resources = await alert_svc.resources_for(db, first_source)
|
|
return await render_template(
|
|
"alerts/rules_form.html",
|
|
rule=None,
|
|
operators=_OPERATORS,
|
|
source_modules=alert_svc.source_modules(),
|
|
initial_source=first_source,
|
|
resources=resources,
|
|
metrics=alert_svc.metric_catalog().get(first_source, []),
|
|
ansible_sources=_ansible_source_names(),
|
|
)
|
|
|
|
|
|
@alerts_bp.get("/rules/<rule_id>/edit")
|
|
@require_role(UserRole.operator)
|
|
async def edit_rule(rule_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule is None:
|
|
return "Not found", 404
|
|
resources = await alert_svc.resources_for(db, rule.source_module)
|
|
return await render_template(
|
|
"alerts/rules_form.html",
|
|
rule=rule,
|
|
operators=_OPERATORS,
|
|
source_modules=alert_svc.source_modules(),
|
|
initial_source=rule.source_module,
|
|
resources=resources,
|
|
metrics=alert_svc.metric_catalog().get(rule.source_module, []),
|
|
ansible_sources=_ansible_source_names(),
|
|
)
|
|
|
|
|
|
@alerts_bp.post("/rules")
|
|
@require_role(UserRole.operator)
|
|
async def create_rule():
|
|
form = await request.form
|
|
user_id = session.get("user_id")
|
|
# Ansible action is admin-only; non-admins simply can't set one.
|
|
ansible_action = None
|
|
if _is_admin():
|
|
ansible_action, err = _parse_ansible_action(form)
|
|
if err:
|
|
return err, 400
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
try:
|
|
fields = await alert_svc.rule_fields(db, **_rule_form(form))
|
|
except ValueError as exc:
|
|
return str(exc), 400
|
|
rule = await alert_svc.add_rule(db, fields, created_by=user_id,
|
|
ansible_action=ansible_action)
|
|
await log_audit(current_app, user_id, session.get("username", ""), "alert_rule.created",
|
|
entity_type="alert_rule", entity_id=rule.name,
|
|
detail={"source": rule.source_module, "resource": rule.resource_name,
|
|
"metric": rule.metric_name})
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>")
|
|
@require_role(UserRole.operator)
|
|
async def update_rule(rule_id: str):
|
|
form = await request.form
|
|
# Validate the admin-only action before opening the transaction. Non-admins
|
|
# leave the existing action untouched (they can still edit everything else).
|
|
apply_action = _is_admin()
|
|
if apply_action:
|
|
ansible_action, err = _parse_ansible_action(form)
|
|
if err:
|
|
return err, 400
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule is None:
|
|
return "Not found", 404
|
|
try:
|
|
fields = await alert_svc.rule_fields(db, **_rule_form(form))
|
|
except ValueError as exc:
|
|
return str(exc), 400
|
|
await alert_svc.apply_rule_change(db, rule, fields)
|
|
if apply_action:
|
|
rule.ansible_action = ansible_action
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"alert_rule.updated", entity_type="alert_rule", entity_id=rule.name)
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>/toggle")
|
|
@require_role(UserRole.operator)
|
|
async def toggle_rule(rule_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule is None:
|
|
return "Not found", 404
|
|
await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled)
|
|
action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
action, entity_type="alert_rule", entity_id=rule.name)
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>/delete")
|
|
@require_role(UserRole.admin)
|
|
async def delete_rule(rule_id: str):
|
|
rule_name = None
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule:
|
|
rule_name = rule.name
|
|
await db.delete(rule)
|
|
if rule_name:
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"alert_rule.deleted", entity_type="alert_rule", entity_id=rule_name)
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.get("/maintenance")
|
|
@require_role(UserRole.viewer)
|
|
async def list_maintenance():
|
|
now = datetime.now(timezone.utc)
|
|
async with current_app.db_sessionmaker() as db:
|
|
windows = await alert_svc.list_maintenance_windows(db)
|
|
return await render_template(
|
|
"alerts/maintenance.html",
|
|
windows=windows,
|
|
now=now,
|
|
source_modules=alert_svc.source_modules(),
|
|
)
|
|
|
|
|
|
@alerts_bp.get("/maintenance/new")
|
|
@require_role(UserRole.operator)
|
|
async def new_maintenance():
|
|
return await render_template(
|
|
"alerts/maintenance_form.html",
|
|
window=None,
|
|
source_modules=alert_svc.source_modules(),
|
|
)
|
|
|
|
|
|
@alerts_bp.post("/maintenance")
|
|
@require_role(UserRole.operator)
|
|
async def create_maintenance():
|
|
form = await request.form
|
|
user_id = session.get("user_id")
|
|
try:
|
|
fields = alert_svc.window_fields(
|
|
name=form.get("name", ""),
|
|
start_at=_form_time(form.get("start_at")), end_at=_form_time(form.get("end_at")),
|
|
scope_source=form.get("scope_source"), scope_resource=form.get("scope_resource"))
|
|
except ValueError as exc:
|
|
return str(exc), 400
|
|
window = MaintenanceWindow(created_by=user_id, **fields)
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
db.add(window)
|
|
scope_desc = alert_svc.window_scope(window)
|
|
await log_audit(current_app, user_id, session.get("username", ""),
|
|
"maintenance_window.created", entity_type="maintenance_window",
|
|
entity_id=window.name, detail={"scope": scope_desc})
|
|
return redirect(url_for("alerts.list_maintenance"))
|
|
|
|
|
|
@alerts_bp.post("/maintenance/<window_id>/delete")
|
|
@require_role(UserRole.operator)
|
|
async def delete_maintenance(window_id: str):
|
|
window_name = None
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(
|
|
select(MaintenanceWindow).where(MaintenanceWindow.id == window_id)
|
|
)
|
|
window = result.scalar_one_or_none()
|
|
if window:
|
|
window_name = window.name
|
|
await db.delete(window)
|
|
if window_name:
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"maintenance_window.deleted", entity_type="maintenance_window",
|
|
entity_id=window_name)
|
|
return redirect(url_for("alerts.list_maintenance"))
|
|
|
|
|
|
@alerts_bp.post("/<rule_id>/acknowledge")
|
|
@require_role(UserRole.operator)
|
|
async def acknowledge(rule_id: str):
|
|
user_id = session.get("user_id")
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
try:
|
|
await alert_svc.acknowledge(db, rule_id, user_id)
|
|
except LookupError:
|
|
return "Not found", 404
|
|
except ValueError as exc:
|
|
return f"Conflict: {exc}", 409
|
|
return redirect(url_for("alerts.list_alerts"))
|