Files
FabledSteward/steward/alerts/routes.py
T
bvandeusenandClaude Opus 5.5 0742b7a2b7 feat(alerts): the metric catalog is written by what emits it (#5061)
#5061 "Metric catalog belongs to what emits: plugins register their
metrics; names match what is recorded".

- The static METRIC_CATALOG is gone. metric_catalog() is core's own sources
  (the four monitor types and "steward", the app's open_fds/open_fds_pct,
  which were recorded but never offered) plus each loaded plugin's
  get_alert_metrics() export. A disabled plugin's sources leave the picker.
- traefik offers what it records: error_rate_4xx_pct and error_rate_5xx_pct.
  The old "error_rate" was never recorded, so a rule on it could not fire.
- tests/core/test_metric_catalog.py walks every MetricBatch.add /
  record_metric call (traefik's tuple loop included) and fails on a recorded
  metric the catalog lacks; it shows on a synthetic writer that it can fail.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 12:55:55 -04:00

329 lines
13 KiB
Python

from __future__ import annotations
from datetime import datetime, timezone
from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
from steward.core.audit import log_audit
from sqlalchemy import select
from steward.auth.middleware import require_role
from steward.alerts import service as alert_svc
from steward.ansible import sources as src_module
from steward.models.alerts import AlertRule, AlertOperator, MaintenanceWindow
from steward.models.users import UserRole
alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")
_OPERATORS = [(op.value, op.value) for op in AlertOperator]
def _rule_form(form) -> dict:
"""The rule form's fields, as alert_svc.rule_fields takes them."""
return {
"name": form.get("name", ""),
"source_module": form.get("source_module", ""),
"resource_name": form.get("resource_name", ""),
"metric_name": form.get("metric_name", ""),
"operator": form.get("operator", ""),
"threshold": form.get("threshold"),
"consecutive_failures_required": form.get("consecutive_failures_required"),
"consecutive_ok_required": form.get("consecutive_ok_required"),
}
def _form_time(value) -> datetime:
"""A datetime-local field, read as UTC."""
try:
return datetime.fromisoformat((value or "").strip()).replace(tzinfo=timezone.utc)
except ValueError:
raise ValueError(f"{value!r} is not a date and time.") from None
def _is_admin() -> bool:
return session.get("user_role") == UserRole.admin.value
def _ansible_source_names() -> list[str]:
try:
return [s["name"] for s in src_module.get_sources(current_app.config.get("ANSIBLE", {}))]
except Exception:
return []
def _parse_ansible_action(form) -> tuple[dict | None, str | None]:
"""Parse + validate the admin-only Ansible action from a rule form.
Returns (action_or_None, error_or_None). action is None when the action
checkbox is unchecked (i.e. the action is being cleared).
"""
if "ansible_enabled" not in form:
return None, None
source_name = (form.get("ansible_source", "") or "").strip()
playbook = (form.get("ansible_playbook", "") or "").strip()
inventory = (form.get("ansible_inventory", "") or "").strip()
if not (source_name and playbook and inventory):
return None, "Ansible action requires source, playbook, and inventory"
try:
srcs = src_module.get_sources(current_app.config.get("ANSIBLE", {}))
except Exception:
return None, "Ansible sources are misconfigured"
source = next((s for s in srcs if s["name"] == source_name), None)
if source is None:
return None, f"Ansible source '{source_name}' not found"
if playbook not in src_module.discover_playbooks(source["path"]):
return None, f"Playbook '{playbook}' not found in source '{source_name}'"
extra_vars: list[str] = []
for line in (form.get("ansible_extra_vars", "") or "").splitlines():
line = line.strip()
if not line:
continue
if "=" not in line:
return None, f"Invalid extra var (expected key=value): {line!r}"
extra_vars.append(line)
action: dict = {"source": source_name, "playbook": playbook, "inventory": inventory}
if extra_vars:
action["extra_vars"] = extra_vars
limit = (form.get("ansible_limit", "") or "").strip()
tags = (form.get("ansible_tags", "") or "").strip()
if limit:
action["limit"] = limit
if tags:
action["tags"] = tags
if "ansible_check" in form:
action["check"] = True
return action, None
@alerts_bp.get("/")
@require_role(UserRole.viewer)
async def list_alerts():
async with current_app.db_sessionmaker() as db:
active = await alert_svc.active_alerts(db)
rules = await alert_svc.list_rules(db)
return await render_template("alerts/list.html", active=active, rules=rules, operators=_OPERATORS)
@alerts_bp.get("/api/rule_fields")
@require_role(UserRole.viewer)
async def rule_fields_api():
source = request.args.get("source_module", "")
current_resource = request.args.get("current_resource", "")
current_metric = request.args.get("current_metric", "")
async with current_app.db_sessionmaker() as db:
resources = await alert_svc.resources_for(db, source) if source else []
async with current_app.db_sessionmaker() as db:
metrics = await alert_svc.metrics_for(db, source)
return await render_template(
"alerts/_rule_fields.html",
resources=resources,
metrics=metrics,
current_resource=current_resource,
current_metric=current_metric,
)
@alerts_bp.get("/rules/new")
@require_role(UserRole.operator)
async def new_rule():
first_source = alert_svc.source_modules()[0]
async with current_app.db_sessionmaker() as db:
resources = await alert_svc.resources_for(db, first_source)
return await render_template(
"alerts/rules_form.html",
rule=None,
operators=_OPERATORS,
source_modules=alert_svc.source_modules(),
initial_source=first_source,
resources=resources,
metrics=alert_svc.metric_catalog().get(first_source, []),
ansible_sources=_ansible_source_names(),
)
@alerts_bp.get("/rules/<rule_id>/edit")
@require_role(UserRole.operator)
async def edit_rule(rule_id: str):
async with current_app.db_sessionmaker() as db:
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if rule is None:
return "Not found", 404
resources = await alert_svc.resources_for(db, rule.source_module)
return await render_template(
"alerts/rules_form.html",
rule=rule,
operators=_OPERATORS,
source_modules=alert_svc.source_modules(),
initial_source=rule.source_module,
resources=resources,
metrics=alert_svc.metric_catalog().get(rule.source_module, []),
ansible_sources=_ansible_source_names(),
)
@alerts_bp.post("/rules")
@require_role(UserRole.operator)
async def create_rule():
form = await request.form
user_id = session.get("user_id")
# Ansible action is admin-only; non-admins simply can't set one.
ansible_action = None
if _is_admin():
ansible_action, err = _parse_ansible_action(form)
if err:
return err, 400
async with current_app.db_sessionmaker() as db:
async with db.begin():
try:
fields = await alert_svc.rule_fields(db, **_rule_form(form))
except ValueError as exc:
return str(exc), 400
rule = await alert_svc.add_rule(db, fields, created_by=user_id,
ansible_action=ansible_action)
await log_audit(current_app, user_id, session.get("username", ""), "alert_rule.created",
entity_type="alert_rule", entity_id=rule.name,
detail={"source": rule.source_module, "resource": rule.resource_name,
"metric": rule.metric_name})
return redirect(url_for("alerts.list_alerts"))
@alerts_bp.post("/rules/<rule_id>")
@require_role(UserRole.operator)
async def update_rule(rule_id: str):
form = await request.form
# Validate the admin-only action before opening the transaction. Non-admins
# leave the existing action untouched (they can still edit everything else).
apply_action = _is_admin()
if apply_action:
ansible_action, err = _parse_ansible_action(form)
if err:
return err, 400
async with current_app.db_sessionmaker() as db:
async with db.begin():
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if rule is None:
return "Not found", 404
try:
fields = await alert_svc.rule_fields(db, **_rule_form(form))
except ValueError as exc:
return str(exc), 400
await alert_svc.apply_rule_change(db, rule, fields)
if apply_action:
rule.ansible_action = ansible_action
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
"alert_rule.updated", entity_type="alert_rule", entity_id=rule.name)
return redirect(url_for("alerts.list_alerts"))
@alerts_bp.post("/rules/<rule_id>/toggle")
@require_role(UserRole.operator)
async def toggle_rule(rule_id: str):
async with current_app.db_sessionmaker() as db:
async with db.begin():
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if rule is None:
return "Not found", 404
await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled)
action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
action, entity_type="alert_rule", entity_id=rule.name)
return redirect(url_for("alerts.list_alerts"))
@alerts_bp.post("/rules/<rule_id>/delete")
@require_role(UserRole.admin)
async def delete_rule(rule_id: str):
rule_name = None
async with current_app.db_sessionmaker() as db:
async with db.begin():
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if rule:
rule_name = rule.name
await db.delete(rule)
if rule_name:
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
"alert_rule.deleted", entity_type="alert_rule", entity_id=rule_name)
return redirect(url_for("alerts.list_alerts"))
@alerts_bp.get("/maintenance")
@require_role(UserRole.viewer)
async def list_maintenance():
now = datetime.now(timezone.utc)
async with current_app.db_sessionmaker() as db:
windows = await alert_svc.list_maintenance_windows(db)
return await render_template(
"alerts/maintenance.html",
windows=windows,
now=now,
source_modules=alert_svc.source_modules(),
)
@alerts_bp.get("/maintenance/new")
@require_role(UserRole.operator)
async def new_maintenance():
return await render_template(
"alerts/maintenance_form.html",
window=None,
source_modules=alert_svc.source_modules(),
)
@alerts_bp.post("/maintenance")
@require_role(UserRole.operator)
async def create_maintenance():
form = await request.form
user_id = session.get("user_id")
try:
fields = alert_svc.window_fields(
name=form.get("name", ""),
start_at=_form_time(form.get("start_at")), end_at=_form_time(form.get("end_at")),
scope_source=form.get("scope_source"), scope_resource=form.get("scope_resource"))
except ValueError as exc:
return str(exc), 400
window = MaintenanceWindow(created_by=user_id, **fields)
async with current_app.db_sessionmaker() as db:
async with db.begin():
db.add(window)
scope_desc = alert_svc.window_scope(window)
await log_audit(current_app, user_id, session.get("username", ""),
"maintenance_window.created", entity_type="maintenance_window",
entity_id=window.name, detail={"scope": scope_desc})
return redirect(url_for("alerts.list_maintenance"))
@alerts_bp.post("/maintenance/<window_id>/delete")
@require_role(UserRole.operator)
async def delete_maintenance(window_id: str):
window_name = None
async with current_app.db_sessionmaker() as db:
async with db.begin():
result = await db.execute(
select(MaintenanceWindow).where(MaintenanceWindow.id == window_id)
)
window = result.scalar_one_or_none()
if window:
window_name = window.name
await db.delete(window)
if window_name:
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
"maintenance_window.deleted", entity_type="maintenance_window",
entity_id=window_name)
return redirect(url_for("alerts.list_maintenance"))
@alerts_bp.post("/<rule_id>/acknowledge")
@require_role(UserRole.operator)
async def acknowledge(rule_id: str):
user_id = session.get("user_id")
async with current_app.db_sessionmaker() as db:
async with db.begin():
try:
await alert_svc.acknowledge(db, rule_id, user_id)
except LookupError:
return "Not found", 404
except ValueError as exc:
return f"Conflict: {exc}", 409
return redirect(url_for("alerts.list_alerts"))