#5212 "Plain labels: no raw ids, scopes, metric keys or unexplained jargon", batch 1: - Audit log: each action reads as a sentence ("paused monitor"), with the code as its tooltip. Detail shows flags as yes/no, lists joined, a run's scope by name, spans as spans, and no ids the entity column already gives. The filter is now a picker of areas. - Alert rules, history and Status say the condition in words: "is down", "Fullest disk >= 85%". Readings use metric_value, and history times use |ago. - Rule form: sources and metrics by name, the threshold's unit, "fires when it is down/up" for an up/down metric instead of 0.5, and a metric reference built from the catalog. - Schedules and Maintenance: viewers no longer see Run now, Pause, Edit, Delete or New. Intervals and next runs read as "every 2h" and "in 3h"; a scope reads "Every alert on Host agent". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
416 lines
17 KiB
Python
416 lines
17 KiB
Python
from __future__ import annotations
|
|
from datetime import datetime, timezone
|
|
from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
|
|
from steward.core.audit import log_audit
|
|
from sqlalchemy import select
|
|
from steward.auth.middleware import require_role
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.paging import PAGE_SIZE, next_page
|
|
from steward.core.time_range import parse_range
|
|
from steward.ansible import sources as src_module
|
|
from steward.models.alerts import AlertRule, AlertOperator, AlertState, MaintenanceWindow
|
|
from steward.models.users import UserRole
|
|
|
|
alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")
|
|
|
|
_OPERATORS = [(op.value, op.value) for op in AlertOperator]
|
|
|
|
def _form_condition(form) -> dict:
|
|
"""The operator and threshold the form asks for. An up/down metric's form
|
|
sends the word it fires on (binary_when) instead: down is < 0.5, up is
|
|
>= 0.5, the midpoint of the 1.0 / 0.0 such a metric records."""
|
|
when = form.get("binary_when")
|
|
if when in ("down", "up") and form.get("metric_name") in ("is_up", "is_healthy"):
|
|
return {"operator": "<" if when == "down" else ">=", "threshold": "0.5"}
|
|
return {"operator": form.get("operator", ""), "threshold": form.get("threshold")}
|
|
|
|
|
|
def _rule_form(form) -> dict:
|
|
"""The rule form's fields, as alert_svc.rule_fields takes them."""
|
|
return {
|
|
"name": form.get("name", ""),
|
|
"source_module": form.get("source_module", ""),
|
|
"resource_name": form.get("resource_name", ""),
|
|
"metric_name": form.get("metric_name", ""),
|
|
**_form_condition(form),
|
|
"consecutive_failures_required": form.get("consecutive_failures_required"),
|
|
"consecutive_ok_required": form.get("consecutive_ok_required"),
|
|
"notify_no_data": "notify_no_data" in form,
|
|
}
|
|
|
|
|
|
def _form_time(value) -> datetime:
|
|
"""A datetime-local field, read as UTC."""
|
|
try:
|
|
return datetime.fromisoformat((value or "").strip()).replace(tzinfo=timezone.utc)
|
|
except ValueError:
|
|
raise ValueError(f"{value!r} is not a date and time.") from None
|
|
|
|
|
|
def _is_admin() -> bool:
|
|
return session.get("user_role") == UserRole.admin.value
|
|
|
|
|
|
def _ansible_source_names() -> list[str]:
|
|
try:
|
|
return [s["name"] for s in src_module.get_sources(current_app.config.get("ANSIBLE", {}))]
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def _parse_ansible_action(form) -> tuple[dict | None, str | None]:
|
|
"""Parse + validate the admin-only Ansible action from a rule form.
|
|
|
|
Returns (action_or_None, error_or_None). action is None when the action
|
|
checkbox is unchecked (i.e. the action is being cleared).
|
|
"""
|
|
if "ansible_enabled" not in form:
|
|
return None, None
|
|
source_name = (form.get("ansible_source", "") or "").strip()
|
|
playbook = (form.get("ansible_playbook", "") or "").strip()
|
|
inventory = (form.get("ansible_inventory", "") or "").strip()
|
|
if not (source_name and playbook and inventory):
|
|
return None, "Ansible action requires source, playbook, and inventory"
|
|
try:
|
|
srcs = src_module.get_sources(current_app.config.get("ANSIBLE", {}))
|
|
except Exception:
|
|
return None, "Ansible sources are misconfigured"
|
|
source = next((s for s in srcs if s["name"] == source_name), None)
|
|
if source is None:
|
|
return None, f"Ansible source '{source_name}' not found"
|
|
if playbook not in src_module.discover_playbooks(source["path"]):
|
|
return None, f"Playbook '{playbook}' not found in source '{source_name}'"
|
|
extra_vars: list[str] = []
|
|
for line in (form.get("ansible_extra_vars", "") or "").splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
if "=" not in line:
|
|
return None, f"Invalid extra var (expected key=value): {line!r}"
|
|
extra_vars.append(line)
|
|
action: dict = {"source": source_name, "playbook": playbook, "inventory": inventory}
|
|
if extra_vars:
|
|
action["extra_vars"] = extra_vars
|
|
limit = (form.get("ansible_limit", "") or "").strip()
|
|
tags = (form.get("ansible_tags", "") or "").strip()
|
|
if limit:
|
|
action["limit"] = limit
|
|
if tags:
|
|
action["tags"] = tags
|
|
if "ansible_check" in form:
|
|
action["check"] = True
|
|
return action, None
|
|
|
|
|
|
@alerts_bp.get("/")
|
|
@require_role(UserRole.viewer)
|
|
async def list_alerts():
|
|
async with current_app.db_sessionmaker() as db:
|
|
active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
|
|
rules = await alert_svc.list_rules(db)
|
|
states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
|
|
|
|
# Every rule with what it is doing now; disabled rules last.
|
|
rules.sort(key=lambda r: (not r.enabled, r.name.lower()))
|
|
rule_state = {r.id: alert_svc.alert_label(states[r.id]) for r in rules if r.id in states}
|
|
return await render_template(
|
|
"alerts/list.html", active=active, rules=rules, rule_state=rule_state,
|
|
operators=_OPERATORS, poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60))
|
|
|
|
|
|
@alerts_bp.get("/active")
|
|
@require_role(UserRole.viewer)
|
|
async def active_fragment():
|
|
"""The Active section alone, which the alerts page polls."""
|
|
async with current_app.db_sessionmaker() as db:
|
|
active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
|
|
return await render_template("alerts/_active.html", active=active)
|
|
|
|
|
|
def _history_row(event, rule) -> dict:
|
|
"""One transition as the history page shows it."""
|
|
from steward.core.alerts import MUTED, RELEASED
|
|
from steward.core.formatting import metric_label, metric_value
|
|
notes, muted_by, released = [], None, False
|
|
for part in (event.note or "").split("; "):
|
|
if part.startswith(MUTED):
|
|
muted_by = part[len(MUTED):].strip() or "a maintenance window"
|
|
elif part == RELEASED:
|
|
released = True
|
|
elif part:
|
|
notes.append(part)
|
|
metric = rule.metric_name
|
|
if event.metric_value is None:
|
|
value = f"No {metric_label(metric)} reading"
|
|
elif metric in ("is_up", "is_healthy"):
|
|
value = metric_value(metric, event.metric_value).capitalize()
|
|
else:
|
|
value = f"{metric_label(metric)} {metric_value(metric, event.metric_value)}"
|
|
return {
|
|
"at": event.transitioned_at, "rule": rule,
|
|
"from_state": event.from_state, "to_state": event.to_state,
|
|
"value": value, "condition": alert_svc.rule_condition(rule),
|
|
"notes": notes, "muted_by": muted_by, "released": released,
|
|
"deliveries": [(label, sent, error) for label, sent, error in (
|
|
("email", event.email_sent, event.email_error),
|
|
("webhook", event.webhook_sent, event.webhook_error)) if sent is not None],
|
|
"ansible_run_id": event.ansible_run_id,
|
|
}
|
|
|
|
|
|
@alerts_bp.get("/history")
|
|
@require_role(UserRole.viewer)
|
|
async def alert_history():
|
|
"""Every alert transition in a range, newest first; one rule's with ?rule=.
|
|
A page at a time as the reader scrolls: ?before=<cursor> is the next
|
|
page's rows alone (core.paging)."""
|
|
since, range_key = parse_range(request.args.get("range") or "7d")
|
|
before = request.args.get("before")
|
|
async with current_app.db_sessionmaker() as db:
|
|
rules = await alert_svc.list_rules(db)
|
|
chosen = next((r for r in rules if r.id == request.args.get("rule")), None)
|
|
rows = await alert_svc.history(db, since, chosen.id if chosen else None, PAGE_SIZE, before)
|
|
rows, after = next_page(rows, lambda row: (row[0].transitioned_at, row[0].id))
|
|
ctx = {"events": [_history_row(e, r) for e, r in rows], "after": after,
|
|
"rules": rules, "chosen": chosen, "current_range": range_key}
|
|
if before:
|
|
return await render_template("alerts/_history_page.html", **ctx)
|
|
if request.headers.get("HX-Request"):
|
|
return await render_template("alerts/_history_rows.html", **ctx)
|
|
return await render_template("alerts/history.html", **ctx)
|
|
|
|
|
|
@alerts_bp.get("/api/rule_fields")
|
|
@require_role(UserRole.viewer)
|
|
async def rule_fields_api():
|
|
source = request.args.get("source_module", "")
|
|
current_resource = request.args.get("current_resource", "")
|
|
current_metric = request.args.get("current_metric", "")
|
|
async with current_app.db_sessionmaker() as db:
|
|
resources = await alert_svc.resources_for(db, source) if source else []
|
|
async with current_app.db_sessionmaker() as db:
|
|
metrics = await alert_svc.metrics_for(db, source)
|
|
return await render_template(
|
|
"alerts/_rule_fields.html",
|
|
resources=resources,
|
|
metrics=metrics,
|
|
current_resource=current_resource,
|
|
current_metric=current_metric,
|
|
)
|
|
|
|
|
|
@alerts_bp.get("/rules/new")
|
|
@require_role(UserRole.operator)
|
|
async def new_rule():
|
|
first_source = alert_svc.source_modules()[0]
|
|
async with current_app.db_sessionmaker() as db:
|
|
resources = await alert_svc.resources_for(db, first_source)
|
|
return await render_template(
|
|
"alerts/rules_form.html",
|
|
rule=None,
|
|
operators=_OPERATORS,
|
|
source_modules=alert_svc.source_modules(),
|
|
initial_source=first_source,
|
|
resources=resources,
|
|
metrics=alert_svc.metric_catalog().get(first_source, []),
|
|
catalog=alert_svc.metric_catalog(),
|
|
ansible_sources=_ansible_source_names(),
|
|
)
|
|
|
|
|
|
@alerts_bp.get("/rules/<rule_id>/edit")
|
|
@require_role(UserRole.operator)
|
|
async def edit_rule(rule_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule is None:
|
|
return "Not found", 404
|
|
resources = await alert_svc.resources_for(db, rule.source_module)
|
|
return await render_template(
|
|
"alerts/rules_form.html",
|
|
rule=rule,
|
|
operators=_OPERATORS,
|
|
source_modules=alert_svc.source_modules(),
|
|
initial_source=rule.source_module,
|
|
resources=resources,
|
|
metrics=alert_svc.metric_catalog().get(rule.source_module, []),
|
|
catalog=alert_svc.metric_catalog(),
|
|
ansible_sources=_ansible_source_names(),
|
|
)
|
|
|
|
|
|
@alerts_bp.post("/rules")
|
|
@require_role(UserRole.operator)
|
|
async def create_rule():
|
|
form = await request.form
|
|
user_id = session.get("user_id")
|
|
# Ansible action is admin-only; non-admins simply can't set one.
|
|
ansible_action = None
|
|
if _is_admin():
|
|
ansible_action, err = _parse_ansible_action(form)
|
|
if err:
|
|
return err, 400
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
try:
|
|
fields = await alert_svc.rule_fields(db, **_rule_form(form))
|
|
except ValueError as exc:
|
|
return str(exc), 400
|
|
rule = await alert_svc.add_rule(db, fields, created_by=user_id,
|
|
ansible_action=ansible_action)
|
|
await log_audit(current_app, user_id, session.get("username", ""), "alert_rule.created",
|
|
entity_type="alert_rule", entity_id=rule.name,
|
|
detail={"source": rule.source_module, "resource": rule.resource_name,
|
|
"metric": rule.metric_name})
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>")
|
|
@require_role(UserRole.operator)
|
|
async def update_rule(rule_id: str):
|
|
form = await request.form
|
|
# Validate the admin-only action before opening the transaction. Non-admins
|
|
# leave the existing action untouched (they can still edit everything else).
|
|
apply_action = _is_admin()
|
|
if apply_action:
|
|
ansible_action, err = _parse_ansible_action(form)
|
|
if err:
|
|
return err, 400
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule is None:
|
|
return "Not found", 404
|
|
try:
|
|
fields = await alert_svc.rule_fields(db, **_rule_form(form))
|
|
except ValueError as exc:
|
|
return str(exc), 400
|
|
await alert_svc.apply_rule_change(db, rule, fields)
|
|
if apply_action:
|
|
rule.ansible_action = ansible_action
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"alert_rule.updated", entity_type="alert_rule", entity_id=rule.name)
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>/toggle")
|
|
@require_role(UserRole.operator)
|
|
async def toggle_rule(rule_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule is None:
|
|
return "Not found", 404
|
|
await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled)
|
|
action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
action, entity_type="alert_rule", entity_id=rule.name)
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>/delete")
|
|
@require_role(UserRole.admin)
|
|
async def delete_rule(rule_id: str):
|
|
rule_name = None
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule:
|
|
rule_name = rule.name
|
|
await db.delete(rule)
|
|
if rule_name:
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"alert_rule.deleted", entity_type="alert_rule", entity_id=rule_name)
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.get("/maintenance")
|
|
@require_role(UserRole.viewer)
|
|
async def list_maintenance():
|
|
now = datetime.now(timezone.utc)
|
|
async with current_app.db_sessionmaker() as db:
|
|
windows = await alert_svc.list_maintenance_windows(db)
|
|
# Active first (ending soonest first), then upcoming (starting soonest
|
|
# first); expired ones are history, newest first, folded away.
|
|
active = sorted((w for w in windows if w.start_at <= now <= w.end_at), key=lambda w: w.end_at)
|
|
upcoming = sorted((w for w in windows if w.start_at > now), key=lambda w: w.start_at)
|
|
expired = sorted((w for w in windows if w.end_at < now), key=lambda w: w.end_at, reverse=True)
|
|
return await render_template(
|
|
"alerts/maintenance.html",
|
|
windows=windows, active=active, upcoming=upcoming, expired=expired,
|
|
now=now,
|
|
source_modules=alert_svc.source_modules(),
|
|
)
|
|
|
|
|
|
@alerts_bp.get("/maintenance/new")
|
|
@require_role(UserRole.operator)
|
|
async def new_maintenance():
|
|
return await render_template(
|
|
"alerts/maintenance_form.html",
|
|
window=None,
|
|
source_modules=alert_svc.source_modules(),
|
|
)
|
|
|
|
|
|
@alerts_bp.post("/maintenance")
|
|
@require_role(UserRole.operator)
|
|
async def create_maintenance():
|
|
form = await request.form
|
|
user_id = session.get("user_id")
|
|
try:
|
|
fields = alert_svc.window_fields(
|
|
name=form.get("name", ""),
|
|
start_at=_form_time(form.get("start_at")), end_at=_form_time(form.get("end_at")),
|
|
scope_source=form.get("scope_source"), scope_resource=form.get("scope_resource"))
|
|
except ValueError as exc:
|
|
return str(exc), 400
|
|
window = MaintenanceWindow(created_by=user_id, **fields)
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
db.add(window)
|
|
scope_desc = alert_svc.window_scope(window)
|
|
await log_audit(current_app, user_id, session.get("username", ""),
|
|
"maintenance_window.created", entity_type="maintenance_window",
|
|
entity_id=window.name, detail={"scope": scope_desc})
|
|
return redirect(url_for("alerts.list_maintenance"))
|
|
|
|
|
|
@alerts_bp.post("/maintenance/<window_id>/delete")
|
|
@require_role(UserRole.operator)
|
|
async def delete_maintenance(window_id: str):
|
|
window_name = None
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(
|
|
select(MaintenanceWindow).where(MaintenanceWindow.id == window_id)
|
|
)
|
|
window = result.scalar_one_or_none()
|
|
if window:
|
|
window_name = window.name
|
|
await db.delete(window)
|
|
if window_name:
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"maintenance_window.deleted", entity_type="maintenance_window",
|
|
entity_id=window_name)
|
|
return redirect(url_for("alerts.list_maintenance"))
|
|
|
|
|
|
@alerts_bp.post("/<rule_id>/acknowledge")
|
|
@require_role(UserRole.operator)
|
|
async def acknowledge(rule_id: str):
|
|
user_id = session.get("user_id")
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
try:
|
|
await alert_svc.acknowledge(db, rule_id, user_id)
|
|
except LookupError:
|
|
return "Not found", 404
|
|
except ValueError as exc:
|
|
return f"Conflict: {exc}", 409
|
|
return redirect(url_for("alerts.list_alerts"))
|