from __future__ import annotations from datetime import datetime, timezone from quart import Blueprint, render_template, request, redirect, url_for, current_app, session from steward.core.audit import log_audit from sqlalchemy import func, select from steward.auth.middleware import require_role from steward.alerts import service as alert_svc from steward.core.paging import PAGE_SIZE, next_page from steward.core.time_range import parse_range from steward.ansible import sources as src_module from steward.models.alerts import AlertEvent, AlertRule, AlertOperator, AlertState, MaintenanceWindow from steward.models.users import UserRole alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts") _OPERATORS = [(op.value, op.value) for op in AlertOperator] def _form_condition(form) -> dict: """The operator and threshold the form asks for. An up/down metric's form sends the word it fires on (binary_when) instead: down is < 0.5, up is >= 0.5, the midpoint of the 1.0 / 0.0 such a metric records.""" when = form.get("binary_when") if when in ("down", "up") and form.get("metric_name") in ("is_up", "is_healthy"): return {"operator": "<" if when == "down" else ">=", "threshold": "0.5"} return {"operator": form.get("operator", ""), "threshold": form.get("threshold")} def _rule_form(form) -> dict: """The rule form's fields, as alert_svc.rule_fields takes them.""" return { "name": form.get("name", ""), "source_module": form.get("source_module", ""), "resource_name": form.get("resource_name", ""), "metric_name": form.get("metric_name", ""), **_form_condition(form), "consecutive_failures_required": form.get("consecutive_failures_required"), "consecutive_ok_required": form.get("consecutive_ok_required"), "notify_no_data": "notify_no_data" in form, "covers": form.get("covers") or "resource", "cover_group": form.get("cover_group", ""), } def _form_time(value) -> datetime: """A datetime-local field, read as UTC.""" try: return datetime.fromisoformat((value or "").strip()).replace(tzinfo=timezone.utc) except ValueError: raise ValueError(f"{value!r} is not a date and time.") from None async def _child_conflict(db, rule: AlertRule): """A 409 for a host's rule under a covering rule, which is changed through its parent (#5464); None for any other rule.""" if not rule.parent_id: return None parent = await db.get(AlertRule, rule.parent_id) return str(alert_svc.PartOfCoveringRule(rule, parent.name if parent else "?")), 409 def _is_admin() -> bool: return session.get("user_role") == UserRole.admin.value def _ansible_source_names() -> list[str]: try: return [s["name"] for s in src_module.get_sources(current_app.config.get("ANSIBLE", {}))] except Exception: return [] def _parse_ansible_action(form) -> tuple[dict | None, str | None]: """Parse + validate the admin-only Ansible action from a rule form. Returns (action_or_None, error_or_None). action is None when the action checkbox is unchecked (i.e. the action is being cleared). """ if "ansible_enabled" not in form: return None, None source_name = (form.get("ansible_source", "") or "").strip() playbook = (form.get("ansible_playbook", "") or "").strip() inventory = (form.get("ansible_inventory", "") or "").strip() if not (source_name and playbook and inventory): return None, "Ansible action requires source, playbook, and inventory" try: srcs = src_module.get_sources(current_app.config.get("ANSIBLE", {})) except Exception: return None, "Ansible sources are misconfigured" source = next((s for s in srcs if s["name"] == source_name), None) if source is None: return None, f"Ansible source '{source_name}' not found" if playbook not in src_module.discover_playbooks(source["path"]): return None, f"Playbook '{playbook}' not found in source '{source_name}'" extra_vars: list[str] = [] for line in (form.get("ansible_extra_vars", "") or "").splitlines(): line = line.strip() if not line: continue if "=" not in line: return None, f"Invalid extra var (expected key=value): {line!r}" extra_vars.append(line) action: dict = {"source": source_name, "playbook": playbook, "inventory": inventory} if extra_vars: action["extra_vars"] = extra_vars limit = (form.get("ansible_limit", "") or "").strip() tags = (form.get("ansible_tags", "") or "").strip() if limit: action["limit"] = limit if tags: action["tags"] = tags if "ansible_check" in form: action["check"] = True return action, None async def _notify() -> list[str]: """Where alerts' notifications go now, from the settings the app runs with and the accounts that get alert email.""" from steward.core.mail import audience async with current_app.db_sessionmaker() as db: accounts = len((await audience(db))["alerts"]) return alert_svc.notify_targets(current_app.config.get("SMTP", {}), current_app.config.get("WEBHOOK", {}), accounts) @alerts_bp.get("/") @require_role(UserRole.viewer) async def list_alerts(): """The verdict and what is active, then every rule with what it is doing now and when it last fired (#5315).""" async with current_app.db_sessionmaker() as db: active = await alert_svc.active_rows(db, datetime.now(timezone.utc)) rules = await alert_svc.list_rules(db) states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()} fired = await alert_svc.last_fired(db) # Every rule with what it is doing now; disabled rules last. A covering # rule's per-host children sit under it (#5464), and it reads as the most # pressing of their states. rule_state = {r.id: alert_svc.alert_label(states[r.id]) for r in rules if r.id in states} children: dict[str, list] = {} uneasy: dict[str, list[str]] = {} for r in rules: if r.parent_id: children.setdefault(r.parent_id, []).append(r) for kids in children.values(): kids.sort(key=lambda r: r.resource_name.lower()) parent_id = kids[0].parent_id labels = [rule_state.get(k.id) for k in kids] rule_state[parent_id] = min( (lb for lb in labels if lb and lb not in ("inactive", "resolved")), key=lambda lb: alert_svc.ACTIVE_ORDER.get(lb, 9), default="inactive") # The hosts that aren't quiet: named on the rule's row, and they keep # its hosts open (#5483). uneasy[parent_id] = [k.resource_name for k, lb in zip(kids, labels) if k.enabled and lb and lb not in ("inactive", "resolved")] fired[parent_id] = max((fired[k.id] for k in kids if k.id in fired), default=None) rules = sorted((r for r in rules if not r.parent_id), key=lambda r: (not r.enabled, r.name.lower())) return await render_template( "alerts/list.html", active=active, rules=rules, children=children, uneasy=uneasy, rule_state=rule_state, last_fired={k: v for k, v in fired.items() if v}, notify=await _notify(), operators=_OPERATORS, poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60)) @alerts_bp.get("/active") @require_role(UserRole.viewer) async def active_fragment(): """The verdict and Active section alone, which the alerts page polls.""" async with current_app.db_sessionmaker() as db: active = await alert_svc.active_rows(db, datetime.now(timezone.utc)) # Rules as people made them: a covering rule counts once, not per host. rules = [r for r in await alert_svc.list_rules(db) if not r.parent_id] return await render_template( "alerts/_active.html", active=active, notify=await _notify(), rules_total=len(rules), rules_off=sum(1 for r in rules if not r.enabled)) def _history_row(event, rule, names: dict | None = None) -> dict: """One transition as the history page shows it. `names` maps user ids to usernames: a transition a person made says who (#5463).""" from steward.core.alerts import MUTED, RELEASED from steward.core.formatting import metric_label, metric_value notes, muted_by, released = [], None, False for part in (event.note or "").split("; "): if part.startswith(MUTED): muted_by = part[len(MUTED):].strip() or "a maintenance window" elif part == RELEASED: released = True elif part: notes.append(part) actor = getattr(event, "actor_id", None) by = (names or {}).get(actor) or ("a deleted user" if actor else None) if event.to_state == "acknowledged": notes.insert(0, f"acknowledged by {by}" if by else "acknowledged") elif by and notes: notes[0] = f"{notes[0]} by {by}" metric = rule.metric_name if event.metric_value is None: value = f"No {metric_label(metric)} reading" elif metric in ("is_up", "is_healthy"): value = metric_value(metric, event.metric_value).capitalize() else: value = f"{metric_label(metric)} {metric_value(metric, event.metric_value)}" return { "at": event.transitioned_at, "rule": rule, "from_state": event.from_state, "to_state": event.to_state, "value": value, "raw": event.metric_value, "condition": alert_svc.rule_condition(rule), "notes": notes, "muted_by": muted_by, "released": released, "deliveries": [(label, sent, error) for label, sent, error in ( ("email", event.email_sent, event.email_error), ("webhook", event.webhook_sent, event.webhook_error)) if sent is not None], "ansible_run_id": event.ansible_run_id, } @alerts_bp.get("/history") @require_role(UserRole.viewer) async def alert_history(): """Every alert episode in a range, newest first; one rule's with ?rule=, which also says how often it fired and for how long in all (#5316). A page of transitions at a time as the reader scrolls: ?before= is the next page's episodes alone (core.paging). The Notified and Action columns show only when something in the range was notified or acted on.""" from steward.alerts.episodes import episodes, summary now = datetime.now(timezone.utc) since, range_key = parse_range(request.args.get("range") or "7d") before = request.args.get("before") async with current_app.db_sessionmaker() as db: # A covering rule has no history of its own; its hosts' rules do. rules = [r for r in await alert_svc.list_rules(db) if r.covers == "resource"] chosen = next((r for r in rules if r.id == request.args.get("rule")), None) rule_id = chosen.id if chosen else None rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before) show_notified, show_action = await alert_svc.history_columns(db, since, rule_id) names = await alert_svc.usernames(db, (e.actor_id for e, _ in rows)) verdict = None if chosen and not before: every = await alert_svc.history(db, since, rule_id, HISTORY_SUMMARY_ROWS) verdict = summary(episodes([_history_row(e, r) for e, r in every], newest_page=True, now=now, since=since)) rows, after = next_page(rows, lambda row: (row[0].transitioned_at, row[0].id)) ctx = {"episodes": episodes([_history_row(e, r, names) for e, r in rows], newest_page=not before, now=now, since=since), "after": after, "rules": rules, "chosen": chosen, "current_range": range_key, "show_notified": show_notified, "show_action": show_action, "verdict": verdict} if before: return await render_template("alerts/_history_page.html", **ctx) if request.headers.get("HX-Request"): return await render_template("alerts/_history_rows.html", **ctx) return await render_template("alerts/history.html", **ctx) # How many of one rule's transitions the history verdict reads ("fired N # times, X in all"); a rule that flapped more often than this in the range # is summed over its newest ones. HISTORY_SUMMARY_ROWS = 2000 @alerts_bp.get("/api/rule_fields") @require_role(UserRole.viewer) async def rule_fields_api(): """Everything the rule watches, for a newly picked source.""" source = request.args.get("source_module", "") async with current_app.db_sessionmaker() as db: ctx = {**await _cover_choices(db, source), **await _target(db, source)} return await render_template("alerts/_rule_fields.html", **ctx) @alerts_bp.get("/api/rule_target") @require_role(UserRole.viewer) async def rule_target_api(): """The host, part and metric pickers and the preview, after a host or a part is picked (#5475).""" async with current_app.db_sessionmaker() as db: ctx = await _target_from_args(db) return await render_template("alerts/_rule_target.html", **ctx) @alerts_bp.get("/api/metric_preview") @require_role(UserRole.viewer) async def metric_preview_api(): """What the picked metric reads now and over the last day (#5475). Its own request, after the page: a cold day of one series can take many seconds to read (#5485), and the page doesn't wait for it.""" async with current_app.db_sessionmaker() as db: ctx = await _target_from_args(db, with_preview=True) return await render_template("alerts/_metric_preview.html", **ctx) @alerts_bp.get("/api/cover_groups") @require_role(UserRole.viewer) async def cover_groups_api(): """The inventory groups, re-read when the reader comes back from making one.""" from steward.hosts.service import group_names async with current_app.db_sessionmaker() as db: groups = await group_names(db) return await render_template("alerts/_cover_groups.html", groups=groups, current_group=request.args.get("cover_group", ""), current_covers=request.args.get("covers", "resource")) async def _target(db, source: str, *, host: str = "", resource: str = "", metric: str = "", covers: str = "resource", keep: bool = False, with_preview: bool = False) -> dict: """The rule form's target pickers (#5475), from the latest-value table; `keep` on a rule's own page, so its resource and metric stay on offer. The chosen metric's preview only `with_preview`: it reads a day of raw samples, so pages and picker swaps leave it to its own request (#5485).""" from steward.core.status import sparkline_svg target = await alert_svc.rule_target(db, source, host=host, resource=resource, metric=metric, keep=keep) preview = None if with_preview and target["resource"] and target["metric"]: preview = await alert_svc.metric_preview(db, source, target["resource"], target["metric"]) if preview: # From zero to the series' own peak (UI & Design ruling 8), so a # disk at 61-62% reads flat rather than as a cliff. preview["spark"] = sparkline_svg(preview["series"], width=120, height=20, stroke="currentColor", floor=1.0) return {**target, "preview": preview, "preview_read": with_preview, "current_covers": covers} async def _target_from_args(db, *, with_preview: bool = False) -> dict: a = request.args return await _target(db, a.get("source_module", ""), host=a.get("pick_host", ""), resource=a.get("resource_name", ""), metric=a.get("metric_name", ""), covers=a.get("covers", "resource"), with_preview=with_preview) async def _cover_choices(db, source: str, rule: AlertRule | None = None) -> dict: """What the rule form offers for covering hosts (#5464): whether this source's resources are hosts, the inventory groups, and the rule's own choice, fixed once it exists.""" from steward.hosts.service import group_names host_source = source in alert_svc.HOST_SOURCES return {"host_source": host_source, "groups": await group_names(db) if host_source else [], "current_covers": rule.covers if rule else "resource", "current_group": (rule.cover_group or "") if rule else "", "covers_locked": rule is not None} @alerts_bp.get("/rules/new") @require_role(UserRole.operator) async def new_rule(): first_source = alert_svc.source_modules()[0] async with current_app.db_sessionmaker() as db: ctx = {**await _cover_choices(db, first_source), **await _target(db, first_source)} return await render_template( "alerts/rules_form.html", rule=None, **ctx, operators=_OPERATORS, source_modules=alert_svc.source_modules(), initial_source=first_source, catalog=alert_svc.metric_catalog(), ansible_sources=_ansible_source_names(), notify=await _notify(), ) @alerts_bp.get("/rules//edit") @require_role(UserRole.operator) async def edit_rule(rule_id: str): async with current_app.db_sessionmaker() as db: result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id)) rule = result.scalar_one_or_none() if rule is None: return "Not found", 404 if rule.parent_id: # A host's rule under a covering rule is changed through it. return redirect(f"/alerts/rules/{rule.parent_id}/edit") ctx = {**await _cover_choices(db, rule.source_module, rule), **await _target(db, rule.source_module, resource=rule.resource_name, metric=rule.metric_name, covers=rule.covers, keep=True)} hosts = sorted((c.resource_name for c in (await db.execute( select(AlertRule).where(AlertRule.parent_id == rule.id))).scalars()), key=str.lower) history_count = (await db.execute( select(func.count()).select_from(AlertEvent).where(AlertEvent.rule_id == rule.id))).scalar_one() fired = (await alert_svc.last_fired(db, rule.id)).get(rule.id) return await render_template( "alerts/rules_form.html", rule=rule, history_count=history_count, fired=fired, notify=await _notify(), covered=hosts, **ctx, operators=_OPERATORS, source_modules=alert_svc.source_modules(), initial_source=rule.source_module, catalog=alert_svc.metric_catalog(), ansible_sources=_ansible_source_names(), ) @alerts_bp.post("/rules") @require_role(UserRole.operator) async def create_rule(): form = await request.form user_id = session.get("user_id") # Ansible action is admin-only; non-admins simply can't set one. ansible_action = None if _is_admin(): ansible_action, err = _parse_ansible_action(form) if err: return err, 400 async with current_app.db_sessionmaker() as db: async with db.begin(): try: fields = await alert_svc.rule_fields(db, **_rule_form(form)) except ValueError as exc: return str(exc), 400 rule = await alert_svc.add_rule(db, fields, created_by=user_id, ansible_action=ansible_action) await log_audit(current_app, user_id, session.get("username", ""), "alert_rule.created", entity_type="alert_rule", entity_id=rule.name, detail={"source": rule.source_module, "resource": rule.resource_name, "metric": rule.metric_name}) return redirect(url_for("alerts.list_alerts")) @alerts_bp.post("/rules/") @require_role(UserRole.operator) async def update_rule(rule_id: str): form = await request.form # Validate the admin-only action before opening the transaction. Non-admins # leave the existing action untouched (they can still edit everything else). apply_action = _is_admin() if apply_action: ansible_action, err = _parse_ansible_action(form) if err: return err, 400 async with current_app.db_sessionmaker() as db: async with db.begin(): result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id)) rule = result.scalar_one_or_none() if rule is None: return "Not found", 404 if (conflict := await _child_conflict(db, rule)): return conflict try: # What a rule covers is fixed once it exists; its group may change. fields = await alert_svc.rule_fields(db, **{**_rule_form(form), "covers": rule.covers}) except ValueError as exc: return str(exc), 400 await alert_svc.apply_rule_change(db, rule, fields, actor_id=session.get("user_id")) if apply_action: rule.ansible_action = ansible_action if rule.covers != "resource": await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id")) await log_audit(current_app, session.get("user_id"), session.get("username", ""), "alert_rule.updated", entity_type="alert_rule", entity_id=rule.name) return redirect(url_for("alerts.list_alerts")) @alerts_bp.post("/rules//toggle") @require_role(UserRole.operator) async def toggle_rule(rule_id: str): async with current_app.db_sessionmaker() as db: async with db.begin(): result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id)) rule = result.scalar_one_or_none() if rule is None: return "Not found", 404 if (conflict := await _child_conflict(db, rule)): return conflict await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled, actor_id=session.get("user_id")) if rule.covers != "resource": await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id")) action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled" await log_audit(current_app, session.get("user_id"), session.get("username", ""), action, entity_type="alert_rule", entity_id=rule.name) return redirect(url_for("alerts.list_alerts")) @alerts_bp.post("/rules//test") @require_role(UserRole.operator) async def test_rule(rule_id: str): """Send test (#5466): a TEST notification for this rule, now, by every channel its alerts use; the result shows under the button.""" try: sent = await alert_svc.send_test(current_app, rule_id) except LookupError: return "Not found", 404 await log_audit(current_app, session.get("user_id"), session.get("username", ""), "alert_rule.tested", entity_type="alert_rule", entity_id=sent["rule"], detail={"rule_id": rule_id, "ok": sent["ok"], "lines": sent["lines"]}) return await render_template("alerts/_test_result.html", ok=sent["ok"], lines=sent["lines"]) @alerts_bp.post("/rules//delete") @require_role(UserRole.admin) async def delete_rule(rule_id: str): rule_name = None async with current_app.db_sessionmaker() as db: async with db.begin(): result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id)) rule = result.scalar_one_or_none() if rule: if (conflict := await _child_conflict(db, rule)): return conflict rule_name = rule.name await db.delete(rule) if rule_name: await log_audit(current_app, session.get("user_id"), session.get("username", ""), "alert_rule.deleted", entity_type="alert_rule", entity_id=rule_name) return redirect(url_for("alerts.list_alerts")) @alerts_bp.get("/maintenance") @require_role(UserRole.viewer) async def list_maintenance(): now = datetime.now(timezone.utc) async with current_app.db_sessionmaker() as db: windows = await alert_svc.list_maintenance_windows(db) # Active first (ending soonest first), then upcoming (starting soonest # first); expired ones are history, newest first, folded away. active = sorted((w for w in windows if w.start_at <= now <= w.end_at), key=lambda w: w.end_at) upcoming = sorted((w for w in windows if w.start_at > now), key=lambda w: w.start_at) expired = sorted((w for w in windows if w.end_at < now), key=lambda w: w.end_at, reverse=True) return await render_template( "alerts/maintenance.html", windows=windows, active=active, upcoming=upcoming, expired=expired, now=now, source_modules=alert_svc.source_modules(), ) @alerts_bp.get("/maintenance/new") @require_role(UserRole.operator) async def new_maintenance(): return await render_template( "alerts/maintenance_form.html", window=None, source_modules=alert_svc.source_modules(), ) @alerts_bp.post("/maintenance") @require_role(UserRole.operator) async def create_maintenance(): form = await request.form user_id = session.get("user_id") try: fields = alert_svc.window_fields( name=form.get("name", ""), start_at=_form_time(form.get("start_at")), end_at=_form_time(form.get("end_at")), scope_source=form.get("scope_source"), scope_resource=form.get("scope_resource")) except ValueError as exc: return str(exc), 400 window = MaintenanceWindow(created_by=user_id, **fields) async with current_app.db_sessionmaker() as db: async with db.begin(): db.add(window) scope_desc = alert_svc.window_scope(window) await log_audit(current_app, user_id, session.get("username", ""), "maintenance_window.created", entity_type="maintenance_window", entity_id=window.name, detail={"scope": scope_desc}) return redirect(url_for("alerts.list_maintenance")) @alerts_bp.post("/maintenance//delete") @require_role(UserRole.operator) async def delete_maintenance(window_id: str): window_name = None async with current_app.db_sessionmaker() as db: async with db.begin(): result = await db.execute( select(MaintenanceWindow).where(MaintenanceWindow.id == window_id) ) window = result.scalar_one_or_none() if window: window_name = window.name await db.delete(window) if window_name: await log_audit(current_app, session.get("user_id"), session.get("username", ""), "maintenance_window.deleted", entity_type="maintenance_window", entity_id=window_name) return redirect(url_for("alerts.list_maintenance")) @alerts_bp.post("//acknowledge") @require_role(UserRole.operator) async def acknowledge(rule_id: str): user_id = session.get("user_id") async with current_app.db_sessionmaker() as db: async with db.begin(): try: await alert_svc.acknowledge(db, rule_id, user_id) except LookupError: return "Not found", 404 except ValueError as exc: return f"Conflict: {exc}", 409 return redirect(url_for("alerts.list_alerts"))