from __future__ import annotations from datetime import datetime, timezone from quart import Blueprint, render_template, request, redirect, url_for, current_app, session from steward.core.audit import log_audit from sqlalchemy import select from steward.auth.middleware import require_role from steward.ansible import sources as src_module from steward.models.alerts import AlertRule, AlertState, AlertStateEnum, AlertOperator, MaintenanceWindow from steward.models.metrics import PluginMetric from steward.models.users import UserRole alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts") _OPERATORS = [(op.value, op.value) for op in AlertOperator] # Static catalog: source_module → available metric names. # Unified monitors emit metrics under their TYPE (icmp/tcp/dns/http), each with # is_up (1.0 up / 0.0 down) and response_ms (where the probe times it). METRIC_CATALOG: dict[str, list[str]] = { "icmp": ["is_up", "response_ms"], "tcp": ["is_up", "response_ms"], "dns": ["is_up", "response_ms"], "http": ["is_up", "response_ms"], "traefik": ["request_rate", "error_rate", "latency_p50_ms", "latency_p95_ms", "latency_p99_ms", "response_bytes_rate", "cert_expiry_days"], "unifi": ["is_up", "latency_ms", "total_clients"], # restart_count = cumulative restarts (alert on crash-looping); is_healthy = # 1.0 healthy / 0.0 unhealthy from the container HEALTHCHECK (alert on <1). "docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy"], "host_agent": [ "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes", "disk_used_pct_worst", "load_1m", "load_5m", "load_15m", "uptime_secs", ], # snmp metrics are user-defined OID labels — populated dynamically from plugin_metrics "snmp": [], } # Ordered list for source module picker _SOURCE_MODULES = list(METRIC_CATALOG.keys()) async def _get_resources(db, source_module: str) -> list[str]: """Return sorted distinct resource names for a source module. For monitor types (icmp/tcp/dns/http), supplements with monitor names so the list is populated even before any metrics have been recorded. """ result = await db.execute( select(PluginMetric.resource_name) .where(PluginMetric.source_module == source_module) .distinct() .order_by(PluginMetric.resource_name) ) resources: set[str] = {r for (r,) in result} if source_module in ("icmp", "tcp", "dns", "http"): from steward.models.monitors import Monitor mon_result = await db.execute( select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name) ) for (name,) in mon_result: resources.add(name) return sorted(resources) def _is_admin() -> bool: return session.get("user_role") == UserRole.admin.value def _ansible_source_names() -> list[str]: try: return [s["name"] for s in src_module.get_sources(current_app.config.get("ANSIBLE", {}))] except Exception: return [] def _parse_ansible_action(form) -> tuple[dict | None, str | None]: """Parse + validate the admin-only Ansible action from a rule form. Returns (action_or_None, error_or_None). action is None when the action checkbox is unchecked (i.e. the action is being cleared). """ if "ansible_enabled" not in form: return None, None source_name = (form.get("ansible_source", "") or "").strip() playbook = (form.get("ansible_playbook", "") or "").strip() inventory = (form.get("ansible_inventory", "") or "").strip() if not (source_name and playbook and inventory): return None, "Ansible action requires source, playbook, and inventory" try: srcs = src_module.get_sources(current_app.config.get("ANSIBLE", {})) except Exception: return None, "Ansible sources are misconfigured" source = next((s for s in srcs if s["name"] == source_name), None) if source is None: return None, f"Ansible source '{source_name}' not found" if playbook not in src_module.discover_playbooks(source["path"]): return None, f"Playbook '{playbook}' not found in source '{source_name}'" extra_vars: list[str] = [] for line in (form.get("ansible_extra_vars", "") or "").splitlines(): line = line.strip() if not line: continue if "=" not in line: return None, f"Invalid extra var (expected key=value): {line!r}" extra_vars.append(line) action: dict = {"source": source_name, "playbook": playbook, "inventory": inventory} if extra_vars: action["extra_vars"] = extra_vars limit = (form.get("ansible_limit", "") or "").strip() tags = (form.get("ansible_tags", "") or "").strip() if limit: action["limit"] = limit if tags: action["tags"] = tags if "ansible_check" in form: action["check"] = True return action, None @alerts_bp.get("/") @require_role(UserRole.viewer) async def list_alerts(): async with current_app.db_sessionmaker() as db: result = await db.execute( select(AlertState, AlertRule) .join(AlertRule, AlertState.rule_id == AlertRule.id) .where(AlertState.state.in_([ AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending ])) .order_by(AlertState.last_transition_at.desc()) ) active = result.all() rules_result = await db.execute( select(AlertRule).order_by(AlertRule.name) ) rules = rules_result.scalars().all() return await render_template("alerts/list.html", active=active, rules=rules, operators=_OPERATORS) @alerts_bp.get("/api/rule_fields") @require_role(UserRole.viewer) async def rule_fields_api(): source = request.args.get("source_module", "") current_resource = request.args.get("current_resource", "") current_metric = request.args.get("current_metric", "") async with current_app.db_sessionmaker() as db: resources = await _get_resources(db, source) if source else [] metrics = METRIC_CATALOG.get(source, []) # Dynamic sources (e.g. snmp) have empty static catalog — query live metric names if source and not metrics: async with current_app.db_sessionmaker() as db: m_result = await db.execute( select(PluginMetric.metric_name) .where(PluginMetric.source_module == source) .distinct() .order_by(PluginMetric.metric_name) ) metrics = [r for (r,) in m_result] return await render_template( "alerts/_rule_fields.html", resources=resources, metrics=metrics, current_resource=current_resource, current_metric=current_metric, ) @alerts_bp.get("/rules/new") @require_role(UserRole.operator) async def new_rule(): first_source = _SOURCE_MODULES[0] async with current_app.db_sessionmaker() as db: resources = await _get_resources(db, first_source) return await render_template( "alerts/rules_form.html", rule=None, operators=_OPERATORS, source_modules=_SOURCE_MODULES, initial_source=first_source, resources=resources, metrics=METRIC_CATALOG.get(first_source, []), ansible_sources=_ansible_source_names(), ) @alerts_bp.get("/rules//edit") @require_role(UserRole.operator) async def edit_rule(rule_id: str): async with current_app.db_sessionmaker() as db: result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id)) rule = result.scalar_one_or_none() if rule is None: return "Not found", 404 resources = await _get_resources(db, rule.source_module) return await render_template( "alerts/rules_form.html", rule=rule, operators=_OPERATORS, source_modules=_SOURCE_MODULES, initial_source=rule.source_module, resources=resources, metrics=METRIC_CATALOG.get(rule.source_module, []), ansible_sources=_ansible_source_names(), ) @alerts_bp.post("/rules") @require_role(UserRole.operator) async def create_rule(): form = await request.form user_id = session.get("user_id") now = datetime.now(timezone.utc) # Ansible action is admin-only; non-admins simply can't set one. ansible_action = None if _is_admin(): ansible_action, err = _parse_ansible_action(form) if err: return err, 400 rule = AlertRule( name=form["name"].strip(), source_module=form["source_module"].strip(), resource_name=form["resource_name"].strip(), metric_name=form["metric_name"].strip(), operator=AlertOperator(form["operator"]), threshold=float(form["threshold"]), consecutive_failures_required=int(form.get("consecutive_failures_required") or 1), enabled=True, ansible_action=ansible_action, created_by=user_id, ) state = AlertState( rule_id=rule.id, state=AlertStateEnum.inactive, last_transition_at=now, ) async with current_app.db_sessionmaker() as db: async with db.begin(): db.add(rule) await db.flush() state.rule_id = rule.id db.add(state) await log_audit(current_app, user_id, session.get("username", ""), "alert_rule.created", entity_type="alert_rule", entity_id=rule.name, detail={"source": rule.source_module, "resource": rule.resource_name, "metric": rule.metric_name}) return redirect(url_for("alerts.list_alerts")) @alerts_bp.post("/rules/") @require_role(UserRole.operator) async def update_rule(rule_id: str): form = await request.form # Validate the admin-only action before opening the transaction. Non-admins # leave the existing action untouched (they can still edit everything else). apply_action = _is_admin() if apply_action: ansible_action, err = _parse_ansible_action(form) if err: return err, 400 async with current_app.db_sessionmaker() as db: async with db.begin(): result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id)) rule = result.scalar_one_or_none() if rule is None: return "Not found", 404 rule.name = form["name"].strip() rule.source_module = form["source_module"].strip() rule.resource_name = form["resource_name"].strip() rule.metric_name = form["metric_name"].strip() rule.operator = AlertOperator(form["operator"]) rule.threshold = float(form["threshold"]) rule.consecutive_failures_required = int(form.get("consecutive_failures_required") or 1) if apply_action: rule.ansible_action = ansible_action await log_audit(current_app, session.get("user_id"), session.get("username", ""), "alert_rule.updated", entity_type="alert_rule", entity_id=rule.name) return redirect(url_for("alerts.list_alerts")) @alerts_bp.post("/rules//toggle") @require_role(UserRole.operator) async def toggle_rule(rule_id: str): async with current_app.db_sessionmaker() as db: async with db.begin(): result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id)) rule = result.scalar_one_or_none() if rule is None: return "Not found", 404 rule.enabled = not rule.enabled action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled" await log_audit(current_app, session.get("user_id"), session.get("username", ""), action, entity_type="alert_rule", entity_id=rule.name) return redirect(url_for("alerts.list_alerts")) @alerts_bp.post("/rules//delete") @require_role(UserRole.admin) async def delete_rule(rule_id: str): rule_name = None async with current_app.db_sessionmaker() as db: async with db.begin(): result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id)) rule = result.scalar_one_or_none() if rule: rule_name = rule.name await db.delete(rule) if rule_name: await log_audit(current_app, session.get("user_id"), session.get("username", ""), "alert_rule.deleted", entity_type="alert_rule", entity_id=rule_name) return redirect(url_for("alerts.list_alerts")) @alerts_bp.get("/maintenance") @require_role(UserRole.viewer) async def list_maintenance(): now = datetime.now(timezone.utc) async with current_app.db_sessionmaker() as db: result = await db.execute( select(MaintenanceWindow).order_by(MaintenanceWindow.start_at.desc()) ) windows = result.scalars().all() return await render_template( "alerts/maintenance.html", windows=windows, now=now, source_modules=_SOURCE_MODULES, ) @alerts_bp.get("/maintenance/new") @require_role(UserRole.operator) async def new_maintenance(): return await render_template( "alerts/maintenance_form.html", window=None, source_modules=_SOURCE_MODULES, ) @alerts_bp.post("/maintenance") @require_role(UserRole.operator) async def create_maintenance(): form = await request.form user_id = session.get("user_id") start_at = datetime.fromisoformat(form["start_at"]).replace(tzinfo=timezone.utc) end_at = datetime.fromisoformat(form["end_at"]).replace(tzinfo=timezone.utc) scope_source = form.get("scope_source", "").strip() or None scope_resource = form.get("scope_resource", "").strip() or None if scope_source is None: scope_resource = None # can't have resource without source window = MaintenanceWindow( name=form["name"].strip(), start_at=start_at, end_at=end_at, scope_source=scope_source, scope_resource=scope_resource, created_by=user_id, ) async with current_app.db_sessionmaker() as db: async with db.begin(): db.add(window) scope_desc = "all" if not scope_source else ( f"{scope_source}/{scope_resource}" if scope_resource else f"{scope_source}/*" ) await log_audit(current_app, user_id, session.get("username", ""), "maintenance_window.created", entity_type="maintenance_window", entity_id=window.name, detail={"scope": scope_desc}) return redirect(url_for("alerts.list_maintenance")) @alerts_bp.post("/maintenance//delete") @require_role(UserRole.operator) async def delete_maintenance(window_id: str): window_name = None async with current_app.db_sessionmaker() as db: async with db.begin(): result = await db.execute( select(MaintenanceWindow).where(MaintenanceWindow.id == window_id) ) window = result.scalar_one_or_none() if window: window_name = window.name await db.delete(window) if window_name: await log_audit(current_app, session.get("user_id"), session.get("username", ""), "maintenance_window.deleted", entity_type="maintenance_window", entity_id=window_name) return redirect(url_for("alerts.list_maintenance")) @alerts_bp.post("//acknowledge") @require_role(UserRole.operator) async def acknowledge(rule_id: str): user_id = session.get("user_id") now = datetime.now(timezone.utc) async with current_app.db_sessionmaker() as db: async with db.begin(): result = await db.execute( select(AlertState).where(AlertState.rule_id == rule_id) ) state = result.scalar_one_or_none() if state is None: return "Not found", 404 if state.state != AlertStateEnum.firing: return "Conflict: alert is not in FIRING state", 409 state.state = AlertStateEnum.acknowledged state.acknowledged_by = user_id state.acknowledged_at = now state.last_transition_at = now return redirect(url_for("alerts.list_alerts"))