from __future__ import annotations import asyncio import logging import uuid from dataclasses import dataclass from datetime import datetime, timezone from typing import TYPE_CHECKING from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from steward.models.alerts import ( AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow, ) from steward.models.metrics import PluginMetric if TYPE_CHECKING: from quart import Quart logger = logging.getLogger(__name__) _app: "Quart | None" = None def init_alerts(app: "Quart") -> None: """Store app reference for use in notification dispatch tasks.""" global _app _app = app def alert_extra_vars(rule: AlertRule, value: float) -> list[str]: """Ansible extra-vars describing the alert that fired, as `key=value` strings. Injected into every alert-triggered playbook run so the playbook can target the exact resource/metric that fired without templating. Pure function. """ return [ f"steward_alert_rule={rule.name}", f"steward_alert_source={rule.source_module}", f"steward_alert_resource={rule.resource_name}", f"steward_alert_metric={rule.metric_name}", f"steward_alert_value={value}", f"steward_alert_threshold={rule.threshold}", f"steward_alert_operator={rule.operator.value}", ] async def record_metric( session: AsyncSession, source_module: str, resource_name: str, metric_name: str, value: float, ) -> None: """One metric sample: record_metrics with a single row.""" await record_metrics(session, [PluginMetric( source_module=source_module, resource_name=resource_name, metric_name=metric_name, value=value)]) class MetricBatch: """Gathers one transaction's samples so they are recorded, and their alert rules evaluated, in a single record_metrics call: batch = MetricBatch() batch.add(source_module="unifi", resource_name="wan", metric_name="is_up", value=1.0) await batch.record(session) """ def __init__(self) -> None: self.rows: list[PluginMetric] = [] def add(self, source_module: str, resource_name: str, metric_name: str, value: float, recorded_at: datetime | None = None) -> None: self.rows.append(PluginMetric(source_module=source_module, resource_name=resource_name, metric_name=metric_name, value=float(value), recorded_at=recorded_at)) async def record(self, session: AsyncSession) -> None: rows, self.rows = self.rows, [] await record_metrics(session, rows) def _in_window(w: MaintenanceWindow, source_module: str, resource_name: str) -> bool: if w.scope_source is None: return True return w.scope_source == source_module and w.scope_resource in (None, resource_name) async def record_metrics(session: AsyncSession, rows: list[PluginMetric]) -> None: """Write metric rows and evaluate the alert rules that watch them. Every ingest path comes through here, so a rule on any metric Steward records can fire. Must be called inside an active transaction: async with session.begin(): await record_metrics(session, rows) Maintenance windows, rules and their states are read once per batch, not once per row. Each rule sees its key's samples oldest first, so a backlog (an agent catching up after an outage) counts consecutive checks as they happened. Notification I/O is deferred via asyncio.create_task() so no network calls occur while the transaction is open. """ if not rows: return now = datetime.now(timezone.utc) for row in rows: if row.recorded_at is None: row.recorded_at = now session.add_all(rows) await session.flush() by_key: dict[tuple[str, str, str], list[PluginMetric]] = {} for row in rows: by_key.setdefault((row.source_module, row.resource_name, row.metric_name), []).append(row) sources = {k[0] for k in by_key} metric_names = {k[2] for k in by_key} rules = (await session.execute( select(AlertRule).where( AlertRule.source_module.in_(sources), AlertRule.metric_name.in_(metric_names), AlertRule.enabled.is_(True), ) )).scalars().all() rules = [r for r in rules if (r.source_module, r.resource_name, r.metric_name) in by_key] if not rules: return windows = (await session.execute( select(MaintenanceWindow).where( MaintenanceWindow.start_at <= now, MaintenanceWindow.end_at >= now) )).scalars().all() states = {st.rule_id: st for st in (await session.execute( select(AlertState).where(AlertState.rule_id.in_([r.id for r in rules])) )).scalars()} for rule in rules: if any(_in_window(w, rule.source_module, rule.resource_name) for w in windows): continue # suppressed by maintenance window alert_state = states.get(rule.id) if alert_state is None: logger.warning("No alert_state row for rule %s, skipping", rule.id) continue samples = sorted(by_key[(rule.source_module, rule.resource_name, rule.metric_name)], key=lambda r: r.recorded_at) for sample in samples: notifications = await _evaluate_rule(session, rule, alert_state, sample.value, sample.recorded_at) for notif_type, event_id in notifications: if _app is not None: asyncio.create_task( _dispatch_notification(_app, notif_type, rule, sample.value, event_id)) # Admin-configured Ansible action runs only on transition-to-firing. if notif_type == "firing" and rule.ansible_action: asyncio.create_task( _run_ansible_action(_app, rule, sample.value, event_id)) def _is_breached(value: float, operator: AlertOperator, threshold: float) -> bool: op = operator.value if op == ">": return value > threshold if op == "<": return value < threshold if op == ">=": return value >= threshold if op == "<=": return value <= threshold if op == "==": return value == threshold if op == "!=": return value != threshold return False @dataclass(frozen=True) class Step: """One sample's effect on an alert: the state after it, the two run counters, and the event to record ("firing" / "resolved") if any.""" state: AlertStateEnum fail_count: int ok_count: int event: str | None = None def next_step(state: AlertStateEnum, fail_count: int, ok_count: int, breached: bool, fail_required: int, ok_required: int) -> Step: """The alert state machine, as a pure function of one sample. inactive → pending → firing when `fail_required` breaching checks run in a row; a non-breaching check while pending goes back to inactive. firing and acknowledged resolve (to inactive, with a resolved event) only after `ok_required` non-breaching checks in a row; a breaching check resets that run. An acknowledgement holds while the alert is still breached: only a resolve clears it. """ fail_required, ok_required = max(1, fail_required), max(1, ok_required) S = AlertStateEnum if state in (S.firing, S.acknowledged): if breached: return Step(state, fail_count + 1, 0) if ok_count + 1 >= ok_required: return Step(S.inactive, 0, 0, "resolved") return Step(state, fail_count, ok_count + 1) # inactive, pending (and the transient resolved, read as inactive) if not breached: return Step(S.inactive, 0, 0) fails = fail_count + 1 if state == S.pending else 1 if fails >= fail_required: return Step(S.firing, fails, 0, "firing") return Step(S.pending, fails, 0) async def _evaluate_rule( session: AsyncSession, rule: AlertRule, state: AlertState, value: float, now: datetime, ) -> list[tuple[str, str]]: """Apply one sample to a rule's state. Returns [(notif_type, event_id)].""" from_state = state.state step = next_step(from_state, state.consecutive_failure_count, state.consecutive_ok_count, _is_breached(value, rule.operator, rule.threshold), rule.consecutive_failures_required, rule.consecutive_ok_required) state.consecutive_failure_count = step.fail_count state.consecutive_ok_count = step.ok_count notifications: list[tuple[str, str]] = [] if step.state != from_state: state.state = step.state state.last_transition_at = now if step.state == AlertStateEnum.inactive: state.acknowledged_by = None state.acknowledged_at = None if step.event: event_id = _add_event(session, rule.id, from_state.value, step.event, value, now) notifications.append((step.event, event_id)) await session.flush() return notifications async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str) -> None: """Put a rule's alert back to inactive because the rule itself changed (what it watches, its threshold, or it was disabled). An alert that was firing or acknowledged gets a resolved event carrying `note`; nobody is notified, since nothing about the watched resource changed.""" state = (await session.execute( select(AlertState).where(AlertState.rule_id == rule.id))).scalar_one_or_none() if state is None: return now = datetime.now(timezone.utc) if state.state in (AlertStateEnum.firing, AlertStateEnum.acknowledged): _add_event(session, rule.id, state.state.value, "resolved", None, now, note=note) if state.state != AlertStateEnum.inactive: state.last_transition_at = now state.state = AlertStateEnum.inactive state.consecutive_failure_count = 0 state.consecutive_ok_count = 0 state.acknowledged_by = None state.acknowledged_at = None def _add_event( session: AsyncSession, rule_id: str, from_state: str, to_state: str, value: float | None, now: datetime, note: str | None = None, ) -> str: """Add an AlertEvent row and return its ID.""" event_id = str(uuid.uuid4()) event = AlertEvent( id=event_id, rule_id=rule_id, from_state=from_state, to_state=to_state, metric_value=value, transitioned_at=now, note=note, ) session.add(event) return event_id async def _dispatch_notification( app: "Quart", notif_type: str, rule: AlertRule, value: float, event_id: str, ) -> None: """Run after the transaction commits. Sends notifications and updates the event row.""" from steward.core.notifications import dispatch_notifications alert_data = { "rule_name": rule.name, "state": notif_type.upper(), "metric": rule.metric_name, "value": value, "threshold": rule.threshold, "resource": rule.resource_name, "source_module": rule.source_module, "timestamp": datetime.now(timezone.utc).isoformat(), } smtp_cfg = app.config.get("SMTP", {}) webhook_cfg = app.config.get("WEBHOOK", {}) results = await dispatch_notifications(smtp_cfg, webhook_cfg, alert_data) # Update event row with delivery results in a new transaction async with app.db_sessionmaker() as db: async with db.begin(): event_result = await db.execute( select(AlertEvent).where(AlertEvent.id == event_id) ) event = event_result.scalar_one_or_none() if event: email_r = results.get("email") if email_r: event.email_sent = email_r["sent"] event.email_error = email_r.get("error") webhook_r = results.get("webhook") if webhook_r: event.webhook_sent = webhook_r["sent"] event.webhook_error = webhook_r.get("error") async def _run_ansible_action( app: "Quart", rule: AlertRule, value: float, event_id: str, ) -> None: """Run the rule's configured Ansible playbook (system-triggered). Runs after the evaluation transaction commits (scheduled via create_task). Creates a system-triggered AnsibleRun, links it to the firing AlertEvent, then executes it with the static params plus injected alert context vars. """ from steward.ansible import executor, sources as src_module from steward.models.ansible import AnsibleRun, AnsibleRunStatus action = rule.ansible_action or {} playbook = action.get("playbook") inventory = action.get("inventory") if not playbook or not inventory: logger.error("Ansible action for rule %r missing playbook/inventory", rule.name) return try: srcs = src_module.get_sources(app.config.get("ANSIBLE", {})) except Exception: logger.exception("Ansible action for rule %r: could not load sources", rule.name) return source = next((s for s in srcs if s["name"] == action.get("source")), None) if source is None: logger.error("Ansible action for rule %r: source %r not configured", rule.name, action.get("source")) return # Static params + auto-injected alert context (all argv, never shell). params: dict = { "extra_vars": list(action.get("extra_vars") or []) + alert_extra_vars(rule, value), } if action.get("limit"): params["limit"] = action["limit"] if action.get("tags"): params["tags"] = action["tags"] if action.get("check"): params["check"] = True run_id = str(uuid.uuid4()) async with app.db_sessionmaker() as db: async with db.begin(): db.add(AnsibleRun( id=run_id, playbook_path=playbook, inventory_path=inventory, source_name=source["name"], triggered_by=None, status=AnsibleRunStatus.running, params=params, )) event = (await db.execute( select(AlertEvent).where(AlertEvent.id == event_id))).scalar_one_or_none() if event: event.ansible_run_id = run_id await executor.start_run(app, run_id, playbook, inventory, source["path"], params)