"""Alert tools: what is firing, what fired before, the rules, and maintenance windows.""" from __future__ import annotations from sqlalchemy import select from steward.alerts import service as alert_svc from steward.models.alerts import AlertEvent, AlertRule, AlertState from steward.models.users import User, UserRole from ._shape import ( age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd, ) def _condition(rule: AlertRule) -> str: n = rule.consecutive_failures_required or 1 tail = f" for {n} consecutive checks" if n > 1 else "" return f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}{tail}" def _action(rule: AlertRule) -> dict | None: """The playbook a rule runs when it fires (alerts/routes.py _parse_ansible_action).""" a = rule.ansible_action if not a: return None return {k: a.get(k) for k in ("source", "playbook", "inventory", "limit", "tags", "extra_vars", "check") if a.get(k) is not None} async def _usernames(db, ids) -> dict[str, str]: ids = {i for i in ids if i} if not ids: return {} return dict((await db.execute(select(User.id, User.username).where(User.id.in_(ids)))).all()) async def list_alerts() -> dict: """Alerts that need attention now: firing, pending, or acknowledged but still breaching. pending: the condition holds but not yet for the rule's required number of consecutive checks. firing: it has. acknowledged: someone has seen it and it still holds. Each carries the rule's condition, the resource and metric it watches, and `since` (its latest state change). Firing, unacknowledged alerts come first. An alert that cleared is not here; get_alert_history shows the transitions. """ async with db_session() as db: active = await alert_svc.active_alerts(db) names = await _usernames(db, [s.acknowledged_by for s, _ in active]) order = {"firing": 0, "pending": 1, "acknowledged": 2} rows = [ { "rule_id": rule.id, "rule_name": rule.name, "state": state.state.value, "source_module": rule.source_module, "resource": rule.resource_name, "metric": rule.metric_name, "condition": _condition(rule), "since": iso(state.last_transition_at), "age_s": age_s(state.last_transition_at), "acknowledged_by": names.get(state.acknowledged_by) if state.acknowledged_by else None, "acknowledged_at": iso(state.acknowledged_at), } for state, rule in active ] rows.sort(key=lambda r: (order.get(r["state"], 3), -(r["age_s"] or 0))) return {"alerts": rows, "total": len(rows)} async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50) -> dict: """State transitions of alert rules, newest first: when each fired, cleared or was acked. rule: one rule's name or id (list_alert_rules shows them); empty for every rule. range: "1h", "6h", "24h", "7d" (default) or "30d". limit: default 50, max 500. Each transition carries the metric value that caused it and whether the email/webhook notification went out (with the error if not), plus the id of any Ansible run the rule's action triggered. """ since, key = lookback(range) limit = clamp_limit(limit, 50, 500) async with db_session() as db: rules = await alert_svc.list_rules(db) stmt = (select(AlertEvent, AlertRule) .join(AlertRule, AlertEvent.rule_id == AlertRule.id) .where(AlertEvent.transitioned_at >= since) .order_by(AlertEvent.transitioned_at.desc()) .limit(limit + 1)) chosen = None if rule: chosen = pick(rule, rules, kind="alert rule", list_tool="list_alert_rules") stmt = stmt.where(AlertEvent.rule_id == chosen.id) rows = (await db.execute(stmt)).all() events = [ { "rule_id": r.id, "rule_name": r.name, "from_state": e.from_state, "to_state": e.to_state, "metric_value": rnd(e.metric_value), "at": iso(e.transitioned_at), "age_s": age_s(e.transitioned_at), "email_sent": e.email_sent, "email_error": e.email_error, "webhook_sent": e.webhook_sent, "webhook_error": e.webhook_error, "ansible_run_id": e.ansible_run_id, } for e, r in rows ] out = {"range": key, "rule": chosen.name if chosen else None} out.update(bounded(events, limit, key="events", more="Pass a larger `limit` (max 500), a shorter `range`, or one `rule`.")) if out["truncated"]: # Fetched limit+1 rows to detect the cut; the true total isn't known. out["total"] = None out["more"] = f"More than {limit} transitions in range. Pass a larger `limit` (max 500), a shorter `range`, or one `rule`." return out async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict: """Every alert rule: what it watches, its condition, current state and automatic action. enabled_only: true to leave out disabled rules. limit: default 200, max 500. `state` is the rule's current state: "inactive" (condition not met), "pending", "firing", "acknowledged", "resolved" (cleared since it last fired), or null if it has never been evaluated. `ansible_action` is the playbook the rule runs when it fires, if any (`check: true` = dry run). """ limit = clamp_limit(limit, 200, 500) async with db_session() as db: rules = await alert_svc.list_rules(db) states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()} if enabled_only: rules = [r for r in rules if r.enabled] rows = [ { "rule_id": r.id, "rule_name": r.name, "enabled": r.enabled, "source_module": r.source_module, "resource": r.resource_name, "metric": r.metric_name, "condition": _condition(r), "state": states[r.id].state.value if r.id in states else None, "state_since": iso(states[r.id].last_transition_at) if r.id in states else None, "ansible_action": _action(r), } for r in rules ] return bounded(rows, limit, key="rules") async def list_maintenance_windows(include_past: bool = False, limit: int = 50) -> dict: """Maintenance windows, during which matching alerts are held back from notifying. include_past: true to include windows that have ended (default: current and upcoming only). limit: default 50, max 500. `scope` is what a window silences: a source module and/or one resource, or everything when both are null. `active` is true while it is in effect. """ limit = clamp_limit(limit, 50, 500) now = now_utc() async with db_session() as db: windows = await alert_svc.list_maintenance_windows(db) rows = [ { "window_id": w.id, "name": w.name, "start_at": iso(w.start_at), "end_at": iso(w.end_at), "active": w.start_at <= now <= w.end_at, "scope": {"source_module": w.scope_source, "resource": w.scope_resource}, } for w in windows if include_past or w.end_at >= now ] return bounded(rows, limit, key="windows") def register(register_tool) -> None: register_tool(list_alerts, role=UserRole.viewer, read_only=True) register_tool(get_alert_history, role=UserRole.viewer, read_only=True) register_tool(list_alert_rules, role=UserRole.viewer, read_only=True) register_tool(list_maintenance_windows, role=UserRole.viewer, read_only=True)