Files
FabledSteward/steward/mcp/tools/alerts.py
T
bvandeusenandClaude Opus 5.5 41c7509366
CI / lint (push) Successful in 2s
CI / integration (push) Successful in 36s
CI / unit (push) Successful in 58s
CI / publish (push) Successful in 1m2s
feat(mcp): diagnostic tools — overview, hosts, monitors, alerts, metrics, audit
MCP-4 of milestone 461 "MCP server: Steward as an LLM-inspectable,
LLM-operable hub". Twelve read-only viewer tools:

- get_overview: what is wrong now, ranked critical-then-oldest. Failing
  monitors with their error and when the run of failures began, firing,
  pending and still-breaching acknowledged alerts, TLS certificates expiring
  within 14 days, agents that stopped reporting, hosts over the configured
  CPU/memory/disk thresholds (Settings > Thresholds), failed Ansible runs in
  the last 24h, and findings from enabled plugins through a new optional
  `mcp_overview(db)` plugin hook. Each finding names the call that shows
  its evidence.
- list_hosts, get_host, get_host_metrics
- list_monitors, get_monitor (HTTP header values and URL passwords are
  redacted)
- list_alerts, get_alert_history, list_alert_rules, list_maintenance_windows
- query_metrics: the whole metric store, narrowed one argument at a time
  (sources > resources > latest values > a series summary)
- list_audit_events

Shared shaping in steward/mcp/tools/_shape.py: UTC ISO times with age_s,
bounded lists that say when and how they were cut, series as
min/avg/max/p95/last plus a ≤60-point series, lookup by id or
case-insensitive name with a did-you-mean, and a range argument that
refuses an unknown key instead of reading it as 24h.

New core reads in steward/core/metric_series.py (sources and resources
via loose index scans, latest values via DISTINCT ON, series across the
raw/hourly-rollup seam). monitors/service.py gains monitor_states,
down_since and recent_failures; agent_overview_by_host now carries the
fullest disk and the last sample time.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 07:39:41 -04:00

192 lines
7.7 KiB
Python

"""Alert tools: what is firing, what fired before, the rules, and maintenance windows."""
from __future__ import annotations
from sqlalchemy import select
from steward.alerts import service as alert_svc
from steward.models.alerts import AlertEvent, AlertRule, AlertState
from steward.models.users import User, UserRole
from ._shape import (
age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd,
)
def _condition(rule: AlertRule) -> str:
n = rule.consecutive_failures_required or 1
tail = f" for {n} consecutive checks" if n > 1 else ""
return f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}{tail}"
def _action(rule: AlertRule) -> dict | None:
"""The playbook a rule runs when it fires (alerts/routes.py _parse_ansible_action)."""
a = rule.ansible_action
if not a:
return None
return {k: a.get(k) for k in ("source", "playbook", "inventory", "limit", "tags",
"extra_vars", "check") if a.get(k) is not None}
async def _usernames(db, ids) -> dict[str, str]:
ids = {i for i in ids if i}
if not ids:
return {}
return dict((await db.execute(select(User.id, User.username).where(User.id.in_(ids)))).all())
async def list_alerts() -> dict:
"""Alerts that need attention now: firing, pending, or acknowledged but still breaching.
pending: the condition holds but not yet for the rule's required number of
consecutive checks. firing: it has. acknowledged: someone has seen it and
it still holds. Each carries the rule's condition, the resource and metric
it watches, and `since` (its latest state change). Firing, unacknowledged
alerts come first. An alert that cleared is not here; get_alert_history
shows the transitions.
"""
async with db_session() as db:
active = await alert_svc.active_alerts(db)
names = await _usernames(db, [s.acknowledged_by for s, _ in active])
order = {"firing": 0, "pending": 1, "acknowledged": 2}
rows = [
{
"rule_id": rule.id,
"rule_name": rule.name,
"state": state.state.value,
"source_module": rule.source_module,
"resource": rule.resource_name,
"metric": rule.metric_name,
"condition": _condition(rule),
"since": iso(state.last_transition_at),
"age_s": age_s(state.last_transition_at),
"acknowledged_by": names.get(state.acknowledged_by) if state.acknowledged_by else None,
"acknowledged_at": iso(state.acknowledged_at),
}
for state, rule in active
]
rows.sort(key=lambda r: (order.get(r["state"], 3), -(r["age_s"] or 0)))
return {"alerts": rows, "total": len(rows)}
async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50) -> dict:
"""State transitions of alert rules, newest first: when each fired, cleared or was acked.
rule: one rule's name or id (list_alert_rules shows them); empty for every rule.
range: "1h", "6h", "24h", "7d" (default) or "30d".
limit: default 50, max 500.
Each transition carries the metric value that caused it and whether the
email/webhook notification went out (with the error if not), plus the id
of any Ansible run the rule's action triggered.
"""
since, key = lookback(range)
limit = clamp_limit(limit, 50, 500)
async with db_session() as db:
rules = await alert_svc.list_rules(db)
stmt = (select(AlertEvent, AlertRule)
.join(AlertRule, AlertEvent.rule_id == AlertRule.id)
.where(AlertEvent.transitioned_at >= since)
.order_by(AlertEvent.transitioned_at.desc())
.limit(limit + 1))
chosen = None
if rule:
chosen = pick(rule, rules, kind="alert rule", list_tool="list_alert_rules")
stmt = stmt.where(AlertEvent.rule_id == chosen.id)
rows = (await db.execute(stmt)).all()
events = [
{
"rule_id": r.id,
"rule_name": r.name,
"from_state": e.from_state,
"to_state": e.to_state,
"metric_value": rnd(e.metric_value),
"at": iso(e.transitioned_at),
"age_s": age_s(e.transitioned_at),
"email_sent": e.email_sent,
"email_error": e.email_error,
"webhook_sent": e.webhook_sent,
"webhook_error": e.webhook_error,
"ansible_run_id": e.ansible_run_id,
}
for e, r in rows
]
out = {"range": key, "rule": chosen.name if chosen else None}
out.update(bounded(events, limit, key="events",
more="Pass a larger `limit` (max 500), a shorter `range`, or one `rule`."))
if out["truncated"]:
# Fetched limit+1 rows to detect the cut; the true total isn't known.
out["total"] = None
out["more"] = f"More than {limit} transitions in range. Pass a larger `limit` (max 500), a shorter `range`, or one `rule`."
return out
async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict:
"""Every alert rule: what it watches, its condition, current state and automatic action.
enabled_only: true to leave out disabled rules.
limit: default 200, max 500.
`state` is the rule's current state: "inactive" (condition not met),
"pending", "firing", "acknowledged", "resolved" (cleared since it last
fired), or null if it has never been evaluated. `ansible_action` is the
playbook the rule runs when it fires, if any (`check: true` = dry run).
"""
limit = clamp_limit(limit, 200, 500)
async with db_session() as db:
rules = await alert_svc.list_rules(db)
states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
if enabled_only:
rules = [r for r in rules if r.enabled]
rows = [
{
"rule_id": r.id,
"rule_name": r.name,
"enabled": r.enabled,
"source_module": r.source_module,
"resource": r.resource_name,
"metric": r.metric_name,
"condition": _condition(r),
"state": states[r.id].state.value if r.id in states else None,
"state_since": iso(states[r.id].last_transition_at) if r.id in states else None,
"ansible_action": _action(r),
}
for r in rules
]
return bounded(rows, limit, key="rules")
async def list_maintenance_windows(include_past: bool = False, limit: int = 50) -> dict:
"""Maintenance windows, during which matching alerts are held back from notifying.
include_past: true to include windows that have ended (default: current
and upcoming only).
limit: default 50, max 500.
`scope` is what a window silences: a source module and/or one resource, or
everything when both are null. `active` is true while it is in effect.
"""
limit = clamp_limit(limit, 50, 500)
now = now_utc()
async with db_session() as db:
windows = await alert_svc.list_maintenance_windows(db)
rows = [
{
"window_id": w.id,
"name": w.name,
"start_at": iso(w.start_at),
"end_at": iso(w.end_at),
"active": w.start_at <= now <= w.end_at,
"scope": {"source_module": w.scope_source, "resource": w.scope_resource},
}
for w in windows
if include_past or w.end_at >= now
]
return bounded(rows, limit, key="windows")
def register(register_tool) -> None:
register_tool(list_alerts, role=UserRole.viewer, read_only=True)
register_tool(get_alert_history, role=UserRole.viewer, read_only=True)
register_tool(list_alert_rules, role=UserRole.viewer, read_only=True)
register_tool(list_maintenance_windows, role=UserRole.viewer, read_only=True)