MCP-4 of milestone 461 "MCP server: Steward as an LLM-inspectable, LLM-operable hub". Twelve read-only viewer tools: - get_overview: what is wrong now, ranked critical-then-oldest. Failing monitors with their error and when the run of failures began, firing, pending and still-breaching acknowledged alerts, TLS certificates expiring within 14 days, agents that stopped reporting, hosts over the configured CPU/memory/disk thresholds (Settings > Thresholds), failed Ansible runs in the last 24h, and findings from enabled plugins through a new optional `mcp_overview(db)` plugin hook. Each finding names the call that shows its evidence. - list_hosts, get_host, get_host_metrics - list_monitors, get_monitor (HTTP header values and URL passwords are redacted) - list_alerts, get_alert_history, list_alert_rules, list_maintenance_windows - query_metrics: the whole metric store, narrowed one argument at a time (sources > resources > latest values > a series summary) - list_audit_events Shared shaping in steward/mcp/tools/_shape.py: UTC ISO times with age_s, bounded lists that say when and how they were cut, series as min/avg/max/p95/last plus a ≤60-point series, lookup by id or case-insensitive name with a did-you-mean, and a range argument that refuses an unknown key instead of reading it as 24h. New core reads in steward/core/metric_series.py (sources and resources via loose index scans, latest values via DISTINCT ON, series across the raw/hourly-rollup seam). monitors/service.py gains monitor_states, down_since and recent_failures; agent_overview_by_host now carries the fullest disk and the last sample time. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
192 lines
7.7 KiB
Python
192 lines
7.7 KiB
Python
"""Alert tools: what is firing, what fired before, the rules, and maintenance windows."""
|
|
from __future__ import annotations
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.alerts import service as alert_svc
|
|
from steward.models.alerts import AlertEvent, AlertRule, AlertState
|
|
from steward.models.users import User, UserRole
|
|
|
|
from ._shape import (
|
|
age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd,
|
|
)
|
|
|
|
|
|
def _condition(rule: AlertRule) -> str:
|
|
n = rule.consecutive_failures_required or 1
|
|
tail = f" for {n} consecutive checks" if n > 1 else ""
|
|
return f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}{tail}"
|
|
|
|
|
|
def _action(rule: AlertRule) -> dict | None:
|
|
"""The playbook a rule runs when it fires (alerts/routes.py _parse_ansible_action)."""
|
|
a = rule.ansible_action
|
|
if not a:
|
|
return None
|
|
return {k: a.get(k) for k in ("source", "playbook", "inventory", "limit", "tags",
|
|
"extra_vars", "check") if a.get(k) is not None}
|
|
|
|
|
|
async def _usernames(db, ids) -> dict[str, str]:
|
|
ids = {i for i in ids if i}
|
|
if not ids:
|
|
return {}
|
|
return dict((await db.execute(select(User.id, User.username).where(User.id.in_(ids)))).all())
|
|
|
|
|
|
async def list_alerts() -> dict:
|
|
"""Alerts that need attention now: firing, pending, or acknowledged but still breaching.
|
|
|
|
pending: the condition holds but not yet for the rule's required number of
|
|
consecutive checks. firing: it has. acknowledged: someone has seen it and
|
|
it still holds. Each carries the rule's condition, the resource and metric
|
|
it watches, and `since` (its latest state change). Firing, unacknowledged
|
|
alerts come first. An alert that cleared is not here; get_alert_history
|
|
shows the transitions.
|
|
"""
|
|
async with db_session() as db:
|
|
active = await alert_svc.active_alerts(db)
|
|
names = await _usernames(db, [s.acknowledged_by for s, _ in active])
|
|
order = {"firing": 0, "pending": 1, "acknowledged": 2}
|
|
rows = [
|
|
{
|
|
"rule_id": rule.id,
|
|
"rule_name": rule.name,
|
|
"state": state.state.value,
|
|
"source_module": rule.source_module,
|
|
"resource": rule.resource_name,
|
|
"metric": rule.metric_name,
|
|
"condition": _condition(rule),
|
|
"since": iso(state.last_transition_at),
|
|
"age_s": age_s(state.last_transition_at),
|
|
"acknowledged_by": names.get(state.acknowledged_by) if state.acknowledged_by else None,
|
|
"acknowledged_at": iso(state.acknowledged_at),
|
|
}
|
|
for state, rule in active
|
|
]
|
|
rows.sort(key=lambda r: (order.get(r["state"], 3), -(r["age_s"] or 0)))
|
|
return {"alerts": rows, "total": len(rows)}
|
|
|
|
|
|
async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50) -> dict:
|
|
"""State transitions of alert rules, newest first: when each fired, cleared or was acked.
|
|
|
|
rule: one rule's name or id (list_alert_rules shows them); empty for every rule.
|
|
range: "1h", "6h", "24h", "7d" (default) or "30d".
|
|
limit: default 50, max 500.
|
|
|
|
Each transition carries the metric value that caused it and whether the
|
|
email/webhook notification went out (with the error if not), plus the id
|
|
of any Ansible run the rule's action triggered.
|
|
"""
|
|
since, key = lookback(range)
|
|
limit = clamp_limit(limit, 50, 500)
|
|
async with db_session() as db:
|
|
rules = await alert_svc.list_rules(db)
|
|
stmt = (select(AlertEvent, AlertRule)
|
|
.join(AlertRule, AlertEvent.rule_id == AlertRule.id)
|
|
.where(AlertEvent.transitioned_at >= since)
|
|
.order_by(AlertEvent.transitioned_at.desc())
|
|
.limit(limit + 1))
|
|
chosen = None
|
|
if rule:
|
|
chosen = pick(rule, rules, kind="alert rule", list_tool="list_alert_rules")
|
|
stmt = stmt.where(AlertEvent.rule_id == chosen.id)
|
|
rows = (await db.execute(stmt)).all()
|
|
events = [
|
|
{
|
|
"rule_id": r.id,
|
|
"rule_name": r.name,
|
|
"from_state": e.from_state,
|
|
"to_state": e.to_state,
|
|
"metric_value": rnd(e.metric_value),
|
|
"at": iso(e.transitioned_at),
|
|
"age_s": age_s(e.transitioned_at),
|
|
"email_sent": e.email_sent,
|
|
"email_error": e.email_error,
|
|
"webhook_sent": e.webhook_sent,
|
|
"webhook_error": e.webhook_error,
|
|
"ansible_run_id": e.ansible_run_id,
|
|
}
|
|
for e, r in rows
|
|
]
|
|
out = {"range": key, "rule": chosen.name if chosen else None}
|
|
out.update(bounded(events, limit, key="events",
|
|
more="Pass a larger `limit` (max 500), a shorter `range`, or one `rule`."))
|
|
if out["truncated"]:
|
|
# Fetched limit+1 rows to detect the cut; the true total isn't known.
|
|
out["total"] = None
|
|
out["more"] = f"More than {limit} transitions in range. Pass a larger `limit` (max 500), a shorter `range`, or one `rule`."
|
|
return out
|
|
|
|
|
|
async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict:
|
|
"""Every alert rule: what it watches, its condition, current state and automatic action.
|
|
|
|
enabled_only: true to leave out disabled rules.
|
|
limit: default 200, max 500.
|
|
|
|
`state` is the rule's current state: "inactive" (condition not met),
|
|
"pending", "firing", "acknowledged", "resolved" (cleared since it last
|
|
fired), or null if it has never been evaluated. `ansible_action` is the
|
|
playbook the rule runs when it fires, if any (`check: true` = dry run).
|
|
"""
|
|
limit = clamp_limit(limit, 200, 500)
|
|
async with db_session() as db:
|
|
rules = await alert_svc.list_rules(db)
|
|
states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
|
|
if enabled_only:
|
|
rules = [r for r in rules if r.enabled]
|
|
rows = [
|
|
{
|
|
"rule_id": r.id,
|
|
"rule_name": r.name,
|
|
"enabled": r.enabled,
|
|
"source_module": r.source_module,
|
|
"resource": r.resource_name,
|
|
"metric": r.metric_name,
|
|
"condition": _condition(r),
|
|
"state": states[r.id].state.value if r.id in states else None,
|
|
"state_since": iso(states[r.id].last_transition_at) if r.id in states else None,
|
|
"ansible_action": _action(r),
|
|
}
|
|
for r in rules
|
|
]
|
|
return bounded(rows, limit, key="rules")
|
|
|
|
|
|
async def list_maintenance_windows(include_past: bool = False, limit: int = 50) -> dict:
|
|
"""Maintenance windows, during which matching alerts are held back from notifying.
|
|
|
|
include_past: true to include windows that have ended (default: current
|
|
and upcoming only).
|
|
limit: default 50, max 500.
|
|
|
|
`scope` is what a window silences: a source module and/or one resource, or
|
|
everything when both are null. `active` is true while it is in effect.
|
|
"""
|
|
limit = clamp_limit(limit, 50, 500)
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
windows = await alert_svc.list_maintenance_windows(db)
|
|
rows = [
|
|
{
|
|
"window_id": w.id,
|
|
"name": w.name,
|
|
"start_at": iso(w.start_at),
|
|
"end_at": iso(w.end_at),
|
|
"active": w.start_at <= now <= w.end_at,
|
|
"scope": {"source_module": w.scope_source, "resource": w.scope_resource},
|
|
}
|
|
for w in windows
|
|
if include_past or w.end_at >= now
|
|
]
|
|
return bounded(rows, limit, key="windows")
|
|
|
|
|
|
def register(register_tool) -> None:
|
|
register_tool(list_alerts, role=UserRole.viewer, read_only=True)
|
|
register_tool(get_alert_history, role=UserRole.viewer, read_only=True)
|
|
register_tool(list_alert_rules, role=UserRole.viewer, read_only=True)
|
|
register_tool(list_maintenance_windows, role=UserRole.viewer, read_only=True)
|