bvandeusenandClaude Opus 5.5
734d313a54
#5463 "Acknowledgements on record: an event and who did it", step 1 of milestone 499 "Alerts you can trust". The live walk in #5461 found that acknowledging changed only the alert's current state: once it resolved, nothing said who had acknowledged it or when. - alert_events.actor_id (migration 0040): the person behind a transition. - Acknowledging writes a firing → acknowledged event by the acknowledger. - A rule edit, disable or enable that resets an alert carries its actor (web and MCP alike). - The history page notes "acknowledged by X" and "rule disabled by X"; MCP get_alert_history returns `by`. - One usernames() helper in alerts.service, used by the active list, the history page and the MCP tools. - docs/core/alerting.md: the state diagram no longer shows an acknowledged alert re-firing, and says what an event records. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
417 lines
20 KiB
Python
417 lines
20 KiB
Python
"""Alert tools: what is firing, what fired before, the rules, and maintenance windows."""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.alerts import service as alert_svc
|
|
from steward.mcp._context import current_caller
|
|
from steward.models.alerts import AlertOperator, AlertRule, AlertState, MaintenanceWindow
|
|
from steward.models.users import UserRole
|
|
|
|
from ._audit import audit_mcp
|
|
from ._shape import (
|
|
age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd,
|
|
)
|
|
|
|
|
|
def _condition(rule: AlertRule) -> str:
|
|
n = rule.consecutive_failures_required or 1
|
|
ok = rule.consecutive_ok_required or 1
|
|
tail = f" for {n} consecutive checks" if n > 1 else ""
|
|
clear = f"; clears after {ok} OK checks" if ok > 1 else ""
|
|
return f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}{tail}{clear}"
|
|
|
|
|
|
def _action(rule: AlertRule) -> dict | None:
|
|
"""The playbook a rule runs when it fires (alerts/routes.py _parse_ansible_action)."""
|
|
a = rule.ansible_action
|
|
if not a:
|
|
return None
|
|
return {k: a.get(k) for k in ("source", "playbook", "inventory", "limit", "tags",
|
|
"extra_vars", "check") if a.get(k) is not None}
|
|
|
|
|
|
async def list_alerts() -> dict:
|
|
"""Alerts that need attention now: firing, pending, acknowledged but still
|
|
breaching, or gone silent (no_data).
|
|
|
|
pending: the condition holds but not yet for the rule's required number of
|
|
consecutive checks. firing: it has. acknowledged: someone has seen it and
|
|
it still holds. no_data: the metric the rule watches has stopped reporting
|
|
(no sample for three of its usual gaps, at least 5 minutes); a firing
|
|
alert that goes silent stays "firing" with `no_data_since` set. Each
|
|
carries the rule's condition, the resource and metric it watches, `since`
|
|
(its latest state change) and `last_sample_at`. Firing, unacknowledged
|
|
alerts come first. An alert that cleared is not here; get_alert_history
|
|
shows the transitions.
|
|
"""
|
|
async with db_session() as db:
|
|
active = await alert_svc.active_alerts(db)
|
|
names = await alert_svc.usernames(db, (s.acknowledged_by for s, _ in active))
|
|
order = {"firing": 0, "no_data": 1, "pending": 2, "acknowledged": 3}
|
|
rows = [
|
|
{
|
|
"rule_id": rule.id,
|
|
"rule_name": rule.name,
|
|
"state": alert_svc.alert_label(state),
|
|
"no_data_since": iso(state.no_data_since),
|
|
"last_sample_at": iso(state.last_sample_at),
|
|
"source_module": rule.source_module,
|
|
"resource": rule.resource_name,
|
|
"metric": rule.metric_name,
|
|
"condition": _condition(rule),
|
|
"since": iso(state.last_transition_at),
|
|
"age_s": age_s(state.last_transition_at),
|
|
"acknowledged_by": names.get(state.acknowledged_by) if state.acknowledged_by else None,
|
|
"acknowledged_at": iso(state.acknowledged_at),
|
|
}
|
|
for state, rule in active
|
|
]
|
|
rows.sort(key=lambda r: (order.get(r["state"], 4), -(r["age_s"] or 0)))
|
|
return {"alerts": rows, "total": len(rows)}
|
|
|
|
|
|
async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50) -> dict:
|
|
"""State transitions of alert rules, newest first: when each fired, cleared or was acked.
|
|
|
|
rule: one rule's name or id (list_alert_rules shows them); empty for every rule.
|
|
range: "1h", "6h", "24h", "7d" (default) or "30d".
|
|
limit: default 50, max 500.
|
|
|
|
Each transition carries the metric value that caused it and whether the
|
|
email/webhook notification went out (with the error if not), plus the id
|
|
of any Ansible run the rule's action triggered. `by` names the person who
|
|
made it happen (an acknowledgement, or a rule edit that reset the alert);
|
|
null when a reading or Steward itself did.
|
|
"""
|
|
since, key = lookback(range)
|
|
limit = clamp_limit(limit, 50, 500)
|
|
async with db_session() as db:
|
|
rules = await alert_svc.list_rules(db)
|
|
chosen = pick(rule, rules, kind="alert rule", list_tool="list_alert_rules") if rule else None
|
|
rows = await alert_svc.history(db, since, chosen.id if chosen else None, limit)
|
|
names = await alert_svc.usernames(db, (e.actor_id for e, _ in rows))
|
|
events = [
|
|
{
|
|
"rule_id": r.id,
|
|
"rule_name": r.name,
|
|
"from_state": e.from_state,
|
|
"to_state": e.to_state,
|
|
"metric_value": rnd(e.metric_value),
|
|
"note": e.note,
|
|
"by": names.get(e.actor_id, e.actor_id),
|
|
"at": iso(e.transitioned_at),
|
|
"age_s": age_s(e.transitioned_at),
|
|
"email_sent": e.email_sent,
|
|
"email_error": e.email_error,
|
|
"webhook_sent": e.webhook_sent,
|
|
"webhook_error": e.webhook_error,
|
|
"ansible_run_id": e.ansible_run_id,
|
|
}
|
|
for e, r in rows
|
|
]
|
|
out = {"range": key, "rule": chosen.name if chosen else None}
|
|
out.update(bounded(events, limit, key="events",
|
|
more="Pass a larger `limit` (max 500), a shorter `range`, or one `rule`."))
|
|
if out["truncated"]:
|
|
# Fetched limit+1 rows to detect the cut; the true total isn't known.
|
|
out["total"] = None
|
|
out["more"] = f"More than {limit} transitions in range. Pass a larger `limit` (max 500), a shorter `range`, or one `rule`."
|
|
return out
|
|
|
|
|
|
async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict:
|
|
"""Every alert rule: what it watches, its condition, current state and automatic action.
|
|
|
|
enabled_only: true to leave out disabled rules.
|
|
limit: default 200, max 500.
|
|
|
|
`state` is the rule's current state: "inactive" (condition not met),
|
|
"pending", "firing", "acknowledged", "resolved" (cleared since it last
|
|
fired), "no_data" (inactive and its metric has stopped reporting), or null
|
|
if it has no state row. `no_data_since` is set while its metric is silent,
|
|
whatever the state. `ansible_action` is the
|
|
playbook the rule runs when it fires, if any (`check: true` = dry run).
|
|
"""
|
|
limit = clamp_limit(limit, 200, 500)
|
|
async with db_session() as db:
|
|
rules = await alert_svc.list_rules(db)
|
|
states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
|
|
if enabled_only:
|
|
rules = [r for r in rules if r.enabled]
|
|
rows = [
|
|
{
|
|
"rule_id": r.id,
|
|
"rule_name": r.name,
|
|
"enabled": r.enabled,
|
|
"source_module": r.source_module,
|
|
"resource": r.resource_name,
|
|
"metric": r.metric_name,
|
|
"condition": _condition(r),
|
|
"state": alert_svc.alert_label(states[r.id]) if r.id in states else None,
|
|
"state_since": iso(states[r.id].last_transition_at) if r.id in states else None,
|
|
"no_data_since": iso(states[r.id].no_data_since) if r.id in states else None,
|
|
"ansible_action": _action(r),
|
|
}
|
|
for r in rules
|
|
]
|
|
return bounded(rows, limit, key="rules")
|
|
|
|
|
|
async def list_maintenance_windows(include_past: bool = False, limit: int = 50) -> dict:
|
|
"""Maintenance windows, during which matching alerts are held back from notifying.
|
|
|
|
include_past: true to include windows that have ended (default: current
|
|
and upcoming only).
|
|
limit: default 50, max 500.
|
|
|
|
`scope` is what a window silences: a source module and/or one resource, or
|
|
everything when both are null. `active` is true while it is in effect.
|
|
"""
|
|
limit = clamp_limit(limit, 50, 500)
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
windows = await alert_svc.list_maintenance_windows(db)
|
|
rows = [
|
|
{
|
|
"window_id": w.id,
|
|
"name": w.name,
|
|
"start_at": iso(w.start_at),
|
|
"end_at": iso(w.end_at),
|
|
"active": w.start_at <= now <= w.end_at,
|
|
"scope": {"source_module": w.scope_source, "resource": w.scope_resource},
|
|
}
|
|
for w in windows
|
|
if include_past or w.end_at >= now
|
|
]
|
|
return bounded(rows, limit, key="windows")
|
|
|
|
|
|
# ── changes ──────────────────────────────────────────────────────────────────
|
|
|
|
async def get_alert_rule_options(source_module: str = "") -> dict:
|
|
"""What an alert rule can watch: the choices create_alert_rule accepts.
|
|
|
|
source_module: empty lists every source and its metrics; a source's name
|
|
also lists its resources (hosts, containers, monitors… whatever that
|
|
source reports under).
|
|
|
|
A monitor reports under its type (icmp/tcp/dns/http) with its name as the
|
|
resource: is_up (1 up, 0 down) and response_ms.
|
|
"""
|
|
async with db_session() as db:
|
|
sources = alert_svc.source_modules()
|
|
if not source_module:
|
|
return {"sources": {src: await alert_svc.metrics_for(db, src) for src in sources},
|
|
"operators": [o.value for o in AlertOperator]}
|
|
if source_module not in sources:
|
|
raise ValueError(f"source_module {source_module!r} is not one of "
|
|
f"{', '.join(sources)}.")
|
|
return {"source_module": source_module,
|
|
"metrics": await alert_svc.metrics_for(db, source_module),
|
|
"resources": await alert_svc.resources_for(db, source_module),
|
|
"operators": [o.value for o in AlertOperator]}
|
|
|
|
|
|
async def _rule(db, rule: str) -> AlertRule:
|
|
return pick(rule, await alert_svc.list_rules(db), kind="alert rule", list_tool="list_alert_rules")
|
|
|
|
|
|
def _rule_row(r: AlertRule) -> dict:
|
|
return {"rule_id": r.id, "rule_name": r.name, "enabled": r.enabled,
|
|
"source_module": r.source_module, "resource": r.resource_name,
|
|
"metric": r.metric_name, "condition": _condition(r),
|
|
"notify_no_data": r.notify_no_data, "ansible_action": _action(r)}
|
|
|
|
|
|
async def create_alert_rule(name: str, source_module: str, resource: str, metric: str,
|
|
operator: str, threshold: float, consecutive_checks: int = 1,
|
|
clear_after_checks: int = 1, notify_no_data: bool = True,
|
|
enabled: bool = True) -> dict:
|
|
"""Add an alert rule: notify when `metric` of `resource` crosses `threshold`.
|
|
|
|
source_module, resource, metric: get_alert_rule_options lists the valid
|
|
choices; a resource must have reported (or, for a monitor, exist).
|
|
operator: ">", "<", ">=", "<=", "==" or "!=".
|
|
consecutive_checks: how many checks in a row must breach before it fires
|
|
(default 1); higher values ride out a single blip.
|
|
clear_after_checks: how many checks in a row must be OK before a firing or
|
|
acknowledged alert resolves (default 1). An acknowledgement holds until
|
|
then, however long the breach lasts.
|
|
notify_no_data: also notify when the metric stops reporting and when it
|
|
resumes (default true). The silence shows in list_alerts either way.
|
|
"Agent offline" is source host_agent, metric seconds_since_seen, "> 180".
|
|
|
|
Notifications go to the channels set in Steward's Settings. The playbook a
|
|
rule runs when it fires is set separately (set_alert_action, admin).
|
|
Recorded in the audit log as "<you> via MCP".
|
|
"""
|
|
caller = current_caller()
|
|
async with db_session() as db, db.begin():
|
|
fields = await alert_svc.rule_fields(
|
|
db, name=name, source_module=source_module, resource_name=resource,
|
|
metric_name=metric, operator=operator, threshold=threshold,
|
|
consecutive_failures_required=consecutive_checks,
|
|
consecutive_ok_required=clear_after_checks, notify_no_data=notify_no_data)
|
|
r = await alert_svc.add_rule(db, fields, created_by=caller.user_id, enabled=enabled)
|
|
await audit_mcp("alert_rule.created", "alert_rule", r.name,
|
|
{"rule_id": r.id, "source": r.source_module, "resource": r.resource_name,
|
|
"metric": r.metric_name})
|
|
return _rule_row(r)
|
|
|
|
|
|
async def update_alert_rule(rule: str, name: str | None = None, source_module: str | None = None,
|
|
resource: str | None = None, metric: str | None = None,
|
|
operator: str | None = None, threshold: float | None = None,
|
|
consecutive_checks: int | None = None,
|
|
clear_after_checks: int | None = None,
|
|
notify_no_data: bool | None = None,
|
|
enabled: bool | None = None) -> dict:
|
|
"""Change an alert rule. Only the arguments you pass change; the rest stay.
|
|
|
|
rule: its name or id (list_alert_rules). Pass enabled=false to silence it
|
|
without deleting it. The changed rule is validated as create_alert_rule
|
|
validates a new one. Changing what it watches (source, resource, metric,
|
|
operator, threshold), or disabling or re-enabling it, puts its alert back
|
|
to inactive; a firing alert records a resolved event saying why. Recorded
|
|
in the audit log.
|
|
"""
|
|
async with db_session() as db, db.begin():
|
|
r = await _rule(db, rule)
|
|
fields = await alert_svc.rule_fields(
|
|
db,
|
|
name=r.name if name is None else name,
|
|
source_module=r.source_module if source_module is None else source_module,
|
|
resource_name=r.resource_name if resource is None else resource,
|
|
metric_name=r.metric_name if metric is None else metric,
|
|
operator=r.operator.value if operator is None else operator,
|
|
threshold=r.threshold if threshold is None else threshold,
|
|
consecutive_failures_required=(r.consecutive_failures_required
|
|
if consecutive_checks is None else consecutive_checks),
|
|
consecutive_ok_required=(r.consecutive_ok_required
|
|
if clear_after_checks is None else clear_after_checks),
|
|
notify_no_data=r.notify_no_data if notify_no_data is None else notify_no_data)
|
|
await alert_svc.apply_rule_change(db, r, fields, enabled=enabled,
|
|
actor_id=current_caller().user_id)
|
|
changed = sorted(k for k, v in {"name": name, "source_module": source_module,
|
|
"resource": resource, "metric": metric, "operator": operator,
|
|
"threshold": threshold, "consecutive_checks": consecutive_checks,
|
|
"clear_after_checks": clear_after_checks,
|
|
"notify_no_data": notify_no_data,
|
|
"enabled": enabled}.items() if v is not None)
|
|
await audit_mcp("alert_rule.updated", "alert_rule", r.name, {"rule_id": r.id, "changed": changed})
|
|
return _rule_row(r)
|
|
|
|
|
|
async def delete_alert_rule(rule: str) -> dict:
|
|
"""Delete an alert rule and its history. To silence it but keep it,
|
|
update_alert_rule(enabled=false). Admin only; recorded in the audit log."""
|
|
async with db_session() as db, db.begin():
|
|
r = await _rule(db, rule)
|
|
await db.delete(r)
|
|
await audit_mcp("alert_rule.deleted", "alert_rule", r.name, {"rule_id": r.id})
|
|
return {"deleted": r.name, "rule_id": r.id}
|
|
|
|
|
|
async def acknowledge_alert(rule: str) -> dict:
|
|
"""Acknowledge a firing alert: you have seen it. It stays "acknowledged"
|
|
while the condition holds and resolves when it clears.
|
|
|
|
rule: the firing rule's name or id (list_alerts). Recorded in the audit log.
|
|
"""
|
|
caller = current_caller()
|
|
async with db_session() as db, db.begin():
|
|
r = await _rule(db, rule)
|
|
try:
|
|
state = await alert_svc.acknowledge(db, r.id, caller.user_id)
|
|
except LookupError:
|
|
raise ValueError(f"{r.name} has never been evaluated, so there is nothing to "
|
|
"acknowledge.") from None
|
|
await audit_mcp("alert_rule.acknowledged", "alert_rule", r.name, {"rule_id": r.id})
|
|
return {"rule_id": r.id, "rule_name": r.name, "state": state.state.value,
|
|
"acknowledged_at": iso(state.acknowledged_at)}
|
|
|
|
|
|
def _when(value: str, what: str) -> datetime:
|
|
try:
|
|
dt = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
|
|
except ValueError:
|
|
raise ValueError(f"{what} {value!r} is not an ISO-8601 time, e.g. 2026-10-06T22:00:00Z.") from None
|
|
return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
|
|
|
|
|
|
def _window_row(w: MaintenanceWindow, now: datetime) -> dict:
|
|
return {"window_id": w.id, "name": w.name, "start_at": iso(w.start_at), "end_at": iso(w.end_at),
|
|
"active": w.start_at <= now <= w.end_at,
|
|
"scope": {"source_module": w.scope_source, "resource": w.scope_resource}}
|
|
|
|
|
|
async def create_maintenance_window(name: str, duration_minutes: int = 0, end_at: str = "",
|
|
start_at: str = "", source_module: str = "",
|
|
resource: str = "") -> dict:
|
|
"""Hold back alert notifications while you work on something.
|
|
|
|
Give the end as duration_minutes OR end_at (ISO-8601; a time with no zone
|
|
is UTC). start_at defaults to now.
|
|
source_module, resource: limit the window to one source, or to one
|
|
resource of it; both empty silences everything.
|
|
|
|
Alerts still change state during a window, so list_alerts shows them;
|
|
only notifications are held. Recorded in the audit log.
|
|
"""
|
|
now = now_utc()
|
|
start = _when(start_at, "start_at") if (start_at or "").strip() else now
|
|
if bool(duration_minutes) == bool((end_at or "").strip()):
|
|
raise ValueError("give exactly one of duration_minutes or end_at.")
|
|
if duration_minutes:
|
|
if duration_minutes < 0:
|
|
raise ValueError("duration_minutes must be positive.")
|
|
end = start + timedelta(minutes=duration_minutes)
|
|
else:
|
|
end = _when(end_at, "end_at")
|
|
fields = alert_svc.window_fields(name=name, start_at=start, end_at=end,
|
|
scope_source=source_module, scope_resource=resource)
|
|
w = MaintenanceWindow(created_by=current_caller().user_id, **fields)
|
|
async with db_session() as db, db.begin():
|
|
db.add(w)
|
|
await audit_mcp("maintenance_window.created", "maintenance_window", w.name,
|
|
{"window_id": w.id, "scope": alert_svc.window_scope(w),
|
|
"start_at": iso(w.start_at), "end_at": iso(w.end_at)})
|
|
return _window_row(w, now)
|
|
|
|
|
|
async def end_maintenance_window(window: str) -> dict:
|
|
"""End a maintenance window now, so notifications resume.
|
|
|
|
window: its name or id (list_maintenance_windows). A window in progress
|
|
ends now and stays on record; one that has not started yet is removed.
|
|
Recorded in the audit log.
|
|
"""
|
|
now = now_utc()
|
|
async with db_session() as db, db.begin():
|
|
windows = await alert_svc.list_maintenance_windows(db)
|
|
w = pick(window, windows, kind="maintenance window", list_tool="list_maintenance_windows")
|
|
if w.end_at <= now:
|
|
raise ValueError(f"{w.name} already ended at {iso(w.end_at)}.")
|
|
if w.start_at > now:
|
|
await db.delete(w)
|
|
outcome = "cancelled"
|
|
else:
|
|
w.end_at = now
|
|
outcome = "ended"
|
|
await audit_mcp(f"maintenance_window.{outcome}", "maintenance_window", w.name,
|
|
{"window_id": w.id})
|
|
return {**_window_row(w, now), "outcome": outcome}
|
|
|
|
|
|
def register(register_tool) -> None:
|
|
for fn in (list_alerts, get_alert_history, list_alert_rules, list_maintenance_windows,
|
|
get_alert_rule_options):
|
|
register_tool(fn, role=UserRole.viewer, read_only=True)
|
|
for fn in (create_alert_rule, update_alert_rule, acknowledge_alert,
|
|
create_maintenance_window, end_maintenance_window):
|
|
register_tool(fn, role=UserRole.operator, read_only=False)
|
|
register_tool(delete_alert_rule, role=UserRole.admin, read_only=False)
|