bvandeusenandClaude Opus 5.5
c368e93e2c
Agent offline is an ordinary rule now. The host_agent scheduler records host_agent/<host>/seconds_since_seen every minute (from last_seen_at, or registration time for an agent that never reported), replacing the task that only logged stale agents. The catalog offers the metric, and picking it on a new rule fills "> 180". No data: each rule's state tracks its last sample and its usual gap, smoothed so that an outage counts as at most three gaps. A core task, alert_silence_check, runs every 60s and marks a rule no_data_since once its series has had no sample for three usual gaps (at least 5 minutes; an hour while the cadence is unknown). A never-sampled rule counts from when it started watching. The mark sits beside the state, so a firing alert that goes silent stays firing. Each silence records one event and sends one notification (rule option notify_no_data, default on; muted in maintenance windows). The next sample clears it with a "data resumed" event and notification. Creating, changing, disabling or re-enabling a rule restarts its watch. Migration 0029 adds the columns and seeds each existing rule's last sample and gap from its last day of data, so an upgrade does not flag every rule. Shown on the Alerts page (a NO DATA badge with its start time), in MCP list_alerts, list_alert_rules and get_host (state "no_data", no_data_since), in get_overview findings (alert_no_data, or a firing alert marked silent) and in the weekly report. The rule form has the notify option, and its metric reference now matches what traefik records and lists host_agent. Emails say "no data" and carry the event's note. The metric-catalog guard also reads positional MetricBatch.add arguments. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
427 lines
20 KiB
Python
427 lines
20 KiB
Python
"""Alert tools: what is firing, what fired before, the rules, and maintenance windows."""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.alerts import service as alert_svc
|
|
from steward.mcp._context import current_caller
|
|
from steward.models.alerts import AlertEvent, AlertOperator, AlertRule, AlertState, MaintenanceWindow
|
|
from steward.models.users import User, UserRole
|
|
|
|
from ._audit import audit_mcp
|
|
from ._shape import (
|
|
age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd,
|
|
)
|
|
|
|
|
|
def _condition(rule: AlertRule) -> str:
|
|
n = rule.consecutive_failures_required or 1
|
|
ok = rule.consecutive_ok_required or 1
|
|
tail = f" for {n} consecutive checks" if n > 1 else ""
|
|
clear = f"; clears after {ok} OK checks" if ok > 1 else ""
|
|
return f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}{tail}{clear}"
|
|
|
|
|
|
def _action(rule: AlertRule) -> dict | None:
|
|
"""The playbook a rule runs when it fires (alerts/routes.py _parse_ansible_action)."""
|
|
a = rule.ansible_action
|
|
if not a:
|
|
return None
|
|
return {k: a.get(k) for k in ("source", "playbook", "inventory", "limit", "tags",
|
|
"extra_vars", "check") if a.get(k) is not None}
|
|
|
|
|
|
async def _usernames(db, ids) -> dict[str, str]:
|
|
ids = {i for i in ids if i}
|
|
if not ids:
|
|
return {}
|
|
return dict((await db.execute(select(User.id, User.username).where(User.id.in_(ids)))).all())
|
|
|
|
|
|
async def list_alerts() -> dict:
|
|
"""Alerts that need attention now: firing, pending, acknowledged but still
|
|
breaching, or gone silent (no_data).
|
|
|
|
pending: the condition holds but not yet for the rule's required number of
|
|
consecutive checks. firing: it has. acknowledged: someone has seen it and
|
|
it still holds. no_data: the metric the rule watches has stopped reporting
|
|
(no sample for three of its usual gaps, at least 5 minutes); a firing
|
|
alert that goes silent stays "firing" with `no_data_since` set. Each
|
|
carries the rule's condition, the resource and metric it watches, `since`
|
|
(its latest state change) and `last_sample_at`. Firing, unacknowledged
|
|
alerts come first. An alert that cleared is not here; get_alert_history
|
|
shows the transitions.
|
|
"""
|
|
async with db_session() as db:
|
|
active = await alert_svc.active_alerts(db)
|
|
names = await _usernames(db, [s.acknowledged_by for s, _ in active])
|
|
order = {"firing": 0, "no_data": 1, "pending": 2, "acknowledged": 3}
|
|
rows = [
|
|
{
|
|
"rule_id": rule.id,
|
|
"rule_name": rule.name,
|
|
"state": alert_svc.alert_label(state),
|
|
"no_data_since": iso(state.no_data_since),
|
|
"last_sample_at": iso(state.last_sample_at),
|
|
"source_module": rule.source_module,
|
|
"resource": rule.resource_name,
|
|
"metric": rule.metric_name,
|
|
"condition": _condition(rule),
|
|
"since": iso(state.last_transition_at),
|
|
"age_s": age_s(state.last_transition_at),
|
|
"acknowledged_by": names.get(state.acknowledged_by) if state.acknowledged_by else None,
|
|
"acknowledged_at": iso(state.acknowledged_at),
|
|
}
|
|
for state, rule in active
|
|
]
|
|
rows.sort(key=lambda r: (order.get(r["state"], 4), -(r["age_s"] or 0)))
|
|
return {"alerts": rows, "total": len(rows)}
|
|
|
|
|
|
async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50) -> dict:
|
|
"""State transitions of alert rules, newest first: when each fired, cleared or was acked.
|
|
|
|
rule: one rule's name or id (list_alert_rules shows them); empty for every rule.
|
|
range: "1h", "6h", "24h", "7d" (default) or "30d".
|
|
limit: default 50, max 500.
|
|
|
|
Each transition carries the metric value that caused it and whether the
|
|
email/webhook notification went out (with the error if not), plus the id
|
|
of any Ansible run the rule's action triggered.
|
|
"""
|
|
since, key = lookback(range)
|
|
limit = clamp_limit(limit, 50, 500)
|
|
async with db_session() as db:
|
|
rules = await alert_svc.list_rules(db)
|
|
stmt = (select(AlertEvent, AlertRule)
|
|
.join(AlertRule, AlertEvent.rule_id == AlertRule.id)
|
|
.where(AlertEvent.transitioned_at >= since)
|
|
.order_by(AlertEvent.transitioned_at.desc())
|
|
.limit(limit + 1))
|
|
chosen = None
|
|
if rule:
|
|
chosen = pick(rule, rules, kind="alert rule", list_tool="list_alert_rules")
|
|
stmt = stmt.where(AlertEvent.rule_id == chosen.id)
|
|
rows = (await db.execute(stmt)).all()
|
|
events = [
|
|
{
|
|
"rule_id": r.id,
|
|
"rule_name": r.name,
|
|
"from_state": e.from_state,
|
|
"to_state": e.to_state,
|
|
"metric_value": rnd(e.metric_value),
|
|
"note": e.note,
|
|
"at": iso(e.transitioned_at),
|
|
"age_s": age_s(e.transitioned_at),
|
|
"email_sent": e.email_sent,
|
|
"email_error": e.email_error,
|
|
"webhook_sent": e.webhook_sent,
|
|
"webhook_error": e.webhook_error,
|
|
"ansible_run_id": e.ansible_run_id,
|
|
}
|
|
for e, r in rows
|
|
]
|
|
out = {"range": key, "rule": chosen.name if chosen else None}
|
|
out.update(bounded(events, limit, key="events",
|
|
more="Pass a larger `limit` (max 500), a shorter `range`, or one `rule`."))
|
|
if out["truncated"]:
|
|
# Fetched limit+1 rows to detect the cut; the true total isn't known.
|
|
out["total"] = None
|
|
out["more"] = f"More than {limit} transitions in range. Pass a larger `limit` (max 500), a shorter `range`, or one `rule`."
|
|
return out
|
|
|
|
|
|
async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict:
|
|
"""Every alert rule: what it watches, its condition, current state and automatic action.
|
|
|
|
enabled_only: true to leave out disabled rules.
|
|
limit: default 200, max 500.
|
|
|
|
`state` is the rule's current state: "inactive" (condition not met),
|
|
"pending", "firing", "acknowledged", "resolved" (cleared since it last
|
|
fired), "no_data" (inactive and its metric has stopped reporting), or null
|
|
if it has no state row. `no_data_since` is set while its metric is silent,
|
|
whatever the state. `ansible_action` is the
|
|
playbook the rule runs when it fires, if any (`check: true` = dry run).
|
|
"""
|
|
limit = clamp_limit(limit, 200, 500)
|
|
async with db_session() as db:
|
|
rules = await alert_svc.list_rules(db)
|
|
states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
|
|
if enabled_only:
|
|
rules = [r for r in rules if r.enabled]
|
|
rows = [
|
|
{
|
|
"rule_id": r.id,
|
|
"rule_name": r.name,
|
|
"enabled": r.enabled,
|
|
"source_module": r.source_module,
|
|
"resource": r.resource_name,
|
|
"metric": r.metric_name,
|
|
"condition": _condition(r),
|
|
"state": alert_svc.alert_label(states[r.id]) if r.id in states else None,
|
|
"state_since": iso(states[r.id].last_transition_at) if r.id in states else None,
|
|
"no_data_since": iso(states[r.id].no_data_since) if r.id in states else None,
|
|
"ansible_action": _action(r),
|
|
}
|
|
for r in rules
|
|
]
|
|
return bounded(rows, limit, key="rules")
|
|
|
|
|
|
async def list_maintenance_windows(include_past: bool = False, limit: int = 50) -> dict:
|
|
"""Maintenance windows, during which matching alerts are held back from notifying.
|
|
|
|
include_past: true to include windows that have ended (default: current
|
|
and upcoming only).
|
|
limit: default 50, max 500.
|
|
|
|
`scope` is what a window silences: a source module and/or one resource, or
|
|
everything when both are null. `active` is true while it is in effect.
|
|
"""
|
|
limit = clamp_limit(limit, 50, 500)
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
windows = await alert_svc.list_maintenance_windows(db)
|
|
rows = [
|
|
{
|
|
"window_id": w.id,
|
|
"name": w.name,
|
|
"start_at": iso(w.start_at),
|
|
"end_at": iso(w.end_at),
|
|
"active": w.start_at <= now <= w.end_at,
|
|
"scope": {"source_module": w.scope_source, "resource": w.scope_resource},
|
|
}
|
|
for w in windows
|
|
if include_past or w.end_at >= now
|
|
]
|
|
return bounded(rows, limit, key="windows")
|
|
|
|
|
|
# ── changes ──────────────────────────────────────────────────────────────────
|
|
|
|
async def get_alert_rule_options(source_module: str = "") -> dict:
|
|
"""What an alert rule can watch: the choices create_alert_rule accepts.
|
|
|
|
source_module: empty lists every source and its metrics; a source's name
|
|
also lists its resources (hosts, containers, monitors… whatever that
|
|
source reports under).
|
|
|
|
A monitor reports under its type (icmp/tcp/dns/http) with its name as the
|
|
resource: is_up (1 up, 0 down) and response_ms.
|
|
"""
|
|
async with db_session() as db:
|
|
sources = alert_svc.source_modules()
|
|
if not source_module:
|
|
return {"sources": {src: await alert_svc.metrics_for(db, src) for src in sources},
|
|
"operators": [o.value for o in AlertOperator]}
|
|
if source_module not in sources:
|
|
raise ValueError(f"source_module {source_module!r} is not one of "
|
|
f"{', '.join(sources)}.")
|
|
return {"source_module": source_module,
|
|
"metrics": await alert_svc.metrics_for(db, source_module),
|
|
"resources": await alert_svc.resources_for(db, source_module),
|
|
"operators": [o.value for o in AlertOperator]}
|
|
|
|
|
|
async def _rule(db, rule: str) -> AlertRule:
|
|
return pick(rule, await alert_svc.list_rules(db), kind="alert rule", list_tool="list_alert_rules")
|
|
|
|
|
|
def _rule_row(r: AlertRule) -> dict:
|
|
return {"rule_id": r.id, "rule_name": r.name, "enabled": r.enabled,
|
|
"source_module": r.source_module, "resource": r.resource_name,
|
|
"metric": r.metric_name, "condition": _condition(r),
|
|
"notify_no_data": r.notify_no_data, "ansible_action": _action(r)}
|
|
|
|
|
|
async def create_alert_rule(name: str, source_module: str, resource: str, metric: str,
|
|
operator: str, threshold: float, consecutive_checks: int = 1,
|
|
clear_after_checks: int = 1, notify_no_data: bool = True,
|
|
enabled: bool = True) -> dict:
|
|
"""Add an alert rule: notify when `metric` of `resource` crosses `threshold`.
|
|
|
|
source_module, resource, metric: get_alert_rule_options lists the valid
|
|
choices; a resource must have reported (or, for a monitor, exist).
|
|
operator: ">", "<", ">=", "<=", "==" or "!=".
|
|
consecutive_checks: how many checks in a row must breach before it fires
|
|
(default 1); higher values ride out a single blip.
|
|
clear_after_checks: how many checks in a row must be OK before a firing or
|
|
acknowledged alert resolves (default 1). An acknowledgement holds until
|
|
then, however long the breach lasts.
|
|
notify_no_data: also notify when the metric stops reporting and when it
|
|
resumes (default true). The silence shows in list_alerts either way.
|
|
"Agent offline" is source host_agent, metric seconds_since_seen, "> 180".
|
|
|
|
Notifications go to the channels set in Steward's Settings. The playbook a
|
|
rule runs when it fires is set separately (set_alert_action, admin).
|
|
Recorded in the audit log as "<you> via MCP".
|
|
"""
|
|
caller = current_caller()
|
|
async with db_session() as db, db.begin():
|
|
fields = await alert_svc.rule_fields(
|
|
db, name=name, source_module=source_module, resource_name=resource,
|
|
metric_name=metric, operator=operator, threshold=threshold,
|
|
consecutive_failures_required=consecutive_checks,
|
|
consecutive_ok_required=clear_after_checks, notify_no_data=notify_no_data)
|
|
r = await alert_svc.add_rule(db, fields, created_by=caller.user_id, enabled=enabled)
|
|
await audit_mcp("alert_rule.created", "alert_rule", r.name,
|
|
{"rule_id": r.id, "source": r.source_module, "resource": r.resource_name,
|
|
"metric": r.metric_name})
|
|
return _rule_row(r)
|
|
|
|
|
|
async def update_alert_rule(rule: str, name: str | None = None, source_module: str | None = None,
|
|
resource: str | None = None, metric: str | None = None,
|
|
operator: str | None = None, threshold: float | None = None,
|
|
consecutive_checks: int | None = None,
|
|
clear_after_checks: int | None = None,
|
|
notify_no_data: bool | None = None,
|
|
enabled: bool | None = None) -> dict:
|
|
"""Change an alert rule. Only the arguments you pass change; the rest stay.
|
|
|
|
rule: its name or id (list_alert_rules). Pass enabled=false to silence it
|
|
without deleting it. The changed rule is validated as create_alert_rule
|
|
validates a new one. Changing what it watches (source, resource, metric,
|
|
operator, threshold), or disabling or re-enabling it, puts its alert back
|
|
to inactive; a firing alert records a resolved event saying why. Recorded
|
|
in the audit log.
|
|
"""
|
|
async with db_session() as db, db.begin():
|
|
r = await _rule(db, rule)
|
|
fields = await alert_svc.rule_fields(
|
|
db,
|
|
name=r.name if name is None else name,
|
|
source_module=r.source_module if source_module is None else source_module,
|
|
resource_name=r.resource_name if resource is None else resource,
|
|
metric_name=r.metric_name if metric is None else metric,
|
|
operator=r.operator.value if operator is None else operator,
|
|
threshold=r.threshold if threshold is None else threshold,
|
|
consecutive_failures_required=(r.consecutive_failures_required
|
|
if consecutive_checks is None else consecutive_checks),
|
|
consecutive_ok_required=(r.consecutive_ok_required
|
|
if clear_after_checks is None else clear_after_checks),
|
|
notify_no_data=r.notify_no_data if notify_no_data is None else notify_no_data)
|
|
await alert_svc.apply_rule_change(db, r, fields, enabled=enabled)
|
|
changed = sorted(k for k, v in {"name": name, "source_module": source_module,
|
|
"resource": resource, "metric": metric, "operator": operator,
|
|
"threshold": threshold, "consecutive_checks": consecutive_checks,
|
|
"clear_after_checks": clear_after_checks,
|
|
"notify_no_data": notify_no_data,
|
|
"enabled": enabled}.items() if v is not None)
|
|
await audit_mcp("alert_rule.updated", "alert_rule", r.name, {"rule_id": r.id, "changed": changed})
|
|
return _rule_row(r)
|
|
|
|
|
|
async def delete_alert_rule(rule: str) -> dict:
|
|
"""Delete an alert rule and its history. To silence it but keep it,
|
|
update_alert_rule(enabled=false). Admin only; recorded in the audit log."""
|
|
async with db_session() as db, db.begin():
|
|
r = await _rule(db, rule)
|
|
await db.delete(r)
|
|
await audit_mcp("alert_rule.deleted", "alert_rule", r.name, {"rule_id": r.id})
|
|
return {"deleted": r.name, "rule_id": r.id}
|
|
|
|
|
|
async def acknowledge_alert(rule: str) -> dict:
|
|
"""Acknowledge a firing alert: you have seen it. It stays "acknowledged"
|
|
while the condition holds and resolves when it clears.
|
|
|
|
rule: the firing rule's name or id (list_alerts). Recorded in the audit log.
|
|
"""
|
|
caller = current_caller()
|
|
async with db_session() as db, db.begin():
|
|
r = await _rule(db, rule)
|
|
try:
|
|
state = await alert_svc.acknowledge(db, r.id, caller.user_id)
|
|
except LookupError:
|
|
raise ValueError(f"{r.name} has never been evaluated, so there is nothing to "
|
|
"acknowledge.") from None
|
|
await audit_mcp("alert_rule.acknowledged", "alert_rule", r.name, {"rule_id": r.id})
|
|
return {"rule_id": r.id, "rule_name": r.name, "state": state.state.value,
|
|
"acknowledged_at": iso(state.acknowledged_at)}
|
|
|
|
|
|
def _when(value: str, what: str) -> datetime:
|
|
try:
|
|
dt = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
|
|
except ValueError:
|
|
raise ValueError(f"{what} {value!r} is not an ISO-8601 time, e.g. 2026-10-06T22:00:00Z.") from None
|
|
return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
|
|
|
|
|
|
def _window_row(w: MaintenanceWindow, now: datetime) -> dict:
|
|
return {"window_id": w.id, "name": w.name, "start_at": iso(w.start_at), "end_at": iso(w.end_at),
|
|
"active": w.start_at <= now <= w.end_at,
|
|
"scope": {"source_module": w.scope_source, "resource": w.scope_resource}}
|
|
|
|
|
|
async def create_maintenance_window(name: str, duration_minutes: int = 0, end_at: str = "",
|
|
start_at: str = "", source_module: str = "",
|
|
resource: str = "") -> dict:
|
|
"""Hold back alert notifications while you work on something.
|
|
|
|
Give the end as duration_minutes OR end_at (ISO-8601; a time with no zone
|
|
is UTC). start_at defaults to now.
|
|
source_module, resource: limit the window to one source, or to one
|
|
resource of it; both empty silences everything.
|
|
|
|
Alerts still change state during a window, so list_alerts shows them;
|
|
only notifications are held. Recorded in the audit log.
|
|
"""
|
|
now = now_utc()
|
|
start = _when(start_at, "start_at") if (start_at or "").strip() else now
|
|
if bool(duration_minutes) == bool((end_at or "").strip()):
|
|
raise ValueError("give exactly one of duration_minutes or end_at.")
|
|
if duration_minutes:
|
|
if duration_minutes < 0:
|
|
raise ValueError("duration_minutes must be positive.")
|
|
end = start + timedelta(minutes=duration_minutes)
|
|
else:
|
|
end = _when(end_at, "end_at")
|
|
fields = alert_svc.window_fields(name=name, start_at=start, end_at=end,
|
|
scope_source=source_module, scope_resource=resource)
|
|
w = MaintenanceWindow(created_by=current_caller().user_id, **fields)
|
|
async with db_session() as db, db.begin():
|
|
db.add(w)
|
|
await audit_mcp("maintenance_window.created", "maintenance_window", w.name,
|
|
{"window_id": w.id, "scope": alert_svc.window_scope(w),
|
|
"start_at": iso(w.start_at), "end_at": iso(w.end_at)})
|
|
return _window_row(w, now)
|
|
|
|
|
|
async def end_maintenance_window(window: str) -> dict:
|
|
"""End a maintenance window now, so notifications resume.
|
|
|
|
window: its name or id (list_maintenance_windows). A window in progress
|
|
ends now and stays on record; one that has not started yet is removed.
|
|
Recorded in the audit log.
|
|
"""
|
|
now = now_utc()
|
|
async with db_session() as db, db.begin():
|
|
windows = await alert_svc.list_maintenance_windows(db)
|
|
w = pick(window, windows, kind="maintenance window", list_tool="list_maintenance_windows")
|
|
if w.end_at <= now:
|
|
raise ValueError(f"{w.name} already ended at {iso(w.end_at)}.")
|
|
if w.start_at > now:
|
|
await db.delete(w)
|
|
outcome = "cancelled"
|
|
else:
|
|
w.end_at = now
|
|
outcome = "ended"
|
|
await audit_mcp(f"maintenance_window.{outcome}", "maintenance_window", w.name,
|
|
{"window_id": w.id})
|
|
return {**_window_row(w, now), "outcome": outcome}
|
|
|
|
|
|
def register(register_tool) -> None:
|
|
for fn in (list_alerts, get_alert_history, list_alert_rules, list_maintenance_windows,
|
|
get_alert_rule_options):
|
|
register_tool(fn, role=UserRole.viewer, read_only=True)
|
|
for fn in (create_alert_rule, update_alert_rule, acknowledge_alert,
|
|
create_maintenance_window, end_maintenance_window):
|
|
register_tool(fn, role=UserRole.operator, read_only=False)
|
|
register_tool(delete_alert_rule, role=UserRole.admin, read_only=False)
|