bvandeusenandClaude Opus 5.5
58bfc7c361
#5060 "One evaluation path: record_metrics batch API; host-agent ingest evaluates rules". - record_metrics(session, rows) writes metric rows and evaluates the rules that watch them, reading rules, states and maintenance windows once per batch. Each rule sees its key's samples oldest first, so an agent catching up after an outage counts consecutive checks as they happened. - Host-agent ingest now goes through it. Before, it wrote plugin_metrics directly and no host_agent rule could ever fire. The catalog gains the per-mount disk_used_pct ("<host>:<mount>"). - MetricBatch gathers a transaction's samples; the docker ingest, traefik, unifi, snmp, monitor and self-monitor writers record one batch each instead of a rule query per sample. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
398 lines
14 KiB
Python
398 lines
14 KiB
Python
from __future__ import annotations
|
|
import asyncio
|
|
import logging
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from steward.models.alerts import (
|
|
AlertEvent,
|
|
AlertOperator,
|
|
AlertRule,
|
|
AlertState,
|
|
AlertStateEnum,
|
|
MaintenanceWindow,
|
|
)
|
|
from steward.models.metrics import PluginMetric
|
|
|
|
if TYPE_CHECKING:
|
|
from quart import Quart
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_app: "Quart | None" = None
|
|
|
|
|
|
def init_alerts(app: "Quart") -> None:
|
|
"""Store app reference for use in notification dispatch tasks."""
|
|
global _app
|
|
_app = app
|
|
|
|
|
|
def alert_extra_vars(rule: AlertRule, value: float) -> list[str]:
|
|
"""Ansible extra-vars describing the alert that fired, as `key=value` strings.
|
|
|
|
Injected into every alert-triggered playbook run so the playbook can target
|
|
the exact resource/metric that fired without templating. Pure function.
|
|
"""
|
|
return [
|
|
f"steward_alert_rule={rule.name}",
|
|
f"steward_alert_source={rule.source_module}",
|
|
f"steward_alert_resource={rule.resource_name}",
|
|
f"steward_alert_metric={rule.metric_name}",
|
|
f"steward_alert_value={value}",
|
|
f"steward_alert_threshold={rule.threshold}",
|
|
f"steward_alert_operator={rule.operator.value}",
|
|
]
|
|
|
|
|
|
async def record_metric(
|
|
session: AsyncSession,
|
|
source_module: str,
|
|
resource_name: str,
|
|
metric_name: str,
|
|
value: float,
|
|
) -> None:
|
|
"""One metric sample: record_metrics with a single row."""
|
|
await record_metrics(session, [PluginMetric(
|
|
source_module=source_module, resource_name=resource_name,
|
|
metric_name=metric_name, value=value)])
|
|
|
|
|
|
class MetricBatch:
|
|
"""Gathers one transaction's samples so they are recorded, and their alert
|
|
rules evaluated, in a single record_metrics call:
|
|
|
|
batch = MetricBatch()
|
|
batch.add(source_module="unifi", resource_name="wan", metric_name="is_up", value=1.0)
|
|
await batch.record(session)
|
|
"""
|
|
|
|
def __init__(self) -> None:
|
|
self.rows: list[PluginMetric] = []
|
|
|
|
def add(self, source_module: str, resource_name: str, metric_name: str, value: float,
|
|
recorded_at: datetime | None = None) -> None:
|
|
self.rows.append(PluginMetric(source_module=source_module, resource_name=resource_name,
|
|
metric_name=metric_name, value=float(value),
|
|
recorded_at=recorded_at))
|
|
|
|
async def record(self, session: AsyncSession) -> None:
|
|
rows, self.rows = self.rows, []
|
|
await record_metrics(session, rows)
|
|
|
|
|
|
def _in_window(w: MaintenanceWindow, source_module: str, resource_name: str) -> bool:
|
|
if w.scope_source is None:
|
|
return True
|
|
return w.scope_source == source_module and w.scope_resource in (None, resource_name)
|
|
|
|
|
|
async def record_metrics(session: AsyncSession, rows: list[PluginMetric]) -> None:
|
|
"""Write metric rows and evaluate the alert rules that watch them.
|
|
|
|
Every ingest path comes through here, so a rule on any metric Steward
|
|
records can fire. Must be called inside an active transaction:
|
|
async with session.begin():
|
|
await record_metrics(session, rows)
|
|
|
|
Maintenance windows, rules and their states are read once per batch, not
|
|
once per row. Each rule sees its key's samples oldest first, so a backlog
|
|
(an agent catching up after an outage) counts consecutive checks as they
|
|
happened. Notification I/O is deferred via asyncio.create_task() so no
|
|
network calls occur while the transaction is open.
|
|
"""
|
|
if not rows:
|
|
return
|
|
now = datetime.now(timezone.utc)
|
|
for row in rows:
|
|
if row.recorded_at is None:
|
|
row.recorded_at = now
|
|
session.add_all(rows)
|
|
await session.flush()
|
|
|
|
by_key: dict[tuple[str, str, str], list[PluginMetric]] = {}
|
|
for row in rows:
|
|
by_key.setdefault((row.source_module, row.resource_name, row.metric_name), []).append(row)
|
|
sources = {k[0] for k in by_key}
|
|
metric_names = {k[2] for k in by_key}
|
|
|
|
rules = (await session.execute(
|
|
select(AlertRule).where(
|
|
AlertRule.source_module.in_(sources),
|
|
AlertRule.metric_name.in_(metric_names),
|
|
AlertRule.enabled.is_(True),
|
|
)
|
|
)).scalars().all()
|
|
rules = [r for r in rules if (r.source_module, r.resource_name, r.metric_name) in by_key]
|
|
if not rules:
|
|
return
|
|
|
|
windows = (await session.execute(
|
|
select(MaintenanceWindow).where(
|
|
MaintenanceWindow.start_at <= now, MaintenanceWindow.end_at >= now)
|
|
)).scalars().all()
|
|
states = {st.rule_id: st for st in (await session.execute(
|
|
select(AlertState).where(AlertState.rule_id.in_([r.id for r in rules]))
|
|
)).scalars()}
|
|
|
|
for rule in rules:
|
|
if any(_in_window(w, rule.source_module, rule.resource_name) for w in windows):
|
|
continue # suppressed by maintenance window
|
|
alert_state = states.get(rule.id)
|
|
if alert_state is None:
|
|
logger.warning("No alert_state row for rule %s, skipping", rule.id)
|
|
continue
|
|
samples = sorted(by_key[(rule.source_module, rule.resource_name, rule.metric_name)],
|
|
key=lambda r: r.recorded_at)
|
|
for sample in samples:
|
|
notifications = await _evaluate_rule(session, rule, alert_state, sample.value,
|
|
sample.recorded_at)
|
|
for notif_type, event_id in notifications:
|
|
if _app is not None:
|
|
asyncio.create_task(
|
|
_dispatch_notification(_app, notif_type, rule, sample.value, event_id))
|
|
# Admin-configured Ansible action runs only on transition-to-firing.
|
|
if notif_type == "firing" and rule.ansible_action:
|
|
asyncio.create_task(
|
|
_run_ansible_action(_app, rule, sample.value, event_id))
|
|
|
|
|
|
def _is_breached(value: float, operator: AlertOperator, threshold: float) -> bool:
|
|
op = operator.value
|
|
if op == ">":
|
|
return value > threshold
|
|
if op == "<":
|
|
return value < threshold
|
|
if op == ">=":
|
|
return value >= threshold
|
|
if op == "<=":
|
|
return value <= threshold
|
|
if op == "==":
|
|
return value == threshold
|
|
if op == "!=":
|
|
return value != threshold
|
|
return False
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Step:
|
|
"""One sample's effect on an alert: the state after it, the two run
|
|
counters, and the event to record ("firing" / "resolved") if any."""
|
|
state: AlertStateEnum
|
|
fail_count: int
|
|
ok_count: int
|
|
event: str | None = None
|
|
|
|
|
|
def next_step(state: AlertStateEnum, fail_count: int, ok_count: int, breached: bool,
|
|
fail_required: int, ok_required: int) -> Step:
|
|
"""The alert state machine, as a pure function of one sample.
|
|
|
|
inactive → pending → firing when `fail_required` breaching checks run in a
|
|
row; a non-breaching check while pending goes back to inactive.
|
|
firing and acknowledged resolve (to inactive, with a resolved event) only
|
|
after `ok_required` non-breaching checks in a row; a breaching check
|
|
resets that run. An acknowledgement holds while the alert is still
|
|
breached: only a resolve clears it.
|
|
"""
|
|
fail_required, ok_required = max(1, fail_required), max(1, ok_required)
|
|
S = AlertStateEnum
|
|
if state in (S.firing, S.acknowledged):
|
|
if breached:
|
|
return Step(state, fail_count + 1, 0)
|
|
if ok_count + 1 >= ok_required:
|
|
return Step(S.inactive, 0, 0, "resolved")
|
|
return Step(state, fail_count, ok_count + 1)
|
|
# inactive, pending (and the transient resolved, read as inactive)
|
|
if not breached:
|
|
return Step(S.inactive, 0, 0)
|
|
fails = fail_count + 1 if state == S.pending else 1
|
|
if fails >= fail_required:
|
|
return Step(S.firing, fails, 0, "firing")
|
|
return Step(S.pending, fails, 0)
|
|
|
|
|
|
async def _evaluate_rule(
|
|
session: AsyncSession,
|
|
rule: AlertRule,
|
|
state: AlertState,
|
|
value: float,
|
|
now: datetime,
|
|
) -> list[tuple[str, str]]:
|
|
"""Apply one sample to a rule's state. Returns [(notif_type, event_id)]."""
|
|
from_state = state.state
|
|
step = next_step(from_state, state.consecutive_failure_count, state.consecutive_ok_count,
|
|
_is_breached(value, rule.operator, rule.threshold),
|
|
rule.consecutive_failures_required, rule.consecutive_ok_required)
|
|
state.consecutive_failure_count = step.fail_count
|
|
state.consecutive_ok_count = step.ok_count
|
|
notifications: list[tuple[str, str]] = []
|
|
if step.state != from_state:
|
|
state.state = step.state
|
|
state.last_transition_at = now
|
|
if step.state == AlertStateEnum.inactive:
|
|
state.acknowledged_by = None
|
|
state.acknowledged_at = None
|
|
if step.event:
|
|
event_id = _add_event(session, rule.id, from_state.value, step.event, value, now)
|
|
notifications.append((step.event, event_id))
|
|
await session.flush()
|
|
return notifications
|
|
|
|
|
|
async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str) -> None:
|
|
"""Put a rule's alert back to inactive because the rule itself changed
|
|
(what it watches, its threshold, or it was disabled). An alert that was
|
|
firing or acknowledged gets a resolved event carrying `note`; nobody is
|
|
notified, since nothing about the watched resource changed."""
|
|
state = (await session.execute(
|
|
select(AlertState).where(AlertState.rule_id == rule.id))).scalar_one_or_none()
|
|
if state is None:
|
|
return
|
|
now = datetime.now(timezone.utc)
|
|
if state.state in (AlertStateEnum.firing, AlertStateEnum.acknowledged):
|
|
_add_event(session, rule.id, state.state.value, "resolved", None, now, note=note)
|
|
if state.state != AlertStateEnum.inactive:
|
|
state.last_transition_at = now
|
|
state.state = AlertStateEnum.inactive
|
|
state.consecutive_failure_count = 0
|
|
state.consecutive_ok_count = 0
|
|
state.acknowledged_by = None
|
|
state.acknowledged_at = None
|
|
|
|
|
|
def _add_event(
|
|
session: AsyncSession,
|
|
rule_id: str,
|
|
from_state: str,
|
|
to_state: str,
|
|
value: float | None,
|
|
now: datetime,
|
|
note: str | None = None,
|
|
) -> str:
|
|
"""Add an AlertEvent row and return its ID."""
|
|
event_id = str(uuid.uuid4())
|
|
event = AlertEvent(
|
|
id=event_id,
|
|
rule_id=rule_id,
|
|
from_state=from_state,
|
|
to_state=to_state,
|
|
metric_value=value,
|
|
transitioned_at=now,
|
|
note=note,
|
|
)
|
|
session.add(event)
|
|
return event_id
|
|
|
|
|
|
async def _dispatch_notification(
|
|
app: "Quart",
|
|
notif_type: str,
|
|
rule: AlertRule,
|
|
value: float,
|
|
event_id: str,
|
|
) -> None:
|
|
"""Run after the transaction commits. Sends notifications and updates the event row."""
|
|
from steward.core.notifications import dispatch_notifications
|
|
|
|
alert_data = {
|
|
"rule_name": rule.name,
|
|
"state": notif_type.upper(),
|
|
"metric": rule.metric_name,
|
|
"value": value,
|
|
"threshold": rule.threshold,
|
|
"resource": rule.resource_name,
|
|
"source_module": rule.source_module,
|
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
|
|
smtp_cfg = app.config.get("SMTP", {})
|
|
webhook_cfg = app.config.get("WEBHOOK", {})
|
|
results = await dispatch_notifications(smtp_cfg, webhook_cfg, alert_data)
|
|
|
|
# Update event row with delivery results in a new transaction
|
|
async with app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
event_result = await db.execute(
|
|
select(AlertEvent).where(AlertEvent.id == event_id)
|
|
)
|
|
event = event_result.scalar_one_or_none()
|
|
if event:
|
|
email_r = results.get("email")
|
|
if email_r:
|
|
event.email_sent = email_r["sent"]
|
|
event.email_error = email_r.get("error")
|
|
webhook_r = results.get("webhook")
|
|
if webhook_r:
|
|
event.webhook_sent = webhook_r["sent"]
|
|
event.webhook_error = webhook_r.get("error")
|
|
|
|
|
|
async def _run_ansible_action(
|
|
app: "Quart",
|
|
rule: AlertRule,
|
|
value: float,
|
|
event_id: str,
|
|
) -> None:
|
|
"""Run the rule's configured Ansible playbook (system-triggered).
|
|
|
|
Runs after the evaluation transaction commits (scheduled via create_task).
|
|
Creates a system-triggered AnsibleRun, links it to the firing AlertEvent,
|
|
then executes it with the static params plus injected alert context vars.
|
|
"""
|
|
from steward.ansible import executor, sources as src_module
|
|
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
|
|
|
action = rule.ansible_action or {}
|
|
playbook = action.get("playbook")
|
|
inventory = action.get("inventory")
|
|
if not playbook or not inventory:
|
|
logger.error("Ansible action for rule %r missing playbook/inventory", rule.name)
|
|
return
|
|
|
|
try:
|
|
srcs = src_module.get_sources(app.config.get("ANSIBLE", {}))
|
|
except Exception:
|
|
logger.exception("Ansible action for rule %r: could not load sources", rule.name)
|
|
return
|
|
source = next((s for s in srcs if s["name"] == action.get("source")), None)
|
|
if source is None:
|
|
logger.error("Ansible action for rule %r: source %r not configured",
|
|
rule.name, action.get("source"))
|
|
return
|
|
|
|
# Static params + auto-injected alert context (all argv, never shell).
|
|
params: dict = {
|
|
"extra_vars": list(action.get("extra_vars") or []) + alert_extra_vars(rule, value),
|
|
}
|
|
if action.get("limit"):
|
|
params["limit"] = action["limit"]
|
|
if action.get("tags"):
|
|
params["tags"] = action["tags"]
|
|
if action.get("check"):
|
|
params["check"] = True
|
|
|
|
run_id = str(uuid.uuid4())
|
|
async with app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
db.add(AnsibleRun(
|
|
id=run_id,
|
|
playbook_path=playbook,
|
|
inventory_path=inventory,
|
|
source_name=source["name"],
|
|
triggered_by=None,
|
|
status=AnsibleRunStatus.running,
|
|
params=params,
|
|
))
|
|
event = (await db.execute(
|
|
select(AlertEvent).where(AlertEvent.id == event_id))).scalar_one_or_none()
|
|
if event:
|
|
event.ansible_run_id = run_id
|
|
|
|
await executor.start_run(app, run_id, playbook, inventory, source["path"], params)
|