bvandeusenandClaude Opus 5.5
c368e93e2c
Agent offline is an ordinary rule now. The host_agent scheduler records host_agent/<host>/seconds_since_seen every minute (from last_seen_at, or registration time for an agent that never reported), replacing the task that only logged stale agents. The catalog offers the metric, and picking it on a new rule fills "> 180". No data: each rule's state tracks its last sample and its usual gap, smoothed so that an outage counts as at most three gaps. A core task, alert_silence_check, runs every 60s and marks a rule no_data_since once its series has had no sample for three usual gaps (at least 5 minutes; an hour while the cadence is unknown). A never-sampled rule counts from when it started watching. The mark sits beside the state, so a firing alert that goes silent stays firing. Each silence records one event and sends one notification (rule option notify_no_data, default on; muted in maintenance windows). The next sample clears it with a "data resumed" event and notification. Creating, changing, disabling or re-enabling a rule restarts its watch. Migration 0029 adds the columns and seeds each existing rule's last sample and gap from its last day of data, so an upgrade does not flag every rule. Shown on the Alerts page (a NO DATA badge with its start time), in MCP list_alerts, list_alert_rules and get_host (state "no_data", no_data_since), in get_overview findings (alert_no_data, or a firing alert marked silent) and in the weekly report. The rule form has the notify option, and its metric reference now matches what traefik records and lists host_agent. Emails say "no data" and carry the event's note. The metric-catalog guard also reads positional MetricBatch.add arguments. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
531 lines
20 KiB
Python
531 lines
20 KiB
Python
from __future__ import annotations
|
|
import asyncio
|
|
import logging
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from steward.models.alerts import (
|
|
AlertEvent,
|
|
AlertOperator,
|
|
AlertRule,
|
|
AlertState,
|
|
AlertStateEnum,
|
|
MaintenanceWindow,
|
|
)
|
|
from steward.models.metrics import PluginMetric
|
|
|
|
if TYPE_CHECKING:
|
|
from quart import Quart
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_app: "Quart | None" = None
|
|
|
|
|
|
def init_alerts(app: "Quart") -> None:
|
|
"""Store app reference for use in notification dispatch tasks."""
|
|
global _app
|
|
_app = app
|
|
|
|
|
|
def alert_extra_vars(rule: AlertRule, value: float) -> list[str]:
|
|
"""Ansible extra-vars describing the alert that fired, as `key=value` strings.
|
|
|
|
Injected into every alert-triggered playbook run so the playbook can target
|
|
the exact resource/metric that fired without templating. Pure function.
|
|
"""
|
|
return [
|
|
f"steward_alert_rule={rule.name}",
|
|
f"steward_alert_source={rule.source_module}",
|
|
f"steward_alert_resource={rule.resource_name}",
|
|
f"steward_alert_metric={rule.metric_name}",
|
|
f"steward_alert_value={value}",
|
|
f"steward_alert_threshold={rule.threshold}",
|
|
f"steward_alert_operator={rule.operator.value}",
|
|
]
|
|
|
|
|
|
async def record_metric(
|
|
session: AsyncSession,
|
|
source_module: str,
|
|
resource_name: str,
|
|
metric_name: str,
|
|
value: float,
|
|
) -> None:
|
|
"""One metric sample: record_metrics with a single row."""
|
|
await record_metrics(session, [PluginMetric(
|
|
source_module=source_module, resource_name=resource_name,
|
|
metric_name=metric_name, value=value)])
|
|
|
|
|
|
class MetricBatch:
|
|
"""Gathers one transaction's samples so they are recorded, and their alert
|
|
rules evaluated, in a single record_metrics call:
|
|
|
|
batch = MetricBatch()
|
|
batch.add(source_module="unifi", resource_name="wan", metric_name="is_up", value=1.0)
|
|
await batch.record(session)
|
|
"""
|
|
|
|
def __init__(self) -> None:
|
|
self.rows: list[PluginMetric] = []
|
|
|
|
def add(self, source_module: str, resource_name: str, metric_name: str, value: float,
|
|
recorded_at: datetime | None = None) -> None:
|
|
self.rows.append(PluginMetric(source_module=source_module, resource_name=resource_name,
|
|
metric_name=metric_name, value=float(value),
|
|
recorded_at=recorded_at))
|
|
|
|
async def record(self, session: AsyncSession) -> None:
|
|
rows, self.rows = self.rows, []
|
|
await record_metrics(session, rows)
|
|
|
|
|
|
def _in_window(w: MaintenanceWindow, source_module: str, resource_name: str) -> bool:
|
|
if w.scope_source is None:
|
|
return True
|
|
return w.scope_source == source_module and w.scope_resource in (None, resource_name)
|
|
|
|
|
|
async def record_metrics(session: AsyncSession, rows: list[PluginMetric]) -> None:
|
|
"""Write metric rows and evaluate the alert rules that watch them.
|
|
|
|
Every ingest path comes through here, so a rule on any metric Steward
|
|
records can fire. Must be called inside an active transaction:
|
|
async with session.begin():
|
|
await record_metrics(session, rows)
|
|
|
|
Maintenance windows, rules and their states are read once per batch, not
|
|
once per row. Each rule sees its key's samples oldest first, so a backlog
|
|
(an agent catching up after an outage) counts consecutive checks as they
|
|
happened. Notification I/O is deferred via asyncio.create_task() so no
|
|
network calls occur while the transaction is open.
|
|
"""
|
|
if not rows:
|
|
return
|
|
now = datetime.now(timezone.utc)
|
|
for row in rows:
|
|
if row.recorded_at is None:
|
|
row.recorded_at = now
|
|
session.add_all(rows)
|
|
await session.flush()
|
|
|
|
by_key: dict[tuple[str, str, str], list[PluginMetric]] = {}
|
|
for row in rows:
|
|
by_key.setdefault((row.source_module, row.resource_name, row.metric_name), []).append(row)
|
|
sources = {k[0] for k in by_key}
|
|
metric_names = {k[2] for k in by_key}
|
|
|
|
rules = (await session.execute(
|
|
select(AlertRule).where(
|
|
AlertRule.source_module.in_(sources),
|
|
AlertRule.metric_name.in_(metric_names),
|
|
AlertRule.enabled.is_(True),
|
|
)
|
|
)).scalars().all()
|
|
rules = [r for r in rules if (r.source_module, r.resource_name, r.metric_name) in by_key]
|
|
if not rules:
|
|
return
|
|
|
|
windows = (await session.execute(
|
|
select(MaintenanceWindow).where(
|
|
MaintenanceWindow.start_at <= now, MaintenanceWindow.end_at >= now)
|
|
)).scalars().all()
|
|
states = {st.rule_id: st for st in (await session.execute(
|
|
select(AlertState).where(AlertState.rule_id.in_([r.id for r in rules]))
|
|
)).scalars()}
|
|
|
|
for rule in rules:
|
|
alert_state = states.get(rule.id)
|
|
if alert_state is None:
|
|
logger.warning("No alert_state row for rule %s, skipping", rule.id)
|
|
continue
|
|
samples = sorted(by_key[(rule.source_module, rule.resource_name, rule.metric_name)],
|
|
key=lambda r: r.recorded_at)
|
|
muted = any(_in_window(w, rule.source_module, rule.resource_name) for w in windows)
|
|
resumed = _track_samples(session, rule, alert_state, samples, now)
|
|
if resumed and rule.notify_no_data and not muted and _app is not None:
|
|
event_id, value, note = resumed
|
|
asyncio.create_task(
|
|
_dispatch_notification(_app, "data_resumed", rule, value, event_id, note))
|
|
if muted:
|
|
continue # suppressed by maintenance window
|
|
for sample in samples:
|
|
notifications = await _evaluate_rule(session, rule, alert_state, sample.value,
|
|
sample.recorded_at)
|
|
for notif_type, event_id in notifications:
|
|
if _app is not None:
|
|
asyncio.create_task(
|
|
_dispatch_notification(_app, notif_type, rule, sample.value, event_id))
|
|
# Admin-configured Ansible action runs only on transition-to-firing.
|
|
if notif_type == "firing" and rule.ansible_action:
|
|
asyncio.create_task(
|
|
_run_ansible_action(_app, rule, sample.value, event_id))
|
|
|
|
|
|
# A series is silent once it has gone this long without a sample, or three of
|
|
# its usual gaps if that is longer. Until its cadence is known (fewer than two
|
|
# samples seen), it gets an hour: a monitor may check only every ten minutes.
|
|
NO_DATA_FLOOR_S = 300.0
|
|
NO_DATA_GAPS = 3
|
|
NO_DATA_UNKNOWN_S = 3600.0
|
|
|
|
|
|
def next_interval(interval: float | None, gap: float) -> float:
|
|
"""A series' usual gap between samples, updated with one more gap.
|
|
|
|
A smoothed average, so one late sample barely moves it. A gap longer than
|
|
the silence threshold counts only as that threshold: an outage is not the
|
|
series' cadence. A real change of cadence still gets learned, a step at a
|
|
time (each step at most 1.5x).
|
|
"""
|
|
gap = max(0.0, gap)
|
|
if interval is None or interval <= 0:
|
|
return gap
|
|
return 0.75 * interval + 0.25 * min(gap, NO_DATA_GAPS * interval)
|
|
|
|
|
|
def no_data_after(interval: float | None) -> float:
|
|
"""Seconds without a sample before a series counts as silent."""
|
|
if not interval:
|
|
return NO_DATA_UNKNOWN_S
|
|
return max(NO_DATA_FLOOR_S, NO_DATA_GAPS * interval)
|
|
|
|
|
|
def silent_since(last_sample_at: datetime | None, interval: float | None,
|
|
watching_since: datetime, now: datetime) -> datetime | None:
|
|
"""When a series went silent, or None while it is still reporting.
|
|
|
|
A series that has never had a sample counts from when the rule started
|
|
watching it (created, changed or re-enabled).
|
|
"""
|
|
last = last_sample_at or watching_since
|
|
limit = timedelta(seconds=no_data_after(interval))
|
|
return last + limit if now - last > limit else None
|
|
|
|
|
|
def _duration(seconds: float) -> str:
|
|
seconds = int(seconds)
|
|
if seconds < 120:
|
|
return f"{seconds}s"
|
|
if seconds < 7200:
|
|
return f"{seconds // 60}m"
|
|
return f"{seconds // 3600}h{(seconds % 3600) // 60:02d}m"
|
|
|
|
|
|
def _track_samples(session: AsyncSession, rule: AlertRule, state: AlertState,
|
|
samples: list[PluginMetric], now: datetime
|
|
) -> tuple[str, float, str] | None:
|
|
"""Note a rule's new samples (oldest first) for silence tracking. When the
|
|
series had been marked no_data, clear it and record a resume event,
|
|
returned as (event_id, value, note) for the caller to notify."""
|
|
for sample in samples:
|
|
if state.last_sample_at is not None:
|
|
if sample.recorded_at <= state.last_sample_at:
|
|
continue # late or duplicate: not a gap
|
|
gap = (sample.recorded_at - state.last_sample_at).total_seconds()
|
|
state.sample_interval_s = next_interval(state.sample_interval_s, gap)
|
|
state.last_sample_at = sample.recorded_at
|
|
if state.no_data_since is None or not samples:
|
|
return None
|
|
note = f"data resumed after {_duration((now - state.no_data_since).total_seconds())} silent"
|
|
value = samples[-1].value
|
|
event_id = _add_event(session, rule.id, "no_data", state.state.value, value, now, note=note)
|
|
state.no_data_since = None
|
|
return event_id, value, note
|
|
|
|
|
|
def _is_breached(value: float, operator: AlertOperator, threshold: float) -> bool:
|
|
op = operator.value
|
|
if op == ">":
|
|
return value > threshold
|
|
if op == "<":
|
|
return value < threshold
|
|
if op == ">=":
|
|
return value >= threshold
|
|
if op == "<=":
|
|
return value <= threshold
|
|
if op == "==":
|
|
return value == threshold
|
|
if op == "!=":
|
|
return value != threshold
|
|
return False
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Step:
|
|
"""One sample's effect on an alert: the state after it, the two run
|
|
counters, and the event to record ("firing" / "resolved") if any."""
|
|
state: AlertStateEnum
|
|
fail_count: int
|
|
ok_count: int
|
|
event: str | None = None
|
|
|
|
|
|
def next_step(state: AlertStateEnum, fail_count: int, ok_count: int, breached: bool,
|
|
fail_required: int, ok_required: int) -> Step:
|
|
"""The alert state machine, as a pure function of one sample.
|
|
|
|
inactive → pending → firing when `fail_required` breaching checks run in a
|
|
row; a non-breaching check while pending goes back to inactive.
|
|
firing and acknowledged resolve (to inactive, with a resolved event) only
|
|
after `ok_required` non-breaching checks in a row; a breaching check
|
|
resets that run. An acknowledgement holds while the alert is still
|
|
breached: only a resolve clears it.
|
|
"""
|
|
fail_required, ok_required = max(1, fail_required), max(1, ok_required)
|
|
S = AlertStateEnum
|
|
if state in (S.firing, S.acknowledged):
|
|
if breached:
|
|
return Step(state, fail_count + 1, 0)
|
|
if ok_count + 1 >= ok_required:
|
|
return Step(S.inactive, 0, 0, "resolved")
|
|
return Step(state, fail_count, ok_count + 1)
|
|
# inactive, pending (and the transient resolved, read as inactive)
|
|
if not breached:
|
|
return Step(S.inactive, 0, 0)
|
|
fails = fail_count + 1 if state == S.pending else 1
|
|
if fails >= fail_required:
|
|
return Step(S.firing, fails, 0, "firing")
|
|
return Step(S.pending, fails, 0)
|
|
|
|
|
|
async def _evaluate_rule(
|
|
session: AsyncSession,
|
|
rule: AlertRule,
|
|
state: AlertState,
|
|
value: float,
|
|
now: datetime,
|
|
) -> list[tuple[str, str]]:
|
|
"""Apply one sample to a rule's state. Returns [(notif_type, event_id)]."""
|
|
from_state = state.state
|
|
step = next_step(from_state, state.consecutive_failure_count, state.consecutive_ok_count,
|
|
_is_breached(value, rule.operator, rule.threshold),
|
|
rule.consecutive_failures_required, rule.consecutive_ok_required)
|
|
state.consecutive_failure_count = step.fail_count
|
|
state.consecutive_ok_count = step.ok_count
|
|
notifications: list[tuple[str, str]] = []
|
|
if step.state != from_state:
|
|
state.state = step.state
|
|
state.last_transition_at = now
|
|
if step.state == AlertStateEnum.inactive:
|
|
state.acknowledged_by = None
|
|
state.acknowledged_at = None
|
|
if step.event:
|
|
event_id = _add_event(session, rule.id, from_state.value, step.event, value, now)
|
|
notifications.append((step.event, event_id))
|
|
await session.flush()
|
|
return notifications
|
|
|
|
|
|
async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str) -> None:
|
|
"""Put a rule's alert back to inactive because the rule itself changed
|
|
(what it watches, its threshold, or it was disabled or re-enabled), and
|
|
start its silence watch over. An alert that was
|
|
firing or acknowledged gets a resolved event carrying `note`; nobody is
|
|
notified, since nothing about the watched resource changed."""
|
|
state = (await session.execute(
|
|
select(AlertState).where(AlertState.rule_id == rule.id))).scalar_one_or_none()
|
|
if state is None:
|
|
return
|
|
now = datetime.now(timezone.utc)
|
|
if state.state in (AlertStateEnum.firing, AlertStateEnum.acknowledged):
|
|
_add_event(session, rule.id, state.state.value, "resolved", None, now, note=note)
|
|
# The watch starts over now: silence is measured from here, against the
|
|
# cadence of whatever the rule watches from now on.
|
|
state.last_transition_at = now
|
|
state.last_sample_at = None
|
|
state.sample_interval_s = None
|
|
state.no_data_since = None
|
|
state.state = AlertStateEnum.inactive
|
|
state.consecutive_failure_count = 0
|
|
state.consecutive_ok_count = 0
|
|
state.acknowledged_by = None
|
|
state.acknowledged_at = None
|
|
|
|
|
|
def _add_event(
|
|
session: AsyncSession,
|
|
rule_id: str,
|
|
from_state: str,
|
|
to_state: str,
|
|
value: float | None,
|
|
now: datetime,
|
|
note: str | None = None,
|
|
) -> str:
|
|
"""Add an AlertEvent row and return its ID."""
|
|
event_id = str(uuid.uuid4())
|
|
event = AlertEvent(
|
|
id=event_id,
|
|
rule_id=rule_id,
|
|
from_state=from_state,
|
|
to_state=to_state,
|
|
metric_value=value,
|
|
transitioned_at=now,
|
|
note=note,
|
|
)
|
|
session.add(event)
|
|
return event_id
|
|
|
|
|
|
async def _dispatch_notification(
|
|
app: "Quart",
|
|
notif_type: str,
|
|
rule: AlertRule,
|
|
value: float | None,
|
|
event_id: str,
|
|
note: str | None = None,
|
|
) -> None:
|
|
"""Run after the transaction commits. Sends notifications and updates the event row."""
|
|
from steward.core.notifications import dispatch_notifications
|
|
|
|
alert_data = {
|
|
"rule_name": rule.name,
|
|
"state": notif_type.upper(),
|
|
"metric": rule.metric_name,
|
|
"value": value,
|
|
"threshold": rule.threshold,
|
|
"resource": rule.resource_name,
|
|
"source_module": rule.source_module,
|
|
"note": note,
|
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
|
|
smtp_cfg = app.config.get("SMTP", {})
|
|
webhook_cfg = app.config.get("WEBHOOK", {})
|
|
results = await dispatch_notifications(smtp_cfg, webhook_cfg, alert_data)
|
|
|
|
# Update event row with delivery results in a new transaction
|
|
async with app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
event_result = await db.execute(
|
|
select(AlertEvent).where(AlertEvent.id == event_id)
|
|
)
|
|
event = event_result.scalar_one_or_none()
|
|
if event:
|
|
email_r = results.get("email")
|
|
if email_r:
|
|
event.email_sent = email_r["sent"]
|
|
event.email_error = email_r.get("error")
|
|
webhook_r = results.get("webhook")
|
|
if webhook_r:
|
|
event.webhook_sent = webhook_r["sent"]
|
|
event.webhook_error = webhook_r.get("error")
|
|
|
|
|
|
async def _run_ansible_action(
|
|
app: "Quart",
|
|
rule: AlertRule,
|
|
value: float,
|
|
event_id: str,
|
|
) -> None:
|
|
"""Run the rule's configured Ansible playbook (system-triggered).
|
|
|
|
Runs after the evaluation transaction commits (scheduled via create_task).
|
|
Creates a system-triggered AnsibleRun, links it to the firing AlertEvent,
|
|
then executes it with the static params plus injected alert context vars.
|
|
"""
|
|
from steward.ansible import executor, sources as src_module
|
|
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
|
|
|
action = rule.ansible_action or {}
|
|
playbook = action.get("playbook")
|
|
inventory = action.get("inventory")
|
|
if not playbook or not inventory:
|
|
logger.error("Ansible action for rule %r missing playbook/inventory", rule.name)
|
|
return
|
|
|
|
try:
|
|
srcs = src_module.get_sources(app.config.get("ANSIBLE", {}))
|
|
except Exception:
|
|
logger.exception("Ansible action for rule %r: could not load sources", rule.name)
|
|
return
|
|
source = next((s for s in srcs if s["name"] == action.get("source")), None)
|
|
if source is None:
|
|
logger.error("Ansible action for rule %r: source %r not configured",
|
|
rule.name, action.get("source"))
|
|
return
|
|
|
|
# Static params + auto-injected alert context (all argv, never shell).
|
|
params: dict = {
|
|
"extra_vars": list(action.get("extra_vars") or []) + alert_extra_vars(rule, value),
|
|
}
|
|
if action.get("limit"):
|
|
params["limit"] = action["limit"]
|
|
if action.get("tags"):
|
|
params["tags"] = action["tags"]
|
|
if action.get("check"):
|
|
params["check"] = True
|
|
|
|
run_id = str(uuid.uuid4())
|
|
async with app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
db.add(AnsibleRun(
|
|
id=run_id,
|
|
playbook_path=playbook,
|
|
inventory_path=inventory,
|
|
source_name=source["name"],
|
|
triggered_by=None,
|
|
status=AnsibleRunStatus.running,
|
|
params=params,
|
|
))
|
|
event = (await db.execute(
|
|
select(AlertEvent).where(AlertEvent.id == event_id))).scalar_one_or_none()
|
|
if event:
|
|
event.ansible_run_id = run_id
|
|
|
|
await executor.start_run(app, run_id, playbook, inventory, source["path"], params)
|
|
|
|
|
|
async def check_silence(app: "Quart") -> None:
|
|
"""Mark the enabled rules whose series has gone silent (no_data).
|
|
|
|
Run every minute by the core scheduler. A rule is silent once its series
|
|
has had no sample for no_data_after(its usual gap); one that never had a
|
|
sample counts from when the rule started watching. The mark sits beside
|
|
the state, so a firing alert that goes silent stays firing. Each silence
|
|
is recorded once, as an event, and notified once (rule.notify_no_data,
|
|
outside maintenance windows), after the transaction commits.
|
|
"""
|
|
now = datetime.now(timezone.utc)
|
|
notify: list[tuple[AlertRule, str, str]] = []
|
|
async with app.db_sessionmaker() as session:
|
|
async with session.begin():
|
|
rows = (await session.execute(
|
|
select(AlertState, AlertRule)
|
|
.join(AlertRule, AlertState.rule_id == AlertRule.id)
|
|
.where(AlertRule.enabled.is_(True), AlertState.no_data_since.is_(None))
|
|
)).all()
|
|
if not rows:
|
|
return
|
|
windows = (await session.execute(
|
|
select(MaintenanceWindow).where(
|
|
MaintenanceWindow.start_at <= now, MaintenanceWindow.end_at >= now)
|
|
)).scalars().all()
|
|
for state, rule in rows:
|
|
watching_since = max(rule.created_at, state.last_transition_at)
|
|
since = silent_since(state.last_sample_at, state.sample_interval_s,
|
|
watching_since, now)
|
|
if since is None:
|
|
continue
|
|
state.no_data_since = since
|
|
if state.last_sample_at is None:
|
|
note = (f"no sample yet, {_duration((now - watching_since).total_seconds())}"
|
|
" after the rule started watching")
|
|
else:
|
|
note = f"no sample for {_duration((now - state.last_sample_at).total_seconds())}"
|
|
if state.sample_interval_s:
|
|
note += f" (usually every {_duration(state.sample_interval_s)})"
|
|
event_id = _add_event(session, rule.id, state.state.value, "no_data", None,
|
|
now, note=note)
|
|
if rule.notify_no_data and not any(
|
|
_in_window(w, rule.source_module, rule.resource_name) for w in windows):
|
|
notify.append((rule, event_id, note))
|
|
for rule, event_id, note in notify:
|
|
asyncio.create_task(_dispatch_notification(app, "no_data", rule, None, event_id, note))
|