bvandeusenandClaude Opus 5.5
0742b7a2b7
#5061 "Metric catalog belongs to what emits: plugins register their metrics; names match what is recorded". - The static METRIC_CATALOG is gone. metric_catalog() is core's own sources (the four monitor types and "steward", the app's open_fds/open_fds_pct, which were recorded but never offered) plus each loaded plugin's get_alert_metrics() export. A disabled plugin's sources leave the picker. - traefik offers what it records: error_rate_4xx_pct and error_rate_5xx_pct. The old "error_rate" was never recorded, so a rule on it could not fire. - tests/core/test_metric_catalog.py walks every MetricBatch.add / record_metric call (traefik's tuple loop included) and fails on a recorded metric the catalog lacks; it shows on a synthetic writer that it can fail. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
261 lines
11 KiB
Python
261 lines
11 KiB
Python
"""Alert queries and validation shared by the alerts pages and the MCP tools.
|
|
|
|
Each function takes an open session (or plain values), with no Quart request
|
|
context, so the routes and the /mcp tools (which run outside Quart's
|
|
pipeline) call the same code and refuse the same input for the same reasons.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.models.alerts import (
|
|
AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
|
|
)
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# What core itself records. Unified monitors emit metrics under their TYPE
|
|
# (icmp/tcp/dns/http), each with is_up (1.0 up / 0.0 down) and response_ms
|
|
# (where the probe times it); "steward" is the app watching itself.
|
|
CORE_METRICS: dict[str, list[str]] = {
|
|
"icmp": ["is_up", "response_ms"],
|
|
"tcp": ["is_up", "response_ms"],
|
|
"dns": ["is_up", "response_ms"],
|
|
"http": ["is_up", "response_ms"],
|
|
"steward": ["open_fds", "open_fds_pct"],
|
|
}
|
|
|
|
|
|
def metric_catalog() -> dict[str, list[str]]:
|
|
"""Every source an alert rule can watch, and its metric names.
|
|
|
|
Core's own sources, then each loaded plugin's get_alert_metrics() export
|
|
({source: [metric names]}), so the catalog is written by whatever emits
|
|
the metrics and a disabled plugin's sources drop out. An empty list means
|
|
the names are dynamic (snmp: the OID labels recorded so far).
|
|
"""
|
|
from steward.core.plugin_manager import get_loaded_plugin_modules
|
|
catalog = {src: list(names) for src, names in CORE_METRICS.items()}
|
|
for name, module in get_loaded_plugin_modules().items():
|
|
hook = getattr(module, "get_alert_metrics", None)
|
|
if hook is None:
|
|
continue
|
|
try:
|
|
contributed = hook() or {}
|
|
except Exception:
|
|
logger.exception("Plugin %r: get_alert_metrics() raised; its metrics are skipped", name)
|
|
continue
|
|
for src, names in contributed.items():
|
|
catalog[str(src)] = [str(n) for n in names]
|
|
return catalog
|
|
|
|
|
|
def source_modules() -> list[str]:
|
|
"""The sources in picker order: core's first, then plugins'."""
|
|
return list(metric_catalog())
|
|
|
|
|
|
# States that need someone's attention: pending (breaching, not yet for long
|
|
# enough), firing, and acknowledged-but-still-breaching.
|
|
ACTIVE_STATES = (AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending)
|
|
|
|
|
|
async def active_alerts(db) -> list:
|
|
"""(AlertState, AlertRule) rows not yet resolved, latest transition first."""
|
|
return list((await db.execute(
|
|
select(AlertState, AlertRule)
|
|
.join(AlertRule, AlertState.rule_id == AlertRule.id)
|
|
.where(AlertState.state.in_(ACTIVE_STATES))
|
|
.order_by(AlertState.last_transition_at.desc())
|
|
)).all())
|
|
|
|
|
|
async def list_rules(db) -> list[AlertRule]:
|
|
return list((await db.execute(select(AlertRule).order_by(AlertRule.name))).scalars())
|
|
|
|
|
|
async def list_maintenance_windows(db) -> list[MaintenanceWindow]:
|
|
"""Every window, latest start first."""
|
|
return list((await db.execute(
|
|
select(MaintenanceWindow).order_by(MaintenanceWindow.start_at.desc())
|
|
)).scalars())
|
|
|
|
|
|
async def resources_for(db, source_module: str) -> list[str]:
|
|
"""Sorted distinct resource names for a source module.
|
|
|
|
For monitor types (icmp/tcp/dns/http), supplements with monitor names so
|
|
the list is populated even before any metrics have been recorded.
|
|
"""
|
|
result = await db.execute(
|
|
select(PluginMetric.resource_name)
|
|
.where(PluginMetric.source_module == source_module)
|
|
.distinct()
|
|
.order_by(PluginMetric.resource_name)
|
|
)
|
|
resources: set[str] = {r for (r,) in result}
|
|
if source_module in MONITOR_TYPE_VALUES:
|
|
mon_result = await db.execute(
|
|
select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name))
|
|
resources.update(name for (name,) in mon_result)
|
|
return sorted(resources)
|
|
|
|
|
|
async def metrics_for(db, source_module: str) -> list[str]:
|
|
"""The metric names a rule on this source can watch: the static catalog, or
|
|
for dynamic sources (snmp) the names recorded so far."""
|
|
metrics = metric_catalog().get(source_module, [])
|
|
if source_module and not metrics:
|
|
result = await db.execute(
|
|
select(PluginMetric.metric_name)
|
|
.where(PluginMetric.source_module == source_module)
|
|
.distinct()
|
|
.order_by(PluginMetric.metric_name))
|
|
metrics = [r for (r,) in result]
|
|
return metrics
|
|
|
|
|
|
def _choices(values: list[str], limit: int = 20) -> str:
|
|
if not values:
|
|
return "none yet"
|
|
shown = ", ".join(values[:limit])
|
|
return shown + (f" (and {len(values) - limit} more)" if len(values) > limit else "")
|
|
|
|
|
|
async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
|
|
operator: str, threshold, consecutive_failures_required=1,
|
|
consecutive_ok_required=1) -> dict:
|
|
"""Validated column values for a new or edited alert rule.
|
|
|
|
The same choices the rule form offers: a known source, a resource that
|
|
source has reported (or, for monitors, a monitor of that type), and a
|
|
metric it emits. ValueError says what was wrong and what is valid.
|
|
"""
|
|
name = (name or "").strip()
|
|
if not name:
|
|
raise ValueError("an alert rule needs a name.")
|
|
source_module = (source_module or "").strip()
|
|
catalog = metric_catalog()
|
|
if source_module not in catalog:
|
|
raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.")
|
|
resource_name = (resource_name or "").strip()
|
|
resources = await resources_for(db, source_module)
|
|
if resource_name not in resources:
|
|
raise ValueError(f"{source_module} has no resource {resource_name!r}. "
|
|
f"Known: {_choices(resources)}.")
|
|
metric_name = (metric_name or "").strip()
|
|
metrics = await metrics_for(db, source_module)
|
|
if metric_name not in metrics:
|
|
raise ValueError(f"{source_module} does not report {metric_name!r}. "
|
|
f"Its metrics: {_choices(metrics)}.")
|
|
try:
|
|
op = AlertOperator((operator or "").strip())
|
|
except ValueError:
|
|
raise ValueError(f"operator {operator!r} is not one of "
|
|
f"{', '.join(o.value for o in AlertOperator)}.") from None
|
|
try:
|
|
threshold = float(threshold)
|
|
except (TypeError, ValueError):
|
|
raise ValueError(f"threshold must be a number, got {threshold!r}.") from None
|
|
try:
|
|
consecutive = (1 if consecutive_failures_required in (None, "")
|
|
else int(consecutive_failures_required))
|
|
except (TypeError, ValueError):
|
|
raise ValueError("consecutive checks must be a whole number.") from None
|
|
if consecutive < 1:
|
|
raise ValueError("consecutive checks must be at least 1.")
|
|
try:
|
|
ok_checks = (1 if consecutive_ok_required in (None, "")
|
|
else int(consecutive_ok_required))
|
|
except (TypeError, ValueError):
|
|
raise ValueError("checks to clear must be a whole number.") from None
|
|
if ok_checks < 1:
|
|
raise ValueError("checks to clear must be at least 1.")
|
|
return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
|
|
"metric_name": metric_name, "operator": op, "threshold": threshold,
|
|
"consecutive_failures_required": consecutive,
|
|
"consecutive_ok_required": ok_checks}
|
|
|
|
|
|
# Changing any of these changes what the rule is watching, so whatever state
|
|
# it had built up no longer means anything.
|
|
WATCHED_FIELDS = ("source_module", "resource_name", "metric_name", "operator", "threshold")
|
|
|
|
|
|
async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool | None = None) -> None:
|
|
"""Apply validated fields (and optionally enabled) to a rule, in the
|
|
caller's transaction. A change to what it watches, or disabling it, puts
|
|
its alert back to inactive (core.alerts.reset_rule_state)."""
|
|
from steward.core.alerts import reset_rule_state
|
|
changed = [k for k in WATCHED_FIELDS if k in fields and getattr(rule, k) != fields[k]]
|
|
for k, v in fields.items():
|
|
setattr(rule, k, v)
|
|
disabling = enabled is False and rule.enabled
|
|
if enabled is not None:
|
|
rule.enabled = bool(enabled)
|
|
if disabling:
|
|
await reset_rule_state(db, rule, "rule disabled")
|
|
elif changed:
|
|
await reset_rule_state(db, rule, "rule changed: " + ", ".join(changed))
|
|
|
|
|
|
async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
|
|
ansible_action: dict | None = None) -> AlertRule:
|
|
"""Insert a rule and its inactive state row, in the caller's transaction."""
|
|
rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
|
|
db.add(rule)
|
|
await db.flush()
|
|
db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
|
|
last_transition_at=datetime.now(timezone.utc)))
|
|
return rule
|
|
|
|
|
|
async def acknowledge(db, rule_id: str, user_id: str) -> AlertState:
|
|
"""Acknowledge a firing alert, in the caller's transaction.
|
|
|
|
LookupError when the rule has no state; ValueError when it is not firing.
|
|
"""
|
|
state = (await db.execute(
|
|
select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one_or_none()
|
|
if state is None:
|
|
raise LookupError(rule_id)
|
|
if state.state != AlertStateEnum.firing:
|
|
raise ValueError(f"the alert is {state.state.value}, not firing; only a firing alert "
|
|
"can be acknowledged.")
|
|
now = datetime.now(timezone.utc)
|
|
state.state = AlertStateEnum.acknowledged
|
|
state.acknowledged_by = user_id
|
|
state.acknowledged_at = now
|
|
state.last_transition_at = now
|
|
return state
|
|
|
|
|
|
def window_fields(*, name: str, start_at: datetime, end_at: datetime,
|
|
scope_source: str | None, scope_resource: str | None) -> dict:
|
|
"""Validated columns for a maintenance window. A resource needs a source."""
|
|
name = (name or "").strip()
|
|
if not name:
|
|
raise ValueError("a maintenance window needs a name.")
|
|
if end_at <= start_at:
|
|
raise ValueError("the window must end after it starts.")
|
|
scope_source = (scope_source or "").strip() or None
|
|
scope_resource = (scope_resource or "").strip() or None
|
|
catalog = metric_catalog()
|
|
if scope_source is not None and scope_source not in catalog:
|
|
raise ValueError(f"source {scope_source!r} is not one of {', '.join(catalog)}.")
|
|
if scope_source is None:
|
|
scope_resource = None # can't have resource without source
|
|
return {"name": name[:128], "start_at": start_at, "end_at": end_at,
|
|
"scope_source": scope_source, "scope_resource": scope_resource}
|
|
|
|
|
|
def window_scope(w) -> str:
|
|
if not w.scope_source:
|
|
return "all"
|
|
return f"{w.scope_source}/{w.scope_resource}" if w.scope_resource else f"{w.scope_source}/*"
|