bvandeusenandClaude Opus 5.5
4fbe93bdf7
The Alerts page's active list now says each alert in one line around the resource it watches, e.g. "cpu_pct on web-1 is 94% (> 90%), firing for 12m", with web-1 linking to its host, monitor or container page (or the plugin's page for Traefik, UniFi and SNMP). Values take their unit from the metric name through formatting.metric_value. A silent series reads "No cpu_pct from web-1 for 20m"; an up/down metric says the word. Actions follow the routes' roles: acknowledge, edit, toggle and New rule for operators and admins, Delete for admins only, nothing for viewers. The sidebar shows how many alerts need attention beside Alerts, from an async context processor that skips htmx fragments, logged-out pages and the database-down page, and renders without the count if the query fails. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
385 lines
17 KiB
Python
385 lines
17 KiB
Python
"""Alert queries and validation shared by the alerts pages and the MCP tools.
|
|
|
|
Each function takes an open session (or plain values), with no Quart request
|
|
context, so the routes and the /mcp tools (which run outside Quart's
|
|
pipeline) call the same code and refuse the same input for the same reasons.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import or_, select
|
|
|
|
from steward.models.alerts import (
|
|
AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
|
|
)
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# What core itself records. Unified monitors emit metrics under their TYPE
|
|
# (icmp/tcp/dns/http), each with is_up (1.0 up / 0.0 down) and response_ms
|
|
# (where the probe times it); "steward" is the app watching itself.
|
|
CORE_METRICS: dict[str, list[str]] = {
|
|
"icmp": ["is_up", "response_ms"],
|
|
"tcp": ["is_up", "response_ms"],
|
|
"dns": ["is_up", "response_ms"],
|
|
"http": ["is_up", "response_ms"],
|
|
"steward": ["open_fds", "open_fds_pct"],
|
|
}
|
|
|
|
|
|
def metric_catalog() -> dict[str, list[str]]:
|
|
"""Every source an alert rule can watch, and its metric names.
|
|
|
|
Core's own sources, then each loaded plugin's get_alert_metrics() export
|
|
({source: [metric names]}), so the catalog is written by whatever emits
|
|
the metrics and a disabled plugin's sources drop out. An empty list means
|
|
the names are dynamic (snmp: the OID labels recorded so far).
|
|
"""
|
|
from steward.core.plugin_manager import get_loaded_plugin_modules
|
|
catalog = {src: list(names) for src, names in CORE_METRICS.items()}
|
|
for name, module in get_loaded_plugin_modules().items():
|
|
hook = getattr(module, "get_alert_metrics", None)
|
|
if hook is None:
|
|
continue
|
|
try:
|
|
contributed = hook() or {}
|
|
except Exception:
|
|
logger.exception("Plugin %r: get_alert_metrics() raised; its metrics are skipped", name)
|
|
continue
|
|
for src, names in contributed.items():
|
|
catalog[str(src)] = [str(n) for n in names]
|
|
return catalog
|
|
|
|
|
|
def source_modules() -> list[str]:
|
|
"""The sources in picker order: core's first, then plugins'."""
|
|
return list(metric_catalog())
|
|
|
|
|
|
# States that need someone's attention: pending (breaching, not yet for long
|
|
# enough), firing, and acknowledged-but-still-breaching.
|
|
ACTIVE_STATES = (AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending)
|
|
|
|
|
|
async def active_alerts(db) -> list:
|
|
"""(AlertState, AlertRule) rows that need attention, latest transition
|
|
first: not yet resolved, or with a series that has gone silent (no_data)."""
|
|
return list((await db.execute(
|
|
select(AlertState, AlertRule)
|
|
.join(AlertRule, AlertState.rule_id == AlertRule.id)
|
|
.where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
|
|
.order_by(AlertState.last_transition_at.desc())
|
|
)).all())
|
|
|
|
|
|
def alert_label(state: AlertState) -> str:
|
|
"""What an alert is, in one word: its state, or "no_data" for a quiet rule
|
|
whose series has gone silent (a firing alert that goes silent stays
|
|
"firing"; check no_data_since for the mark)."""
|
|
if state.no_data_since is not None and state.state == AlertStateEnum.inactive:
|
|
return "no_data"
|
|
return state.state.value
|
|
|
|
|
|
async def latest_value(db, rule: AlertRule) -> float | None:
|
|
"""The newest sample of the series a rule watches, or None before the first."""
|
|
return (await db.execute(
|
|
select(PluginMetric.value)
|
|
.where(PluginMetric.source_module == rule.source_module,
|
|
PluginMetric.resource_name == rule.resource_name,
|
|
PluginMetric.metric_name == rule.metric_name)
|
|
.order_by(PluginMetric.recorded_at.desc()).limit(1)
|
|
)).scalar_one_or_none()
|
|
|
|
|
|
# Sources whose resources have no page of their own: the plugin's page lists them.
|
|
PLUGIN_PAGES = {"traefik": "/plugins/traefik/", "unifi": "/plugins/unifi/",
|
|
"snmp": "/plugins/snmp/"}
|
|
|
|
|
|
async def resource_url(db, rule: AlertRule) -> str | None:
|
|
"""Where the thing a rule watches is shown, or None when nothing shows it.
|
|
|
|
host_agent resources are the host's name, or "<host>:<mount|nic|core>";
|
|
docker's are "<host>/<container>"; a monitor's is its name under its type.
|
|
"""
|
|
from steward.models.hosts import Host
|
|
source, resource = rule.source_module, rule.resource_name
|
|
if source == "host_agent":
|
|
name = resource.split(":", 1)[0]
|
|
host_id = (await db.execute(
|
|
select(Host.id).where(Host.name == name).limit(1))).scalar_one_or_none()
|
|
return f"/hosts/{host_id}" if host_id else None
|
|
if source in MONITOR_TYPE_VALUES:
|
|
monitor_id = (await db.execute(
|
|
select(Monitor.id).where(Monitor.type == source, Monitor.name == resource).limit(1)
|
|
)).scalar_one_or_none()
|
|
return f"/monitors/{monitor_id}" if monitor_id else None
|
|
if source == "docker" and "/" in resource:
|
|
host_name, container = resource.split("/", 1)
|
|
host_id = (await db.execute(
|
|
select(Host.id).where(Host.name == host_name).limit(1))).scalar_one_or_none()
|
|
return f"/plugins/docker/container/{host_id}/{container}" if host_id else None
|
|
return PLUGIN_PAGES.get(source)
|
|
|
|
|
|
# Metrics whose value is a word: the sentence says the word, not the condition.
|
|
_BINARY = ("is_up", "is_healthy")
|
|
|
|
|
|
def alert_sentence(rule: AlertRule, label: str, value: float | None,
|
|
since: datetime | None, now: datetime) -> dict:
|
|
"""An alert as one sentence around its resource: {"lead", "resource",
|
|
"tail"}, read lead + resource + tail, e.g. "cpu_pct on " + "web-1" +
|
|
" is 94% (> 90%), firing for 12m". label is alert_label(state); since is
|
|
when that label began (no_data_since for no_data)."""
|
|
from steward.core.formatting import human_duration, metric_value
|
|
metric = rule.metric_name
|
|
held = human_duration((now - since).total_seconds()) if since else None
|
|
if label == "no_data":
|
|
return {"lead": f"No {metric} from ", "resource": rule.resource_name,
|
|
"tail": f" for {held}" if held else ""}
|
|
for_how_long = f", {label} for {held}" if held else f", {label}"
|
|
if value is None:
|
|
reading = " has no reading yet"
|
|
elif metric in _BINARY:
|
|
return {"lead": "", "resource": rule.resource_name,
|
|
"tail": f" is {metric_value(metric, value)}{for_how_long}"}
|
|
else:
|
|
reading = f" is {metric_value(metric, value)}"
|
|
condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})"
|
|
return {"lead": f"{metric} on ", "resource": rule.resource_name,
|
|
"tail": f"{reading}{condition}{for_how_long}"}
|
|
|
|
|
|
async def active_rows(db, now: datetime) -> list[dict]:
|
|
"""The alerts page's active list: each alert's state, rule, sentence and
|
|
the link to what it watches."""
|
|
rows = []
|
|
for state, rule in await active_alerts(db):
|
|
label = alert_label(state)
|
|
since = state.no_data_since if label == "no_data" else state.last_transition_at
|
|
rows.append({
|
|
"state": state, "rule": rule, "label": label,
|
|
"says": alert_sentence(rule, label, await latest_value(db, rule), since, now),
|
|
"url": await resource_url(db, rule),
|
|
})
|
|
return rows
|
|
|
|
|
|
async def active_count(db) -> int:
|
|
"""How many alerts need attention: the sidebar's number."""
|
|
from sqlalchemy import func
|
|
return (await db.execute(
|
|
select(func.count()).select_from(AlertState)
|
|
.where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
|
|
)).scalar_one()
|
|
|
|
|
|
async def list_rules(db) -> list[AlertRule]:
|
|
return list((await db.execute(select(AlertRule).order_by(AlertRule.name))).scalars())
|
|
|
|
|
|
async def history(db, since: datetime, rule_id: str | None = None,
|
|
limit: int = 200) -> list[tuple[AlertEvent, AlertRule]]:
|
|
"""Alert transitions since `since`, newest first, with their rule; one
|
|
rule's when `rule_id` is given. Returns up to limit+1 rows so a caller can
|
|
tell the list was cut."""
|
|
stmt = (select(AlertEvent, AlertRule)
|
|
.join(AlertRule, AlertEvent.rule_id == AlertRule.id)
|
|
.where(AlertEvent.transitioned_at >= since)
|
|
.order_by(AlertEvent.transitioned_at.desc())
|
|
.limit(limit + 1))
|
|
if rule_id:
|
|
stmt = stmt.where(AlertEvent.rule_id == rule_id)
|
|
return [tuple(row) for row in (await db.execute(stmt)).all()]
|
|
|
|
|
|
async def list_maintenance_windows(db) -> list[MaintenanceWindow]:
|
|
"""Every window, latest start first."""
|
|
return list((await db.execute(
|
|
select(MaintenanceWindow).order_by(MaintenanceWindow.start_at.desc())
|
|
)).scalars())
|
|
|
|
|
|
async def resources_for(db, source_module: str) -> list[str]:
|
|
"""Sorted distinct resource names for a source module.
|
|
|
|
For monitor types (icmp/tcp/dns/http), supplements with monitor names so
|
|
the list is populated even before any metrics have been recorded.
|
|
"""
|
|
result = await db.execute(
|
|
select(PluginMetric.resource_name)
|
|
.where(PluginMetric.source_module == source_module)
|
|
.distinct()
|
|
.order_by(PluginMetric.resource_name)
|
|
)
|
|
resources: set[str] = {r for (r,) in result}
|
|
if source_module in MONITOR_TYPE_VALUES:
|
|
mon_result = await db.execute(
|
|
select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name))
|
|
resources.update(name for (name,) in mon_result)
|
|
return sorted(resources)
|
|
|
|
|
|
async def metrics_for(db, source_module: str) -> list[str]:
|
|
"""The metric names a rule on this source can watch: the static catalog, or
|
|
for dynamic sources (snmp) the names recorded so far."""
|
|
metrics = metric_catalog().get(source_module, [])
|
|
if source_module and not metrics:
|
|
result = await db.execute(
|
|
select(PluginMetric.metric_name)
|
|
.where(PluginMetric.source_module == source_module)
|
|
.distinct()
|
|
.order_by(PluginMetric.metric_name))
|
|
metrics = [r for (r,) in result]
|
|
return metrics
|
|
|
|
|
|
def _choices(values: list[str], limit: int = 20) -> str:
|
|
if not values:
|
|
return "none yet"
|
|
shown = ", ".join(values[:limit])
|
|
return shown + (f" (and {len(values) - limit} more)" if len(values) > limit else "")
|
|
|
|
|
|
async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
|
|
operator: str, threshold, consecutive_failures_required=1,
|
|
consecutive_ok_required=1, notify_no_data=True) -> dict:
|
|
"""Validated column values for a new or edited alert rule.
|
|
|
|
The same choices the rule form offers: a known source, a resource that
|
|
source has reported (or, for monitors, a monitor of that type), and a
|
|
metric it emits. ValueError says what was wrong and what is valid.
|
|
"""
|
|
name = (name or "").strip()
|
|
if not name:
|
|
raise ValueError("an alert rule needs a name.")
|
|
source_module = (source_module or "").strip()
|
|
catalog = metric_catalog()
|
|
if source_module not in catalog:
|
|
raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.")
|
|
resource_name = (resource_name or "").strip()
|
|
resources = await resources_for(db, source_module)
|
|
if resource_name not in resources:
|
|
raise ValueError(f"{source_module} has no resource {resource_name!r}. "
|
|
f"Known: {_choices(resources)}.")
|
|
metric_name = (metric_name or "").strip()
|
|
metrics = await metrics_for(db, source_module)
|
|
if metric_name not in metrics:
|
|
raise ValueError(f"{source_module} does not report {metric_name!r}. "
|
|
f"Its metrics: {_choices(metrics)}.")
|
|
try:
|
|
op = AlertOperator((operator or "").strip())
|
|
except ValueError:
|
|
raise ValueError(f"operator {operator!r} is not one of "
|
|
f"{', '.join(o.value for o in AlertOperator)}.") from None
|
|
try:
|
|
threshold = float(threshold)
|
|
except (TypeError, ValueError):
|
|
raise ValueError(f"threshold must be a number, got {threshold!r}.") from None
|
|
try:
|
|
consecutive = (1 if consecutive_failures_required in (None, "")
|
|
else int(consecutive_failures_required))
|
|
except (TypeError, ValueError):
|
|
raise ValueError("consecutive checks must be a whole number.") from None
|
|
if consecutive < 1:
|
|
raise ValueError("consecutive checks must be at least 1.")
|
|
try:
|
|
ok_checks = (1 if consecutive_ok_required in (None, "")
|
|
else int(consecutive_ok_required))
|
|
except (TypeError, ValueError):
|
|
raise ValueError("checks to clear must be a whole number.") from None
|
|
if ok_checks < 1:
|
|
raise ValueError("checks to clear must be at least 1.")
|
|
return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
|
|
"metric_name": metric_name, "operator": op, "threshold": threshold,
|
|
"consecutive_failures_required": consecutive,
|
|
"consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data)}
|
|
|
|
|
|
# Changing any of these changes what the rule is watching, so whatever state
|
|
# it had built up no longer means anything.
|
|
WATCHED_FIELDS = ("source_module", "resource_name", "metric_name", "operator", "threshold")
|
|
|
|
|
|
async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool | None = None) -> None:
|
|
"""Apply validated fields (and optionally enabled) to a rule, in the
|
|
caller's transaction. A change to what it watches, or disabling or
|
|
re-enabling it, puts its alert back to inactive and starts its silence
|
|
watch over (core.alerts.reset_rule_state)."""
|
|
from steward.core.alerts import reset_rule_state
|
|
changed = [k for k in WATCHED_FIELDS if k in fields and getattr(rule, k) != fields[k]]
|
|
for k, v in fields.items():
|
|
setattr(rule, k, v)
|
|
disabling = enabled is False and rule.enabled
|
|
enabling = enabled is True and not rule.enabled
|
|
if enabled is not None:
|
|
rule.enabled = bool(enabled)
|
|
if disabling:
|
|
await reset_rule_state(db, rule, "rule disabled")
|
|
elif enabling:
|
|
await reset_rule_state(db, rule, "rule enabled")
|
|
elif changed:
|
|
await reset_rule_state(db, rule, "rule changed: " + ", ".join(changed))
|
|
|
|
|
|
async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
|
|
ansible_action: dict | None = None) -> AlertRule:
|
|
"""Insert a rule and its inactive state row, in the caller's transaction."""
|
|
rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
|
|
db.add(rule)
|
|
await db.flush()
|
|
db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
|
|
last_transition_at=datetime.now(timezone.utc)))
|
|
return rule
|
|
|
|
|
|
async def acknowledge(db, rule_id: str, user_id: str) -> AlertState:
|
|
"""Acknowledge a firing alert, in the caller's transaction.
|
|
|
|
LookupError when the rule has no state; ValueError when it is not firing.
|
|
"""
|
|
state = (await db.execute(
|
|
select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one_or_none()
|
|
if state is None:
|
|
raise LookupError(rule_id)
|
|
if state.state != AlertStateEnum.firing:
|
|
raise ValueError(f"the alert is {state.state.value}, not firing; only a firing alert "
|
|
"can be acknowledged.")
|
|
now = datetime.now(timezone.utc)
|
|
state.state = AlertStateEnum.acknowledged
|
|
state.acknowledged_by = user_id
|
|
state.acknowledged_at = now
|
|
state.last_transition_at = now
|
|
return state
|
|
|
|
|
|
def window_fields(*, name: str, start_at: datetime, end_at: datetime,
|
|
scope_source: str | None, scope_resource: str | None) -> dict:
|
|
"""Validated columns for a maintenance window. A resource needs a source."""
|
|
name = (name or "").strip()
|
|
if not name:
|
|
raise ValueError("a maintenance window needs a name.")
|
|
if end_at <= start_at:
|
|
raise ValueError("the window must end after it starts.")
|
|
scope_source = (scope_source or "").strip() or None
|
|
scope_resource = (scope_resource or "").strip() or None
|
|
catalog = metric_catalog()
|
|
if scope_source is not None and scope_source not in catalog:
|
|
raise ValueError(f"source {scope_source!r} is not one of {', '.join(catalog)}.")
|
|
if scope_source is None:
|
|
scope_resource = None # can't have resource without source
|
|
return {"name": name[:128], "start_at": start_at, "end_at": end_at,
|
|
"scope_source": scope_source, "scope_resource": scope_resource}
|
|
|
|
|
|
def window_scope(w) -> str:
|
|
if not w.scope_source:
|
|
return "all"
|
|
return f"{w.scope_source}/{w.scope_resource}" if w.scope_resource else f"{w.scope_source}/*"
|