bvandeusenandClaude Opus 5.5
3a772716ae
#5212 "Plain labels: no raw ids, scopes, metric keys or unexplained jargon", batch 1: - Audit log: each action reads as a sentence ("paused monitor"), with the code as its tooltip. Detail shows flags as yes/no, lists joined, a run's scope by name, spans as spans, and no ids the entity column already gives. The filter is now a picker of areas. - Alert rules, history and Status say the condition in words: "is down", "Fullest disk >= 85%". Readings use metric_value, and history times use |ago. - Rule form: sources and metrics by name, the threshold's unit, "fires when it is down/up" for an up/down metric instead of 0.5, and a metric reference built from the catalog. - Schedules and Maintenance: viewers no longer see Run now, Pause, Edit, Delete or New. Intervals and next runs read as "every 2h" and "in 3h"; a scope reads "Every alert on Host agent". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
457 lines
20 KiB
Python
457 lines
20 KiB
Python
"""Alert queries and validation shared by the alerts pages and the MCP tools.
|
|
|
|
Each function takes an open session (or plain values), with no Quart request
|
|
context, so the routes and the /mcp tools (which run outside Quart's
|
|
pipeline) call the same code and refuse the same input for the same reasons.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import func, or_, select
|
|
|
|
from steward.models.alerts import (
|
|
AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
|
|
)
|
|
from steward.models.metrics import PluginMetric, PluginMetricLatest
|
|
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# What core itself records. Unified monitors emit metrics under their TYPE
|
|
# (icmp/tcp/dns/http), each with is_up (1.0 up / 0.0 down) and response_ms
|
|
# (where the probe times it); "steward" is the app watching itself.
|
|
CORE_METRICS: dict[str, list[str]] = {
|
|
"icmp": ["is_up", "response_ms"],
|
|
"tcp": ["is_up", "response_ms"],
|
|
"dns": ["is_up", "response_ms"],
|
|
"http": ["is_up", "response_ms"],
|
|
"steward": ["open_fds", "open_fds_pct"],
|
|
}
|
|
|
|
|
|
def metric_catalog() -> dict[str, list[str]]:
|
|
"""Every source an alert rule can watch, and its metric names.
|
|
|
|
Core's own sources, then each loaded plugin's get_alert_metrics() export
|
|
({source: [metric names]}), so the catalog is written by whatever emits
|
|
the metrics and a disabled plugin's sources drop out. An empty list means
|
|
the names are dynamic (snmp: the OID labels recorded so far).
|
|
"""
|
|
from steward.core.plugin_manager import get_loaded_plugin_modules
|
|
catalog = {src: list(names) for src, names in CORE_METRICS.items()}
|
|
for name, module in get_loaded_plugin_modules().items():
|
|
hook = getattr(module, "get_alert_metrics", None)
|
|
if hook is None:
|
|
continue
|
|
try:
|
|
contributed = hook() or {}
|
|
except Exception:
|
|
logger.exception("Plugin %r: get_alert_metrics() raised; its metrics are skipped", name)
|
|
continue
|
|
for src, names in contributed.items():
|
|
catalog[str(src)] = [str(n) for n in names]
|
|
return catalog
|
|
|
|
|
|
def source_modules() -> list[str]:
|
|
"""The sources in picker order: core's first, then plugins'."""
|
|
return list(metric_catalog())
|
|
|
|
|
|
# States that need someone's attention: pending (breaching, not yet for long
|
|
# enough), firing, and acknowledged-but-still-breaching.
|
|
ACTIVE_STATES = (AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending)
|
|
|
|
|
|
async def active_alerts(db) -> list:
|
|
"""(AlertState, AlertRule) rows that need attention, latest transition
|
|
first: not yet resolved, or with a series that has gone silent (no_data)."""
|
|
return list((await db.execute(
|
|
select(AlertState, AlertRule)
|
|
.join(AlertRule, AlertState.rule_id == AlertRule.id)
|
|
.where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
|
|
.order_by(AlertState.last_transition_at.desc())
|
|
)).all())
|
|
|
|
|
|
def alert_label(state: AlertState) -> str:
|
|
"""What an alert is, in one word: its state, or "no_data" for a quiet rule
|
|
whose series has gone silent (a firing alert that goes silent stays
|
|
"firing"; check no_data_since for the mark)."""
|
|
if state.no_data_since is not None and state.state == AlertStateEnum.inactive:
|
|
return "no_data"
|
|
return state.state.value
|
|
|
|
|
|
async def latest_value(db, rule: AlertRule) -> float | None:
|
|
"""The newest sample of the series a rule watches, or None before the first
|
|
(a primary-key read of the latest-value table, #5208)."""
|
|
return (await db.execute(
|
|
select(PluginMetricLatest.value)
|
|
.where(PluginMetricLatest.source_module == rule.source_module,
|
|
PluginMetricLatest.resource_name == rule.resource_name,
|
|
PluginMetricLatest.metric_name == rule.metric_name)
|
|
)).scalar_one_or_none()
|
|
|
|
|
|
# Sources whose resources have no page of their own: the plugin's page lists them.
|
|
PLUGIN_PAGES = {"traefik": "/plugins/traefik/", "unifi": "/plugins/unifi/",
|
|
"snmp": "/plugins/snmp/"}
|
|
|
|
|
|
async def resource_url(db, rule: AlertRule) -> str | None:
|
|
"""Where the thing a rule watches is shown, or None when nothing shows it.
|
|
|
|
host_agent resources are the host's name, or "<host>:<mount|nic|core>";
|
|
docker's are "<host>/<container>"; a monitor's is its name under its type.
|
|
"""
|
|
from steward.models.hosts import Host
|
|
source, resource = rule.source_module, rule.resource_name
|
|
if source == "host_agent":
|
|
name = resource.split(":", 1)[0]
|
|
host_id = (await db.execute(
|
|
select(Host.id).where(Host.name == name).limit(1))).scalar_one_or_none()
|
|
return f"/hosts/{host_id}" if host_id else None
|
|
if source in MONITOR_TYPE_VALUES:
|
|
monitor_id = (await db.execute(
|
|
select(Monitor.id).where(Monitor.type == source, Monitor.name == resource).limit(1)
|
|
)).scalar_one_or_none()
|
|
return f"/monitors/{monitor_id}" if monitor_id else None
|
|
if source == "docker" and "/" in resource:
|
|
host_name, container = resource.split("/", 1)
|
|
host_id = (await db.execute(
|
|
select(Host.id).where(Host.name == host_name).limit(1))).scalar_one_or_none()
|
|
return f"/plugins/docker/container/{host_id}/{container}" if host_id else None
|
|
return PLUGIN_PAGES.get(source)
|
|
|
|
|
|
# Metrics whose value is a word: the sentence says the word, not the condition.
|
|
_BINARY = ("is_up", "is_healthy")
|
|
|
|
|
|
def rule_condition(rule: AlertRule) -> str:
|
|
"""What a rule fires on, in words: "CPU > 90%", "Fullest disk >= 85%",
|
|
or "is down" for an up/down metric rather than "is_up < 0.5"."""
|
|
from steward.core.alerts import _is_breached
|
|
from steward.core.formatting import metric_label, metric_value
|
|
metric = rule.metric_name
|
|
if metric in _BINARY:
|
|
fires_on = [metric_value(metric, v) for v in (1.0, 0.0)
|
|
if _is_breached(v, rule.operator, rule.threshold)]
|
|
if len(fires_on) == 1:
|
|
return f"is {fires_on[0]}"
|
|
return f"{metric_label(metric)} {rule.operator.value} {metric_value(metric, rule.threshold)}"
|
|
|
|
|
|
def alert_sentence(rule: AlertRule, label: str, value: float | None,
|
|
since: datetime | None, now: datetime) -> dict:
|
|
"""An alert as one sentence around its resource: {"lead", "resource",
|
|
"tail"}, read lead + resource + tail, e.g. "CPU on " + "web-1" +
|
|
" is 94% (> 90%), firing for 12m". label is alert_label(state); since is
|
|
when that label began (no_data_since for no_data)."""
|
|
from steward.core.formatting import human_duration, metric_label, metric_value
|
|
metric = rule.metric_name
|
|
said = metric_label(metric)
|
|
held = human_duration((now - since).total_seconds()) if since else None
|
|
if label == "no_data":
|
|
return {"lead": f"No {said} reading from ", "resource": rule.resource_name,
|
|
"tail": f" for {held}" if held else ""}
|
|
# An acknowledged alert is still breaching: say for how long it has been,
|
|
# not how long ago someone acknowledged it (the page says that apart).
|
|
word = "breaching" if label == "acknowledged" else label
|
|
for_how_long = f", {word} for {held}" if held else f", {word}"
|
|
if value is None:
|
|
reading = " has no reading yet"
|
|
elif metric in _BINARY:
|
|
return {"lead": "", "resource": rule.resource_name,
|
|
"tail": f" is {metric_value(metric, value)}{for_how_long}"}
|
|
else:
|
|
reading = f" is {metric_value(metric, value)}"
|
|
condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})"
|
|
return {"lead": f"{said} on ", "resource": rule.resource_name,
|
|
"tail": f"{reading}{condition}{for_how_long}"}
|
|
|
|
|
|
# The order the active list reads in: what is breaching and unhandled, then
|
|
# what has gone silent, then what someone has taken, then what may yet clear.
|
|
ACTIVE_ORDER = {"firing": 0, "no_data": 1, "acknowledged": 2, "pending": 3}
|
|
|
|
_BREACHING = (AlertStateEnum.pending.value, AlertStateEnum.firing.value,
|
|
AlertStateEnum.acknowledged.value)
|
|
|
|
|
|
async def breach_started(db, rule_ids) -> dict[str, datetime]:
|
|
"""{rule_id: when its current breach began}: the newest transition into
|
|
pending or firing from a quiet state. Acknowledging moves an alert's
|
|
last_transition_at, so that column cannot say this."""
|
|
if not rule_ids:
|
|
return {}
|
|
return dict((await db.execute(
|
|
select(AlertEvent.rule_id, func.max(AlertEvent.transitioned_at))
|
|
.where(AlertEvent.rule_id.in_(list(rule_ids)),
|
|
AlertEvent.from_state.not_in(_BREACHING),
|
|
AlertEvent.to_state.in_((AlertStateEnum.pending.value, AlertStateEnum.firing.value)))
|
|
.group_by(AlertEvent.rule_id)
|
|
)).all())
|
|
|
|
|
|
async def active_rows(db, now: datetime) -> list[dict]:
|
|
"""The alerts page's active list, firing first, then no data, then
|
|
acknowledged, then pending; within each the longest-running first. Each
|
|
row: its state, rule, label, sentence, the link to what it watches, when
|
|
the breach began, and who acknowledged it."""
|
|
from steward.models.users import User
|
|
alerts = await active_alerts(db)
|
|
started = await breach_started(db, {rule.id for state, rule in alerts
|
|
if state.state.value in _BREACHING})
|
|
ackers = {s.acknowledged_by for s, _ in alerts if s.acknowledged_by}
|
|
names = dict((await db.execute(
|
|
select(User.id, User.username).where(User.id.in_(ackers)))).all()) if ackers else {}
|
|
rows = []
|
|
for state, rule in alerts:
|
|
label = alert_label(state)
|
|
if label == "no_data":
|
|
since = state.no_data_since
|
|
elif label == "acknowledged":
|
|
since = started.get(rule.id, state.last_transition_at)
|
|
else:
|
|
since = state.last_transition_at
|
|
rows.append({
|
|
"state": state, "rule": rule, "label": label, "since": since,
|
|
"breach_since": started.get(rule.id) if label != "no_data" else None,
|
|
"acked_by": names.get(state.acknowledged_by, state.acknowledged_by),
|
|
"says": alert_sentence(rule, label, await latest_value(db, rule), since, now),
|
|
"url": await resource_url(db, rule),
|
|
})
|
|
epoch = datetime.max.replace(tzinfo=timezone.utc)
|
|
rows.sort(key=lambda r: (ACTIVE_ORDER.get(r["label"], 9), r["since"] or epoch))
|
|
return rows
|
|
|
|
|
|
def watches_host(rule: AlertRule, host_name: str, monitor_keys: set) -> bool:
|
|
"""Whether a rule watches something on this host: its agent series
|
|
("<host>" or "<host>:<part>"), one of its containers ("<host>/<name>"),
|
|
or one of its monitors ((type, name) in monitor_keys)."""
|
|
source, resource = rule.source_module, rule.resource_name
|
|
if source == "host_agent":
|
|
return resource == host_name or resource.startswith(host_name + ":")
|
|
if source == "docker":
|
|
return resource.startswith(host_name + "/")
|
|
return (source, resource) in monitor_keys
|
|
|
|
|
|
async def active_count(db) -> int:
|
|
"""How many alerts need attention: the sidebar's number."""
|
|
from sqlalchemy import func
|
|
return (await db.execute(
|
|
select(func.count()).select_from(AlertState)
|
|
.where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
|
|
)).scalar_one()
|
|
|
|
|
|
async def list_rules(db) -> list[AlertRule]:
|
|
return list((await db.execute(select(AlertRule).order_by(AlertRule.name))).scalars())
|
|
|
|
|
|
async def history(db, since: datetime, rule_id: str | None = None,
|
|
limit: int = 200, before: str | None = None) -> list[tuple[AlertEvent, AlertRule]]:
|
|
"""Alert transitions since `since`, newest first, with their rule; one
|
|
rule's when `rule_id` is given; only those older than the `before` cursor
|
|
when given (core.paging). Returns up to limit+1 rows so a caller can tell
|
|
the list was cut."""
|
|
from steward.core.paging import keyset
|
|
stmt = (select(AlertEvent, AlertRule)
|
|
.join(AlertRule, AlertEvent.rule_id == AlertRule.id)
|
|
.where(AlertEvent.transitioned_at >= since))
|
|
if rule_id:
|
|
stmt = stmt.where(AlertEvent.rule_id == rule_id)
|
|
stmt = keyset(stmt, AlertEvent.transitioned_at, AlertEvent.id, before, limit)
|
|
return [tuple(row) for row in (await db.execute(stmt)).all()]
|
|
|
|
|
|
async def list_maintenance_windows(db) -> list[MaintenanceWindow]:
|
|
"""Every window, latest start first."""
|
|
return list((await db.execute(
|
|
select(MaintenanceWindow).order_by(MaintenanceWindow.start_at.desc())
|
|
)).scalars())
|
|
|
|
|
|
async def resources_for(db, source_module: str) -> list[str]:
|
|
"""Sorted distinct resource names for a source module.
|
|
|
|
For monitor types (icmp/tcp/dns/http), supplements with monitor names so
|
|
the list is populated even before any metrics have been recorded.
|
|
"""
|
|
result = await db.execute(
|
|
select(PluginMetric.resource_name)
|
|
.where(PluginMetric.source_module == source_module)
|
|
.distinct()
|
|
.order_by(PluginMetric.resource_name)
|
|
)
|
|
resources: set[str] = {r for (r,) in result}
|
|
if source_module in MONITOR_TYPE_VALUES:
|
|
mon_result = await db.execute(
|
|
select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name))
|
|
resources.update(name for (name,) in mon_result)
|
|
return sorted(resources)
|
|
|
|
|
|
async def metrics_for(db, source_module: str) -> list[str]:
|
|
"""The metric names a rule on this source can watch: the static catalog, or
|
|
for dynamic sources (snmp) the names recorded so far."""
|
|
metrics = metric_catalog().get(source_module, [])
|
|
if source_module and not metrics:
|
|
result = await db.execute(
|
|
select(PluginMetric.metric_name)
|
|
.where(PluginMetric.source_module == source_module)
|
|
.distinct()
|
|
.order_by(PluginMetric.metric_name))
|
|
metrics = [r for (r,) in result]
|
|
return metrics
|
|
|
|
|
|
def _choices(values: list[str], limit: int = 20) -> str:
|
|
if not values:
|
|
return "none yet"
|
|
shown = ", ".join(values[:limit])
|
|
return shown + (f" (and {len(values) - limit} more)" if len(values) > limit else "")
|
|
|
|
|
|
async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
|
|
operator: str, threshold, consecutive_failures_required=1,
|
|
consecutive_ok_required=1, notify_no_data=True) -> dict:
|
|
"""Validated column values for a new or edited alert rule.
|
|
|
|
The same choices the rule form offers: a known source, a resource that
|
|
source has reported (or, for monitors, a monitor of that type), and a
|
|
metric it emits. ValueError says what was wrong and what is valid.
|
|
"""
|
|
name = (name or "").strip()
|
|
if not name:
|
|
raise ValueError("an alert rule needs a name.")
|
|
source_module = (source_module or "").strip()
|
|
catalog = metric_catalog()
|
|
if source_module not in catalog:
|
|
raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.")
|
|
resource_name = (resource_name or "").strip()
|
|
resources = await resources_for(db, source_module)
|
|
if resource_name not in resources:
|
|
raise ValueError(f"{source_module} has no resource {resource_name!r}. "
|
|
f"Known: {_choices(resources)}.")
|
|
metric_name = (metric_name or "").strip()
|
|
metrics = await metrics_for(db, source_module)
|
|
if metric_name not in metrics:
|
|
raise ValueError(f"{source_module} does not report {metric_name!r}. "
|
|
f"Its metrics: {_choices(metrics)}.")
|
|
try:
|
|
op = AlertOperator((operator or "").strip())
|
|
except ValueError:
|
|
raise ValueError(f"operator {operator!r} is not one of "
|
|
f"{', '.join(o.value for o in AlertOperator)}.") from None
|
|
try:
|
|
threshold = float(threshold)
|
|
except (TypeError, ValueError):
|
|
raise ValueError(f"threshold must be a number, got {threshold!r}.") from None
|
|
try:
|
|
consecutive = (1 if consecutive_failures_required in (None, "")
|
|
else int(consecutive_failures_required))
|
|
except (TypeError, ValueError):
|
|
raise ValueError("consecutive checks must be a whole number.") from None
|
|
if consecutive < 1:
|
|
raise ValueError("consecutive checks must be at least 1.")
|
|
try:
|
|
ok_checks = (1 if consecutive_ok_required in (None, "")
|
|
else int(consecutive_ok_required))
|
|
except (TypeError, ValueError):
|
|
raise ValueError("checks to clear must be a whole number.") from None
|
|
if ok_checks < 1:
|
|
raise ValueError("checks to clear must be at least 1.")
|
|
return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
|
|
"metric_name": metric_name, "operator": op, "threshold": threshold,
|
|
"consecutive_failures_required": consecutive,
|
|
"consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data)}
|
|
|
|
|
|
# Changing any of these changes what the rule is watching, so whatever state
|
|
# it had built up no longer means anything.
|
|
WATCHED_FIELDS = ("source_module", "resource_name", "metric_name", "operator", "threshold")
|
|
|
|
|
|
async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool | None = None) -> None:
|
|
"""Apply validated fields (and optionally enabled) to a rule, in the
|
|
caller's transaction. A change to what it watches, or disabling or
|
|
re-enabling it, puts its alert back to inactive and starts its silence
|
|
watch over (core.alerts.reset_rule_state)."""
|
|
from steward.core.alerts import reset_rule_state
|
|
changed = [k for k in WATCHED_FIELDS if k in fields and getattr(rule, k) != fields[k]]
|
|
for k, v in fields.items():
|
|
setattr(rule, k, v)
|
|
disabling = enabled is False and rule.enabled
|
|
enabling = enabled is True and not rule.enabled
|
|
if enabled is not None:
|
|
rule.enabled = bool(enabled)
|
|
if disabling:
|
|
await reset_rule_state(db, rule, "rule disabled")
|
|
elif enabling:
|
|
await reset_rule_state(db, rule, "rule enabled")
|
|
elif changed:
|
|
await reset_rule_state(db, rule, "rule changed: " + ", ".join(changed))
|
|
|
|
|
|
async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
|
|
ansible_action: dict | None = None) -> AlertRule:
|
|
"""Insert a rule and its inactive state row, in the caller's transaction."""
|
|
rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
|
|
db.add(rule)
|
|
await db.flush()
|
|
db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
|
|
last_transition_at=datetime.now(timezone.utc)))
|
|
return rule
|
|
|
|
|
|
async def acknowledge(db, rule_id: str, user_id: str) -> AlertState:
|
|
"""Acknowledge a firing alert, in the caller's transaction.
|
|
|
|
LookupError when the rule has no state; ValueError when it is not firing.
|
|
"""
|
|
state = (await db.execute(
|
|
select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one_or_none()
|
|
if state is None:
|
|
raise LookupError(rule_id)
|
|
if state.state != AlertStateEnum.firing:
|
|
raise ValueError(f"the alert is {state.state.value}, not firing; only a firing alert "
|
|
"can be acknowledged.")
|
|
now = datetime.now(timezone.utc)
|
|
state.state = AlertStateEnum.acknowledged
|
|
state.acknowledged_by = user_id
|
|
state.acknowledged_at = now
|
|
state.last_transition_at = now
|
|
return state
|
|
|
|
|
|
def window_fields(*, name: str, start_at: datetime, end_at: datetime,
|
|
scope_source: str | None, scope_resource: str | None) -> dict:
|
|
"""Validated columns for a maintenance window. A resource needs a source."""
|
|
name = (name or "").strip()
|
|
if not name:
|
|
raise ValueError("a maintenance window needs a name.")
|
|
if end_at <= start_at:
|
|
raise ValueError("the window must end after it starts.")
|
|
scope_source = (scope_source or "").strip() or None
|
|
scope_resource = (scope_resource or "").strip() or None
|
|
catalog = metric_catalog()
|
|
if scope_source is not None and scope_source not in catalog:
|
|
raise ValueError(f"source {scope_source!r} is not one of {', '.join(catalog)}.")
|
|
if scope_source is None:
|
|
scope_resource = None # can't have resource without source
|
|
return {"name": name[:128], "start_at": start_at, "end_at": end_at,
|
|
"scope_source": scope_source, "scope_resource": scope_resource}
|
|
|
|
|
|
def window_scope(w) -> str:
|
|
if not w.scope_source:
|
|
return "all"
|
|
return f"{w.scope_source}/{w.scope_resource}" if w.scope_resource else f"{w.scope_source}/*"
|