bvandeusenandClaude Opus 5.5
c8dbe811f8
#5467 "Steward's own health: metrics, rules can watch them, a page that shows them". The self-monitor tick now records, under source "steward", resource "process": ingest_posts / ingest_failed / ingest_slowest_ms (the host-agent posts since the last tick, tallied in steward/core/health.py), db_roundtrip_ms (a timed SELECT 1), and unwatched_secs (the gap before this tick past watch.GAP_S, from Steward's own last sample). The hourly cleanup records cleanup_secs. The open-file pair is still recorded, and is now only skipped on its own where /proc is absent. All of these are in CORE_METRICS, so an alert rule can watch them under "Steward itself". Settings → System (/settings/system/, admin) shows a verdict first (failed posts, a gap in watching, open files near the limit), then each measure now and over the last day, with a sparkline scaled to its own peak. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
837 lines
40 KiB
Python
837 lines
40 KiB
Python
"""Alert queries and validation shared by the alerts pages and the MCP tools.
|
|
|
|
Each function takes an open session (or plain values), with no Quart request
|
|
context, so the routes and the /mcp tools (which run outside Quart's
|
|
pipeline) call the same code and refuse the same input for the same reasons.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import func, or_, select
|
|
|
|
from steward.models.alerts import (
|
|
AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
|
|
)
|
|
from steward.models.metrics import PluginMetricLatest
|
|
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# What core itself records. Unified monitors emit metrics under their TYPE
|
|
# (icmp/tcp/dns/http), each with is_up (1.0 up / 0.0 down) and response_ms
|
|
# (where the probe times it); "steward" is the app watching itself.
|
|
CORE_METRICS: dict[str, list[str]] = {
|
|
"icmp": ["is_up", "response_ms"],
|
|
"tcp": ["is_up", "response_ms"],
|
|
"dns": ["is_up", "response_ms"],
|
|
"http": ["is_up", "response_ms"],
|
|
"steward": ["ingest_posts", "ingest_failed", "ingest_slowest_ms", "db_roundtrip_ms",
|
|
"unwatched_secs", "cleanup_secs", "open_fds", "open_fds_pct"],
|
|
}
|
|
|
|
|
|
def metric_catalog() -> dict[str, list[str]]:
|
|
"""Every source an alert rule can watch, and its metric names.
|
|
|
|
Core's own sources, then each loaded plugin's get_alert_metrics() export
|
|
({source: [metric names]}), so the catalog is written by whatever emits
|
|
the metrics and a disabled plugin's sources drop out. An empty list means
|
|
the names are dynamic (snmp: the OID labels recorded so far).
|
|
"""
|
|
from steward.core.plugin_manager import get_loaded_plugin_modules
|
|
catalog = {src: list(names) for src, names in CORE_METRICS.items()}
|
|
for name, module in get_loaded_plugin_modules().items():
|
|
hook = getattr(module, "get_alert_metrics", None)
|
|
if hook is None:
|
|
continue
|
|
try:
|
|
contributed = hook() or {}
|
|
except Exception:
|
|
logger.exception("Plugin %r: get_alert_metrics() raised; its metrics are skipped", name)
|
|
continue
|
|
for src, names in contributed.items():
|
|
catalog[str(src)] = [str(n) for n in names]
|
|
return catalog
|
|
|
|
|
|
def source_modules() -> list[str]:
|
|
"""The sources in picker order: core's first, then plugins'."""
|
|
return list(metric_catalog())
|
|
|
|
|
|
# States that need someone's attention: pending (breaching, not yet for long
|
|
# enough), firing, and acknowledged-but-still-breaching.
|
|
ACTIVE_STATES = (AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending)
|
|
|
|
|
|
async def active_alerts(db) -> list:
|
|
"""(AlertState, AlertRule) rows that need attention, latest transition
|
|
first: not yet resolved, or with a series that has gone silent (no_data)."""
|
|
return list((await db.execute(
|
|
select(AlertState, AlertRule)
|
|
.join(AlertRule, AlertState.rule_id == AlertRule.id)
|
|
.where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
|
|
.order_by(AlertState.last_transition_at.desc())
|
|
)).all())
|
|
|
|
|
|
def alert_label(state: AlertState) -> str:
|
|
"""What an alert is, in one word: its state, or "no_data" for a quiet rule
|
|
whose series has gone silent (a firing alert that goes silent stays
|
|
"firing"; check no_data_since for the mark)."""
|
|
if state.no_data_since is not None and state.state == AlertStateEnum.inactive:
|
|
return "no_data"
|
|
return state.state.value
|
|
|
|
|
|
async def latest_value(db, rule: AlertRule) -> float | None:
|
|
"""The newest sample of the series a rule watches, or None before the first
|
|
(a primary-key read of the latest-value table, #5208)."""
|
|
return (await db.execute(
|
|
select(PluginMetricLatest.value)
|
|
.where(PluginMetricLatest.source_module == rule.source_module,
|
|
PluginMetricLatest.resource_name == rule.resource_name,
|
|
PluginMetricLatest.metric_name == rule.metric_name)
|
|
)).scalar_one_or_none()
|
|
|
|
|
|
# Sources whose resources have no page of their own: the plugin's page lists them.
|
|
PLUGIN_PAGES = {"traefik": "/plugins/traefik/", "unifi": "/plugins/unifi/",
|
|
"snmp": "/plugins/snmp/"}
|
|
|
|
|
|
async def resource_url(db, rule: AlertRule) -> str | None:
|
|
"""Where the thing a rule watches is shown, or None when nothing shows it.
|
|
|
|
host_agent resources are the host's name, or "<host>:<mount|nic|core>";
|
|
docker's are "<host>/<container>"; a monitor's is its name under its type.
|
|
"""
|
|
from steward.models.hosts import Host
|
|
source, resource = rule.source_module, rule.resource_name
|
|
if source == "host_agent":
|
|
name = resource.split(":", 1)[0]
|
|
host_id = (await db.execute(
|
|
select(Host.id).where(Host.name == name).limit(1))).scalar_one_or_none()
|
|
return f"/hosts/{host_id}" if host_id else None
|
|
if source in MONITOR_TYPE_VALUES:
|
|
monitor_id = (await db.execute(
|
|
select(Monitor.id).where(Monitor.type == source, Monitor.name == resource).limit(1)
|
|
)).scalar_one_or_none()
|
|
return f"/monitors/{monitor_id}" if monitor_id else None
|
|
if source == "docker" and "/" in resource:
|
|
host_name, container = resource.split("/", 1)
|
|
host_id = (await db.execute(
|
|
select(Host.id).where(Host.name == host_name).limit(1))).scalar_one_or_none()
|
|
return f"/plugins/docker/container/{host_id}/{container}" if host_id else None
|
|
return PLUGIN_PAGES.get(source)
|
|
|
|
|
|
# Metrics whose value is a word: the sentence says the word, not the condition.
|
|
_BINARY = ("is_up", "is_healthy")
|
|
|
|
|
|
def rule_condition(rule: AlertRule) -> str:
|
|
"""What a rule fires on, in words: "CPU > 90%", "Fullest disk >= 85%",
|
|
or "is down" for an up/down metric rather than "is_up < 0.5"."""
|
|
return condition_text(rule.metric_name, rule.operator, rule.threshold)
|
|
|
|
|
|
def condition_text(metric: str, operator: AlertOperator | str, threshold: float) -> str:
|
|
"""rule_condition from a rule's parts, for an alert carried as a dict (an
|
|
email); the operator may be its value, ">=", rather than the enum."""
|
|
from steward.core.alerts import _is_breached
|
|
from steward.core.formatting import metric_label, metric_value
|
|
operator = AlertOperator(operator)
|
|
if metric in _BINARY:
|
|
fires_on = [metric_value(metric, v) for v in (1.0, 0.0)
|
|
if _is_breached(v, operator, threshold)]
|
|
if len(fires_on) == 1:
|
|
return f"is {fires_on[0]}"
|
|
return f"{metric_label(metric)} {operator.value} {metric_value(metric, threshold)}"
|
|
|
|
|
|
def alert_sentence(rule: AlertRule, label: str, value: float | None,
|
|
since: datetime | None, now: datetime) -> dict:
|
|
"""An alert as one sentence around its resource: {"lead", "resource",
|
|
"tail"}, read lead + resource + tail, e.g. "CPU on " + "web-1" +
|
|
" is 94% (> 90%), firing for 12m". label is alert_label(state); since is
|
|
when that label began (no_data_since for no_data)."""
|
|
from steward.core.formatting import human_duration, metric_label, metric_value
|
|
metric = rule.metric_name
|
|
said = metric_label(metric)
|
|
held = human_duration((now - since).total_seconds()) if since else None
|
|
if label == "no_data":
|
|
return {"lead": f"No {said} reading from ", "resource": rule.resource_name,
|
|
"tail": f" for {held}" if held else ""}
|
|
# An acknowledged alert is still breaching: say for how long it has been,
|
|
# not how long ago someone acknowledged it (the page says that apart).
|
|
word = "breaching" if label == "acknowledged" else label
|
|
for_how_long = f", {word} for {held}" if held else f", {word}"
|
|
if value is None:
|
|
reading = " has no reading yet"
|
|
elif metric in _BINARY:
|
|
return {"lead": "", "resource": rule.resource_name,
|
|
"tail": f" is {metric_value(metric, value)}{for_how_long}"}
|
|
else:
|
|
reading = f" is {metric_value(metric, value)}"
|
|
condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})"
|
|
return {"lead": f"{said} on ", "resource": rule.resource_name,
|
|
"tail": f"{reading}{condition}{for_how_long}"}
|
|
|
|
|
|
# The order the active list reads in: what is breaching and unhandled, then
|
|
# what has gone silent, then what someone has taken, then what may yet clear.
|
|
ACTIVE_ORDER = {"firing": 0, "no_data": 1, "acknowledged": 2, "pending": 3}
|
|
|
|
_BREACHING = (AlertStateEnum.pending.value, AlertStateEnum.firing.value,
|
|
AlertStateEnum.acknowledged.value)
|
|
|
|
|
|
async def breach_started(db, rule_ids) -> dict[str, datetime]:
|
|
"""{rule_id: when its current breach began}: the newest transition into
|
|
pending or firing from a quiet state. Acknowledging moves an alert's
|
|
last_transition_at, so that column cannot say this."""
|
|
if not rule_ids:
|
|
return {}
|
|
return dict((await db.execute(
|
|
select(AlertEvent.rule_id, func.max(AlertEvent.transitioned_at))
|
|
.where(AlertEvent.rule_id.in_(list(rule_ids)),
|
|
AlertEvent.from_state.not_in(_BREACHING),
|
|
AlertEvent.to_state.in_((AlertStateEnum.pending.value, AlertStateEnum.firing.value)))
|
|
.group_by(AlertEvent.rule_id)
|
|
)).all())
|
|
|
|
|
|
async def usernames(db, ids) -> dict[str, str]:
|
|
"""{user id: username} for the given ids (None and repeats ignored)."""
|
|
from steward.models.users import User
|
|
ids = {i for i in ids if i}
|
|
if not ids:
|
|
return {}
|
|
return dict((await db.execute(select(User.id, User.username).where(User.id.in_(ids)))).all())
|
|
|
|
|
|
async def active_rows(db, now: datetime) -> list[dict]:
|
|
"""The alerts page's active list, firing first, then no data, then
|
|
acknowledged, then pending; within each the longest-running first. Each
|
|
row: its state, rule, label, sentence, the link to what it watches, when
|
|
the breach began, and who acknowledged it."""
|
|
alerts = await active_alerts(db)
|
|
started = await breach_started(db, {rule.id for state, rule in alerts
|
|
if state.state.value in _BREACHING})
|
|
names = await usernames(db, (s.acknowledged_by for s, _ in alerts))
|
|
rows = []
|
|
for state, rule in alerts:
|
|
label = alert_label(state)
|
|
if label == "no_data":
|
|
since = state.no_data_since
|
|
elif label == "acknowledged":
|
|
since = started.get(rule.id, state.last_transition_at)
|
|
else:
|
|
since = state.last_transition_at
|
|
rows.append({
|
|
"state": state, "rule": rule, "label": label, "since": since,
|
|
"breach_since": started.get(rule.id) if label != "no_data" else None,
|
|
"acked_by": names.get(state.acknowledged_by, state.acknowledged_by),
|
|
"says": alert_sentence(rule, label, await latest_value(db, rule), since, now),
|
|
"url": await resource_url(db, rule),
|
|
})
|
|
epoch = datetime.max.replace(tzinfo=timezone.utc)
|
|
rows.sort(key=lambda r: (ACTIVE_ORDER.get(r["label"], 9), r["since"] or epoch))
|
|
return rows
|
|
|
|
|
|
def watches_host(rule: AlertRule, host_name: str, monitor_keys: set) -> bool:
|
|
"""Whether a rule watches something on this host: its agent series
|
|
("<host>" or "<host>:<part>"), one of its containers ("<host>/<name>"),
|
|
or one of its monitors ((type, name) in monitor_keys)."""
|
|
source, resource = rule.source_module, rule.resource_name
|
|
if source == "host_agent":
|
|
return resource == host_name or resource.startswith(host_name + ":")
|
|
if source == "docker":
|
|
return resource.startswith(host_name + "/")
|
|
return (source, resource) in monitor_keys
|
|
|
|
|
|
async def active_count(db) -> int:
|
|
"""How many alerts need attention: the sidebar's number."""
|
|
from sqlalchemy import func
|
|
return (await db.execute(
|
|
select(func.count()).select_from(AlertState)
|
|
.where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
|
|
)).scalar_one()
|
|
|
|
|
|
def notify_targets(smtp: dict, webhook: dict, accounts: int = 0) -> list[str]:
|
|
"""Where an alert's notifications go, in words, by the same tests
|
|
core.notifications.dispatch_notifications sends by: email when a server
|
|
is set and `accounts` (that get alerts) or Also send to addresses exist,
|
|
a webhook when its URL is. Empty: nobody (#5315, milestone 495)."""
|
|
from urllib.parse import urlsplit
|
|
out = []
|
|
recipients = smtp.get("recipients") or []
|
|
if isinstance(recipients, str):
|
|
recipients = [r.strip() for r in recipients.split(",") if r.strip()]
|
|
who = ([f"{accounts} account{'s' if accounts != 1 else ''}"] if accounts else []) + recipients
|
|
if smtp.get("host") and who:
|
|
out.append("email to " + ", ".join(who))
|
|
if url := webhook.get("url"):
|
|
out.append("a webhook to " + (urlsplit(url).hostname or url))
|
|
return out
|
|
|
|
|
|
async def last_fired(db, rule_id: str | None = None) -> dict[str, datetime]:
|
|
"""{rule id: when it last went to firing}, for the rules that ever have;
|
|
one rule's when `rule_id` is given."""
|
|
stmt = (select(AlertEvent.rule_id, func.max(AlertEvent.transitioned_at))
|
|
.where(AlertEvent.to_state == AlertStateEnum.firing.value)
|
|
.group_by(AlertEvent.rule_id))
|
|
if rule_id:
|
|
stmt = stmt.where(AlertEvent.rule_id == rule_id)
|
|
rows = await db.execute(stmt)
|
|
return {rid: at for rid, at in rows.all()}
|
|
|
|
|
|
async def list_rules(db) -> list[AlertRule]:
|
|
return list((await db.execute(select(AlertRule).order_by(AlertRule.name))).scalars())
|
|
|
|
|
|
async def history(db, since: datetime, rule_id: str | None = None,
|
|
limit: int = 200, before: str | None = None) -> list[tuple[AlertEvent, AlertRule]]:
|
|
"""Alert transitions since `since`, newest first, with their rule; one
|
|
rule's when `rule_id` is given; only those older than the `before` cursor
|
|
when given (core.paging). Returns up to limit+1 rows so a caller can tell
|
|
the list was cut."""
|
|
from steward.core.paging import keyset
|
|
stmt = (select(AlertEvent, AlertRule)
|
|
.join(AlertRule, AlertEvent.rule_id == AlertRule.id)
|
|
.where(AlertEvent.transitioned_at >= since))
|
|
if rule_id:
|
|
stmt = stmt.where(AlertEvent.rule_id == rule_id)
|
|
stmt = keyset(stmt, AlertEvent.transitioned_at, AlertEvent.id, before, limit)
|
|
return [tuple(row) for row in (await db.execute(stmt)).all()]
|
|
|
|
|
|
async def history_columns(db, since: datetime, rule_id: str | None = None) -> tuple[bool, bool]:
|
|
"""(whether any transition in the range notified someone or was muted,
|
|
whether any started an Ansible run): the history page shows its Notified
|
|
and Action columns only then (#5316)."""
|
|
from steward.core.alerts import MUTED
|
|
base = select(AlertEvent.id).where(AlertEvent.transitioned_at >= since)
|
|
if rule_id:
|
|
base = base.where(AlertEvent.rule_id == rule_id)
|
|
notified = await db.scalar(select(base.where(or_(
|
|
AlertEvent.email_sent.is_not(None), AlertEvent.webhook_sent.is_not(None),
|
|
AlertEvent.note.like(f"%{MUTED}%"))).exists()))
|
|
acted = await db.scalar(select(base.where(AlertEvent.ansible_run_id.is_not(None)).exists()))
|
|
return bool(notified), bool(acted)
|
|
|
|
|
|
async def list_maintenance_windows(db) -> list[MaintenanceWindow]:
|
|
"""Every window, latest start first."""
|
|
return list((await db.execute(
|
|
select(MaintenanceWindow).order_by(MaintenanceWindow.start_at.desc())
|
|
)).scalars())
|
|
|
|
|
|
async def resources_for(db, source_module: str) -> list[str]:
|
|
"""Sorted distinct resource names for a source module.
|
|
|
|
Read from the latest-value table, one row per series: a DISTINCT over
|
|
plugin_metrics reads every raw sample (Postgres has no skip scan), which
|
|
held the rule page for up to 15 s on host_agent (#5208's trap again).
|
|
|
|
For monitor types (icmp/tcp/dns/http), supplements with monitor names so
|
|
the list is populated even before any metrics have been recorded.
|
|
"""
|
|
result = await db.execute(
|
|
select(PluginMetricLatest.resource_name)
|
|
.where(PluginMetricLatest.source_module == source_module)
|
|
.distinct()
|
|
)
|
|
resources: set[str] = {r for (r,) in result}
|
|
if source_module in MONITOR_TYPE_VALUES:
|
|
mon_result = await db.execute(
|
|
select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name))
|
|
resources.update(name for (name,) in mon_result)
|
|
return sorted(resources)
|
|
|
|
|
|
# A host's parts, by the shape of the name after "<host>:" (#5475): the order
|
|
# and heading each kind takes in the rule form's Part picker.
|
|
PART_KINDS = (("mount", "Mounts"), ("core", "CPU cores"), ("disk", "Disks"),
|
|
("net", "Interfaces"), ("temp", "Temperatures"), ("other", "Other"))
|
|
|
|
|
|
def part_kind(part: str) -> tuple[str, str]:
|
|
"""(kind, label) for a host part: "/var" is a mount, "core3" a core,
|
|
"diskio:sda" a disk, "net:eth0" an interface, "temp:Core 0" a sensor."""
|
|
if part.startswith("/"):
|
|
return "mount", part
|
|
if part.startswith("core") and part[4:].isdigit():
|
|
return "core", f"core {part[4:]}"
|
|
for prefix, kind in (("diskio:", "disk"), ("net:", "net"), ("temp:", "temp")):
|
|
if part.startswith(prefix):
|
|
return kind, part[len(prefix):]
|
|
return "other", part
|
|
|
|
|
|
def _part_sort(part: str):
|
|
kind, label = part_kind(part)
|
|
return (int(part[4:]), "") if kind == "core" else (0, label.lower())
|
|
|
|
|
|
def alertable(source_module: str, reported: set[str]) -> list[str]:
|
|
"""The metrics a rule on a resource can watch, from what it reports: the
|
|
source's catalog, in its order, narrowed to those; every reported name
|
|
for a source with no catalog (snmp)."""
|
|
catalog = metric_catalog().get(source_module, [])
|
|
return [m for m in catalog if m in reported] if catalog else sorted(reported)
|
|
|
|
|
|
async def rule_target(db, source_module: str, *, host: str = "", resource: str = "",
|
|
metric: str = "", keep: bool = False) -> dict:
|
|
"""What the rule form's target pickers offer (#5475), from the latest-value
|
|
table: the resources that report something a rule can watch, and for the
|
|
chosen one the metrics it reports. For a host source the resources split
|
|
into a Host picker and a Part picker ("the whole host", then the host's
|
|
mounts, cores and so on). With `keep` (a rule's own page), its resource
|
|
and metric stay on offer even after they go quiet, so opening a rule
|
|
never drops them; a metric carried over from another part is dropped.
|
|
|
|
Returns host_source, hosts, host, parts [(heading, [(value, label)])],
|
|
resources, resource, metrics, metric."""
|
|
catalog = metric_catalog().get(source_module, [])
|
|
seen: dict[str, set[str]] = {}
|
|
for res, name in (await db.execute(
|
|
select(PluginMetricLatest.resource_name, PluginMetricLatest.metric_name)
|
|
.where(PluginMetricLatest.source_module == source_module))).all():
|
|
seen.setdefault(res, set()).add(name)
|
|
pickable = {r: ms for r, names in seen.items() if (ms := alertable(source_module, names))}
|
|
if source_module in MONITOR_TYPE_VALUES:
|
|
for (name,) in (await db.execute(select(Monitor.name).where(Monitor.type == source_module))).all():
|
|
pickable.setdefault(name, list(catalog))
|
|
if resource and resource not in pickable:
|
|
pickable[resource] = list(catalog) or ([metric] if metric else [])
|
|
|
|
out: dict = {"host_source": source_module in HOST_SOURCES, "hosts": [], "host": "",
|
|
"parts": [], "resources": [], "resource": resource}
|
|
if out["host_source"]:
|
|
hosts = sorted({r.split(":", 1)[0] for r in pickable}, key=str.lower)
|
|
host = host or resource.split(":", 1)[0]
|
|
if host not in hosts:
|
|
host = hosts[0] if hosts else ""
|
|
mine = [r for r in pickable if r == host or r.startswith(host + ":")]
|
|
if resource not in mine:
|
|
resource = host if host in pickable else (sorted(mine)[0] if mine else "")
|
|
by_kind: dict[str, list] = {}
|
|
for r in sorted((r for r in mine if r != host), key=lambda r: _part_sort(r[len(host) + 1:])):
|
|
kind, label = part_kind(r[len(host) + 1:])
|
|
by_kind.setdefault(kind, []).append((r, label))
|
|
parts = [("", [(host, "The whole host")])] if host in pickable else []
|
|
parts += [(heading, by_kind[k]) for k, heading in PART_KINDS if k in by_kind]
|
|
out.update(hosts=hosts, host=host, parts=parts, resource=resource)
|
|
else:
|
|
out["resources"] = sorted(pickable, key=str.lower)
|
|
metrics = list(pickable.get(out["resource"]) or catalog)
|
|
if keep and metric and metric not in metrics and (metric in catalog or not catalog):
|
|
metrics.append(metric)
|
|
out.update(metrics=metrics, metric=metric if metric in metrics else "")
|
|
return out
|
|
|
|
|
|
# How far back the rule form's metric preview looks: a day shows the daily
|
|
# swing (backups, batch jobs) that a threshold has to sit above.
|
|
PREVIEW_HOURS = 24
|
|
|
|
|
|
async def metric_preview(db, source_module: str, resource: str, metric: str,
|
|
now: datetime | None = None) -> dict | None:
|
|
"""What the chosen metric reads (#5475), so a threshold is set against real
|
|
numbers: the latest reading and when, and the last PREVIEW_HOURS as hourly
|
|
averages with the lowest and highest. Reads one series: the latest-value
|
|
row, then a recorded_at range on plugin_metrics' series index. None when
|
|
it has no reading."""
|
|
from datetime import timedelta
|
|
from steward.models.metrics import PluginMetric
|
|
latest = (await db.execute(select(PluginMetricLatest).where(
|
|
PluginMetricLatest.source_module == source_module,
|
|
PluginMetricLatest.resource_name == resource,
|
|
PluginMetricLatest.metric_name == metric))).scalar_one_or_none()
|
|
if latest is None:
|
|
return None
|
|
now = now or datetime.now(timezone.utc)
|
|
hour = func.date_trunc("hour", PluginMetric.recorded_at)
|
|
rows = (await db.execute(
|
|
select(hour, func.avg(PluginMetric.value), func.min(PluginMetric.value),
|
|
func.max(PluginMetric.value))
|
|
.where(PluginMetric.source_module == source_module,
|
|
PluginMetric.resource_name == resource,
|
|
PluginMetric.metric_name == metric,
|
|
PluginMetric.recorded_at >= now - timedelta(hours=PREVIEW_HOURS))
|
|
.group_by(hour).order_by(hour))).all()
|
|
return {"now": latest.value, "at": latest.recorded_at,
|
|
"series": [avg for _, avg, _, _ in rows],
|
|
"low": min((lo for _, _, lo, _ in rows), default=None),
|
|
"high": max((hi for _, _, _, hi in rows), default=None)}
|
|
|
|
|
|
async def metrics_for(db, source_module: str) -> list[str]:
|
|
"""The metric names a rule on this source can watch: the static catalog, or
|
|
for dynamic sources (snmp) the names recorded so far."""
|
|
metrics = metric_catalog().get(source_module, [])
|
|
if source_module and not metrics:
|
|
result = await db.execute(
|
|
select(PluginMetricLatest.metric_name)
|
|
.where(PluginMetricLatest.source_module == source_module)
|
|
.distinct()
|
|
.order_by(PluginMetricLatest.metric_name))
|
|
metrics = [r for (r,) in result]
|
|
return metrics
|
|
|
|
|
|
def _choices(values: list[str], limit: int = 20) -> str:
|
|
if not values:
|
|
return "none yet"
|
|
shown = ", ".join(values[:limit])
|
|
return shown + (f" (and {len(values) - limit} more)" if len(values) > limit else "")
|
|
|
|
|
|
async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
|
|
operator: str, threshold, consecutive_failures_required=1,
|
|
consecutive_ok_required=1, notify_no_data=True,
|
|
covers: str = "resource", cover_group: str = "") -> dict:
|
|
"""Validated column values for a new or edited alert rule.
|
|
|
|
The same choices the rule form offers: a known source, a resource that
|
|
source has reported (or, for monitors, a monitor of that type), and a
|
|
metric it emits. A covering rule (`covers` "all_hosts", or "group" with
|
|
an inventory group's name in `cover_group`, #5464) names no resource: it
|
|
is for a source whose resources are hosts. ValueError says what was wrong
|
|
and what is valid.
|
|
"""
|
|
name = (name or "").strip()
|
|
if not name:
|
|
raise ValueError("an alert rule needs a name.")
|
|
source_module = (source_module or "").strip()
|
|
catalog = metric_catalog()
|
|
if source_module not in catalog:
|
|
raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.")
|
|
covers = (covers or "resource").strip()
|
|
if covers not in COVERS:
|
|
raise ValueError(f"covers {covers!r} is not one of {', '.join(COVERS)}.")
|
|
cover_group = (cover_group or "").strip()
|
|
if covers == "resource":
|
|
resource_name, cover_group = (resource_name or "").strip(), None
|
|
resources = await resources_for(db, source_module)
|
|
if resource_name not in resources:
|
|
raise ValueError(f"{source_module} has no resource {resource_name!r}. "
|
|
f"Known: {_choices(resources)}.")
|
|
else:
|
|
if source_module not in HOST_SOURCES:
|
|
raise ValueError(f"only a rule on {', '.join(HOST_SOURCES)} can cover hosts; "
|
|
f"{source_module}'s resources are not hosts.")
|
|
resource_name = ""
|
|
if covers == "group":
|
|
from steward.hosts.service import group_names
|
|
groups = await group_names(db)
|
|
if cover_group not in groups:
|
|
raise ValueError(f"there is no inventory group {cover_group!r}. "
|
|
f"Groups: {_choices(groups)}.")
|
|
else:
|
|
cover_group = None
|
|
metric_name = (metric_name or "").strip()
|
|
metrics = await metrics_for(db, source_module)
|
|
if metric_name not in metrics:
|
|
raise ValueError(f"{source_module} does not report {metric_name!r}. "
|
|
f"Its metrics: {_choices(metrics)}.")
|
|
try:
|
|
op = AlertOperator((operator or "").strip())
|
|
except ValueError:
|
|
raise ValueError(f"operator {operator!r} is not one of "
|
|
f"{', '.join(o.value for o in AlertOperator)}.") from None
|
|
try:
|
|
threshold = float(threshold)
|
|
except (TypeError, ValueError):
|
|
raise ValueError(f"threshold must be a number, got {threshold!r}.") from None
|
|
try:
|
|
consecutive = (1 if consecutive_failures_required in (None, "")
|
|
else int(consecutive_failures_required))
|
|
except (TypeError, ValueError):
|
|
raise ValueError("consecutive checks must be a whole number.") from None
|
|
if consecutive < 1:
|
|
raise ValueError("consecutive checks must be at least 1.")
|
|
try:
|
|
ok_checks = (1 if consecutive_ok_required in (None, "")
|
|
else int(consecutive_ok_required))
|
|
except (TypeError, ValueError):
|
|
raise ValueError("checks to clear must be a whole number.") from None
|
|
if ok_checks < 1:
|
|
raise ValueError("checks to clear must be at least 1.")
|
|
return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
|
|
"metric_name": metric_name, "operator": op, "threshold": threshold,
|
|
"consecutive_failures_required": consecutive,
|
|
"consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data),
|
|
"covers": covers, "cover_group": cover_group}
|
|
|
|
|
|
# Changing any of these changes what the rule is watching, so whatever state
|
|
# it had built up no longer means anything.
|
|
WATCHED_FIELDS = ("source_module", "resource_name", "metric_name", "operator", "threshold")
|
|
|
|
|
|
async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool | None = None,
|
|
actor_id: str | None = None) -> None:
|
|
"""Apply validated fields (and optionally enabled) to a rule, in the
|
|
caller's transaction. A change to what it watches, or disabling or
|
|
re-enabling it, puts its alert back to inactive and starts its silence
|
|
watch over (core.alerts.reset_rule_state); the event says who did it."""
|
|
from steward.core.alerts import reset_rule_state
|
|
changed = [k for k in WATCHED_FIELDS if k in fields and getattr(rule, k) != fields[k]]
|
|
for k, v in fields.items():
|
|
setattr(rule, k, v)
|
|
disabling = enabled is False and rule.enabled
|
|
enabling = enabled is True and not rule.enabled
|
|
if enabled is not None:
|
|
rule.enabled = bool(enabled)
|
|
if disabling:
|
|
await reset_rule_state(db, rule, "rule disabled", actor_id)
|
|
elif enabling:
|
|
await reset_rule_state(db, rule, "rule enabled", actor_id)
|
|
elif changed:
|
|
await reset_rule_state(db, rule, "rule changed: " + ", ".join(changed), actor_id)
|
|
|
|
|
|
async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
|
|
ansible_action: dict | None = None) -> AlertRule:
|
|
"""Insert a rule and its inactive state row, in the caller's transaction.
|
|
A covering rule gets no state; its per-host children are made here."""
|
|
rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
|
|
db.add(rule)
|
|
await db.flush()
|
|
if rule.covers != "resource":
|
|
await reconcile_children(db, rule)
|
|
return rule
|
|
db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
|
|
last_transition_at=datetime.now(timezone.utc)))
|
|
return rule
|
|
|
|
|
|
# ── Covering rules (#5464) ───────────────────────────────────────────────────
|
|
# A rule can cover every host, or every host in an inventory group (inventory
|
|
# groups are Steward's grouping system: the Hosts ruling, #5294). Steward keeps
|
|
# one ordinary child rule per covered host, so each host's alert has its own
|
|
# state, history and acknowledgement, and a host that appears later is covered
|
|
# without anyone copying the rule.
|
|
|
|
COVERS = ("resource", "all_hosts", "group")
|
|
# Sources whose bare resource names are hosts ("<host>", parts are "<host>:…").
|
|
HOST_SOURCES = ("host_agent",)
|
|
# A host counts as present while it has reported within this long: the raw
|
|
# metrics window, so a decommissioned host's children go once its data does.
|
|
COVER_SEEN_DAYS = 7
|
|
|
|
|
|
def covers_text(rule: AlertRule) -> str:
|
|
"""What a covering rule spans, in words: "every host", "every host in web"."""
|
|
if rule.covers == "group":
|
|
return f"every host in {rule.cover_group}"
|
|
return "every host" if rule.covers == "all_hosts" else rule.resource_name
|
|
|
|
|
|
async def covered_hosts(db, rule: AlertRule, now: datetime | None = None) -> list[str]:
|
|
"""The host names a covering rule spans now: hosts that have reported
|
|
under its source within COVER_SEEN_DAYS, all of them or those whose
|
|
linked Ansible target is in its group."""
|
|
from datetime import timedelta
|
|
from steward.hosts.service import host_groups
|
|
from steward.models.hosts import Host
|
|
now = now or datetime.now(timezone.utc)
|
|
reported = set((await db.execute(
|
|
select(PluginMetricLatest.resource_name)
|
|
.where(PluginMetricLatest.source_module == rule.source_module,
|
|
PluginMetricLatest.recorded_at >= now - timedelta(days=COVER_SEEN_DAYS),
|
|
~PluginMetricLatest.resource_name.contains(":"))
|
|
.distinct())).scalars())
|
|
if rule.covers != "group":
|
|
return sorted(reported, key=str.lower)
|
|
hosts = (await db.execute(select(Host.id, Host.name).where(Host.name.in_(reported)))).all()
|
|
groups = await host_groups(db, [hid for hid, _ in hosts])
|
|
return sorted((name for hid, name in hosts if rule.cover_group in groups.get(hid, [])),
|
|
key=str.lower)
|
|
|
|
|
|
def child_fields(parent: AlertRule, host: str) -> dict:
|
|
"""A covering rule's child for one host: the parent's condition on that
|
|
host, named "<parent>: <host>"."""
|
|
return {"name": f"{parent.name}: {host}"[:128], "source_module": parent.source_module,
|
|
"resource_name": host, "metric_name": parent.metric_name,
|
|
"operator": parent.operator, "threshold": parent.threshold,
|
|
"consecutive_failures_required": parent.consecutive_failures_required,
|
|
"consecutive_ok_required": parent.consecutive_ok_required,
|
|
"notify_no_data": parent.notify_no_data}
|
|
|
|
|
|
async def reconcile_children(db, parent: AlertRule, *, actor_id: str | None = None,
|
|
now: datetime | None = None) -> tuple[list[str], list[str]]:
|
|
"""Bring a covering rule's children in line with it, in the caller's
|
|
transaction: one per covered host, each carrying the parent's condition,
|
|
action and on/off. A host newly covered gets a child; a child whose host
|
|
is no longer covered is deleted, its history with it. Returns (hosts
|
|
added, hosts removed)."""
|
|
hosts = set(await covered_hosts(db, parent, now))
|
|
children = {c.resource_name: c for c in (await db.execute(
|
|
select(AlertRule).where(AlertRule.parent_id == parent.id))).scalars()}
|
|
added = sorted(hosts - children.keys(), key=str.lower)
|
|
removed = sorted(children.keys() - hosts, key=str.lower)
|
|
for host in added:
|
|
await add_rule(db, {**child_fields(parent, host), "parent_id": parent.id},
|
|
created_by=parent.created_by, enabled=parent.enabled,
|
|
ansible_action=parent.ansible_action)
|
|
for host in removed:
|
|
await db.delete(children[host])
|
|
for host, child in children.items():
|
|
if host in hosts:
|
|
fields = child_fields(parent, host)
|
|
if (any(getattr(child, k) != v for k, v in fields.items())
|
|
or child.enabled != parent.enabled):
|
|
await apply_rule_change(db, child, fields, enabled=parent.enabled,
|
|
actor_id=actor_id)
|
|
if child.ansible_action != parent.ansible_action:
|
|
child.ansible_action = parent.ansible_action
|
|
return added, removed
|
|
|
|
|
|
async def reconcile_covering_rules(app) -> None:
|
|
"""Every covering rule's children brought in line with the hosts there are
|
|
now. Run each minute by the core scheduler (alert_reconcile), so a host
|
|
that joins a group, or starts reporting, is covered within a minute."""
|
|
async with app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
parents = (await db.execute(
|
|
select(AlertRule).where(AlertRule.covers != "resource"))).scalars().all()
|
|
for parent in parents:
|
|
added, removed = await reconcile_children(db, parent)
|
|
if added or removed:
|
|
logger.info("Covering rule %r: added %s, removed %s", parent.name,
|
|
", ".join(added) or "none", ", ".join(removed) or "none")
|
|
|
|
|
|
def part_of(rule: AlertRule, rules) -> AlertRule | None:
|
|
"""The covering rule a child belongs to, from a list of rules; None for a
|
|
rule that stands alone."""
|
|
return next((r for r in rules if r.id == rule.parent_id), None) if rule.parent_id else None
|
|
|
|
|
|
async def test_alert(db, rule: AlertRule) -> dict:
|
|
"""What a Send test carries for `rule` (#5466): a TEST alert with the
|
|
rule's name and condition, on its resource at the latest reading. A rule
|
|
covering hosts tests on its first host."""
|
|
target = rule
|
|
if rule.covers != "resource":
|
|
target = (await db.execute(
|
|
select(AlertRule).where(AlertRule.parent_id == rule.id)
|
|
.order_by(AlertRule.resource_name).limit(1))).scalar_one_or_none() or rule
|
|
return {"rule_name": rule.name, "state": "TEST", "metric": rule.metric_name,
|
|
"value": await latest_value(db, target) if target.resource_name else None,
|
|
"operator": rule.operator.value, "threshold": rule.threshold,
|
|
"resource": target.resource_name or covers_text(rule),
|
|
"source_module": rule.source_module,
|
|
"note": "A test of this rule's notifications. Nothing has fired.",
|
|
"timestamp": datetime.now(timezone.utc).isoformat()}
|
|
|
|
|
|
async def send_test(app, rule_id: str) -> dict:
|
|
"""Send a TEST notification for a rule now, by every channel a real alert
|
|
of it would use: email to whoever gets alerts, each in their own zone, and
|
|
the webhook. Not held for a burst, and not recorded in the rule's history.
|
|
|
|
Returns {"rule", "ok", "lines"}: ok when something was sent and nothing
|
|
failed; lines say what happened to each channel. LookupError for no rule.
|
|
"""
|
|
from urllib.parse import urlsplit
|
|
from steward.core.mail import recipients
|
|
from steward.core.notifications import dispatch_notifications
|
|
smtp, webhook = app.config.get("SMTP", {}), app.config.get("WEBHOOK", {})
|
|
async with app.db_sessionmaker() as db:
|
|
rule = await db.get(AlertRule, rule_id)
|
|
if rule is None:
|
|
raise LookupError(rule_id)
|
|
alert = await test_alert(db, rule)
|
|
to = await recipients(db, "alerts", smtp) if smtp.get("host") else []
|
|
results = await dispatch_notifications(smtp, webhook, alert, to)
|
|
lines = []
|
|
if email := results.get("email"):
|
|
who = ", ".join(r.address for r in to)
|
|
lines.append(f"Email sent to {who}." if email["sent"]
|
|
else f"Email to {who} failed: {email['error']}")
|
|
if hook := results.get("webhook"):
|
|
host = urlsplit(webhook["url"]).hostname or webhook["url"]
|
|
lines.append(f"Webhook to {host} delivered." if hook["sent"]
|
|
else f"Webhook to {host} failed: {hook['error']}")
|
|
if not lines:
|
|
lines.append("Nothing is set up to send to: no email server with someone to send to, "
|
|
"and no webhook.")
|
|
return {"rule": rule.name, "ok": bool(results) and all(r["sent"] for r in results.values()),
|
|
"lines": lines, "email": results.get("email"), "webhook": results.get("webhook")}
|
|
|
|
|
|
class PartOfCoveringRule(ValueError):
|
|
"""A child of a covering rule is changed through its parent, never alone."""
|
|
|
|
def __init__(self, child: AlertRule, parent_name: str):
|
|
super().__init__(f"{child.name} is part of the rule {parent_name!r}, which keeps one "
|
|
f"rule per host; change or disable it there.")
|
|
|
|
|
|
async def acknowledge(db, rule_id: str, user_id: str | None) -> AlertState:
|
|
"""Acknowledge a firing alert, in the caller's transaction, and record it
|
|
in the alert's history as a firing → acknowledged event by `user_id`, so
|
|
who saw it outlives the alert (#5463).
|
|
|
|
LookupError when the rule has no state; ValueError when it is not firing.
|
|
"""
|
|
from steward.core.alerts import _add_event
|
|
state = (await db.execute(
|
|
select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one_or_none()
|
|
if state is None:
|
|
raise LookupError(rule_id)
|
|
if state.state != AlertStateEnum.firing:
|
|
raise ValueError(f"the alert is {state.state.value}, not firing; only a firing alert "
|
|
"can be acknowledged.")
|
|
now = datetime.now(timezone.utc)
|
|
state.state = AlertStateEnum.acknowledged
|
|
state.acknowledged_by = user_id
|
|
state.acknowledged_at = now
|
|
state.last_transition_at = now
|
|
_add_event(db, rule_id, AlertStateEnum.firing.value, AlertStateEnum.acknowledged.value,
|
|
None, now, actor_id=user_id)
|
|
return state
|
|
|
|
|
|
def window_fields(*, name: str, start_at: datetime, end_at: datetime,
|
|
scope_source: str | None, scope_resource: str | None) -> dict:
|
|
"""Validated columns for a maintenance window. A resource needs a source."""
|
|
name = (name or "").strip()
|
|
if not name:
|
|
raise ValueError("a maintenance window needs a name.")
|
|
if end_at <= start_at:
|
|
raise ValueError("the window must end after it starts.")
|
|
scope_source = (scope_source or "").strip() or None
|
|
scope_resource = (scope_resource or "").strip() or None
|
|
catalog = metric_catalog()
|
|
if scope_source is not None and scope_source not in catalog:
|
|
raise ValueError(f"source {scope_source!r} is not one of {', '.join(catalog)}.")
|
|
if scope_source is None:
|
|
scope_resource = None # can't have resource without source
|
|
return {"name": name[:128], "start_at": start_at, "end_at": end_at,
|
|
"scope_source": scope_source, "scope_resource": scope_resource}
|
|
|
|
|
|
def window_scope(w) -> str:
|
|
if not w.scope_source:
|
|
return "all"
|
|
return f"{w.scope_source}/{w.scope_resource}" if w.scope_resource else f"{w.scope_source}/*"
|