bvandeusen/FabledSteward
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(alerts): the metric catalog is written by what emits it (#5061)

Browse Source 
#5061 "Metric catalog belongs to what emits: plugins register their
metrics; names match what is recorded".

- The static METRIC_CATALOG is gone. metric_catalog() is core's own sources
  (the four monitor types and "steward", the app's open_fds/open_fds_pct,
  which were recorded but never offered) plus each loaded plugin's
  get_alert_metrics() export. A disabled plugin's sources leave the picker.
- traefik offers what it records: error_rate_4xx_pct and error_rate_5xx_pct.
  The old "error_rate" was never recorded, so a rule on it could not fire.
- tests/core/test_metric_catalog.py walks every MetricBatch.add /
  record_metric call (traefik's tuple loop included) and fails on a recorded
  metric the catalog lacks; it shows on a synthetic writer that it can fail.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-10-06 12:55:55 -04:00
co-authored by Claude Opus 5.5
parent 58bfc7c361
commit 0742b7a2b7
10 changed files with 197 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
steward/alerts/routes.py
+7 -7
View File
@@ -122,17 +122,17 @@ async def rule_fields_api():
@alerts_bp.get("/rules/new")
@require_role(UserRole.operator)
async def new_rule():
first_source = alert_svc.SOURCE_MODULES[0]
first_source = alert_svc.source_modules()[0]
async with current_app.db_sessionmaker() as db:
resources = await alert_svc.resources_for(db, first_source)
return await render_template(
"alerts/rules_form.html",
rule=None,
operators=_OPERATORS,
source_modules=alert_svc.SOURCE_MODULES,
source_modules=alert_svc.source_modules(),
initial_source=first_source,
resources=resources,
metrics=alert_svc.METRIC_CATALOG.get(first_source, []),
metrics=alert_svc.metric_catalog().get(first_source, []),
ansible_sources=_ansible_source_names(),
)
@@ -150,10 +150,10 @@ async def edit_rule(rule_id: str):
"alerts/rules_form.html",
rule=rule,
operators=_OPERATORS,
source_modules=alert_svc.SOURCE_MODULES,
source_modules=alert_svc.source_modules(),
initial_source=rule.source_module,
resources=resources,
metrics=alert_svc.METRIC_CATALOG.get(rule.source_module, []),
metrics=alert_svc.metric_catalog().get(rule.source_module, []),
ansible_sources=_ansible_source_names(),
)
@@ -256,7 +256,7 @@ async def list_maintenance():
"alerts/maintenance.html",
windows=windows,
now=now,
source_modules=alert_svc.SOURCE_MODULES,
source_modules=alert_svc.source_modules(),
)
@@ -266,7 +266,7 @@ async def new_maintenance():
return await render_template(
"alerts/maintenance_form.html",
window=None,
source_modules=alert_svc.SOURCE_MODULES,
source_modules=alert_svc.source_modules(),
)
steward/alerts/service.py
+44 -25
View File
@@ -6,6 +6,7 @@ pipeline) call the same code and refuse the same input for the same reasons.
"""
from __future__ import annotations
import logging
from datetime import datetime, timezone
from sqlalchemy import select
@@ -16,32 +17,48 @@ from steward.models.alerts import (
from steward.models.metrics import PluginMetric
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
# Static catalog: source_module → available metric names.
# Unified monitors emit metrics under their TYPE (icmp/tcp/dns/http), each with
# is_up (1.0 up / 0.0 down) and response_ms (where the probe times it).
METRIC_CATALOG: dict[str, list[str]] = {
logger = logging.getLogger(__name__)
# What core itself records. Unified monitors emit metrics under their TYPE
# (icmp/tcp/dns/http), each with is_up (1.0 up / 0.0 down) and response_ms
# (where the probe times it); "steward" is the app watching itself.
CORE_METRICS: dict[str, list[str]] = {
"icmp": ["is_up", "response_ms"],
"tcp": ["is_up", "response_ms"],
"dns": ["is_up", "response_ms"],
"http": ["is_up", "response_ms"],
"traefik": ["request_rate", "error_rate", "latency_p50_ms", "latency_p95_ms",
"latency_p99_ms", "response_bytes_rate", "cert_expiry_days"],
"unifi": ["is_up", "latency_ms", "total_clients"],
# restart_count = cumulative restarts (alert on crash-looping); is_healthy =
# 1.0 healthy / 0.0 unhealthy from the container HEALTHCHECK (alert on <1).
"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy"],
"host_agent": [
"cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
"disk_used_pct_worst", "load_1m", "load_5m", "load_15m", "uptime_secs",
# Per mount: the resource is "<host>:<mount>".
"disk_used_pct",
],
# snmp metrics are user-defined OID labels — populated dynamically from plugin_metrics
"snmp": [],
"steward": ["open_fds", "open_fds_pct"],
}
# Ordered list for the source module picker.
SOURCE_MODULES = list(METRIC_CATALOG.keys())
def metric_catalog() -> dict[str, list[str]]:
"""Every source an alert rule can watch, and its metric names.
Core's own sources, then each loaded plugin's get_alert_metrics() export
({source: [metric names]}), so the catalog is written by whatever emits
the metrics and a disabled plugin's sources drop out. An empty list means
the names are dynamic (snmp: the OID labels recorded so far).
"""
from steward.core.plugin_manager import get_loaded_plugin_modules
catalog = {src: list(names) for src, names in CORE_METRICS.items()}
for name, module in get_loaded_plugin_modules().items():
hook = getattr(module, "get_alert_metrics", None)
if hook is None:
continue
try:
contributed = hook() or {}
except Exception:
logger.exception("Plugin %r: get_alert_metrics() raised; its metrics are skipped", name)
continue
for src, names in contributed.items():
catalog[str(src)] = [str(n) for n in names]
return catalog
def source_modules() -> list[str]:
"""The sources in picker order: core's first, then plugins'."""
return list(metric_catalog())
# States that need someone's attention: pending (breaching, not yet for long
# enough), firing, and acknowledged-but-still-breaching.
@@ -92,7 +109,7 @@ async def resources_for(db, source_module: str) -> list[str]:
async def metrics_for(db, source_module: str) -> list[str]:
"""The metric names a rule on this source can watch: the static catalog, or
for dynamic sources (snmp) the names recorded so far."""
metrics = METRIC_CATALOG.get(source_module, [])
metrics = metric_catalog().get(source_module, [])
if source_module and not metrics:
result = await db.execute(
select(PluginMetric.metric_name)
@@ -123,8 +140,9 @@ async def rule_fields(db, *, name: str, source_module: str, resource_name: str,
if not name:
raise ValueError("an alert rule needs a name.")
source_module = (source_module or "").strip()
if source_module not in METRIC_CATALOG:
raise ValueError(f"source_module {source_module!r} is not one of {', '.join(SOURCE_MODULES)}.")
catalog = metric_catalog()
if source_module not in catalog:
raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.")
resource_name = (resource_name or "").strip()
resources = await resources_for(db, source_module)
if resource_name not in resources:
@@ -227,8 +245,9 @@ def window_fields(*, name: str, start_at: datetime, end_at: datetime,
raise ValueError("the window must end after it starts.")
scope_source = (scope_source or "").strip() or None
scope_resource = (scope_resource or "").strip() or None
if scope_source is not None and scope_source not in METRIC_CATALOG:
raise ValueError(f"source {scope_source!r} is not one of {', '.join(SOURCE_MODULES)}.")
catalog = metric_catalog()
if scope_source is not None and scope_source not in catalog:
raise ValueError(f"source {scope_source!r} is not one of {', '.join(catalog)}.")
if scope_source is None:
scope_resource = None # can't have resource without source
return {"name": name[:128], "start_at": start_at, "end_at": end_at,
steward/mcp/tools/alerts.py
+4 -4
View File
@@ -204,13 +204,13 @@ async def get_alert_rule_options(source_module: str = "") -> dict:
resource: is_up (1 up, 0 down) and response_ms.
"""
async with db_session() as db:
sources = alert_svc.source_modules()
if not source_module:
return {"sources": {src: await alert_svc.metrics_for(db, src)
for src in alert_svc.SOURCE_MODULES},
return {"sources": {src: await alert_svc.metrics_for(db, src) for src in sources},
"operators": [o.value for o in AlertOperator]}
if source_module not in alert_svc.METRIC_CATALOG:
if source_module not in sources:
raise ValueError(f"source_module {source_module!r} is not one of "
f"{', '.join(alert_svc.SOURCE_MODULES)}.")
f"{', '.join(sources)}.")
return {"source_module": source_module,
"metrics": await alert_svc.metrics_for(db, source_module),
"resources": await alert_svc.resources_for(db, source_module),