bvandeusen/FabledSteward
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(mcp): monitoring-config write tools — monitors, alert rules, acks, maintenance
CI / lint (push) Successful in 2s
Details
CI / integration (push) Successful in 41s
Details
CI / unit (push) Successful in 1m0s
Details
CI / publish (push) Successful in 58s
Details

Browse Source 
MCP-7 #4971 "Monitoring-config write tools: monitors, alert rules, acks,
maintenance".

The validation the forms did inline (or not at all) moves to the services,
and both the routes and the tools call it, so bad input is refused the same
way with the same reason:
- monitors/service: monitor_fields, build_config, normalise_target. Port
  1-65535, status 100-599, timeout 1-60, method GET/HEAD/POST, headers an
  object of strings, a key the type does not take is named. The forms now
  answer 400 with the reason instead of redirecting silently or a 500.
- alerts/service: METRIC_CATALOG and SOURCE_MODULES, resources_for,
  metrics_for, rule_fields (source, resource and metric from the same
  choices the rule form offers), add_rule, acknowledge, window_fields.

Tools: create_monitor, update_monitor (config merged over the stored one;
a rename warns about rules still watching the old name),
set_monitor_enabled, create_alert_rule, update_alert_rule,
acknowledge_alert, create_maintenance_window (duration or end),
end_maintenance_window (ends one in progress, cancels an upcoming one) —
operator. delete_monitor, delete_alert_rule — admin. get_alert_rule_options
— viewer. Every write is audited as "<user> via MCP".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-10-06 08:19:54 -04:00
co-authored by Claude Opus 5.5
parent 2ebfdc8cdf
commit 9abe0beecc
9 changed files with 1008 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files
steward/alerts/routes.py
+56 -128
View File
@@ -6,63 +6,32 @@ from sqlalchemy import select
from steward.auth.middleware import require_role
from steward.alerts import service as alert_svc
from steward.ansible import sources as src_module
from steward.models.alerts import AlertRule, AlertState, AlertStateEnum, AlertOperator, MaintenanceWindow
from steward.models.metrics import PluginMetric
from steward.models.alerts import AlertRule, AlertOperator, MaintenanceWindow
from steward.models.users import UserRole
alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")
_OPERATORS = [(op.value, op.value) for op in AlertOperator]
# Static catalog: source_module → available metric names.
# Unified monitors emit metrics under their TYPE (icmp/tcp/dns/http), each with
# is_up (1.0 up / 0.0 down) and response_ms (where the probe times it).
METRIC_CATALOG: dict[str, list[str]] = {
"icmp": ["is_up", "response_ms"],
"tcp": ["is_up", "response_ms"],
"dns": ["is_up", "response_ms"],
"http": ["is_up", "response_ms"],
"traefik": ["request_rate", "error_rate", "latency_p50_ms", "latency_p95_ms",
"latency_p99_ms", "response_bytes_rate", "cert_expiry_days"],
"unifi": ["is_up", "latency_ms", "total_clients"],
# restart_count = cumulative restarts (alert on crash-looping); is_healthy =
# 1.0 healthy / 0.0 unhealthy from the container HEALTHCHECK (alert on <1).
"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy"],
"host_agent": [
"cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
"disk_used_pct_worst", "load_1m", "load_5m", "load_15m", "uptime_secs",
],
# snmp metrics are user-defined OID labels — populated dynamically from plugin_metrics
"snmp": [],
}
# Ordered list for source module picker
_SOURCE_MODULES = list(METRIC_CATALOG.keys())
def _rule_form(form) -> dict:
"""The rule form's fields, as alert_svc.rule_fields takes them."""
return {
"name": form.get("name", ""),
"source_module": form.get("source_module", ""),
"resource_name": form.get("resource_name", ""),
"metric_name": form.get("metric_name", ""),
"operator": form.get("operator", ""),
"threshold": form.get("threshold"),
"consecutive_failures_required": form.get("consecutive_failures_required"),
}
async def _get_resources(db, source_module: str) -> list[str]:
"""Return sorted distinct resource names for a source module.
For monitor types (icmp/tcp/dns/http), supplements with monitor names so
the list is populated even before any metrics have been recorded.
"""
result = await db.execute(
select(PluginMetric.resource_name)
.where(PluginMetric.source_module == source_module)
.distinct()
.order_by(PluginMetric.resource_name)
)
resources: set[str] = {r for (r,) in result}
if source_module in ("icmp", "tcp", "dns", "http"):
from steward.models.monitors import Monitor
mon_result = await db.execute(
select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name)
)
for (name,) in mon_result:
resources.add(name)
return sorted(resources)
def _form_time(value) -> datetime:
"""A datetime-local field, read as UTC."""
try:
return datetime.fromisoformat((value or "").strip()).replace(tzinfo=timezone.utc)
except ValueError:
raise ValueError(f"{value!r} is not a date and time.") from None
def _is_admin() -> bool:
@@ -137,18 +106,9 @@ async def rule_fields_api():
current_resource = request.args.get("current_resource", "")
current_metric = request.args.get("current_metric", "")
async with current_app.db_sessionmaker() as db:
resources = await _get_resources(db, source) if source else []
metrics = METRIC_CATALOG.get(source, [])
# Dynamic sources (e.g. snmp) have empty static catalog — query live metric names
if source and not metrics:
async with current_app.db_sessionmaker() as db:
m_result = await db.execute(
select(PluginMetric.metric_name)
.where(PluginMetric.source_module == source)
.distinct()
.order_by(PluginMetric.metric_name)
)
metrics = [r for (r,) in m_result]
resources = await alert_svc.resources_for(db, source) if source else []
async with current_app.db_sessionmaker() as db:
metrics = await alert_svc.metrics_for(db, source)
return await render_template(
"alerts/_rule_fields.html",
resources=resources,
@@ -161,17 +121,17 @@ async def rule_fields_api():
@alerts_bp.get("/rules/new")
@require_role(UserRole.operator)
async def new_rule():
first_source = _SOURCE_MODULES[0]
first_source = alert_svc.SOURCE_MODULES[0]
async with current_app.db_sessionmaker() as db:
resources = await _get_resources(db, first_source)
resources = await alert_svc.resources_for(db, first_source)
return await render_template(
"alerts/rules_form.html",
rule=None,
operators=_OPERATORS,
source_modules=_SOURCE_MODULES,
source_modules=alert_svc.SOURCE_MODULES,
initial_source=first_source,
resources=resources,
metrics=METRIC_CATALOG.get(first_source, []),
metrics=alert_svc.METRIC_CATALOG.get(first_source, []),
ansible_sources=_ansible_source_names(),
)
@@ -184,15 +144,15 @@ async def edit_rule(rule_id: str):
rule = result.scalar_one_or_none()
if rule is None:
return "Not found", 404
resources = await _get_resources(db, rule.source_module)
resources = await alert_svc.resources_for(db, rule.source_module)
return await render_template(
"alerts/rules_form.html",
rule=rule,
operators=_OPERATORS,
source_modules=_SOURCE_MODULES,
source_modules=alert_svc.SOURCE_MODULES,
initial_source=rule.source_module,
resources=resources,
metrics=METRIC_CATALOG.get(rule.source_module, []),
metrics=alert_svc.METRIC_CATALOG.get(rule.source_module, []),
ansible_sources=_ansible_source_names(),
)
@@ -202,36 +162,20 @@ async def edit_rule(rule_id: str):
async def create_rule():
form = await request.form
user_id = session.get("user_id")
now = datetime.now(timezone.utc)
# Ansible action is admin-only; non-admins simply can't set one.
ansible_action = None
if _is_admin():
ansible_action, err = _parse_ansible_action(form)
if err:
return err, 400
rule = AlertRule(
name=form["name"].strip(),
source_module=form["source_module"].strip(),
resource_name=form["resource_name"].strip(),
metric_name=form["metric_name"].strip(),
operator=AlertOperator(form["operator"]),
threshold=float(form["threshold"]),
consecutive_failures_required=int(form.get("consecutive_failures_required") or 1),
enabled=True,
ansible_action=ansible_action,
created_by=user_id,
)
state = AlertState(
rule_id=rule.id,
state=AlertStateEnum.inactive,
last_transition_at=now,
)
async with current_app.db_sessionmaker() as db:
async with db.begin():
db.add(rule)
await db.flush()
state.rule_id = rule.id
db.add(state)
try:
fields = await alert_svc.rule_fields(db, **_rule_form(form))
except ValueError as exc:
return str(exc), 400
rule = await alert_svc.add_rule(db, fields, created_by=user_id,
ansible_action=ansible_action)
await log_audit(current_app, user_id, session.get("username", ""), "alert_rule.created",
entity_type="alert_rule", entity_id=rule.name,
detail={"source": rule.source_module, "resource": rule.resource_name,
@@ -256,13 +200,12 @@ async def update_rule(rule_id: str):
rule = result.scalar_one_or_none()
if rule is None:
return "Not found", 404
rule.name = form["name"].strip()
rule.source_module = form["source_module"].strip()
rule.resource_name = form["resource_name"].strip()
rule.metric_name = form["metric_name"].strip()
rule.operator = AlertOperator(form["operator"])
rule.threshold = float(form["threshold"])
rule.consecutive_failures_required = int(form.get("consecutive_failures_required") or 1)
try:
fields = await alert_svc.rule_fields(db, **_rule_form(form))
except ValueError as exc:
return str(exc), 400
for k, v in fields.items():
setattr(rule, k, v)
if apply_action:
rule.ansible_action = ansible_action
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
@@ -313,7 +256,7 @@ async def list_maintenance():
"alerts/maintenance.html",
windows=windows,
now=now,
source_modules=_SOURCE_MODULES,
source_modules=alert_svc.SOURCE_MODULES,
)
@@ -323,7 +266,7 @@ async def new_maintenance():
return await render_template(
"alerts/maintenance_form.html",
window=None,
source_modules=_SOURCE_MODULES,
source_modules=alert_svc.SOURCE_MODULES,
)
@@ -332,26 +275,18 @@ async def new_maintenance():
async def create_maintenance():
form = await request.form
user_id = session.get("user_id")
start_at = datetime.fromisoformat(form["start_at"]).replace(tzinfo=timezone.utc)
end_at = datetime.fromisoformat(form["end_at"]).replace(tzinfo=timezone.utc)
scope_source = form.get("scope_source", "").strip() or None
scope_resource = form.get("scope_resource", "").strip() or None
if scope_source is None:
scope_resource = None # can't have resource without source
window = MaintenanceWindow(
name=form["name"].strip(),
start_at=start_at,
end_at=end_at,
scope_source=scope_source,
scope_resource=scope_resource,
created_by=user_id,
)
try:
fields = alert_svc.window_fields(
name=form.get("name", ""),
start_at=_form_time(form.get("start_at")), end_at=_form_time(form.get("end_at")),
scope_source=form.get("scope_source"), scope_resource=form.get("scope_resource"))
except ValueError as exc:
return str(exc), 400
window = MaintenanceWindow(created_by=user_id, **fields)
async with current_app.db_sessionmaker() as db:
async with db.begin():
db.add(window)
scope_desc = "all" if not scope_source else (
f"{scope_source}/{scope_resource}" if scope_resource else f"{scope_source}/*"
)
scope_desc = alert_svc.window_scope(window)
await log_audit(current_app, user_id, session.get("username", ""),
"maintenance_window.created", entity_type="maintenance_window",
entity_id=window.name, detail={"scope": scope_desc})
@@ -382,19 +317,12 @@ async def delete_maintenance(window_id: str):
@require_role(UserRole.operator)
async def acknowledge(rule_id: str):
user_id = session.get("user_id")
now = datetime.now(timezone.utc)
async with current_app.db_sessionmaker() as db:
async with db.begin():
result = await db.execute(
select(AlertState).where(AlertState.rule_id == rule_id)
)
state = result.scalar_one_or_none()
if state is None:
try:
await alert_svc.acknowledge(db, rule_id, user_id)
except LookupError:
return "Not found", 404
if state.state != AlertStateEnum.firing:
return "Conflict: alert is not in FIRING state", 409
state.state = AlertStateEnum.acknowledged
state.acknowledged_by = user_id
state.acknowledged_at = now
state.last_transition_at = now
except ValueError as exc:
return f"Conflict: {exc}", 409
return redirect(url_for("alerts.list_alerts"))
steward/alerts/service.py
+177 -5
View File
@@ -1,14 +1,45 @@
"""Alert queries shared by the alerts pages and the MCP tools.
"""Alert queries and validation shared by the alerts pages and the MCP tools.
Each function takes an open session and returns rows, with no Quart request
context, so the routes and the /mcp tools (which run outside Quart's pipeline)
call the same code.
Each function takes an open session (or plain values), with no Quart request
context, so the routes and the /mcp tools (which run outside Quart's
pipeline) call the same code and refuse the same input for the same reasons.
"""
from __future__ import annotations
from datetime import datetime, timezone
from sqlalchemy import select
from steward.models.alerts import AlertRule, AlertState, AlertStateEnum, MaintenanceWindow
from steward.models.alerts import (
AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
)
from steward.models.metrics import PluginMetric
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
# Static catalog: source_module → available metric names.
# Unified monitors emit metrics under their TYPE (icmp/tcp/dns/http), each with
# is_up (1.0 up / 0.0 down) and response_ms (where the probe times it).
METRIC_CATALOG: dict[str, list[str]] = {
"icmp": ["is_up", "response_ms"],
"tcp": ["is_up", "response_ms"],
"dns": ["is_up", "response_ms"],
"http": ["is_up", "response_ms"],
"traefik": ["request_rate", "error_rate", "latency_p50_ms", "latency_p95_ms",
"latency_p99_ms", "response_bytes_rate", "cert_expiry_days"],
"unifi": ["is_up", "latency_ms", "total_clients"],
# restart_count = cumulative restarts (alert on crash-looping); is_healthy =
# 1.0 healthy / 0.0 unhealthy from the container HEALTHCHECK (alert on <1).
"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy"],
"host_agent": [
"cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
"disk_used_pct_worst", "load_1m", "load_5m", "load_15m", "uptime_secs",
],
# snmp metrics are user-defined OID labels — populated dynamically from plugin_metrics
"snmp": [],
}
# Ordered list for the source module picker.
SOURCE_MODULES = list(METRIC_CATALOG.keys())
# States that need someone's attention: pending (breaching, not yet for long
# enough), firing, and acknowledged-but-still-breaching.
@@ -34,3 +65,144 @@ async def list_maintenance_windows(db) -> list[MaintenanceWindow]:
return list((await db.execute(
select(MaintenanceWindow).order_by(MaintenanceWindow.start_at.desc())
)).scalars())
async def resources_for(db, source_module: str) -> list[str]:
"""Sorted distinct resource names for a source module.
For monitor types (icmp/tcp/dns/http), supplements with monitor names so
the list is populated even before any metrics have been recorded.
"""
result = await db.execute(
select(PluginMetric.resource_name)
.where(PluginMetric.source_module == source_module)
.distinct()
.order_by(PluginMetric.resource_name)
)
resources: set[str] = {r for (r,) in result}
if source_module in MONITOR_TYPE_VALUES:
mon_result = await db.execute(
select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name))
resources.update(name for (name,) in mon_result)
return sorted(resources)
async def metrics_for(db, source_module: str) -> list[str]:
"""The metric names a rule on this source can watch: the static catalog, or
for dynamic sources (snmp) the names recorded so far."""
metrics = METRIC_CATALOG.get(source_module, [])
if source_module and not metrics:
result = await db.execute(
select(PluginMetric.metric_name)
.where(PluginMetric.source_module == source_module)
.distinct()
.order_by(PluginMetric.metric_name))
metrics = [r for (r,) in result]
return metrics
def _choices(values: list[str], limit: int = 20) -> str:
if not values:
return "none yet"
shown = ", ".join(values[:limit])
return shown + (f" (and {len(values) - limit} more)" if len(values) > limit else "")
async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
operator: str, threshold, consecutive_failures_required=1) -> dict:
"""Validated column values for a new or edited alert rule.
The same choices the rule form offers: a known source, a resource that
source has reported (or, for monitors, a monitor of that type), and a
metric it emits. ValueError says what was wrong and what is valid.
"""
name = (name or "").strip()
if not name:
raise ValueError("an alert rule needs a name.")
source_module = (source_module or "").strip()
if source_module not in METRIC_CATALOG:
raise ValueError(f"source_module {source_module!r} is not one of {', '.join(SOURCE_MODULES)}.")
resource_name = (resource_name or "").strip()
resources = await resources_for(db, source_module)
if resource_name not in resources:
raise ValueError(f"{source_module} has no resource {resource_name!r}. "
f"Known: {_choices(resources)}.")
metric_name = (metric_name or "").strip()
metrics = await metrics_for(db, source_module)
if metric_name not in metrics:
raise ValueError(f"{source_module} does not report {metric_name!r}. "
f"Its metrics: {_choices(metrics)}.")
try:
op = AlertOperator((operator or "").strip())
except ValueError:
raise ValueError(f"operator {operator!r} is not one of "
f"{', '.join(o.value for o in AlertOperator)}.") from None
try:
threshold = float(threshold)
except (TypeError, ValueError):
raise ValueError(f"threshold must be a number, got {threshold!r}.") from None
try:
consecutive = (1 if consecutive_failures_required in (None, "")
else int(consecutive_failures_required))
except (TypeError, ValueError):
raise ValueError("consecutive checks must be a whole number.") from None
if consecutive < 1:
raise ValueError("consecutive checks must be at least 1.")
return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
"metric_name": metric_name, "operator": op, "threshold": threshold,
"consecutive_failures_required": consecutive}
async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
ansible_action: dict | None = None) -> AlertRule:
"""Insert a rule and its inactive state row, in the caller's transaction."""
rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
db.add(rule)
await db.flush()
db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
last_transition_at=datetime.now(timezone.utc)))
return rule
async def acknowledge(db, rule_id: str, user_id: str) -> AlertState:
"""Acknowledge a firing alert, in the caller's transaction.
LookupError when the rule has no state; ValueError when it is not firing.
"""
state = (await db.execute(
select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one_or_none()
if state is None:
raise LookupError(rule_id)
if state.state != AlertStateEnum.firing:
raise ValueError(f"the alert is {state.state.value}, not firing; only a firing alert "
"can be acknowledged.")
now = datetime.now(timezone.utc)
state.state = AlertStateEnum.acknowledged
state.acknowledged_by = user_id
state.acknowledged_at = now
state.last_transition_at = now
return state
def window_fields(*, name: str, start_at: datetime, end_at: datetime,
scope_source: str | None, scope_resource: str | None) -> dict:
"""Validated columns for a maintenance window. A resource needs a source."""
name = (name or "").strip()
if not name:
raise ValueError("a maintenance window needs a name.")
if end_at <= start_at:
raise ValueError("the window must end after it starts.")
scope_source = (scope_source or "").strip() or None
scope_resource = (scope_resource or "").strip() or None
if scope_source is not None and scope_source not in METRIC_CATALOG:
raise ValueError(f"source {scope_source!r} is not one of {', '.join(SOURCE_MODULES)}.")
if scope_source is None:
scope_resource = None # can't have resource without source
return {"name": name[:128], "start_at": start_at, "end_at": end_at,
"scope_source": scope_source, "scope_resource": scope_resource}
def window_scope(w) -> str:
if not w.scope_source:
return "all"
return f"{w.scope_source}/{w.scope_resource}" if w.scope_resource else f"{w.scope_source}/*"