MCP-7 #4971 "Monitoring-config write tools: monitors, alert rules, acks, maintenance". The validation the forms did inline (or not at all) moves to the services, and both the routes and the tools call it, so bad input is refused the same way with the same reason: - monitors/service: monitor_fields, build_config, normalise_target. Port 1-65535, status 100-599, timeout 1-60, method GET/HEAD/POST, headers an object of strings, a key the type does not take is named. The forms now answer 400 with the reason instead of redirecting silently or a 500. - alerts/service: METRIC_CATALOG and SOURCE_MODULES, resources_for, metrics_for, rule_fields (source, resource and metric from the same choices the rule form offers), add_rule, acknowledge, window_fields. Tools: create_monitor, update_monitor (config merged over the stored one; a rename warns about rules still watching the old name), set_monitor_enabled, create_alert_rule, update_alert_rule, acknowledge_alert, create_maintenance_window (duration or end), end_maintenance_window (ends one in progress, cancels an upcoming one) — operator. delete_monitor, delete_alert_rule — admin. get_alert_rule_options — viewer. Every write is audited as "<user> via MCP". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
277 lines
12 KiB
Python
277 lines
12 KiB
Python
"""Monitor tools: every uptime check, and one check's failures in depth."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.models.alerts import AlertRule
|
|
from steward.models.hosts import Host
|
|
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
|
|
from steward.models.users import UserRole
|
|
from steward.monitors import service as monitor_svc
|
|
|
|
from ._audit import audit_mcp
|
|
from ._shape import (
|
|
age_s, bounded, clamp_limit, db_session, iso, now_utc, pick, redact_url, rnd,
|
|
)
|
|
|
|
_STATES = ("up", "down", "pending", "disabled")
|
|
|
|
|
|
def _state(m: Monitor, latest) -> str:
|
|
if not m.enabled:
|
|
return "disabled"
|
|
if latest is None:
|
|
return "pending"
|
|
return "up" if latest.is_up else "down"
|
|
|
|
|
|
def _tls_days(latest) -> float | None:
|
|
if latest is None or latest.tls_expires_at is None:
|
|
return None
|
|
return rnd((latest.tls_expires_at - now_utc()).total_seconds() / 86400.0, 1)
|
|
|
|
|
|
def _row(m: Monitor, st: dict, host_names: dict[str, str]) -> dict:
|
|
latest = st["latest"]
|
|
return {
|
|
"monitor_id": m.id,
|
|
"monitor_name": m.name,
|
|
"type": m.type,
|
|
"target": redact_url(m.target),
|
|
"host_id": m.host_id,
|
|
"host_name": host_names.get(m.host_id) if m.host_id else None,
|
|
"state": _state(m, latest),
|
|
"down_since": iso(st["down_since"]),
|
|
"down_for_s": age_s(st["down_since"]),
|
|
"last_checked_at": iso(latest.checked_at) if latest else None,
|
|
"response_ms": rnd(latest.response_ms) if latest else None,
|
|
"error": latest.error_msg if latest is not None and not latest.is_up else None,
|
|
"uptime_pct": st["uptime"] or {"24h": None, "7d": None, "30d": None},
|
|
"tls_expires_in_days": _tls_days(latest),
|
|
}
|
|
|
|
|
|
async def _load(db):
|
|
monitors = list((await db.execute(select(Monitor).order_by(Monitor.name))).scalars())
|
|
host_names = dict((await db.execute(select(Host.id, Host.name))).all())
|
|
return monitors, host_names
|
|
|
|
|
|
async def list_monitors(state: str = "", type: str = "", limit: int = 100) -> dict:
|
|
"""Every uptime monitor (ping, TCP, DNS, HTTP) with its current state.
|
|
|
|
state: "down", "up", "pending" (never checked yet) or "disabled"; empty for all.
|
|
type: "icmp", "tcp", "dns" or "http"; empty for all.
|
|
limit: default 100, max 500.
|
|
|
|
Down monitors come first, each with `down_since` (when its current run of
|
|
failures began) and the latest `error`. `uptime_pct` covers 24h/7d/30d;
|
|
`tls_expires_in_days` is set for HTTPS checks. get_monitor shows one
|
|
monitor's configuration and recent failures.
|
|
"""
|
|
if state and state not in _STATES:
|
|
raise ValueError(f"state {state!r} is not one of {', '.join(_STATES)}.")
|
|
if type and type not in MONITOR_TYPE_VALUES:
|
|
raise ValueError(f"type {type!r} is not one of {', '.join(MONITOR_TYPE_VALUES)}.")
|
|
limit = clamp_limit(limit, 100, 500)
|
|
async with db_session() as db:
|
|
monitors, host_names = await _load(db)
|
|
if type:
|
|
monitors = [m for m in monitors if m.type == type]
|
|
states = await monitor_svc.monitor_states(db, monitors)
|
|
rows = [_row(m, states[m.id], host_names) for m in monitors]
|
|
if state:
|
|
rows = [r for r in rows if r["state"] == state]
|
|
order = {"down": 0, "pending": 1, "up": 2, "disabled": 3}
|
|
rows.sort(key=lambda r: (order[r["state"]], r["monitor_name"].lower()))
|
|
counts = {s: 0 for s in _STATES}
|
|
for r in rows:
|
|
counts[r["state"]] += 1
|
|
return {"counts": counts, **bounded(rows, limit, key="monitors",
|
|
more="Pass a larger `limit`, or filter by `state` or `type`.")}
|
|
|
|
|
|
def _config(m: Monitor) -> dict:
|
|
try:
|
|
cfg = json.loads(m.config_json or "{}")
|
|
except ValueError:
|
|
return {}
|
|
if isinstance(cfg.get("headers"), dict):
|
|
# Header values are often credentials (Authorization, API keys).
|
|
cfg["headers"] = {k: "<redacted>" for k in cfg["headers"]}
|
|
return cfg
|
|
|
|
|
|
async def get_monitor(monitor: str, failures: int = 10) -> dict:
|
|
"""One monitor in depth: configuration, current state, uptime and recent failures.
|
|
|
|
monitor: the monitor's name (case-insensitive) or id, as list_monitors shows.
|
|
failures: how many of its latest failed checks to return (default 10, max 50),
|
|
newest first, each with its error message.
|
|
|
|
`heartbeat` is its last 30 checks oldest-first as a string, "+" up and "-"
|
|
down, to show a flapping check at a glance. Header values in an HTTP
|
|
monitor's configuration are redacted.
|
|
"""
|
|
failures = clamp_limit(failures, 10, 50)
|
|
async with db_session() as db:
|
|
monitors, host_names = await _load(db)
|
|
m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors")
|
|
st = (await monitor_svc.monitor_states(db, [m]))[m.id]
|
|
failed = await monitor_svc.recent_failures(db, m.id, failures)
|
|
return {
|
|
**_row(m, st, host_names),
|
|
"check_interval_s": m.check_interval_seconds or None,
|
|
"config": _config(m),
|
|
"heartbeat": "".join("+" if r.is_up else "-" for r in st["recent"]),
|
|
"recent_failures": [
|
|
{
|
|
"checked_at": iso(r.checked_at),
|
|
"age_s": age_s(r.checked_at),
|
|
"error": r.error_msg,
|
|
"status_code": r.status_code,
|
|
"response_ms": rnd(r.response_ms),
|
|
}
|
|
for r in failed
|
|
],
|
|
}
|
|
|
|
|
|
# ── changes ──────────────────────────────────────────────────────────────────
|
|
|
|
async def _host_id(db, host: str) -> str | None:
|
|
if not (host or "").strip():
|
|
return None
|
|
hosts = (await db.execute(select(Host))).scalars()
|
|
return pick(host, hosts, kind="host", list_tool="list_hosts").id
|
|
|
|
|
|
def _saved(m: Monitor, host_names: dict[str, str]) -> dict:
|
|
return {
|
|
"monitor_id": m.id,
|
|
"monitor_name": m.name,
|
|
"type": m.type,
|
|
"target": redact_url(m.target),
|
|
"host_name": host_names.get(m.host_id) if m.host_id else None,
|
|
"enabled": m.enabled,
|
|
"check_interval_s": m.check_interval_seconds or None,
|
|
"config": _config(m),
|
|
"next": f'get_monitor(monitor="{m.id}")',
|
|
}
|
|
|
|
|
|
async def create_monitor(name: str, type: str, target: str, host: str = "",
|
|
config: dict | None = None, check_interval_s: int = 0,
|
|
enabled: bool = True) -> dict:
|
|
"""Add an uptime check. Recorded in the audit log as "<you> via MCP".
|
|
|
|
type: "icmp" (ping), "tcp" (connect to a port), "dns" (resolve a name)
|
|
or "http" (request a URL).
|
|
target: an address or hostname; for http a URL (a bare host gets https://).
|
|
host: a Steward host's name or id to file the check under; empty for a
|
|
free-standing destination.
|
|
config, by type (every key optional, defaults shown):
|
|
tcp: {"port": 80}
|
|
dns: {"expected_ip": null} (set to fail when the name resolves elsewhere)
|
|
http: {"method": "GET" | "HEAD" | "POST", "expected_status": 200,
|
|
"content_match": "", "headers": {}, "timeout_seconds": 10 (1-60),
|
|
"follow_redirects": true, "verify_ssl": true}
|
|
icmp takes none.
|
|
check_interval_s: seconds between checks; 0 (default) uses the global
|
|
interval from Settings.
|
|
|
|
The check's metrics are reported under source_module = its type and
|
|
resource = its name, which is what an alert rule on it watches.
|
|
"""
|
|
fields = monitor_svc.monitor_fields(type, target=target, name=name,
|
|
check_interval_seconds=check_interval_s, opts=config or {})
|
|
async with db_session() as db, db.begin():
|
|
m = Monitor(host_id=await _host_id(db, host), enabled=enabled, **fields)
|
|
db.add(m)
|
|
_monitors, host_names = await _load(db)
|
|
await audit_mcp("monitor.created", "monitor", m.name,
|
|
{"monitor_id": m.id, "type": m.type, "target": redact_url(m.target)})
|
|
return _saved(m, host_names)
|
|
|
|
|
|
async def update_monitor(monitor: str, name: str | None = None, target: str | None = None,
|
|
host: str | None = None, config: dict | None = None,
|
|
check_interval_s: int | None = None) -> dict:
|
|
"""Change a monitor. Only the arguments you pass change; the rest stay.
|
|
|
|
monitor: its name or id (list_monitors).
|
|
host: a host's name or id to file it under, or "" to unlink it.
|
|
config: the keys to change, merged over its current config (see
|
|
create_monitor for the keys of each type). A monitor's type cannot
|
|
change; delete it and create another.
|
|
|
|
Alert rules watch a monitor by its name, so a rename returns `warning`
|
|
naming the rules still on the old name.
|
|
|
|
Recorded in the audit log.
|
|
"""
|
|
async with db_session() as db, db.begin():
|
|
monitors, host_names = await _load(db)
|
|
m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors")
|
|
current = {k: v for k, v in m.config.items() if k in monitor_svc.CONFIG_KEYS[m.type]}
|
|
fields = monitor_svc.monitor_fields(
|
|
m.type, target=m.target if target is None else target,
|
|
name=m.name if name is None else name,
|
|
check_interval_seconds=(m.check_interval_seconds if check_interval_s is None
|
|
else check_interval_s),
|
|
opts={**current, **(config or {})})
|
|
old_name = m.name
|
|
for k, v in fields.items():
|
|
setattr(m, k, v)
|
|
if host is not None:
|
|
m.host_id = await _host_id(db, host)
|
|
stranded = []
|
|
if m.name != old_name:
|
|
stranded = list((await db.execute(
|
|
select(AlertRule.name).where(AlertRule.source_module == m.type,
|
|
AlertRule.resource_name == old_name))).scalars())
|
|
changed = sorted(k for k, v in {"name": name, "target": target, "host": host, "config": config,
|
|
"check_interval_s": check_interval_s}.items() if v is not None)
|
|
await audit_mcp("monitor.updated", "monitor", m.name, {"monitor_id": m.id, "changed": changed})
|
|
out = _saved(m, host_names)
|
|
if stranded:
|
|
out["warning"] = (f"Alert rules watch this monitor by name and still watch {old_name!r}: "
|
|
f"{', '.join(stranded)}. update_alert_rule(resource=\"{m.name}\") "
|
|
"points each at the new name.")
|
|
return out
|
|
|
|
|
|
async def set_monitor_enabled(monitor: str, enabled: bool) -> dict:
|
|
"""Pause (enabled=false) or resume a monitor. A paused monitor runs no checks
|
|
and raises no alerts. Recorded in the audit log."""
|
|
async with db_session() as db, db.begin():
|
|
monitors, host_names = await _load(db)
|
|
m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors")
|
|
m.enabled = bool(enabled)
|
|
await audit_mcp("monitor.enabled" if m.enabled else "monitor.disabled", "monitor", m.name,
|
|
{"monitor_id": m.id})
|
|
return _saved(m, host_names)
|
|
|
|
|
|
async def delete_monitor(monitor: str) -> dict:
|
|
"""Delete a monitor and its check history. To stop it but keep its history,
|
|
set_monitor_enabled(enabled=false) instead. Admin only; recorded in the
|
|
audit log."""
|
|
async with db_session() as db, db.begin():
|
|
monitors, _host_names = await _load(db)
|
|
m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors")
|
|
await db.delete(m)
|
|
await audit_mcp("monitor.deleted", "monitor", m.name,
|
|
{"monitor_id": m.id, "type": m.type, "target": redact_url(m.target)})
|
|
return {"deleted": m.name, "monitor_id": m.id}
|
|
|
|
|
|
def register(register_tool) -> None:
|
|
register_tool(list_monitors, role=UserRole.viewer, read_only=True)
|
|
register_tool(get_monitor, role=UserRole.viewer, read_only=True)
|
|
for fn in (create_monitor, update_monitor, set_monitor_enabled):
|
|
register_tool(fn, role=UserRole.operator, read_only=False)
|
|
register_tool(delete_monitor, role=UserRole.admin, read_only=False)
|