Files
FabledSteward/steward/mcp/tools/monitors.py
T
bvandeusenandClaude Opus 5.5 c9f5c049db
CI / lint (push) Successful in 3s
CI / integration (push) Successful in 50s
CI / unit (push) Successful in 1m6s
CI / publish (push) Successful in 1m2s
feat(alerts): renames carry their alerts — monitor and host renames move rules, windows and history (#5065)
#5065 "Renames carry their alerts". Alert rules, maintenance-window scopes
and metric history key a thing by (source, name), so a rename used to leave
the rules watching a name nothing reported under, and split the history.

- core/renames.carry_rename moves AlertRule.resource_name,
  MaintenanceWindow.scope_resource, plugin_metrics and plugin_metrics_hourly
  to the new name in the caller's transaction. A host rename also carries
  its parts: host_agent "<host>:…" and docker "<host>/…" (HOST_SOURCES).
  Matching is exact (a prefix compare, not LIKE or a collation range).
  History left under the new name by something deleted is dropped first
  rather than merged in.
- Web monitor edit, MCP update_monitor and web host edit call it; MCP
  answers with `carried` (it used to return a warning naming stranded rules).
- A name another monitor of the same type, or another host, already has is
  refused on create and rename: two would share one history.
- The rename is audited (monitor.updated / host.renamed) with what moved.

Tests: an integration test renames a firing monitor through the web route
and finds its rule still firing and cleared by a sample on the new name,
its window and raw+hourly history moved and an orphan's history dropped; a
host rename moves agent and container parts but not a longer-named host's
or another source's; a guard that every host_agent resource is "<host>" or
"<host>:…".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 14:02:43 -04:00

279 lines
12 KiB
Python

"""Monitor tools: every uptime check, and one check's failures in depth."""
from __future__ import annotations
import json
from sqlalchemy import select
from steward.core import renames
from steward.models.hosts import Host
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
from steward.models.users import UserRole
from steward.monitors import service as monitor_svc
from ._audit import audit_mcp
from ._shape import (
age_s, bounded, clamp_limit, db_session, iso, now_utc, pick, redact_url, rnd,
)
_STATES = ("up", "down", "pending", "disabled")
def _state(m: Monitor, latest) -> str:
if not m.enabled:
return "disabled"
if latest is None:
return "pending"
return "up" if latest.is_up else "down"
def _tls_days(latest) -> float | None:
if latest is None or latest.tls_expires_at is None:
return None
return rnd((latest.tls_expires_at - now_utc()).total_seconds() / 86400.0, 1)
def _row(m: Monitor, st: dict, host_names: dict[str, str]) -> dict:
latest = st["latest"]
return {
"monitor_id": m.id,
"monitor_name": m.name,
"type": m.type,
"target": redact_url(m.target),
"host_id": m.host_id,
"host_name": host_names.get(m.host_id) if m.host_id else None,
"state": _state(m, latest),
"down_since": iso(st["down_since"]),
"down_for_s": age_s(st["down_since"]),
"last_checked_at": iso(latest.checked_at) if latest else None,
"response_ms": rnd(latest.response_ms) if latest else None,
"error": latest.error_msg if latest is not None and not latest.is_up else None,
"uptime_pct": st["uptime"] or {"24h": None, "7d": None, "30d": None},
"tls_expires_in_days": _tls_days(latest),
}
async def _load(db):
monitors = list((await db.execute(select(Monitor).order_by(Monitor.name))).scalars())
host_names = dict((await db.execute(select(Host.id, Host.name))).all())
return monitors, host_names
async def list_monitors(state: str = "", type: str = "", limit: int = 100) -> dict:
"""Every uptime monitor (ping, TCP, DNS, HTTP) with its current state.
state: "down", "up", "pending" (never checked yet) or "disabled"; empty for all.
type: "icmp", "tcp", "dns" or "http"; empty for all.
limit: default 100, max 500.
Down monitors come first, each with `down_since` (when its current run of
failures began) and the latest `error`. `uptime_pct` covers 24h/7d/30d;
`tls_expires_in_days` is set for HTTPS checks. get_monitor shows one
monitor's configuration and recent failures.
"""
if state and state not in _STATES:
raise ValueError(f"state {state!r} is not one of {', '.join(_STATES)}.")
if type and type not in MONITOR_TYPE_VALUES:
raise ValueError(f"type {type!r} is not one of {', '.join(MONITOR_TYPE_VALUES)}.")
limit = clamp_limit(limit, 100, 500)
async with db_session() as db:
monitors, host_names = await _load(db)
if type:
monitors = [m for m in monitors if m.type == type]
states = await monitor_svc.monitor_states(db, monitors)
rows = [_row(m, states[m.id], host_names) for m in monitors]
if state:
rows = [r for r in rows if r["state"] == state]
order = {"down": 0, "pending": 1, "up": 2, "disabled": 3}
rows.sort(key=lambda r: (order[r["state"]], r["monitor_name"].lower()))
counts = {s: 0 for s in _STATES}
for r in rows:
counts[r["state"]] += 1
return {"counts": counts, **bounded(rows, limit, key="monitors",
more="Pass a larger `limit`, or filter by `state` or `type`.")}
def _config(m: Monitor) -> dict:
try:
cfg = json.loads(m.config_json or "{}")
except ValueError:
return {}
if isinstance(cfg.get("headers"), dict):
# Header values are often credentials (Authorization, API keys).
cfg["headers"] = {k: "<redacted>" for k in cfg["headers"]}
return cfg
async def get_monitor(monitor: str, failures: int = 10) -> dict:
"""One monitor in depth: configuration, current state, uptime and recent failures.
monitor: the monitor's name (case-insensitive) or id, as list_monitors shows.
failures: how many of its latest failed checks to return (default 10, max 50),
newest first, each with its error message.
`heartbeat` is its last 30 checks oldest-first as a string, "+" up and "-"
down, to show a flapping check at a glance. Header values in an HTTP
monitor's configuration are redacted.
"""
failures = clamp_limit(failures, 10, 50)
async with db_session() as db:
monitors, host_names = await _load(db)
m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors")
st = (await monitor_svc.monitor_states(db, [m]))[m.id]
failed = await monitor_svc.recent_failures(db, m.id, failures)
return {
**_row(m, st, host_names),
"check_interval_s": m.check_interval_seconds or None,
"config": _config(m),
"heartbeat": "".join("+" if r.is_up else "-" for r in st["recent"]),
"recent_failures": [
{
"checked_at": iso(r.checked_at),
"age_s": age_s(r.checked_at),
"error": r.error_msg,
"status_code": r.status_code,
"response_ms": rnd(r.response_ms),
}
for r in failed
],
}
# ── changes ──────────────────────────────────────────────────────────────────
async def _host_id(db, host: str) -> str | None:
if not (host or "").strip():
return None
hosts = (await db.execute(select(Host))).scalars()
return pick(host, hosts, kind="host", list_tool="list_hosts").id
def _saved(m: Monitor, host_names: dict[str, str]) -> dict:
return {
"monitor_id": m.id,
"monitor_name": m.name,
"type": m.type,
"target": redact_url(m.target),
"host_name": host_names.get(m.host_id) if m.host_id else None,
"enabled": m.enabled,
"check_interval_s": m.check_interval_seconds or None,
"config": _config(m),
"next": f'get_monitor(monitor="{m.id}")',
}
async def create_monitor(name: str, type: str, target: str, host: str = "",
config: dict | None = None, check_interval_s: int = 0,
enabled: bool = True) -> dict:
"""Add an uptime check. Recorded in the audit log as "<you> via MCP".
type: "icmp" (ping), "tcp" (connect to a port), "dns" (resolve a name)
or "http" (request a URL).
target: an address or hostname; for http a URL (a bare host gets https://).
host: a Steward host's name or id to file the check under; empty for a
free-standing destination.
config, by type (every key optional, defaults shown):
tcp: {"port": 80}
dns: {"expected_ip": null} (set to fail when the name resolves elsewhere)
http: {"method": "GET" | "HEAD" | "POST", "expected_status": 200,
"content_match": "", "headers": {}, "timeout_seconds": 10 (1-60),
"follow_redirects": true, "verify_ssl": true}
icmp takes none.
check_interval_s: seconds between checks; 0 (default) uses the global
interval from Settings.
The check's metrics are reported under source_module = its type and
resource = its name, which is what an alert rule on it watches.
"""
fields = monitor_svc.monitor_fields(type, target=target, name=name,
check_interval_seconds=check_interval_s, opts=config or {})
async with db_session() as db, db.begin():
await monitor_svc.check_name_free(db, fields["type"], fields["name"])
m = Monitor(host_id=await _host_id(db, host), enabled=enabled, **fields)
db.add(m)
_monitors, host_names = await _load(db)
await audit_mcp("monitor.created", "monitor", m.name,
{"monitor_id": m.id, "type": m.type, "target": redact_url(m.target)})
return _saved(m, host_names)
async def update_monitor(monitor: str, name: str | None = None, target: str | None = None,
host: str | None = None, config: dict | None = None,
check_interval_s: int | None = None) -> dict:
"""Change a monitor. Only the arguments you pass change; the rest stay.
monitor: its name or id (list_monitors).
host: a host's name or id to file it under, or "" to unlink it.
config: the keys to change, merged over its current config (see
create_monitor for the keys of each type). A monitor's type cannot
change; delete it and create another.
A rename carries the monitor's alert rules, maintenance windows and
check history to the new name; `carried` says what moved. A name another
monitor of the same type has is refused.
Recorded in the audit log.
"""
async with db_session() as db, db.begin():
monitors, host_names = await _load(db)
m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors")
current = {k: v for k, v in m.config.items() if k in monitor_svc.CONFIG_KEYS[m.type]}
fields = monitor_svc.monitor_fields(
m.type, target=m.target if target is None else target,
name=m.name if name is None else name,
check_interval_seconds=(m.check_interval_seconds if check_interval_s is None
else check_interval_s),
opts={**current, **(config or {})})
old_name = m.name
if fields["name"] != old_name:
await monitor_svc.check_name_free(db, m.type, fields["name"], exclude_id=m.id)
for k, v in fields.items():
setattr(m, k, v)
if host is not None:
m.host_id = await _host_id(db, host)
moved = await renames.carry_rename(db, ((m.type, None),), old_name, m.name)
changed = sorted(k for k, v in {"name": name, "target": target, "host": host, "config": config,
"check_interval_s": check_interval_s}.items() if v is not None)
detail = {"monitor_id": m.id, "changed": changed}
if moved:
detail["renamed_from"] = old_name
detail["carried"] = moved
await audit_mcp("monitor.updated", "monitor", m.name, detail)
out = _saved(m, host_names)
if moved:
out["carried"] = {"from": old_name, **moved}
return out
async def set_monitor_enabled(monitor: str, enabled: bool) -> dict:
"""Pause (enabled=false) or resume a monitor. A paused monitor runs no checks
and raises no alerts. Recorded in the audit log."""
async with db_session() as db, db.begin():
monitors, host_names = await _load(db)
m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors")
m.enabled = bool(enabled)
await audit_mcp("monitor.enabled" if m.enabled else "monitor.disabled", "monitor", m.name,
{"monitor_id": m.id})
return _saved(m, host_names)
async def delete_monitor(monitor: str) -> dict:
"""Delete a monitor and its check history. To stop it but keep its history,
set_monitor_enabled(enabled=false) instead. Admin only; recorded in the
audit log."""
async with db_session() as db, db.begin():
monitors, _host_names = await _load(db)
m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors")
await db.delete(m)
await audit_mcp("monitor.deleted", "monitor", m.name,
{"monitor_id": m.id, "type": m.type, "target": redact_url(m.target)})
return {"deleted": m.name, "monitor_id": m.id}
def register(register_tool) -> None:
register_tool(list_monitors, role=UserRole.viewer, read_only=True)
register_tool(get_monitor, role=UserRole.viewer, read_only=True)
for fn in (create_monitor, update_monitor, set_monitor_enabled):
register_tool(fn, role=UserRole.operator, read_only=False)
register_tool(delete_monitor, role=UserRole.admin, read_only=False)