bvandeusenandClaude Opus 5.5
8edc0c30a3
check_interval_s was `m.check_interval_seconds or None`, so a monitor following monitors.poll_interval_seconds read as null and a reader couldn't tell how often it ran. Monitor results now carry the effective interval and `interval_from: "monitor" | "global"`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
293 lines
13 KiB
Python
293 lines
13 KiB
Python
"""Monitor tools: every uptime check, and one check's failures in depth."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.core import renames
|
|
from steward.mcp._context import mcp_app
|
|
from steward.models.hosts import Host
|
|
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
|
|
from steward.models.users import UserRole
|
|
from steward.monitors import service as monitor_svc
|
|
|
|
from ._audit import audit_mcp
|
|
from ._shape import (
|
|
age_s, bounded, clamp_limit, db_session, iso, now_utc, pick, redact_url, rnd,
|
|
)
|
|
|
|
_STATES = ("up", "down", "pending", "disabled")
|
|
|
|
|
|
def _state(m: Monitor, latest) -> str:
|
|
if not m.enabled:
|
|
return "disabled"
|
|
if latest is None:
|
|
return "pending"
|
|
return "up" if latest.is_up else "down"
|
|
|
|
|
|
def _interval(m: Monitor) -> dict:
|
|
"""How often a monitor runs: its own interval, or the global one it
|
|
follows when it has none (#5433), and which."""
|
|
if m.check_interval_seconds:
|
|
return {"check_interval_s": m.check_interval_seconds, "interval_from": "monitor"}
|
|
return {"check_interval_s": int(mcp_app().config.get("MONITORS_POLL_INTERVAL", 60)),
|
|
"interval_from": "global"}
|
|
|
|
|
|
def _tls_days(latest) -> float | None:
|
|
if latest is None or latest.tls_expires_at is None:
|
|
return None
|
|
return rnd((latest.tls_expires_at - now_utc()).total_seconds() / 86400.0, 1)
|
|
|
|
|
|
def _row(m: Monitor, st: dict, host_names: dict[str, str]) -> dict:
|
|
latest = st["latest"]
|
|
return {
|
|
"monitor_id": m.id,
|
|
"monitor_name": m.name,
|
|
"type": m.type,
|
|
"target": redact_url(m.target),
|
|
"host_id": m.host_id,
|
|
"host_name": host_names.get(m.host_id) if m.host_id else None,
|
|
"state": _state(m, latest),
|
|
"down_since": iso(st["down_since"]),
|
|
"down_for_s": age_s(st["down_since"]),
|
|
"last_checked_at": iso(latest.checked_at) if latest else None,
|
|
"response_ms": rnd(latest.response_ms) if latest else None,
|
|
"error": latest.error_msg if latest is not None and not latest.is_up else None,
|
|
"uptime_pct": st["uptime"] or {"24h": None, "7d": None, "30d": None},
|
|
"tls_expires_in_days": _tls_days(latest),
|
|
}
|
|
|
|
|
|
async def _load(db):
|
|
monitors = list((await db.execute(select(Monitor).order_by(Monitor.name))).scalars())
|
|
host_names = dict((await db.execute(select(Host.id, Host.name))).all())
|
|
return monitors, host_names
|
|
|
|
|
|
async def list_monitors(state: str = "", type: str = "", limit: int = 100) -> dict:
|
|
"""Every uptime monitor (ping, TCP, DNS, HTTP) with its current state.
|
|
|
|
state: "down", "up", "pending" (never checked yet) or "disabled"; empty for all.
|
|
type: "icmp", "tcp", "dns" or "http"; empty for all.
|
|
limit: default 100, max 500.
|
|
|
|
Down monitors come first, each with `down_since` (when its current run of
|
|
failures began) and the latest `error`. `uptime_pct` covers 24h/7d/30d;
|
|
`tls_expires_in_days` is set for HTTPS checks. get_monitor shows one
|
|
monitor's configuration and recent failures.
|
|
"""
|
|
if state and state not in _STATES:
|
|
raise ValueError(f"state {state!r} is not one of {', '.join(_STATES)}.")
|
|
if type and type not in MONITOR_TYPE_VALUES:
|
|
raise ValueError(f"type {type!r} is not one of {', '.join(MONITOR_TYPE_VALUES)}.")
|
|
limit = clamp_limit(limit, 100, 500)
|
|
async with db_session() as db:
|
|
monitors, host_names = await _load(db)
|
|
if type:
|
|
monitors = [m for m in monitors if m.type == type]
|
|
states = await monitor_svc.monitor_states(db, monitors)
|
|
rows = [_row(m, states[m.id], host_names) for m in monitors]
|
|
if state:
|
|
rows = [r for r in rows if r["state"] == state]
|
|
order = {"down": 0, "pending": 1, "up": 2, "disabled": 3}
|
|
rows.sort(key=lambda r: (order[r["state"]], r["monitor_name"].lower()))
|
|
counts = {s: 0 for s in _STATES}
|
|
for r in rows:
|
|
counts[r["state"]] += 1
|
|
return {"counts": counts, **bounded(rows, limit, key="monitors",
|
|
more="Pass a larger `limit`, or filter by `state` or `type`.")}
|
|
|
|
|
|
def _config(m: Monitor) -> dict:
|
|
try:
|
|
cfg = json.loads(m.config_json or "{}")
|
|
except ValueError:
|
|
return {}
|
|
if isinstance(cfg.get("headers"), dict):
|
|
# Header values are often credentials (Authorization, API keys).
|
|
cfg["headers"] = {k: "<redacted>" for k in cfg["headers"]}
|
|
return cfg
|
|
|
|
|
|
async def get_monitor(monitor: str, failures: int = 10) -> dict:
|
|
"""One monitor in depth: configuration, current state, uptime and recent failures.
|
|
|
|
monitor: the monitor's name (case-insensitive) or id, as list_monitors shows.
|
|
failures: how many of its latest failed checks to return (default 10, max 50),
|
|
newest first, each with its error message.
|
|
|
|
`heartbeat` is its last 30 checks oldest-first as a string, "+" up and "-"
|
|
down, to show a flapping check at a glance. `check_interval_s` is how
|
|
often it runs; `interval_from` says whether that is its own setting
|
|
("monitor") or the global monitors interval it follows ("global").
|
|
Header values in an HTTP monitor's configuration are redacted.
|
|
"""
|
|
failures = clamp_limit(failures, 10, 50)
|
|
async with db_session() as db:
|
|
monitors, host_names = await _load(db)
|
|
m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors")
|
|
st = (await monitor_svc.monitor_states(db, [m]))[m.id]
|
|
failed = await monitor_svc.recent_failures(db, m.id, failures)
|
|
return {
|
|
**_row(m, st, host_names),
|
|
**_interval(m),
|
|
"config": _config(m),
|
|
"heartbeat": "".join("+" if r.is_up else "-" for r in st["recent"]),
|
|
"recent_failures": [
|
|
{
|
|
"checked_at": iso(r.checked_at),
|
|
"age_s": age_s(r.checked_at),
|
|
"error": r.error_msg,
|
|
"status_code": r.status_code,
|
|
"response_ms": rnd(r.response_ms),
|
|
}
|
|
for r in failed
|
|
],
|
|
}
|
|
|
|
|
|
# ── changes ──────────────────────────────────────────────────────────────────
|
|
|
|
async def _host_id(db, host: str) -> str | None:
|
|
if not (host or "").strip():
|
|
return None
|
|
hosts = (await db.execute(select(Host))).scalars()
|
|
return pick(host, hosts, kind="host", list_tool="list_hosts").id
|
|
|
|
|
|
def _saved(m: Monitor, host_names: dict[str, str]) -> dict:
|
|
return {
|
|
"monitor_id": m.id,
|
|
"monitor_name": m.name,
|
|
"type": m.type,
|
|
"target": redact_url(m.target),
|
|
"host_name": host_names.get(m.host_id) if m.host_id else None,
|
|
"enabled": m.enabled,
|
|
**_interval(m),
|
|
"config": _config(m),
|
|
"next": f'get_monitor(monitor="{m.id}")',
|
|
}
|
|
|
|
|
|
async def create_monitor(name: str, type: str, target: str, host: str = "",
|
|
config: dict | None = None, check_interval_s: int = 0,
|
|
enabled: bool = True) -> dict:
|
|
"""Add an uptime check. Recorded in the audit log as "<you> via MCP".
|
|
|
|
type: "icmp" (ping), "tcp" (connect to a port), "dns" (resolve a name)
|
|
or "http" (request a URL).
|
|
target: an address or hostname; for http a URL (a bare host gets https://).
|
|
host: a Steward host's name or id to file the check under; empty for a
|
|
free-standing destination.
|
|
config, by type (every key optional, defaults shown):
|
|
tcp: {"port": 80}
|
|
dns: {"expected_ip": null, "resolver": null} (expected_ip: fail when the
|
|
name resolves elsewhere; resolver: a DNS server's IP to ask
|
|
directly, timing its answer; null asks the container's own)
|
|
http: {"method": "GET" | "HEAD" | "POST", "expected_status": 200,
|
|
"content_match": "", "headers": {}, "timeout_seconds": 10 (1-60),
|
|
"follow_redirects": true, "verify_ssl": true}
|
|
icmp takes none.
|
|
check_interval_s: seconds between checks; 0 (default) uses the global
|
|
interval from Settings.
|
|
|
|
The check's metrics are reported under source_module = its type and
|
|
resource = its name, which is what an alert rule on it watches.
|
|
"""
|
|
fields = monitor_svc.monitor_fields(type, target=target, name=name,
|
|
check_interval_seconds=check_interval_s, opts=config or {})
|
|
async with db_session() as db, db.begin():
|
|
await monitor_svc.check_name_free(db, fields["type"], fields["name"])
|
|
m = Monitor(host_id=await _host_id(db, host), enabled=enabled, **fields)
|
|
db.add(m)
|
|
_monitors, host_names = await _load(db)
|
|
await audit_mcp("monitor.created", "monitor", m.name,
|
|
{"monitor_id": m.id, "type": m.type, "target": redact_url(m.target)})
|
|
return _saved(m, host_names)
|
|
|
|
|
|
async def update_monitor(monitor: str, name: str | None = None, target: str | None = None,
|
|
host: str | None = None, config: dict | None = None,
|
|
check_interval_s: int | None = None) -> dict:
|
|
"""Change a monitor. Only the arguments you pass change; the rest stay.
|
|
|
|
monitor: its name or id (list_monitors).
|
|
host: a host's name or id to file it under, or "" to unlink it.
|
|
config: the keys to change, merged over its current config (see
|
|
create_monitor for the keys of each type). A monitor's type cannot
|
|
change; delete it and create another.
|
|
|
|
A rename carries the monitor's alert rules, maintenance windows and
|
|
check history to the new name; `carried` says what moved. A name another
|
|
monitor of the same type has is refused.
|
|
|
|
Recorded in the audit log.
|
|
"""
|
|
async with db_session() as db, db.begin():
|
|
monitors, host_names = await _load(db)
|
|
m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors")
|
|
current = {k: v for k, v in m.config.items() if k in monitor_svc.CONFIG_KEYS[m.type]}
|
|
fields = monitor_svc.monitor_fields(
|
|
m.type, target=m.target if target is None else target,
|
|
name=m.name if name is None else name,
|
|
check_interval_seconds=(m.check_interval_seconds if check_interval_s is None
|
|
else check_interval_s),
|
|
opts={**current, **(config or {})})
|
|
old_name = m.name
|
|
if fields["name"] != old_name:
|
|
await monitor_svc.check_name_free(db, m.type, fields["name"], exclude_id=m.id)
|
|
for k, v in fields.items():
|
|
setattr(m, k, v)
|
|
if host is not None:
|
|
m.host_id = await _host_id(db, host)
|
|
moved = await renames.carry_rename(db, ((m.type, None),), old_name, m.name)
|
|
changed = sorted(k for k, v in {"name": name, "target": target, "host": host, "config": config,
|
|
"check_interval_s": check_interval_s}.items() if v is not None)
|
|
detail = {"monitor_id": m.id, "changed": changed}
|
|
if moved:
|
|
detail["renamed_from"] = old_name
|
|
detail["carried"] = moved
|
|
await audit_mcp("monitor.updated", "monitor", m.name, detail)
|
|
out = _saved(m, host_names)
|
|
if moved:
|
|
out["carried"] = {"from": old_name, **moved}
|
|
return out
|
|
|
|
|
|
async def set_monitor_enabled(monitor: str, enabled: bool) -> dict:
|
|
"""Pause (enabled=false) or resume a monitor. A paused monitor runs no checks
|
|
and raises no alerts. Recorded in the audit log."""
|
|
async with db_session() as db, db.begin():
|
|
monitors, host_names = await _load(db)
|
|
m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors")
|
|
m.enabled = bool(enabled)
|
|
await audit_mcp("monitor.enabled" if m.enabled else "monitor.disabled", "monitor", m.name,
|
|
{"monitor_id": m.id})
|
|
return _saved(m, host_names)
|
|
|
|
|
|
async def delete_monitor(monitor: str) -> dict:
|
|
"""Delete a monitor and its check history. To stop it but keep its history,
|
|
set_monitor_enabled(enabled=false) instead. Admin only; recorded in the
|
|
audit log."""
|
|
async with db_session() as db, db.begin():
|
|
monitors, _host_names = await _load(db)
|
|
m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors")
|
|
await db.delete(m)
|
|
await audit_mcp("monitor.deleted", "monitor", m.name,
|
|
{"monitor_id": m.id, "type": m.type, "target": redact_url(m.target)})
|
|
return {"deleted": m.name, "monitor_id": m.id}
|
|
|
|
|
|
def register(register_tool) -> None:
|
|
register_tool(list_monitors, role=UserRole.viewer, read_only=True)
|
|
register_tool(get_monitor, role=UserRole.viewer, read_only=True)
|
|
for fn in (create_monitor, update_monitor, set_monitor_enabled):
|
|
register_tool(fn, role=UserRole.operator, read_only=False)
|
|
register_tool(delete_monitor, role=UserRole.admin, read_only=False)
|