"""Monitor tools: every uptime check, and one check's failures in depth.""" from __future__ import annotations import json from sqlalchemy import select from steward.core import renames from steward.mcp._context import mcp_app from steward.models.hosts import Host from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor from steward.models.users import UserRole from steward.monitors import service as monitor_svc from ._audit import audit_mcp from ._shape import ( age_s, bounded, clamp_limit, db_session, iso, now_utc, pick, redact_url, rnd, ) _STATES = ("up", "down", "pending", "disabled") def _state(m: Monitor, latest) -> str: if not m.enabled: return "disabled" if latest is None: return "pending" return "up" if latest.is_up else "down" def _interval(m: Monitor) -> dict: """How often a monitor runs: its own interval, or the global one it follows when it has none (#5433), and which.""" if m.check_interval_seconds: return {"check_interval_s": m.check_interval_seconds, "interval_from": "monitor"} return {"check_interval_s": int(mcp_app().config.get("MONITORS_POLL_INTERVAL", 60)), "interval_from": "global"} def _tls_days(latest) -> float | None: if latest is None or latest.tls_expires_at is None: return None return rnd((latest.tls_expires_at - now_utc()).total_seconds() / 86400.0, 1) def _row(m: Monitor, st: dict, host_names: dict[str, str]) -> dict: latest = st["latest"] return { "monitor_id": m.id, "monitor_name": m.name, "type": m.type, "target": redact_url(m.target), "host_id": m.host_id, "host_name": host_names.get(m.host_id) if m.host_id else None, "state": _state(m, latest), "down_since": iso(st["down_since"]), "down_for_s": age_s(st["down_since"]), "last_checked_at": iso(latest.checked_at) if latest else None, "response_ms": rnd(latest.response_ms) if latest else None, "error": latest.error_msg if latest is not None and not latest.is_up else None, "uptime_pct": st["uptime"] or {"24h": None, "7d": None, "30d": None}, "tls_expires_in_days": _tls_days(latest), } async def _load(db): monitors = list((await db.execute(select(Monitor).order_by(Monitor.name))).scalars()) host_names = dict((await db.execute(select(Host.id, Host.name))).all()) return monitors, host_names async def list_monitors(state: str = "", type: str = "", limit: int = 100) -> dict: """Every uptime monitor (ping, TCP, DNS, HTTP) with its current state. state: "down", "up", "pending" (never checked yet) or "disabled"; empty for all. type: "icmp", "tcp", "dns" or "http"; empty for all. limit: default 100, max 500. Down monitors come first, each with `down_since` (when its current run of failures began) and the latest `error`. `uptime_pct` covers 24h/7d/30d; `tls_expires_in_days` is set for HTTPS checks. get_monitor shows one monitor's configuration and recent failures. """ if state and state not in _STATES: raise ValueError(f"state {state!r} is not one of {', '.join(_STATES)}.") if type and type not in MONITOR_TYPE_VALUES: raise ValueError(f"type {type!r} is not one of {', '.join(MONITOR_TYPE_VALUES)}.") limit = clamp_limit(limit, 100, 500) async with db_session() as db: monitors, host_names = await _load(db) if type: monitors = [m for m in monitors if m.type == type] states = await monitor_svc.monitor_states(db, monitors) rows = [_row(m, states[m.id], host_names) for m in monitors] if state: rows = [r for r in rows if r["state"] == state] order = {"down": 0, "pending": 1, "up": 2, "disabled": 3} rows.sort(key=lambda r: (order[r["state"]], r["monitor_name"].lower())) counts = {s: 0 for s in _STATES} for r in rows: counts[r["state"]] += 1 return {"counts": counts, **bounded(rows, limit, key="monitors", more="Pass a larger `limit`, or filter by `state` or `type`.")} def _config(m: Monitor) -> dict: try: cfg = json.loads(m.config_json or "{}") except ValueError: return {} if isinstance(cfg.get("headers"), dict): # Header values are often credentials (Authorization, API keys). cfg["headers"] = {k: "" for k in cfg["headers"]} return cfg async def get_monitor(monitor: str, failures: int = 10) -> dict: """One monitor in depth: configuration, current state, uptime and recent failures. monitor: the monitor's name (case-insensitive) or id, as list_monitors shows. failures: how many of its latest failed checks to return (default 10, max 50), newest first, each with its error message. `heartbeat` is its last 30 checks oldest-first as a string, "+" up and "-" down, to show a flapping check at a glance. `check_interval_s` is how often it runs; `interval_from` says whether that is its own setting ("monitor") or the global monitors interval it follows ("global"). Header values in an HTTP monitor's configuration are redacted. """ failures = clamp_limit(failures, 10, 50) async with db_session() as db: monitors, host_names = await _load(db) m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors") st = (await monitor_svc.monitor_states(db, [m]))[m.id] failed = await monitor_svc.recent_failures(db, m.id, failures) return { **_row(m, st, host_names), **_interval(m), "config": _config(m), "heartbeat": "".join("+" if r.is_up else "-" for r in st["recent"]), "recent_failures": [ { "checked_at": iso(r.checked_at), "age_s": age_s(r.checked_at), "error": r.error_msg, "status_code": r.status_code, "response_ms": rnd(r.response_ms), } for r in failed ], } # ── changes ────────────────────────────────────────────────────────────────── async def _host_id(db, host: str) -> str | None: if not (host or "").strip(): return None hosts = (await db.execute(select(Host))).scalars() return pick(host, hosts, kind="host", list_tool="list_hosts").id def _saved(m: Monitor, host_names: dict[str, str]) -> dict: return { "monitor_id": m.id, "monitor_name": m.name, "type": m.type, "target": redact_url(m.target), "host_name": host_names.get(m.host_id) if m.host_id else None, "enabled": m.enabled, **_interval(m), "config": _config(m), "next": f'get_monitor(monitor="{m.id}")', } async def create_monitor(name: str, type: str, target: str, host: str = "", config: dict | None = None, check_interval_s: int = 0, enabled: bool = True) -> dict: """Add an uptime check. Recorded in the audit log as " via MCP". type: "icmp" (ping), "tcp" (connect to a port), "dns" (resolve a name) or "http" (request a URL). target: an address or hostname; for http a URL (a bare host gets https://). host: a Steward host's name or id to file the check under; empty for a free-standing destination. config, by type (every key optional, defaults shown): tcp: {"port": 80} dns: {"expected_ip": null, "resolver": null} (expected_ip: fail when the name resolves elsewhere; resolver: a DNS server's IP to ask directly, timing its answer; null asks the container's own) http: {"method": "GET" | "HEAD" | "POST", "expected_status": 200, "content_match": "", "headers": {}, "timeout_seconds": 10 (1-60), "follow_redirects": true, "verify_ssl": true} icmp takes none. check_interval_s: seconds between checks; 0 (default) uses the global interval from Settings. The check's metrics are reported under source_module = its type and resource = its name, which is what an alert rule on it watches. """ fields = monitor_svc.monitor_fields(type, target=target, name=name, check_interval_seconds=check_interval_s, opts=config or {}) async with db_session() as db, db.begin(): await monitor_svc.check_name_free(db, fields["type"], fields["name"]) m = Monitor(host_id=await _host_id(db, host), enabled=enabled, **fields) db.add(m) _monitors, host_names = await _load(db) await audit_mcp("monitor.created", "monitor", m.name, {"monitor_id": m.id, "type": m.type, "target": redact_url(m.target)}) return _saved(m, host_names) async def update_monitor(monitor: str, name: str | None = None, target: str | None = None, host: str | None = None, config: dict | None = None, check_interval_s: int | None = None) -> dict: """Change a monitor. Only the arguments you pass change; the rest stay. monitor: its name or id (list_monitors). host: a host's name or id to file it under, or "" to unlink it. config: the keys to change, merged over its current config (see create_monitor for the keys of each type). A monitor's type cannot change; delete it and create another. A rename carries the monitor's alert rules, maintenance windows and check history to the new name; `carried` says what moved. A name another monitor of the same type has is refused. Recorded in the audit log. """ async with db_session() as db, db.begin(): monitors, host_names = await _load(db) m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors") current = {k: v for k, v in m.config.items() if k in monitor_svc.CONFIG_KEYS[m.type]} fields = monitor_svc.monitor_fields( m.type, target=m.target if target is None else target, name=m.name if name is None else name, check_interval_seconds=(m.check_interval_seconds if check_interval_s is None else check_interval_s), opts={**current, **(config or {})}) old_name = m.name if fields["name"] != old_name: await monitor_svc.check_name_free(db, m.type, fields["name"], exclude_id=m.id) for k, v in fields.items(): setattr(m, k, v) if host is not None: m.host_id = await _host_id(db, host) moved = await renames.carry_rename(db, ((m.type, None),), old_name, m.name) changed = sorted(k for k, v in {"name": name, "target": target, "host": host, "config": config, "check_interval_s": check_interval_s}.items() if v is not None) detail = {"monitor_id": m.id, "changed": changed} if moved: detail["renamed_from"] = old_name detail["carried"] = moved await audit_mcp("monitor.updated", "monitor", m.name, detail) out = _saved(m, host_names) if moved: out["carried"] = {"from": old_name, **moved} return out async def set_monitor_enabled(monitor: str, enabled: bool) -> dict: """Pause (enabled=false) or resume a monitor. A paused monitor runs no checks and raises no alerts. Recorded in the audit log.""" async with db_session() as db, db.begin(): monitors, host_names = await _load(db) m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors") m.enabled = bool(enabled) await audit_mcp("monitor.enabled" if m.enabled else "monitor.disabled", "monitor", m.name, {"monitor_id": m.id}) return _saved(m, host_names) async def delete_monitor(monitor: str) -> dict: """Delete a monitor and its check history. To stop it but keep its history, set_monitor_enabled(enabled=false) instead. Admin only; recorded in the audit log.""" async with db_session() as db, db.begin(): monitors, _host_names = await _load(db) m = pick(monitor, monitors, kind="monitor", list_tool="list_monitors") await db.delete(m) await audit_mcp("monitor.deleted", "monitor", m.name, {"monitor_id": m.id, "type": m.type, "target": redact_url(m.target)}) return {"deleted": m.name, "monitor_id": m.id} def register(register_tool) -> None: register_tool(list_monitors, role=UserRole.viewer, read_only=True) register_tool(get_monitor, role=UserRole.viewer, read_only=True) for fn in (create_monitor, update_monitor, set_monitor_enabled): register_tool(fn, role=UserRole.operator, read_only=False) register_tool(delete_monitor, role=UserRole.admin, read_only=False)