MCP-4 of milestone 461 "MCP server: Steward as an LLM-inspectable, LLM-operable hub". Twelve read-only viewer tools: - get_overview: what is wrong now, ranked critical-then-oldest. Failing monitors with their error and when the run of failures began, firing, pending and still-breaching acknowledged alerts, TLS certificates expiring within 14 days, agents that stopped reporting, hosts over the configured CPU/memory/disk thresholds (Settings > Thresholds), failed Ansible runs in the last 24h, and findings from enabled plugins through a new optional `mcp_overview(db)` plugin hook. Each finding names the call that shows its evidence. - list_hosts, get_host, get_host_metrics - list_monitors, get_monitor (HTTP header values and URL passwords are redacted) - list_alerts, get_alert_history, list_alert_rules, list_maintenance_windows - query_metrics: the whole metric store, narrowed one argument at a time (sources > resources > latest values > a series summary) - list_audit_events Shared shaping in steward/mcp/tools/_shape.py: UTC ISO times with age_s, bounded lists that say when and how they were cut, series as min/avg/max/p95/last plus a ≤60-point series, lookup by id or case-insensitive name with a did-you-mean, and a range argument that refuses an unknown key instead of reading it as 24h. New core reads in steward/core/metric_series.py (sources and resources via loose index scans, latest values via DISTINCT ON, series across the raw/hourly-rollup seam). monitors/service.py gains monitor_states, down_since and recent_failures; agent_overview_by_host now carries the fullest disk and the last sample time. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
210 lines
8.9 KiB
Python
210 lines
8.9 KiB
Python
"""Host tools: the fleet, one host in depth, and a host's agent metrics."""
|
|
from __future__ import annotations
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import selectinload
|
|
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core import metric_series
|
|
from steward.hosts import service as host_svc
|
|
from steward.mcp._context import mcp_app
|
|
from steward.models.ansible_inventory import AnsibleTarget
|
|
from steward.models.hosts import Host
|
|
from steward.models.users import UserRole
|
|
|
|
from ._shape import (
|
|
age_s, bounded, clamp_limit, db_session, iso, lookback, pick, rnd, summarize,
|
|
)
|
|
|
|
AGENT_SOURCE = "host_agent"
|
|
|
|
|
|
async def _all_hosts(db) -> list[Host]:
|
|
return list((await db.execute(select(Host).order_by(Host.name))).scalars())
|
|
|
|
|
|
def _agent_glance(a: dict | None) -> dict | None:
|
|
if not a:
|
|
return None
|
|
return {
|
|
"reporting": a.get("fresh", False),
|
|
"last_seen_at": iso(a.get("last_at")),
|
|
"last_seen_age_s": age_s(a.get("last_at")),
|
|
"cpu_pct": rnd(a.get("cpu_pct")),
|
|
"mem_used_pct": rnd(a.get("mem_used_pct")),
|
|
"disk_used_pct_worst": rnd(a.get("disk_used_pct_worst")),
|
|
"disk_root_used_pct": rnd(a.get("disk_root")),
|
|
}
|
|
|
|
|
|
async def list_hosts(limit: int = 100) -> dict:
|
|
"""Every registered host with its health at a glance.
|
|
|
|
For each host: `status` rolled up from its linked monitors ("down" if any
|
|
monitor's latest check failed, "up", "pending" when none has run yet,
|
|
"unmonitored" when it has no monitors), the best latency, uptime % over
|
|
24h/7d/30d, and — when a host agent reports for it — CPU, memory and disk
|
|
utilisation and whether the agent is still reporting (`agent.reporting`;
|
|
false means its last sample is older than the stale window).
|
|
|
|
Use get_host for one host's monitors, alerts and Ansible link, and
|
|
get_host_metrics for a metric's history. limit: default 100, max 500.
|
|
"""
|
|
limit = clamp_limit(limit, 100, 500)
|
|
async with db_session() as db:
|
|
hosts = await _all_hosts(db)
|
|
status = await host_svc.host_status(db, [h.id for h in hosts])
|
|
uptime = await host_svc.compute_uptime(db)
|
|
agent = await host_svc.agent_overview_by_host(
|
|
db, [h.name for h in hosts], host_svc.agent_stale_after_seconds(mcp_app().config))
|
|
rows = []
|
|
for h in hosts:
|
|
st = status.get(h.id)
|
|
rows.append({
|
|
"host_id": h.id,
|
|
"host_name": h.name,
|
|
"address": h.address,
|
|
"status": st["state"] if st else "unmonitored",
|
|
"monitor_count": st["count"] if st else 0,
|
|
"best_latency_ms": rnd(st["latency_ms"]) if st else None,
|
|
"uptime_pct": uptime.get(h.id) or {"24h": None, "7d": None, "30d": None},
|
|
"agent": _agent_glance(agent.get(h.name)),
|
|
})
|
|
return bounded(rows, limit, key="hosts")
|
|
|
|
|
|
async def get_host(host: str) -> dict:
|
|
"""One host in depth: its monitors, agent snapshot, active alerts and Ansible link.
|
|
|
|
host: the host's name (case-insensitive) or id, as list_hosts shows.
|
|
|
|
Returns the host's monitors (latest result, error, uptime), the agent's
|
|
latest values — host-level metrics plus each sub-resource such as a mount
|
|
or interface under `agent.parts` — alerts currently pending, firing or
|
|
acknowledged on this host's resources, and the Ansible inventory target
|
|
linked to it with its groups (`ansible_target`: null when unlinked).
|
|
"""
|
|
async with db_session() as db:
|
|
h = pick(host, await _all_hosts(db), kind="host", list_tool="list_hosts")
|
|
monitors = await host_svc.host_monitors(db, h.id)
|
|
uptime = (await host_svc.compute_uptime(db)).get(h.id)
|
|
status = (await host_svc.host_status(db, [h.id])).get(h.id)
|
|
glance = (await host_svc.agent_overview_by_host(
|
|
db, [h.name], host_svc.agent_stale_after_seconds(mcp_app().config))).get(h.name)
|
|
samples = await metric_series.latest_values(db, AGENT_SOURCE, h.name, include_children=True)
|
|
active = await alert_svc.active_alerts(db)
|
|
target = (await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.host_id == h.id)
|
|
.options(selectinload(AnsibleTarget.groups))
|
|
)).scalar_one_or_none()
|
|
|
|
agent = None
|
|
if samples:
|
|
host_level: dict = {}
|
|
parts: dict[str, dict] = {}
|
|
for s in samples:
|
|
if s.resource_name == h.name:
|
|
host_level[s.metric_name] = rnd(s.value)
|
|
else:
|
|
parts.setdefault(s.resource_name[len(h.name) + 1:], {})[s.metric_name] = rnd(s.value)
|
|
agent = {
|
|
**(_agent_glance(glance) or {}),
|
|
"metrics": host_level,
|
|
"parts": dict(list(sorted(parts.items()))[:40]),
|
|
"parts_total": len(parts),
|
|
}
|
|
|
|
prefix = h.name + ":"
|
|
alerts = [
|
|
{
|
|
"rule_id": rule.id, "rule_name": rule.name, "state": state.state.value,
|
|
"resource": rule.resource_name, "metric": rule.metric_name,
|
|
"condition": f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}",
|
|
"since": iso(state.last_transition_at), "age_s": age_s(state.last_transition_at),
|
|
}
|
|
for state, rule in active
|
|
if rule.resource_name == h.name or rule.resource_name.startswith(prefix)
|
|
]
|
|
|
|
return {
|
|
"host_id": h.id,
|
|
"host_name": h.name,
|
|
"address": h.address,
|
|
"status": status["state"] if status else "unmonitored",
|
|
"uptime_pct": uptime or {"24h": None, "7d": None, "30d": None},
|
|
"monitors": [
|
|
{
|
|
"monitor_id": d["monitor"].id,
|
|
"monitor_name": d["monitor"].name,
|
|
"type": d["monitor"].type,
|
|
"target": d["monitor"].target,
|
|
"enabled": d["monitor"].enabled,
|
|
"state": ("pending" if d["latest"] is None
|
|
else "up" if d["latest"].is_up else "down"),
|
|
"last_checked_at": iso(d["latest"].checked_at) if d["latest"] else None,
|
|
"response_ms": rnd(d["latest"].response_ms) if d["latest"] else None,
|
|
"error": d["latest"].error_msg if d["latest"] and not d["latest"].is_up else None,
|
|
"uptime_24h_pct": d["uptime_pct"],
|
|
}
|
|
for d in monitors
|
|
],
|
|
"agent": agent,
|
|
"active_alerts": alerts,
|
|
"ansible_target": {
|
|
"target_id": target.id,
|
|
"target_name": target.name,
|
|
"address": target.address,
|
|
"groups": sorted(g.name for g in target.groups),
|
|
} if target else None,
|
|
}
|
|
|
|
|
|
async def get_host_metrics(host: str, metric: str = "", range: str = "24h") -> dict:
|
|
"""A host agent metric over time: summary statistics plus a ≤60-point series.
|
|
|
|
host: name or id, as list_hosts shows.
|
|
metric: e.g. "cpu_pct", "mem_used_pct", "disk_used_pct_worst", "load_1m",
|
|
"net_rx_bps", "temp_c_max". Empty lists the metrics this host reports,
|
|
with their latest values. For a mount or interface, use query_metrics
|
|
with source_module="host_agent" and resource="<host>:<part>" (get_host
|
|
lists the parts).
|
|
range: "1h", "6h", "24h" (default), "7d" or "30d".
|
|
|
|
The summary carries min, avg, max, p95 and last over the range; each series
|
|
point is [time, value].
|
|
"""
|
|
since, key = lookback(range)
|
|
async with db_session() as db:
|
|
h = pick(host, await _all_hosts(db), kind="host", list_tool="list_hosts")
|
|
if not metric:
|
|
latest = await metric_series.latest_values(db, AGENT_SOURCE, h.name)
|
|
if not latest:
|
|
raise ValueError(f"no agent metrics recorded for host {h.name!r}; "
|
|
"is a host agent installed and reporting? list_hosts shows `agent`.")
|
|
return {
|
|
"host_id": h.id, "host_name": h.name,
|
|
"metrics": [
|
|
{"metric": s.metric_name, "last": rnd(s.value),
|
|
"last_at": iso(s.recorded_at), "age_s": age_s(s.recorded_at)}
|
|
for s in latest
|
|
],
|
|
"next": "Pass one of these as `metric` for its history.",
|
|
}
|
|
pts = await metric_series.points(db, AGENT_SOURCE, h.name, metric, since)
|
|
if not pts:
|
|
names = [s.metric_name for s in await metric_series.latest_values(db, AGENT_SOURCE, h.name)]
|
|
if metric not in names:
|
|
hint = (f" It reports: {', '.join(names)}." if names
|
|
else " No agent metrics are recorded for it at all.")
|
|
raise ValueError(f"host {h.name!r} has no metric {metric!r}.{hint}")
|
|
return {
|
|
"host_id": h.id, "host_name": h.name, "metric": metric, "range": key,
|
|
**summarize(pts),
|
|
}
|
|
|
|
|
|
def register(register_tool) -> None:
|
|
register_tool(list_hosts, role=UserRole.viewer, read_only=True)
|
|
register_tool(get_host, role=UserRole.viewer, read_only=True)
|
|
register_tool(get_host_metrics, role=UserRole.viewer, read_only=True)
|