"""Host tools: the fleet, one host in depth, and a host's agent metrics.""" from __future__ import annotations from sqlalchemy import select from sqlalchemy.orm import selectinload from steward.alerts import service as alert_svc from steward.core import metric_series from steward.hosts import service as host_svc from steward.mcp._context import mcp_app from steward.models.ansible_inventory import AnsibleTarget from steward.models.hosts import Host from steward.models.users import UserRole from ._shape import ( age_s, bounded, clamp_limit, db_session, iso, lookback, pick, rnd, summarize, ) AGENT_SOURCE = "host_agent" async def _all_hosts(db) -> list[Host]: return list((await db.execute(select(Host).order_by(Host.name))).scalars()) def _agent_glance(a: dict | None) -> dict | None: if not a: return None return { "reporting": a.get("fresh", False), "last_seen_at": iso(a.get("last_at")), "last_seen_age_s": age_s(a.get("last_at")), "cpu_pct": rnd(a.get("cpu_pct")), "mem_used_pct": rnd(a.get("mem_used_pct")), "disk_used_pct_worst": rnd(a.get("disk_used_pct_worst")), "disk_root_used_pct": rnd(a.get("disk_root")), } _PART_GROUPS = ( # (prefix, group, the field that names one part) ("net:", "interfaces", "interface"), ("diskio:", "disk_io", "device"), ("drive:", "drive_health", "device"), ("temp:", "temperatures", "sensor"), ) # Sort each group by what makes a part worth a look first. _PART_ORDER = { "mounts": lambda r: -(r.get("disk_used_pct") or 0), "interfaces": lambda r: -((r.get("net_rx_bps") or 0) + (r.get("net_tx_bps") or 0)), "disk_io": lambda r: -((r.get("disk_read_bps") or 0) + (r.get("disk_write_bps") or 0)), "drive_health": lambda r: (-(r.get("drive_failing") or 0), -(r.get("drive_bad_sectors") or 0)), "temperatures": lambda r: -(r.get("temp_c") or 0), "other": lambda r: r["part"], } def agent_parts(parts: dict[str, dict], host_name: str, limit: int) -> dict: """A host's sub-resources grouped by kind, each group bounded on its own. `parts` maps the part name (the resource after ":") to its latest metrics. Per-core CPU is summarised, not listed: the host's cpu_pct already says how busy it is, and 30+ cores would crowd out the rest. """ groups: dict[str, list] = {"mounts": [], "interfaces": [], "disk_io": [], "drive_health": [], "temperatures": [], "other": []} cores = [] for name, metrics in parts.items(): if name.startswith("core") and name[4:].isdigit(): if metrics.get("cpu_pct") is not None: cores.append((int(name[4:]), metrics["cpu_pct"])) continue for prefix, group, field in _PART_GROUPS: if name.startswith(prefix): groups[group].append({field: name[len(prefix):], "part": name, **metrics}) break else: if any(m.startswith("disk_") for m in metrics): groups["mounts"].append({"mount": name, "part": name, **metrics}) else: groups["other"].append({"part": name, **metrics}) more = (f"Pass a larger `parts_limit` (max 200), or read one part with query_metrics(" f'source_module="host_agent", resource="{host_name}:").') out: dict = {} for group, rows in groups.items(): if rows or group != "other": rows.sort(key=_PART_ORDER[group]) out[group] = bounded(rows, limit, key="items", more=more) if cores: pcts = [pct for _, pct in cores] busiest = sorted(cores, key=lambda c: (-c[1], c[0]))[:3] out["cpu_cores"] = { "count": len(cores), "min_pct": rnd(min(pcts)), "avg_pct": rnd(sum(pcts) / len(pcts)), "max_pct": rnd(max(pcts)), "busiest": [{"core": i, "cpu_pct": rnd(pct)} for i, pct in busiest], } return out async def list_hosts(limit: int = 100) -> dict: """Every registered host with its health at a glance. For each host: `status` rolled up from its linked monitors ("down" if any monitor's latest check failed, "up", "pending" when none has run yet, "unmonitored" when it has no monitors), the best latency, uptime % over 24h/7d/30d, and — when a host agent reports for it — CPU, memory and disk utilisation and whether the agent is still reporting (`agent.reporting`; false means its last sample is older than the stale window). Use get_host for one host's monitors, alerts and Ansible link, and get_host_metrics for a metric's history. limit: default 100, max 500. """ limit = clamp_limit(limit, 100, 500) async with db_session() as db: hosts = await _all_hosts(db) status = await host_svc.host_status(db, [h.id for h in hosts]) uptime = await host_svc.compute_uptime(db) agent = await host_svc.agent_overview_by_host( db, [h.name for h in hosts], host_svc.agent_stale_after_seconds(mcp_app().config)) rows = [] for h in hosts: st = status.get(h.id) rows.append({ "host_id": h.id, "host_name": h.name, "address": h.address, "status": st["state"] if st else "unmonitored", "monitor_count": st["count"] if st else 0, "best_latency_ms": rnd(st["latency_ms"]) if st else None, "uptime_pct": uptime.get(h.id) or {"24h": None, "7d": None, "30d": None}, "agent": _agent_glance(agent.get(h.name)), }) return bounded(rows, limit, key="hosts") async def get_host(host: str, parts_limit: int = 20) -> dict: """One host in depth: its monitors, agent snapshot, active alerts and Ansible link. host: the host's name (case-insensitive) or id, as list_hosts shows. parts_limit: how many of each kind of sub-resource to list (default 20, max 200). Returns the host's monitors (latest result, error, uptime), the agent's latest values, alerts currently pending, firing or acknowledged on this host's resources, and the Ansible inventory target linked to it with its groups (`ansible_target`: null when unlinked). The agent's values are `agent.metrics` (host-level) and its sub-resources grouped by kind: `mounts` (fullest first), `interfaces` and `disk_io` (busiest first), `temperatures` (hottest first), each with `items`, `total` and `truncated`; `cpu_cores` summarises per-core CPU (count, min/avg/max, the three busiest). One part's history is query_metrics with source_module="host_agent" and resource=":", where `part` is the item's `part`. """ parts_limit = clamp_limit(parts_limit, 20, 200) async with db_session() as db: h = pick(host, await _all_hosts(db), kind="host", list_tool="list_hosts") monitors = await host_svc.host_monitors(db, h.id) uptime = (await host_svc.compute_uptime(db)).get(h.id) status = (await host_svc.host_status(db, [h.id])).get(h.id) glance = (await host_svc.agent_overview_by_host( db, [h.name], host_svc.agent_stale_after_seconds(mcp_app().config))).get(h.name) samples = await metric_series.latest_values(db, AGENT_SOURCE, h.name, include_children=True) active = await alert_svc.active_alerts(db) target = (await db.execute( select(AnsibleTarget).where(AnsibleTarget.host_id == h.id) .options(selectinload(AnsibleTarget.groups)) )).scalar_one_or_none() agent = None if samples: host_level: dict = {} parts: dict[str, dict] = {} for s in samples: if s.resource_name == h.name: host_level[s.metric_name] = rnd(s.value) else: parts.setdefault(s.resource_name[len(h.name) + 1:], {})[s.metric_name] = rnd(s.value) agent = { **(_agent_glance(glance) or {}), "metrics": host_level, **agent_parts(parts, h.name, parts_limit), } prefix = h.name + ":" alerts = [ { "rule_id": rule.id, "rule_name": rule.name, "state": alert_svc.alert_label(state), "resource": rule.resource_name, "metric": rule.metric_name, "condition": f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}", "since": iso(state.last_transition_at), "age_s": age_s(state.last_transition_at), } for state, rule in active if rule.resource_name == h.name or rule.resource_name.startswith(prefix) ] return { "host_id": h.id, "host_name": h.name, "address": h.address, "status": status["state"] if status else "unmonitored", "uptime_pct": uptime or {"24h": None, "7d": None, "30d": None}, "monitors": [ { "monitor_id": d["monitor"].id, "monitor_name": d["monitor"].name, "type": d["monitor"].type, "target": d["monitor"].target, "enabled": d["monitor"].enabled, "state": ("pending" if d["latest"] is None else "up" if d["latest"].is_up else "down"), "last_checked_at": iso(d["latest"].checked_at) if d["latest"] else None, "response_ms": rnd(d["latest"].response_ms) if d["latest"] else None, "error": d["latest"].error_msg if d["latest"] and not d["latest"].is_up else None, "uptime_24h_pct": d["uptime_pct"], } for d in monitors ], "agent": agent, "active_alerts": alerts, "ansible_target": { "target_id": target.id, "target_name": target.name, "address": target.address, "groups": sorted(g.name for g in target.groups), } if target else None, } async def get_host_metrics(host: str, metric: str = "", range: str = "24h") -> dict: """A host agent metric over time: summary statistics plus a ≤60-point series. host: name or id, as list_hosts shows. metric: e.g. "cpu_pct", "mem_used_pct", "disk_used_pct_worst", "load_1m", "net_rx_bps", "temp_c_max". Empty lists the metrics this host reports, with their latest values. For a mount or interface, use query_metrics with source_module="host_agent" and resource=":" (get_host lists the parts). range: "1h", "6h", "24h" (default), "7d" or "30d". The summary carries min, avg, max, p95 and last over the range; each series point is [time, value]. """ since, key = lookback(range) async with db_session() as db: h = pick(host, await _all_hosts(db), kind="host", list_tool="list_hosts") if not metric: latest = await metric_series.latest_values(db, AGENT_SOURCE, h.name) if not latest: raise ValueError(f"no agent metrics recorded for host {h.name!r}; " "is a host agent installed and reporting? list_hosts shows `agent`.") return { "host_id": h.id, "host_name": h.name, "metrics": [ {"metric": s.metric_name, "last": rnd(s.value), "last_at": iso(s.recorded_at), "age_s": age_s(s.recorded_at)} for s in latest ], "next": "Pass one of these as `metric` for its history.", } pts = await metric_series.points(db, AGENT_SOURCE, h.name, metric, since) if not pts: names = [s.metric_name for s in await metric_series.latest_values(db, AGENT_SOURCE, h.name)] if metric not in names: hint = (f" It reports: {', '.join(names)}." if names else " No agent metrics are recorded for it at all.") raise ValueError(f"host {h.name!r} has no metric {metric!r}.{hint}") return { "host_id": h.id, "host_name": h.name, "metric": metric, "range": key, **summarize(pts), } def register(register_tool) -> None: register_tool(list_hosts, role=UserRole.viewer, read_only=True) register_tool(get_host, role=UserRole.viewer, read_only=True) register_tool(get_host_metrics, role=UserRole.viewer, read_only=True)