bvandeusenandClaude Opus 5.5
6eeb4039d7
- The host page's live strip gains a Temperature tile: temp_c_max (real thermometers only), its last 6 h and the sensor setting it, graded by Settings → Thresholds and counted in the header's worst reading, in °C. - MCP get_host: each temperature carries `counted`; an unwired input is `counted: false` with `not_counted_because`, listed after the real ones. host_agent registers the judge (judge_sensors), so core imports no plugin. - The Monitors card's "Add monitor" wrapped into two lines beside the uptime; it reads "Add", titled "Add a monitor for this host". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
304 lines
14 KiB
Python
304 lines
14 KiB
Python
"""Host tools: the fleet, one host in depth, and a host's agent metrics."""
|
|
from __future__ import annotations
|
|
|
|
from typing import Callable
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import selectinload
|
|
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core import metric_series
|
|
from steward.hosts import service as host_svc
|
|
from steward.mcp._context import mcp_app
|
|
from steward.models.ansible_inventory import AnsibleTarget
|
|
from steward.models.hosts import Host
|
|
from steward.models.users import UserRole
|
|
|
|
from ._shape import (
|
|
age_s, bounded, clamp_limit, db_session, iso, lookback, pick, rnd, summarize,
|
|
)
|
|
from .tickets import open_tickets_for
|
|
|
|
AGENT_SOURCE = "host_agent"
|
|
|
|
# Which of a host's sensors don't count as its temperature, and why: unwired
|
|
# inputs read a constant or move against the CPU. The plugin that knows its
|
|
# sensors registers the judge; with none, every sensor counts.
|
|
_UNCOUNTED_SENSORS: Callable[[str, dict[str, float]], dict[str, str]] | None = None
|
|
|
|
|
|
def judge_sensors(judge: Callable[[str, dict[str, float]], dict[str, str]]) -> None:
|
|
"""`judge(host_name, {sensor: celsius})` → {sensor: why} for each sensor
|
|
that is not a thermometer; get_host marks those `counted: false`."""
|
|
global _UNCOUNTED_SENSORS
|
|
_UNCOUNTED_SENSORS = judge
|
|
|
|
|
|
async def _all_hosts(db) -> list[Host]:
|
|
return list((await db.execute(select(Host).order_by(Host.name))).scalars())
|
|
|
|
|
|
def _agent_glance(a: dict | None) -> dict | None:
|
|
if not a:
|
|
return None
|
|
return {
|
|
"reporting": a.get("fresh", False),
|
|
"last_seen_at": iso(a.get("last_at")),
|
|
"last_seen_age_s": age_s(a.get("last_at")),
|
|
"cpu_pct": rnd(a.get("cpu_pct")),
|
|
"mem_used_pct": rnd(a.get("mem_used_pct")),
|
|
"disk_used_pct_worst": rnd(a.get("disk_used_pct_worst")),
|
|
"disk_root_used_pct": rnd(a.get("disk_root")),
|
|
}
|
|
|
|
|
|
_PART_GROUPS = ( # (prefix, group, the field that names one part)
|
|
("net:", "interfaces", "interface"),
|
|
("diskio:", "disk_io", "device"),
|
|
("drive:", "drive_health", "device"),
|
|
("temp:", "temperatures", "sensor"),
|
|
)
|
|
# Sort each group by what makes a part worth a look first.
|
|
_PART_ORDER = {
|
|
"mounts": lambda r: -(r.get("disk_used_pct") or 0),
|
|
"interfaces": lambda r: -((r.get("net_rx_bps") or 0) + (r.get("net_tx_bps") or 0)),
|
|
"disk_io": lambda r: -((r.get("disk_read_bps") or 0) + (r.get("disk_write_bps") or 0)),
|
|
"drive_health": lambda r: (-(r.get("drive_failing") or 0), -(r.get("drive_bad_sectors") or 0)),
|
|
"temperatures": lambda r: (not r.get("counted", True), -(r.get("temp_c") or 0)),
|
|
"other": lambda r: r["part"],
|
|
}
|
|
|
|
|
|
def agent_parts(parts: dict[str, dict], host_name: str, limit: int) -> dict:
|
|
"""A host's sub-resources grouped by kind, each group bounded on its own.
|
|
|
|
`parts` maps the part name (the resource after "<host>:") to its latest
|
|
metrics. Per-core CPU is summarised, not listed: the host's cpu_pct
|
|
already says how busy it is, and 30+ cores would crowd out the rest.
|
|
"""
|
|
groups: dict[str, list] = {"mounts": [], "interfaces": [], "disk_io": [], "drive_health": [],
|
|
"temperatures": [], "other": []}
|
|
cores = []
|
|
for name, metrics in parts.items():
|
|
if name.startswith("core") and name[4:].isdigit():
|
|
if metrics.get("cpu_pct") is not None:
|
|
cores.append((int(name[4:]), metrics["cpu_pct"]))
|
|
continue
|
|
for prefix, group, field in _PART_GROUPS:
|
|
if name.startswith(prefix):
|
|
groups[group].append({field: name[len(prefix):], "part": name, **metrics})
|
|
break
|
|
else:
|
|
if any(m.startswith("disk_") for m in metrics):
|
|
groups["mounts"].append({"mount": name, "part": name, **metrics})
|
|
else:
|
|
groups["other"].append({"part": name, **metrics})
|
|
temps = {r["sensor"]: r["temp_c"] for r in groups["temperatures"] if r.get("temp_c") is not None}
|
|
uncounted = _UNCOUNTED_SENSORS(host_name, temps) if _UNCOUNTED_SENSORS and temps else {}
|
|
for r in groups["temperatures"]:
|
|
r["counted"] = r["sensor"] not in uncounted
|
|
if not r["counted"]:
|
|
r["not_counted_because"] = uncounted[r["sensor"]]
|
|
more = (f"Pass a larger `parts_limit` (max 200), or read one part with query_metrics("
|
|
f'source_module="host_agent", resource="{host_name}:<part>").')
|
|
out: dict = {}
|
|
for group, rows in groups.items():
|
|
if rows or group != "other":
|
|
rows.sort(key=_PART_ORDER[group])
|
|
out[group] = bounded(rows, limit, key="items", more=more)
|
|
if cores:
|
|
pcts = [pct for _, pct in cores]
|
|
busiest = sorted(cores, key=lambda c: (-c[1], c[0]))[:3]
|
|
out["cpu_cores"] = {
|
|
"count": len(cores), "min_pct": rnd(min(pcts)),
|
|
"avg_pct": rnd(sum(pcts) / len(pcts)), "max_pct": rnd(max(pcts)),
|
|
"busiest": [{"core": i, "cpu_pct": rnd(pct)} for i, pct in busiest],
|
|
}
|
|
return out
|
|
|
|
|
|
async def list_hosts(limit: int = 100) -> dict:
|
|
"""Every registered host with its health at a glance.
|
|
|
|
For each host: `status` rolled up from its linked monitors ("down" if any
|
|
monitor's latest check failed, "up", "pending" when none has run yet,
|
|
"unmonitored" when it has no monitors), the best latency, uptime % over
|
|
24h/7d/30d, and — when a host agent reports for it — CPU, memory and disk
|
|
utilisation and whether the agent is still reporting (`agent.reporting`;
|
|
false means its last sample is older than the stale window).
|
|
|
|
Use get_host for one host's monitors, alerts and Ansible link, and
|
|
get_host_metrics for a metric's history. limit: default 100, max 500.
|
|
"""
|
|
limit = clamp_limit(limit, 100, 500)
|
|
async with db_session() as db:
|
|
hosts = await _all_hosts(db)
|
|
status = await host_svc.host_status(db, [h.id for h in hosts])
|
|
uptime = await host_svc.compute_uptime(db)
|
|
agent = await host_svc.agent_overview_by_host(
|
|
db, [h.name for h in hosts], host_svc.agent_stale_after_seconds(mcp_app().config))
|
|
rows = []
|
|
for h in hosts:
|
|
st = status.get(h.id)
|
|
rows.append({
|
|
"host_id": h.id,
|
|
"host_name": h.name,
|
|
"address": h.address,
|
|
"status": st["state"] if st else "unmonitored",
|
|
"monitor_count": st["count"] if st else 0,
|
|
"best_latency_ms": rnd(st["latency_ms"]) if st else None,
|
|
"uptime_pct": uptime.get(h.id) or {"24h": None, "7d": None, "30d": None},
|
|
"agent": _agent_glance(agent.get(h.name)),
|
|
})
|
|
return bounded(rows, limit, key="hosts")
|
|
|
|
|
|
async def get_host(host: str, parts_limit: int = 20) -> dict:
|
|
"""One host in depth: its monitors, agent snapshot, active alerts and Ansible link.
|
|
|
|
host: the host's name (case-insensitive) or id, as list_hosts shows.
|
|
parts_limit: how many of each kind of sub-resource to list (default 20,
|
|
max 200).
|
|
|
|
Returns the host's monitors (latest result, error, uptime), the agent's
|
|
latest values, alerts currently pending, firing or acknowledged on this
|
|
host's resources, its open tickets (`open_tickets`: known problems someone
|
|
recorded, list_tickets / get_ticket), and the Ansible inventory target
|
|
linked to it with its groups (`ansible_target`: null when unlinked).
|
|
|
|
The agent's values are `agent.metrics` (host-level) and its sub-resources
|
|
grouped by kind: `mounts` (fullest first), `interfaces` and `disk_io`
|
|
(busiest first), `temperatures` (the ones that count as the host's
|
|
temperature hottest first, then those left out of it: `counted: false`
|
|
with `not_counted_because`, likely nothing wired to them), each with `items`,
|
|
`total` and `truncated`; `cpu_cores` summarises per-core CPU (count,
|
|
min/avg/max, the three busiest). One part's history is query_metrics with
|
|
source_module="host_agent" and resource="<host>:<part>", where `part` is
|
|
the item's `part`.
|
|
"""
|
|
parts_limit = clamp_limit(parts_limit, 20, 200)
|
|
async with db_session() as db:
|
|
h = pick(host, await _all_hosts(db), kind="host", list_tool="list_hosts")
|
|
monitors = await host_svc.host_monitors(db, h.id)
|
|
uptime = (await host_svc.compute_uptime(db)).get(h.id)
|
|
status = (await host_svc.host_status(db, [h.id])).get(h.id)
|
|
glance = (await host_svc.agent_overview_by_host(
|
|
db, [h.name], host_svc.agent_stale_after_seconds(mcp_app().config))).get(h.name)
|
|
samples = await metric_series.latest_values(db, AGENT_SOURCE, h.name, include_children=True)
|
|
active = await alert_svc.active_alerts(db)
|
|
target = (await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.host_id == h.id)
|
|
.options(selectinload(AnsibleTarget.groups))
|
|
)).scalar_one_or_none()
|
|
tickets = await open_tickets_for(db, host=h.name)
|
|
|
|
agent = None
|
|
if samples:
|
|
host_level: dict = {}
|
|
parts: dict[str, dict] = {}
|
|
for s in samples:
|
|
if s.resource_name == h.name:
|
|
host_level[s.metric_name] = rnd(s.value)
|
|
else:
|
|
parts.setdefault(s.resource_name[len(h.name) + 1:], {})[s.metric_name] = rnd(s.value)
|
|
agent = {
|
|
**(_agent_glance(glance) or {}),
|
|
"metrics": host_level,
|
|
**agent_parts(parts, h.name, parts_limit),
|
|
}
|
|
|
|
prefix = h.name + ":"
|
|
alerts = [
|
|
{
|
|
"rule_id": rule.id, "rule_name": rule.name, "state": alert_svc.alert_label(state),
|
|
"resource": rule.resource_name, "metric": rule.metric_name,
|
|
"condition": f"{rule.metric_name} {rule.operator.said} {rule.threshold:g}",
|
|
"since": iso(state.last_transition_at), "age_s": age_s(state.last_transition_at),
|
|
}
|
|
for state, rule in active
|
|
if rule.resource_name == h.name or rule.resource_name.startswith(prefix)
|
|
]
|
|
|
|
return {
|
|
"host_id": h.id,
|
|
"host_name": h.name,
|
|
"address": h.address,
|
|
"status": status["state"] if status else "unmonitored",
|
|
"uptime_pct": uptime or {"24h": None, "7d": None, "30d": None},
|
|
"monitors": [
|
|
{
|
|
"monitor_id": d["monitor"].id,
|
|
"monitor_name": d["monitor"].name,
|
|
"type": d["monitor"].type,
|
|
"target": d["monitor"].target,
|
|
"enabled": d["monitor"].enabled,
|
|
"state": ("pending" if d["latest"] is None
|
|
else "up" if d["latest"].is_up else "down"),
|
|
"last_checked_at": iso(d["latest"].checked_at) if d["latest"] else None,
|
|
"response_ms": rnd(d["latest"].response_ms) if d["latest"] else None,
|
|
"error": d["latest"].error_msg if d["latest"] and not d["latest"].is_up else None,
|
|
"uptime_24h_pct": d["uptime_pct"],
|
|
}
|
|
for d in monitors
|
|
],
|
|
"agent": agent,
|
|
"active_alerts": alerts,
|
|
"open_tickets": tickets,
|
|
"ansible_target": {
|
|
"target_id": target.id,
|
|
"target_name": target.name,
|
|
"address": target.address,
|
|
"groups": sorted(g.name for g in target.groups),
|
|
} if target else None,
|
|
}
|
|
|
|
|
|
async def get_host_metrics(host: str, metric: str = "", range: str = "24h") -> dict:
|
|
"""A host agent metric over time: summary statistics plus a ≤60-point series.
|
|
|
|
host: name or id, as list_hosts shows.
|
|
metric: e.g. "cpu_pct", "mem_used_pct", "disk_used_pct_worst", "load_1m",
|
|
"net_rx_bps", "temp_c_max". Empty lists the metrics this host reports,
|
|
with their latest values. For a mount or interface, use query_metrics
|
|
with source_module="host_agent" and resource="<host>:<part>" (get_host
|
|
lists the parts).
|
|
range: "1h", "6h", "24h" (default), "7d" or "30d".
|
|
|
|
The summary carries min, avg, max, p95 and last over the range; each series
|
|
point is [time, value].
|
|
"""
|
|
since, key = lookback(range)
|
|
async with db_session() as db:
|
|
h = pick(host, await _all_hosts(db), kind="host", list_tool="list_hosts")
|
|
if not metric:
|
|
latest = await metric_series.latest_values(db, AGENT_SOURCE, h.name)
|
|
if not latest:
|
|
raise ValueError(f"no agent metrics recorded for host {h.name!r}; "
|
|
"is a host agent installed and reporting? list_hosts shows `agent`.")
|
|
return {
|
|
"host_id": h.id, "host_name": h.name,
|
|
"metrics": [
|
|
{"metric": s.metric_name, "last": rnd(s.value),
|
|
"last_at": iso(s.recorded_at), "age_s": age_s(s.recorded_at)}
|
|
for s in latest
|
|
],
|
|
"next": "Pass one of these as `metric` for its history.",
|
|
}
|
|
pts = await metric_series.points(db, AGENT_SOURCE, h.name, metric, since)
|
|
if not pts:
|
|
names = [s.metric_name for s in await metric_series.latest_values(db, AGENT_SOURCE, h.name)]
|
|
if metric not in names:
|
|
hint = (f" It reports: {', '.join(names)}." if names
|
|
else " No agent metrics are recorded for it at all.")
|
|
raise ValueError(f"host {h.name!r} has no metric {metric!r}.{hint}")
|
|
return {
|
|
"host_id": h.id, "host_name": h.name, "metric": metric, "range": key,
|
|
**summarize(pts),
|
|
}
|
|
|
|
|
|
def register(register_tool) -> None:
|
|
register_tool(list_hosts, role=UserRole.viewer, read_only=True)
|
|
register_tool(get_host, role=UserRole.viewer, read_only=True)
|
|
register_tool(get_host_metrics, role=UserRole.viewer, read_only=True)
|