bvandeusenandClaude Opus 5.5
c368e93e2c
Agent offline is an ordinary rule now. The host_agent scheduler records host_agent/<host>/seconds_since_seen every minute (from last_seen_at, or registration time for an agent that never reported), replacing the task that only logged stale agents. The catalog offers the metric, and picking it on a new rule fills "> 180". No data: each rule's state tracks its last sample and its usual gap, smoothed so that an outage counts as at most three gaps. A core task, alert_silence_check, runs every 60s and marks a rule no_data_since once its series has had no sample for three usual gaps (at least 5 minutes; an hour while the cadence is unknown). A never-sampled rule counts from when it started watching. The mark sits beside the state, so a firing alert that goes silent stays firing. Each silence records one event and sends one notification (rule option notify_no_data, default on; muted in maintenance windows). The next sample clears it with a "data resumed" event and notification. Creating, changing, disabling or re-enabling a rule restarts its watch. Migration 0029 adds the columns and seeds each existing rule's last sample and gap from its last day of data, so an upgrade does not flag every rule. Shown on the Alerts page (a NO DATA badge with its start time), in MCP list_alerts, list_alert_rules and get_host (state "no_data", no_data_since), in get_overview findings (alert_no_data, or a firing alert marked silent) and in the weekly report. The rule form has the notify option, and its metric reference now matches what traefik records and lists host_agent. Emails say "no data" and carry the event's note. The metric-catalog guard also reads positional MetricBatch.add arguments. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
276 lines
12 KiB
Python
276 lines
12 KiB
Python
"""Host tools: the fleet, one host in depth, and a host's agent metrics."""
|
|
from __future__ import annotations
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import selectinload
|
|
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core import metric_series
|
|
from steward.hosts import service as host_svc
|
|
from steward.mcp._context import mcp_app
|
|
from steward.models.ansible_inventory import AnsibleTarget
|
|
from steward.models.hosts import Host
|
|
from steward.models.users import UserRole
|
|
|
|
from ._shape import (
|
|
age_s, bounded, clamp_limit, db_session, iso, lookback, pick, rnd, summarize,
|
|
)
|
|
|
|
AGENT_SOURCE = "host_agent"
|
|
|
|
|
|
async def _all_hosts(db) -> list[Host]:
|
|
return list((await db.execute(select(Host).order_by(Host.name))).scalars())
|
|
|
|
|
|
def _agent_glance(a: dict | None) -> dict | None:
|
|
if not a:
|
|
return None
|
|
return {
|
|
"reporting": a.get("fresh", False),
|
|
"last_seen_at": iso(a.get("last_at")),
|
|
"last_seen_age_s": age_s(a.get("last_at")),
|
|
"cpu_pct": rnd(a.get("cpu_pct")),
|
|
"mem_used_pct": rnd(a.get("mem_used_pct")),
|
|
"disk_used_pct_worst": rnd(a.get("disk_used_pct_worst")),
|
|
"disk_root_used_pct": rnd(a.get("disk_root")),
|
|
}
|
|
|
|
|
|
_PART_GROUPS = ( # (prefix, group, the field that names one part)
|
|
("net:", "interfaces", "interface"),
|
|
("diskio:", "disk_io", "device"),
|
|
("temp:", "temperatures", "sensor"),
|
|
)
|
|
# Sort each group by what makes a part worth a look first.
|
|
_PART_ORDER = {
|
|
"mounts": lambda r: -(r.get("disk_used_pct") or 0),
|
|
"interfaces": lambda r: -((r.get("net_rx_bps") or 0) + (r.get("net_tx_bps") or 0)),
|
|
"disk_io": lambda r: -((r.get("disk_read_bps") or 0) + (r.get("disk_write_bps") or 0)),
|
|
"temperatures": lambda r: -(r.get("temp_c") or 0),
|
|
"other": lambda r: r["part"],
|
|
}
|
|
|
|
|
|
def agent_parts(parts: dict[str, dict], host_name: str, limit: int) -> dict:
|
|
"""A host's sub-resources grouped by kind, each group bounded on its own.
|
|
|
|
`parts` maps the part name (the resource after "<host>:") to its latest
|
|
metrics. Per-core CPU is summarised, not listed: the host's cpu_pct
|
|
already says how busy it is, and 30+ cores would crowd out the rest.
|
|
"""
|
|
groups: dict[str, list] = {"mounts": [], "interfaces": [], "disk_io": [],
|
|
"temperatures": [], "other": []}
|
|
cores = []
|
|
for name, metrics in parts.items():
|
|
if name.startswith("core") and name[4:].isdigit():
|
|
if metrics.get("cpu_pct") is not None:
|
|
cores.append((int(name[4:]), metrics["cpu_pct"]))
|
|
continue
|
|
for prefix, group, field in _PART_GROUPS:
|
|
if name.startswith(prefix):
|
|
groups[group].append({field: name[len(prefix):], "part": name, **metrics})
|
|
break
|
|
else:
|
|
if any(m.startswith("disk_") for m in metrics):
|
|
groups["mounts"].append({"mount": name, "part": name, **metrics})
|
|
else:
|
|
groups["other"].append({"part": name, **metrics})
|
|
more = (f"Pass a larger `parts_limit` (max 200), or read one part with query_metrics("
|
|
f'source_module="host_agent", resource="{host_name}:<part>").')
|
|
out: dict = {}
|
|
for group, rows in groups.items():
|
|
if rows or group != "other":
|
|
rows.sort(key=_PART_ORDER[group])
|
|
out[group] = bounded(rows, limit, key="items", more=more)
|
|
if cores:
|
|
pcts = [pct for _, pct in cores]
|
|
busiest = sorted(cores, key=lambda c: (-c[1], c[0]))[:3]
|
|
out["cpu_cores"] = {
|
|
"count": len(cores), "min_pct": rnd(min(pcts)),
|
|
"avg_pct": rnd(sum(pcts) / len(pcts)), "max_pct": rnd(max(pcts)),
|
|
"busiest": [{"core": i, "cpu_pct": rnd(pct)} for i, pct in busiest],
|
|
}
|
|
return out
|
|
|
|
|
|
async def list_hosts(limit: int = 100) -> dict:
|
|
"""Every registered host with its health at a glance.
|
|
|
|
For each host: `status` rolled up from its linked monitors ("down" if any
|
|
monitor's latest check failed, "up", "pending" when none has run yet,
|
|
"unmonitored" when it has no monitors), the best latency, uptime % over
|
|
24h/7d/30d, and — when a host agent reports for it — CPU, memory and disk
|
|
utilisation and whether the agent is still reporting (`agent.reporting`;
|
|
false means its last sample is older than the stale window).
|
|
|
|
Use get_host for one host's monitors, alerts and Ansible link, and
|
|
get_host_metrics for a metric's history. limit: default 100, max 500.
|
|
"""
|
|
limit = clamp_limit(limit, 100, 500)
|
|
async with db_session() as db:
|
|
hosts = await _all_hosts(db)
|
|
status = await host_svc.host_status(db, [h.id for h in hosts])
|
|
uptime = await host_svc.compute_uptime(db)
|
|
agent = await host_svc.agent_overview_by_host(
|
|
db, [h.name for h in hosts], host_svc.agent_stale_after_seconds(mcp_app().config))
|
|
rows = []
|
|
for h in hosts:
|
|
st = status.get(h.id)
|
|
rows.append({
|
|
"host_id": h.id,
|
|
"host_name": h.name,
|
|
"address": h.address,
|
|
"status": st["state"] if st else "unmonitored",
|
|
"monitor_count": st["count"] if st else 0,
|
|
"best_latency_ms": rnd(st["latency_ms"]) if st else None,
|
|
"uptime_pct": uptime.get(h.id) or {"24h": None, "7d": None, "30d": None},
|
|
"agent": _agent_glance(agent.get(h.name)),
|
|
})
|
|
return bounded(rows, limit, key="hosts")
|
|
|
|
|
|
async def get_host(host: str, parts_limit: int = 20) -> dict:
|
|
"""One host in depth: its monitors, agent snapshot, active alerts and Ansible link.
|
|
|
|
host: the host's name (case-insensitive) or id, as list_hosts shows.
|
|
parts_limit: how many of each kind of sub-resource to list (default 20,
|
|
max 200).
|
|
|
|
Returns the host's monitors (latest result, error, uptime), the agent's
|
|
latest values, alerts currently pending, firing or acknowledged on this
|
|
host's resources, and the Ansible inventory target linked to it with its
|
|
groups (`ansible_target`: null when unlinked).
|
|
|
|
The agent's values are `agent.metrics` (host-level) and its sub-resources
|
|
grouped by kind: `mounts` (fullest first), `interfaces` and `disk_io`
|
|
(busiest first), `temperatures` (hottest first), each with `items`,
|
|
`total` and `truncated`; `cpu_cores` summarises per-core CPU (count,
|
|
min/avg/max, the three busiest). One part's history is query_metrics with
|
|
source_module="host_agent" and resource="<host>:<part>", where `part` is
|
|
the item's `part`.
|
|
"""
|
|
parts_limit = clamp_limit(parts_limit, 20, 200)
|
|
async with db_session() as db:
|
|
h = pick(host, await _all_hosts(db), kind="host", list_tool="list_hosts")
|
|
monitors = await host_svc.host_monitors(db, h.id)
|
|
uptime = (await host_svc.compute_uptime(db)).get(h.id)
|
|
status = (await host_svc.host_status(db, [h.id])).get(h.id)
|
|
glance = (await host_svc.agent_overview_by_host(
|
|
db, [h.name], host_svc.agent_stale_after_seconds(mcp_app().config))).get(h.name)
|
|
samples = await metric_series.latest_values(db, AGENT_SOURCE, h.name, include_children=True)
|
|
active = await alert_svc.active_alerts(db)
|
|
target = (await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.host_id == h.id)
|
|
.options(selectinload(AnsibleTarget.groups))
|
|
)).scalar_one_or_none()
|
|
|
|
agent = None
|
|
if samples:
|
|
host_level: dict = {}
|
|
parts: dict[str, dict] = {}
|
|
for s in samples:
|
|
if s.resource_name == h.name:
|
|
host_level[s.metric_name] = rnd(s.value)
|
|
else:
|
|
parts.setdefault(s.resource_name[len(h.name) + 1:], {})[s.metric_name] = rnd(s.value)
|
|
agent = {
|
|
**(_agent_glance(glance) or {}),
|
|
"metrics": host_level,
|
|
**agent_parts(parts, h.name, parts_limit),
|
|
}
|
|
|
|
prefix = h.name + ":"
|
|
alerts = [
|
|
{
|
|
"rule_id": rule.id, "rule_name": rule.name, "state": alert_svc.alert_label(state),
|
|
"resource": rule.resource_name, "metric": rule.metric_name,
|
|
"condition": f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}",
|
|
"since": iso(state.last_transition_at), "age_s": age_s(state.last_transition_at),
|
|
}
|
|
for state, rule in active
|
|
if rule.resource_name == h.name or rule.resource_name.startswith(prefix)
|
|
]
|
|
|
|
return {
|
|
"host_id": h.id,
|
|
"host_name": h.name,
|
|
"address": h.address,
|
|
"status": status["state"] if status else "unmonitored",
|
|
"uptime_pct": uptime or {"24h": None, "7d": None, "30d": None},
|
|
"monitors": [
|
|
{
|
|
"monitor_id": d["monitor"].id,
|
|
"monitor_name": d["monitor"].name,
|
|
"type": d["monitor"].type,
|
|
"target": d["monitor"].target,
|
|
"enabled": d["monitor"].enabled,
|
|
"state": ("pending" if d["latest"] is None
|
|
else "up" if d["latest"].is_up else "down"),
|
|
"last_checked_at": iso(d["latest"].checked_at) if d["latest"] else None,
|
|
"response_ms": rnd(d["latest"].response_ms) if d["latest"] else None,
|
|
"error": d["latest"].error_msg if d["latest"] and not d["latest"].is_up else None,
|
|
"uptime_24h_pct": d["uptime_pct"],
|
|
}
|
|
for d in monitors
|
|
],
|
|
"agent": agent,
|
|
"active_alerts": alerts,
|
|
"ansible_target": {
|
|
"target_id": target.id,
|
|
"target_name": target.name,
|
|
"address": target.address,
|
|
"groups": sorted(g.name for g in target.groups),
|
|
} if target else None,
|
|
}
|
|
|
|
|
|
async def get_host_metrics(host: str, metric: str = "", range: str = "24h") -> dict:
|
|
"""A host agent metric over time: summary statistics plus a ≤60-point series.
|
|
|
|
host: name or id, as list_hosts shows.
|
|
metric: e.g. "cpu_pct", "mem_used_pct", "disk_used_pct_worst", "load_1m",
|
|
"net_rx_bps", "temp_c_max". Empty lists the metrics this host reports,
|
|
with their latest values. For a mount or interface, use query_metrics
|
|
with source_module="host_agent" and resource="<host>:<part>" (get_host
|
|
lists the parts).
|
|
range: "1h", "6h", "24h" (default), "7d" or "30d".
|
|
|
|
The summary carries min, avg, max, p95 and last over the range; each series
|
|
point is [time, value].
|
|
"""
|
|
since, key = lookback(range)
|
|
async with db_session() as db:
|
|
h = pick(host, await _all_hosts(db), kind="host", list_tool="list_hosts")
|
|
if not metric:
|
|
latest = await metric_series.latest_values(db, AGENT_SOURCE, h.name)
|
|
if not latest:
|
|
raise ValueError(f"no agent metrics recorded for host {h.name!r}; "
|
|
"is a host agent installed and reporting? list_hosts shows `agent`.")
|
|
return {
|
|
"host_id": h.id, "host_name": h.name,
|
|
"metrics": [
|
|
{"metric": s.metric_name, "last": rnd(s.value),
|
|
"last_at": iso(s.recorded_at), "age_s": age_s(s.recorded_at)}
|
|
for s in latest
|
|
],
|
|
"next": "Pass one of these as `metric` for its history.",
|
|
}
|
|
pts = await metric_series.points(db, AGENT_SOURCE, h.name, metric, since)
|
|
if not pts:
|
|
names = [s.metric_name for s in await metric_series.latest_values(db, AGENT_SOURCE, h.name)]
|
|
if metric not in names:
|
|
hint = (f" It reports: {', '.join(names)}." if names
|
|
else " No agent metrics are recorded for it at all.")
|
|
raise ValueError(f"host {h.name!r} has no metric {metric!r}.{hint}")
|
|
return {
|
|
"host_id": h.id, "host_name": h.name, "metric": metric, "range": key,
|
|
**summarize(pts),
|
|
}
|
|
|
|
|
|
def register(register_tool) -> None:
|
|
register_tool(list_hosts, role=UserRole.viewer, read_only=True)
|
|
register_tool(get_host, role=UserRole.viewer, read_only=True)
|
|
register_tool(get_host_metrics, role=UserRole.viewer, read_only=True)
|