MCP-4 of milestone 461 "MCP server: Steward as an LLM-inspectable, LLM-operable hub". Twelve read-only viewer tools: - get_overview: what is wrong now, ranked critical-then-oldest. Failing monitors with their error and when the run of failures began, firing, pending and still-breaching acknowledged alerts, TLS certificates expiring within 14 days, agents that stopped reporting, hosts over the configured CPU/memory/disk thresholds (Settings > Thresholds), failed Ansible runs in the last 24h, and findings from enabled plugins through a new optional `mcp_overview(db)` plugin hook. Each finding names the call that shows its evidence. - list_hosts, get_host, get_host_metrics - list_monitors, get_monitor (HTTP header values and URL passwords are redacted) - list_alerts, get_alert_history, list_alert_rules, list_maintenance_windows - query_metrics: the whole metric store, narrowed one argument at a time (sources > resources > latest values > a series summary) - list_audit_events Shared shaping in steward/mcp/tools/_shape.py: UTC ISO times with age_s, bounded lists that say when and how they were cut, series as min/avg/max/p95/last plus a ≤60-point series, lookup by id or case-insensitive name with a did-you-mean, and a range argument that refuses an unknown key instead of reading it as 24h. New core reads in steward/core/metric_series.py (sources and resources via loose index scans, latest values via DISTINCT ON, series across the raw/hourly-rollup seam). monitors/service.py gains monitor_states, down_since and recent_failures; agent_overview_by_host now carries the fullest disk and the last sample time. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
248 lines
9.7 KiB
Python
248 lines
9.7 KiB
Python
"""Host queries shared by the Hosts pages, dashboard widgets and MCP tools.
|
|
|
|
Every function takes an open session and returns plain data; none reads
|
|
Quart's request context, so the web routes and the /mcp tools (which run
|
|
outside Quart's pipeline) call the same code.
|
|
"""
|
|
from __future__ import annotations
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import and_, case, func, select
|
|
|
|
from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor
|
|
from steward.models.monitors import Monitor, MonitorResult
|
|
|
|
|
|
# Host-level agent metrics read for the at-a-glance views (plus the root
|
|
# mount's disk_used_pct, read from the "<host>:/" sub-resource).
|
|
AGENT_GLANCE_METRICS = ("cpu_pct", "mem_used_pct", "disk_used_pct", "disk_used_pct_worst")
|
|
|
|
|
|
def agent_stale_after_seconds(config) -> int:
|
|
"""The host_agent plugin's freshness window from app config (default 180s)."""
|
|
return int(((config.get("PLUGINS") or {}).get("host_agent") or {})
|
|
.get("stale_after_seconds", 180))
|
|
|
|
|
|
async def compute_uptime(db) -> dict[str, dict]:
|
|
"""Per-host uptime % for 24h/7d/30d, aggregated over the host's monitors.
|
|
|
|
Returns {host_id: {"24h": float|None, "7d": float|None, "30d": float|None}}.
|
|
"""
|
|
now = datetime.now(timezone.utc)
|
|
cutoff_30d = now - timedelta(days=30)
|
|
cutoff_7d = now - timedelta(days=7)
|
|
cutoff_24h = now - timedelta(hours=24)
|
|
|
|
result = await db.execute(
|
|
select(
|
|
Monitor.host_id,
|
|
func.count(MonitorResult.id).label("total_30d"),
|
|
func.sum(case((MonitorResult.is_up.is_(True), 1), else_=0)).label("up_30d"),
|
|
func.sum(case((MonitorResult.checked_at >= cutoff_7d, 1), else_=0)).label("total_7d"),
|
|
func.sum(case(
|
|
(and_(MonitorResult.checked_at >= cutoff_7d, MonitorResult.is_up.is_(True)), 1),
|
|
else_=0,
|
|
)).label("up_7d"),
|
|
func.sum(case((MonitorResult.checked_at >= cutoff_24h, 1), else_=0)).label("total_24h"),
|
|
func.sum(case(
|
|
(and_(MonitorResult.checked_at >= cutoff_24h, MonitorResult.is_up.is_(True)), 1),
|
|
else_=0,
|
|
)).label("up_24h"),
|
|
)
|
|
.join(Monitor, Monitor.id == MonitorResult.monitor_id)
|
|
.where(Monitor.host_id.isnot(None), MonitorResult.checked_at >= cutoff_30d)
|
|
.group_by(Monitor.host_id)
|
|
)
|
|
|
|
def _pct(up, total):
|
|
return round(float(up) / float(total) * 100, 2) if total else None
|
|
|
|
stats: dict[str, dict] = {}
|
|
for row in result:
|
|
stats[row.host_id] = {
|
|
"24h": _pct(row.up_24h, row.total_24h),
|
|
"7d": _pct(row.up_7d, row.total_7d),
|
|
"30d": _pct(row.up_30d, row.total_30d),
|
|
}
|
|
return stats
|
|
|
|
|
|
async def host_status(db, host_ids: list[str]) -> dict[str, dict]:
|
|
"""Per-host status rollup from the latest result of each linked monitor.
|
|
|
|
{host_id: {"state": "up"|"down"|"pending", "latency_ms": float|None,
|
|
"count": int}}. State is down if ANY linked monitor's latest
|
|
result is down (worst-case), up if at least one is up, else pending.
|
|
"""
|
|
if not host_ids:
|
|
return {}
|
|
rn = func.row_number().over(
|
|
partition_by=MonitorResult.monitor_id, order_by=MonitorResult.checked_at.desc()
|
|
).label("rn")
|
|
subq = (
|
|
select(MonitorResult.id, rn)
|
|
.join(Monitor, Monitor.id == MonitorResult.monitor_id)
|
|
.where(Monitor.host_id.in_(host_ids), Monitor.enabled.is_(True))
|
|
.subquery()
|
|
)
|
|
rows = (await db.execute(
|
|
select(MonitorResult, Monitor.host_id)
|
|
.join(subq, MonitorResult.id == subq.c.id)
|
|
.join(Monitor, Monitor.id == MonitorResult.monitor_id)
|
|
.where(subq.c.rn == 1)
|
|
)).all()
|
|
out: dict[str, dict] = {}
|
|
for res, host_id in rows:
|
|
d = out.setdefault(host_id, {"state": "pending", "latency_ms": None, "count": 0})
|
|
d["count"] += 1
|
|
if not res.is_up:
|
|
d["state"] = "down"
|
|
elif d["state"] != "down":
|
|
d["state"] = "up"
|
|
if res.is_up and res.response_ms is not None:
|
|
d["latency_ms"] = (res.response_ms if d["latency_ms"] is None
|
|
else min(d["latency_ms"], res.response_ms))
|
|
return out
|
|
|
|
|
|
async def host_monitors(db, host_id: str) -> list[dict]:
|
|
"""Display rows for the monitors linked to one host (host hub section)."""
|
|
monitors = (await db.execute(
|
|
select(Monitor).where(Monitor.host_id == host_id).order_by(Monitor.name)
|
|
)).scalars().all()
|
|
ids = [m.id for m in monitors]
|
|
recent = await _last_n_results(db, ids)
|
|
uptime = await _uptime_by_monitor(db, ids)
|
|
data = []
|
|
for m in monitors:
|
|
mrows = recent.get(m.id, [])
|
|
latest = mrows[-1] if mrows else None
|
|
data.append({
|
|
"monitor": m, "latest": latest,
|
|
"heartbeat": [{"state": "up" if r.is_up else "down",
|
|
"title": _heartbeat_title(m.type, r)} for r in mrows],
|
|
"uptime_pct": uptime.get(m.id, {}).get("24h"),
|
|
})
|
|
return data
|
|
|
|
|
|
async def agent_metrics_by_host(db, host_names: list[str]) -> dict[str, dict]:
|
|
"""Latest agent cpu/mem per host name from the generic PluginMetric table.
|
|
|
|
Read directly (PluginMetric is a core model) so the Hosts list can show an
|
|
at-a-glance agent column without importing the host_agent plugin.
|
|
"""
|
|
from steward.models.metrics import PluginMetric
|
|
if not host_names:
|
|
return {}
|
|
wanted = ("cpu_pct", "mem_used_pct")
|
|
subq = (
|
|
select(
|
|
PluginMetric.resource_name, PluginMetric.metric_name,
|
|
func.max(PluginMetric.recorded_at).label("m"),
|
|
)
|
|
.where(
|
|
PluginMetric.source_module == "host_agent",
|
|
PluginMetric.metric_name.in_(wanted),
|
|
PluginMetric.resource_name.in_(host_names),
|
|
)
|
|
.group_by(PluginMetric.resource_name, PluginMetric.metric_name)
|
|
).subquery()
|
|
rows = (await db.execute(
|
|
select(PluginMetric).join(
|
|
subq,
|
|
(PluginMetric.resource_name == subq.c.resource_name)
|
|
& (PluginMetric.metric_name == subq.c.metric_name)
|
|
& (PluginMetric.recorded_at == subq.c.m),
|
|
)
|
|
)).scalars().all()
|
|
out: dict[str, dict] = {}
|
|
for r in rows:
|
|
out.setdefault(r.resource_name, {})[r.metric_name] = r.value
|
|
return out
|
|
|
|
|
|
async def agent_overview_by_host(
|
|
db, host_names: list[str], stale_after_seconds: int = 180,
|
|
) -> dict[str, dict]:
|
|
"""Per-host agent glance (cpu/mem/disk + freshness) from PluginMetric.
|
|
|
|
{host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root,
|
|
last_at, fresh}} for each host with any agent samples.
|
|
|
|
Core-safe (no host_agent import). Freshness compares the latest host-level
|
|
sample against the host_agent plugin's stale window (stale_after_seconds;
|
|
callers pass agent_stale_after_seconds(app.config)).
|
|
"""
|
|
from steward.models.metrics import PluginMetric
|
|
if not host_names:
|
|
return {}
|
|
now = datetime.now(timezone.utc)
|
|
stale_after = int(stale_after_seconds)
|
|
resources = list(host_names) + [h + ":/" for h in host_names]
|
|
subq = (
|
|
select(
|
|
PluginMetric.resource_name, PluginMetric.metric_name,
|
|
func.max(PluginMetric.recorded_at).label("m"),
|
|
)
|
|
.where(
|
|
PluginMetric.source_module == "host_agent",
|
|
PluginMetric.resource_name.in_(resources),
|
|
PluginMetric.metric_name.in_(AGENT_GLANCE_METRICS),
|
|
)
|
|
.group_by(PluginMetric.resource_name, PluginMetric.metric_name)
|
|
).subquery()
|
|
rows = (await db.execute(
|
|
select(PluginMetric).join(
|
|
subq,
|
|
(PluginMetric.resource_name == subq.c.resource_name)
|
|
& (PluginMetric.metric_name == subq.c.metric_name)
|
|
& (PluginMetric.recorded_at == subq.c.m),
|
|
).where(PluginMetric.source_module == "host_agent")
|
|
)).scalars().all()
|
|
out: dict[str, dict] = {}
|
|
for r in rows:
|
|
if r.resource_name.endswith(":/") and r.metric_name == "disk_used_pct":
|
|
out.setdefault(r.resource_name[:-2], {})["disk_root"] = r.value
|
|
elif r.resource_name in host_names:
|
|
d = out.setdefault(r.resource_name, {})
|
|
d[r.metric_name] = r.value
|
|
if d.get("_ts") is None or r.recorded_at > d["_ts"]:
|
|
d["_ts"] = r.recorded_at
|
|
for d in out.values():
|
|
ts = d.pop("_ts", None)
|
|
d["last_at"] = ts
|
|
d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
|
|
return out
|
|
|
|
|
|
async def agent_cpu_sparks_by_host(db, host_names: list[str], hours: int = 1) -> dict[str, str]:
|
|
"""{host_name: inline-SVG cpu sparkline} over the last `hours` (core-safe).
|
|
|
|
A tiny at-a-glance trend in each row — the host-view sparkline, on the widget.
|
|
"""
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.core.status import sparkline_svg
|
|
if not host_names:
|
|
return {}
|
|
cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
|
|
rows = (await db.execute(
|
|
select(PluginMetric.resource_name, PluginMetric.value, PluginMetric.recorded_at)
|
|
.where(
|
|
PluginMetric.source_module == "host_agent",
|
|
PluginMetric.metric_name == "cpu_pct",
|
|
PluginMetric.resource_name.in_(host_names),
|
|
PluginMetric.recorded_at >= cutoff,
|
|
)
|
|
.order_by(PluginMetric.recorded_at)
|
|
)).all()
|
|
by_host: dict[str, list[float]] = {}
|
|
for r in rows:
|
|
by_host.setdefault(r.resource_name, []).append(r.value)
|
|
# Amber line to read as "CPU"; only draw when there are a couple of points.
|
|
return {
|
|
name: sparkline_svg(vals[-40:], width=64, height=18, stroke="#c8a840")
|
|
for name, vals in by_host.items() if len(vals) >= 2
|
|
}
|