bvandeusenandClaude Opus 5.5
bfe3618e88
Hosts list: - CPU and Memory are their own labelled columns (#5279); a silent agent's last readings are greyed, and Status says "agent silent for 7m". - One uptime figure, 30 days, with 24h and 7d on hover. The SLA tab (/hosts/uptime) repeated it and is removed (#5280; one home per fact). - Status is a word with how long ("Down for 12m"); host_status now carries down_since. Edit and Delete live on the host's page only (#5281). - Figures are right-aligned with tabular digits (.table .num), not a monospace face. Agent fleet (#5282): its add-host form is gone, so hosts are added only on Hosts -> Add host. That form was also the only way to get a token for a manual curl install, so that moves to the host's Agent card as "Install manually" (POST /fleet/<host_id>/register), shown on the fleet page as before. "Delete" there reads "Unregister". Add host (#5283): laid out like the add-monitor form; a name clash comes back inline with what was typed instead of a bare text 400; the address is optional, as the host page already allowed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
256 lines
11 KiB
Python
256 lines
11 KiB
Python
"""Host queries shared by the Hosts pages, dashboard widgets and MCP tools.
|
|
|
|
Every function takes an open session and returns plain data; none reads
|
|
Quart's request context, so the web routes and the /mcp tools (which run
|
|
outside Quart's pipeline) call the same code.
|
|
"""
|
|
from __future__ import annotations
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import and_, case, func, select
|
|
|
|
from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor
|
|
from steward.models.hosts import Host
|
|
from steward.models.monitors import Monitor, MonitorResult
|
|
|
|
|
|
# Host-level agent metrics read for the at-a-glance views (plus the root
|
|
# mount's disk_used_pct, read from the "<host>:/" sub-resource).
|
|
AGENT_GLANCE_METRICS = ("cpu_pct", "mem_used_pct", "disk_used_pct", "disk_used_pct_worst", "load_1m")
|
|
|
|
|
|
async def check_name_free(db, name: str, exclude_id: str | None = None) -> None:
|
|
"""ValueError when another host already has this name, or it is blank.
|
|
|
|
Agent metrics, container metrics and the alert rules on them go by the
|
|
host's name, so two hosts sharing one would share one history.
|
|
"""
|
|
if not name:
|
|
raise ValueError("a host needs a name.")
|
|
q = select(Host.id).where(Host.name == name)
|
|
if exclude_id:
|
|
q = q.where(Host.id != exclude_id)
|
|
if (await db.execute(q.limit(1))).first():
|
|
raise ValueError(f"another host is already named {name!r}; pick a different name.")
|
|
|
|
|
|
def agent_stale_after_seconds(config) -> int:
|
|
"""The host_agent plugin's freshness window from app config (default 180s)."""
|
|
return int(((config.get("PLUGINS") or {}).get("host_agent") or {})
|
|
.get("stale_after_seconds", 180))
|
|
|
|
|
|
async def compute_uptime(db) -> dict[str, dict]:
|
|
"""Per-host uptime % for 24h/7d/30d, aggregated over the host's monitors.
|
|
|
|
Returns {host_id: {"24h": float|None, "7d": float|None, "30d": float|None}}.
|
|
"""
|
|
now = datetime.now(timezone.utc)
|
|
cutoff_30d = now - timedelta(days=30)
|
|
cutoff_7d = now - timedelta(days=7)
|
|
cutoff_24h = now - timedelta(hours=24)
|
|
|
|
result = await db.execute(
|
|
select(
|
|
Monitor.host_id,
|
|
func.count(MonitorResult.id).label("total_30d"),
|
|
func.sum(case((MonitorResult.is_up.is_(True), 1), else_=0)).label("up_30d"),
|
|
func.sum(case((MonitorResult.checked_at >= cutoff_7d, 1), else_=0)).label("total_7d"),
|
|
func.sum(case(
|
|
(and_(MonitorResult.checked_at >= cutoff_7d, MonitorResult.is_up.is_(True)), 1),
|
|
else_=0,
|
|
)).label("up_7d"),
|
|
func.sum(case((MonitorResult.checked_at >= cutoff_24h, 1), else_=0)).label("total_24h"),
|
|
func.sum(case(
|
|
(and_(MonitorResult.checked_at >= cutoff_24h, MonitorResult.is_up.is_(True)), 1),
|
|
else_=0,
|
|
)).label("up_24h"),
|
|
)
|
|
.join(Monitor, Monitor.id == MonitorResult.monitor_id)
|
|
.where(Monitor.host_id.isnot(None), MonitorResult.checked_at >= cutoff_30d)
|
|
.group_by(Monitor.host_id)
|
|
)
|
|
|
|
def _pct(up, total):
|
|
return round(float(up) / float(total) * 100, 2) if total else None
|
|
|
|
stats: dict[str, dict] = {}
|
|
for row in result:
|
|
stats[row.host_id] = {
|
|
"24h": _pct(row.up_24h, row.total_24h),
|
|
"7d": _pct(row.up_7d, row.total_7d),
|
|
"30d": _pct(row.up_30d, row.total_30d),
|
|
}
|
|
return stats
|
|
|
|
|
|
async def host_status(db, host_ids: list[str]) -> dict[str, dict]:
|
|
"""Per-host status rollup from the latest result of each linked monitor.
|
|
|
|
{host_id: {"state": "up"|"down"|"pending", "latency_ms": float|None,
|
|
"count": int, "down_since": datetime|None}}. State is down if
|
|
ANY linked monitor's latest result is down (worst-case), up if at least
|
|
one is up, else pending. down_since is when the longest-down of its down
|
|
monitors went down.
|
|
"""
|
|
if not host_ids:
|
|
return {}
|
|
monitors = (await db.execute(
|
|
select(Monitor.id, Monitor.host_id)
|
|
.where(Monitor.host_id.in_(host_ids), Monitor.enabled.is_(True))
|
|
)).all()
|
|
latest = await _last_n_results(db, [m.id for m in monitors], n=1)
|
|
rows = [(latest[mid][-1], host_id, mid) for mid, host_id in monitors if latest.get(mid)]
|
|
out: dict[str, dict] = {}
|
|
down: dict[str, list[str]] = {}
|
|
for res, host_id, mid in rows:
|
|
d = out.setdefault(host_id, {"state": "pending", "latency_ms": None, "count": 0,
|
|
"down_since": None})
|
|
d["count"] += 1
|
|
if not res.is_up:
|
|
d["state"] = "down"
|
|
down.setdefault(host_id, []).append(mid)
|
|
elif d["state"] != "down":
|
|
d["state"] = "up"
|
|
if res.is_up and res.response_ms is not None:
|
|
d["latency_ms"] = (res.response_ms if d["latency_ms"] is None
|
|
else min(d["latency_ms"], res.response_ms))
|
|
if down:
|
|
from steward.monitors.service import down_since
|
|
for host_id, mids in down.items():
|
|
starts = [t for t in [await down_since(db, mid) for mid in mids] if t]
|
|
out[host_id]["down_since"] = min(starts) if starts else None
|
|
return out
|
|
|
|
|
|
def host_rank(status: dict | None, agent: dict | None) -> int:
|
|
"""Where a host sorts in the Hosts list: a down monitor first, then an
|
|
agent gone quiet, then monitors not yet checked, then the healthy; a host
|
|
with neither monitors nor agent last."""
|
|
if status and status["state"] == "down":
|
|
return 0
|
|
if agent and not agent.get("fresh"):
|
|
return 1
|
|
if status and status["state"] == "pending":
|
|
return 2
|
|
if status or agent:
|
|
return 3
|
|
return 4
|
|
|
|
|
|
async def host_monitors(db, host_id: str) -> list[dict]:
|
|
"""Display rows for the monitors linked to one host (host hub section)."""
|
|
monitors = (await db.execute(
|
|
select(Monitor).where(Monitor.host_id == host_id).order_by(Monitor.name)
|
|
)).scalars().all()
|
|
ids = [m.id for m in monitors]
|
|
recent = await _last_n_results(db, ids)
|
|
uptime = await _uptime_by_monitor(db, ids)
|
|
data = []
|
|
from steward.monitors.service import down_reason, down_since
|
|
for m in monitors:
|
|
mrows = recent.get(m.id, [])
|
|
latest = mrows[-1] if mrows else None
|
|
state = ("paused" if not m.enabled else "pending" if latest is None
|
|
else "up" if latest.is_up else "down")
|
|
data.append({
|
|
"monitor": m, "latest": latest, "state": state,
|
|
"heartbeat": [{"state": "up" if r.is_up else "down",
|
|
"title": _heartbeat_title(m.type, r)} for r in mrows],
|
|
"uptime_pct": uptime.get(m.id, {}).get("24h"),
|
|
"down_since": await down_since(db, m.id) if state == "down" else None,
|
|
"down_reason": down_reason(latest) if state == "down" else None,
|
|
})
|
|
# Problems first: down, then not yet checked, then up, paused last.
|
|
order = {"down": 0, "pending": 1, "up": 2, "paused": 3}
|
|
return sorted(data, key=lambda d: (order[d["state"]], d["monitor"].name.lower()))
|
|
|
|
|
|
async def agent_overview_by_host(
|
|
db, host_names: list[str], stale_after_seconds: int = 180,
|
|
) -> dict[str, dict]:
|
|
"""Per-host agent glance (cpu/mem/disk/load + freshness) from plugin_metric_latest.
|
|
|
|
{host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root, load_1m,
|
|
cores, load_per_core, last_at, fresh}} for each host with any
|
|
agent samples. Cores are counted from the per-core CPU sub-resources
|
|
("<host>:core<N>"), as on the host page; load_per_core is load_1m as a %
|
|
of them, the figure Settings → Thresholds grades "load" by.
|
|
|
|
Core-safe (no host_agent import). Freshness compares the latest host-level
|
|
sample against the host_agent plugin's stale window (stale_after_seconds;
|
|
callers pass agent_stale_after_seconds(app.config)). A host whose agent has
|
|
gone quiet keeps its last values with fresh=False (#5208).
|
|
"""
|
|
from steward.core.metric_series import latest_by_resource
|
|
if not host_names:
|
|
return {}
|
|
now = datetime.now(timezone.utc)
|
|
stale_after = int(stale_after_seconds)
|
|
latest = await latest_by_resource(
|
|
db, "host_agent", list(host_names) + [h + ":/" for h in host_names],
|
|
set(AGENT_GLANCE_METRICS) | {"disk_used_pct"})
|
|
from steward.models.metrics import PluginMetricLatest
|
|
cores: dict[str, int] = {}
|
|
for res in (await db.execute(
|
|
select(PluginMetricLatest.resource_name).where(
|
|
PluginMetricLatest.source_module == "host_agent",
|
|
PluginMetricLatest.metric_name == "cpu_pct",
|
|
PluginMetricLatest.resource_name.like("%:core%"))
|
|
)).scalars():
|
|
host = res.rsplit(":core", 1)[0]
|
|
cores[host] = cores.get(host, 0) + 1
|
|
out: dict[str, dict] = {}
|
|
for name in host_names:
|
|
own = {m: r for m, r in latest.get(name, {}).items() if m in AGENT_GLANCE_METRICS}
|
|
root = latest.get(name + ":/", {}).get("disk_used_pct")
|
|
if not own and root is None:
|
|
continue
|
|
d = {m: r.value for m, r in own.items()}
|
|
if root is not None:
|
|
d["disk_root"] = root.value
|
|
ts = max((r.recorded_at for r in own.values()), default=None)
|
|
d["last_at"] = ts
|
|
d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
|
|
d["cores"] = cores.get(name)
|
|
load = d.get("load_1m")
|
|
d["load_per_core"] = round(load / d["cores"] * 100) if (load is not None and d["cores"]) else None
|
|
out[name] = d
|
|
return out
|
|
|
|
|
|
GLANCE_ORDER = {"down": 0, "crit": 1, "stale": 2, "warn": 3, "up": 4, "unknown": 5}
|
|
|
|
|
|
def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict]:
|
|
"""The Hosts — Overview rows, worst first (#5251).
|
|
|
|
Each is {host, state, worst, agent}. State is "down" when a linked monitor
|
|
is down, "stale" when the agent has stopped reporting, else the grade of
|
|
the reading furthest past its cutoff ("crit" / "warn": CPU, memory, disk /
|
|
or load per core, against Settings → Thresholds), else "up"; "unknown" for
|
|
a host with neither monitors nor agent. `worst` is (label, value, grade)
|
|
or None, from formatting.worst_reading."""
|
|
from steward.core.formatting import worst_reading
|
|
rows = []
|
|
for h in hosts:
|
|
st, a = status.get(h.id), agent.get(h.name)
|
|
worst = None
|
|
if a and a.get("fresh"):
|
|
worst = worst_reading((
|
|
("CPU", a.get("cpu_pct"), "cpu"), ("Memory", a.get("mem_used_pct"), "mem"),
|
|
("Disk /", a.get("disk_root"), "disk"), ("Load", a.get("load_per_core"), "load")),
|
|
thresholds)
|
|
if st and st["state"] == "down":
|
|
state = "down"
|
|
elif a and not a.get("fresh"):
|
|
state = "stale"
|
|
elif worst:
|
|
state = worst[2]
|
|
elif st or a:
|
|
state = "up"
|
|
else:
|
|
state = "unknown"
|
|
rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}})
|
|
return sorted(rows, key=lambda r: (GLANCE_ORDER[r["state"]], r["host"].name.lower()))
|