Files
FabledSteward/steward/hosts/service.py
T
bvandeusenandClaude Opus 5.5 5a14e7c693
CI / lint (push) Successful in 3s
CI / integration (push) Successful in 1m8s
CI / unit (push) Successful in 1m25s
CI / publish (push) Successful in 1m17s
feat(ui): alerts, monitors, hosts, schedules and maintenance lists put problems first, with since-times (#5210)
#5210 "Problems first, with since-time, across every list" (milestone 477),
first batch.

- Alerts: the Active list reads firing, no data, acknowledged, pending, the
  longest-running first, and re-polls itself. An acknowledged alert says how
  long it has been breaching (from the alert's events; acknowledging moves
  last_transition_at) and, apart from that, who acknowledged it and when.
  Rules show what each is doing now; disabled rules sort last. The current
  value reads the latest-value table (#5208) instead of plugin_metrics.
- Monitors: a down monitor says "Down 12m · connection refused" in the list.
- Hosts: sorted down, agent gone quiet, pending, healthy. The agent column
  shows "—" for a missing reading (was 0%) and a Stale badge with its age.
  agent_metrics_by_host is gone; the list reads agent_overview_by_host.
- Schedules: failing first (failed or interrupted run, or a firing that could
  not start one), then running, then the rest, paused last; status badge
  from the shared states.
- Maintenance windows: active (ending soonest), then upcoming (starting
  soonest), with expired folded away; each says when it ends/starts/ended.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 23:58:56 -04:00

217 lines
8.7 KiB
Python

"""Host queries shared by the Hosts pages, dashboard widgets and MCP tools.
Every function takes an open session and returns plain data; none reads
Quart's request context, so the web routes and the /mcp tools (which run
outside Quart's pipeline) call the same code.
"""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from sqlalchemy import and_, case, func, select
from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor
from steward.models.hosts import Host
from steward.models.monitors import Monitor, MonitorResult
# Host-level agent metrics read for the at-a-glance views (plus the root
# mount's disk_used_pct, read from the "<host>:/" sub-resource).
AGENT_GLANCE_METRICS = ("cpu_pct", "mem_used_pct", "disk_used_pct", "disk_used_pct_worst")
async def check_name_free(db, name: str, exclude_id: str | None = None) -> None:
"""ValueError when another host already has this name, or it is blank.
Agent metrics, container metrics and the alert rules on them go by the
host's name, so two hosts sharing one would share one history.
"""
if not name:
raise ValueError("a host needs a name.")
q = select(Host.id).where(Host.name == name)
if exclude_id:
q = q.where(Host.id != exclude_id)
if (await db.execute(q.limit(1))).first():
raise ValueError(f"another host is already named {name!r}; pick a different name.")
def agent_stale_after_seconds(config) -> int:
"""The host_agent plugin's freshness window from app config (default 180s)."""
return int(((config.get("PLUGINS") or {}).get("host_agent") or {})
.get("stale_after_seconds", 180))
async def compute_uptime(db) -> dict[str, dict]:
"""Per-host uptime % for 24h/7d/30d, aggregated over the host's monitors.
Returns {host_id: {"24h": float|None, "7d": float|None, "30d": float|None}}.
"""
now = datetime.now(timezone.utc)
cutoff_30d = now - timedelta(days=30)
cutoff_7d = now - timedelta(days=7)
cutoff_24h = now - timedelta(hours=24)
result = await db.execute(
select(
Monitor.host_id,
func.count(MonitorResult.id).label("total_30d"),
func.sum(case((MonitorResult.is_up.is_(True), 1), else_=0)).label("up_30d"),
func.sum(case((MonitorResult.checked_at >= cutoff_7d, 1), else_=0)).label("total_7d"),
func.sum(case(
(and_(MonitorResult.checked_at >= cutoff_7d, MonitorResult.is_up.is_(True)), 1),
else_=0,
)).label("up_7d"),
func.sum(case((MonitorResult.checked_at >= cutoff_24h, 1), else_=0)).label("total_24h"),
func.sum(case(
(and_(MonitorResult.checked_at >= cutoff_24h, MonitorResult.is_up.is_(True)), 1),
else_=0,
)).label("up_24h"),
)
.join(Monitor, Monitor.id == MonitorResult.monitor_id)
.where(Monitor.host_id.isnot(None), MonitorResult.checked_at >= cutoff_30d)
.group_by(Monitor.host_id)
)
def _pct(up, total):
return round(float(up) / float(total) * 100, 2) if total else None
stats: dict[str, dict] = {}
for row in result:
stats[row.host_id] = {
"24h": _pct(row.up_24h, row.total_24h),
"7d": _pct(row.up_7d, row.total_7d),
"30d": _pct(row.up_30d, row.total_30d),
}
return stats
async def host_status(db, host_ids: list[str]) -> dict[str, dict]:
"""Per-host status rollup from the latest result of each linked monitor.
{host_id: {"state": "up"|"down"|"pending", "latency_ms": float|None,
"count": int}}. State is down if ANY linked monitor's latest
result is down (worst-case), up if at least one is up, else pending.
"""
if not host_ids:
return {}
monitors = (await db.execute(
select(Monitor.id, Monitor.host_id)
.where(Monitor.host_id.in_(host_ids), Monitor.enabled.is_(True))
)).all()
latest = await _last_n_results(db, [m.id for m in monitors], n=1)
rows = [(latest[mid][-1], host_id) for mid, host_id in monitors if latest.get(mid)]
out: dict[str, dict] = {}
for res, host_id in rows:
d = out.setdefault(host_id, {"state": "pending", "latency_ms": None, "count": 0})
d["count"] += 1
if not res.is_up:
d["state"] = "down"
elif d["state"] != "down":
d["state"] = "up"
if res.is_up and res.response_ms is not None:
d["latency_ms"] = (res.response_ms if d["latency_ms"] is None
else min(d["latency_ms"], res.response_ms))
return out
def host_rank(status: dict | None, agent: dict | None) -> int:
"""Where a host sorts in the Hosts list: a down monitor first, then an
agent gone quiet, then monitors not yet checked, then the healthy; a host
with neither monitors nor agent last."""
if status and status["state"] == "down":
return 0
if agent and not agent.get("fresh"):
return 1
if status and status["state"] == "pending":
return 2
if status or agent:
return 3
return 4
async def host_monitors(db, host_id: str) -> list[dict]:
"""Display rows for the monitors linked to one host (host hub section)."""
monitors = (await db.execute(
select(Monitor).where(Monitor.host_id == host_id).order_by(Monitor.name)
)).scalars().all()
ids = [m.id for m in monitors]
recent = await _last_n_results(db, ids)
uptime = await _uptime_by_monitor(db, ids)
data = []
for m in monitors:
mrows = recent.get(m.id, [])
latest = mrows[-1] if mrows else None
data.append({
"monitor": m, "latest": latest,
"heartbeat": [{"state": "up" if r.is_up else "down",
"title": _heartbeat_title(m.type, r)} for r in mrows],
"uptime_pct": uptime.get(m.id, {}).get("24h"),
})
return data
async def agent_overview_by_host(
db, host_names: list[str], stale_after_seconds: int = 180,
) -> dict[str, dict]:
"""Per-host agent glance (cpu/mem/disk + freshness) from plugin_metric_latest.
{host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root,
last_at, fresh}} for each host with any agent samples.
Core-safe (no host_agent import). Freshness compares the latest host-level
sample against the host_agent plugin's stale window (stale_after_seconds;
callers pass agent_stale_after_seconds(app.config)). A host whose agent has
gone quiet keeps its last values with fresh=False (#5208).
"""
from steward.core.metric_series import latest_by_resource
if not host_names:
return {}
now = datetime.now(timezone.utc)
stale_after = int(stale_after_seconds)
latest = await latest_by_resource(
db, "host_agent", list(host_names) + [h + ":/" for h in host_names],
set(AGENT_GLANCE_METRICS) | {"disk_used_pct"})
out: dict[str, dict] = {}
for name in host_names:
own = {m: r for m, r in latest.get(name, {}).items() if m in AGENT_GLANCE_METRICS}
root = latest.get(name + ":/", {}).get("disk_used_pct")
if not own and root is None:
continue
d = {m: r.value for m, r in own.items()}
if root is not None:
d["disk_root"] = root.value
ts = max((r.recorded_at for r in own.values()), default=None)
d["last_at"] = ts
d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
out[name] = d
return out
async def agent_cpu_sparks_by_host(db, host_names: list[str], hours: int = 1) -> dict[str, str]:
"""{host_name: inline-SVG cpu sparkline} over the last `hours` (core-safe).
A tiny at-a-glance trend in each row — the host-view sparkline, on the widget.
"""
from steward.models.metrics import PluginMetric
from steward.core.status import sparkline_svg
if not host_names:
return {}
cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
rows = (await db.execute(
select(PluginMetric.resource_name, PluginMetric.value, PluginMetric.recorded_at)
.where(
PluginMetric.source_module == "host_agent",
PluginMetric.metric_name == "cpu_pct",
PluginMetric.resource_name.in_(host_names),
PluginMetric.recorded_at >= cutoff,
)
.order_by(PluginMetric.recorded_at)
)).all()
by_host: dict[str, list[float]] = {}
for r in rows:
by_host.setdefault(r.resource_name, []).append(r.value)
# Amber line to read as "CPU"; only draw when there are a couple of points.
return {
name: sparkline_svg(vals[-40:], width=64, height=18, stroke="#c8a840")
for name, vals in by_host.items() if len(vals) >= 2
}