"""Host queries shared by the Hosts pages, dashboard widgets and MCP tools. Every function takes an open session and returns plain data; none reads Quart's request context, so the web routes and the /mcp tools (which run outside Quart's pipeline) call the same code. """ from __future__ import annotations from datetime import datetime, timedelta, timezone from sqlalchemy import and_, case, func, select from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor from steward.models.hosts import Host from steward.models.monitors import Monitor, MonitorResult # Host-level agent metrics read for the at-a-glance views (plus the root # mount's disk_used_pct, read from the ":/" sub-resource). AGENT_GLANCE_METRICS = ("cpu_pct", "mem_used_pct", "disk_used_pct", "disk_used_pct_worst", "load_1m") async def check_name_free(db, name: str, exclude_id: str | None = None) -> None: """ValueError when another host already has this name, or it is blank. Agent metrics, container metrics and the alert rules on them go by the host's name, so two hosts sharing one would share one history. """ if not name: raise ValueError("a host needs a name.") q = select(Host.id).where(Host.name == name) if exclude_id: q = q.where(Host.id != exclude_id) if (await db.execute(q.limit(1))).first(): raise ValueError(f"another host is already named {name!r}; pick a different name.") def agent_stale_after_seconds(config) -> int: """The host_agent plugin's freshness window from app config (default 180s).""" return int(((config.get("PLUGINS") or {}).get("host_agent") or {}) .get("stale_after_seconds", 180)) async def compute_uptime(db) -> dict[str, dict]: """Per-host uptime % for 24h/7d/30d, aggregated over the host's monitors. Returns {host_id: {"24h": float|None, "7d": float|None, "30d": float|None}}. """ now = datetime.now(timezone.utc) cutoff_30d = now - timedelta(days=30) cutoff_7d = now - timedelta(days=7) cutoff_24h = now - timedelta(hours=24) result = await db.execute( select( Monitor.host_id, func.count(MonitorResult.id).label("total_30d"), func.sum(case((MonitorResult.is_up.is_(True), 1), else_=0)).label("up_30d"), func.sum(case((MonitorResult.checked_at >= cutoff_7d, 1), else_=0)).label("total_7d"), func.sum(case( (and_(MonitorResult.checked_at >= cutoff_7d, MonitorResult.is_up.is_(True)), 1), else_=0, )).label("up_7d"), func.sum(case((MonitorResult.checked_at >= cutoff_24h, 1), else_=0)).label("total_24h"), func.sum(case( (and_(MonitorResult.checked_at >= cutoff_24h, MonitorResult.is_up.is_(True)), 1), else_=0, )).label("up_24h"), ) .join(Monitor, Monitor.id == MonitorResult.monitor_id) .where(Monitor.host_id.isnot(None), MonitorResult.checked_at >= cutoff_30d) .group_by(Monitor.host_id) ) def _pct(up, total): return round(float(up) / float(total) * 100, 2) if total else None stats: dict[str, dict] = {} for row in result: stats[row.host_id] = { "24h": _pct(row.up_24h, row.total_24h), "7d": _pct(row.up_7d, row.total_7d), "30d": _pct(row.up_30d, row.total_30d), } return stats async def host_status(db, host_ids: list[str]) -> dict[str, dict]: """Per-host status rollup from the latest result of each linked monitor. {host_id: {"state": "up"|"down"|"pending", "latency_ms": float|None, "count": int}}. State is down if ANY linked monitor's latest result is down (worst-case), up if at least one is up, else pending. """ if not host_ids: return {} monitors = (await db.execute( select(Monitor.id, Monitor.host_id) .where(Monitor.host_id.in_(host_ids), Monitor.enabled.is_(True)) )).all() latest = await _last_n_results(db, [m.id for m in monitors], n=1) rows = [(latest[mid][-1], host_id) for mid, host_id in monitors if latest.get(mid)] out: dict[str, dict] = {} for res, host_id in rows: d = out.setdefault(host_id, {"state": "pending", "latency_ms": None, "count": 0}) d["count"] += 1 if not res.is_up: d["state"] = "down" elif d["state"] != "down": d["state"] = "up" if res.is_up and res.response_ms is not None: d["latency_ms"] = (res.response_ms if d["latency_ms"] is None else min(d["latency_ms"], res.response_ms)) return out def worst_uptime_first(hosts, uptime: dict) -> list: """Hosts by their 24h uptime, worst first, then 7d and 30d to break ties; hosts with no uptime yet (no monitors) last, by name.""" def key(h): u = uptime.get(h.id) or {} windows = [u.get(w) for w in ("24h", "7d", "30d")] known = [w for w in windows if w is not None] return (not known, *[101.0 if w is None else w for w in windows], h.name.lower()) return sorted(hosts, key=key) def host_rank(status: dict | None, agent: dict | None) -> int: """Where a host sorts in the Hosts list: a down monitor first, then an agent gone quiet, then monitors not yet checked, then the healthy; a host with neither monitors nor agent last.""" if status and status["state"] == "down": return 0 if agent and not agent.get("fresh"): return 1 if status and status["state"] == "pending": return 2 if status or agent: return 3 return 4 async def host_monitors(db, host_id: str) -> list[dict]: """Display rows for the monitors linked to one host (host hub section).""" monitors = (await db.execute( select(Monitor).where(Monitor.host_id == host_id).order_by(Monitor.name) )).scalars().all() ids = [m.id for m in monitors] recent = await _last_n_results(db, ids) uptime = await _uptime_by_monitor(db, ids) data = [] from steward.monitors.service import down_reason, down_since for m in monitors: mrows = recent.get(m.id, []) latest = mrows[-1] if mrows else None state = ("paused" if not m.enabled else "pending" if latest is None else "up" if latest.is_up else "down") data.append({ "monitor": m, "latest": latest, "state": state, "heartbeat": [{"state": "up" if r.is_up else "down", "title": _heartbeat_title(m.type, r)} for r in mrows], "uptime_pct": uptime.get(m.id, {}).get("24h"), "down_since": await down_since(db, m.id) if state == "down" else None, "down_reason": down_reason(latest) if state == "down" else None, }) # Problems first: down, then not yet checked, then up, paused last. order = {"down": 0, "pending": 1, "up": 2, "paused": 3} return sorted(data, key=lambda d: (order[d["state"]], d["monitor"].name.lower())) async def agent_overview_by_host( db, host_names: list[str], stale_after_seconds: int = 180, ) -> dict[str, dict]: """Per-host agent glance (cpu/mem/disk/load + freshness) from plugin_metric_latest. {host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root, load_1m, cores, load_per_core, last_at, fresh}} for each host with any agent samples. Cores are counted from the per-core CPU sub-resources (":core"), as on the host page; load_per_core is load_1m as a % of them, the figure Settings → Thresholds grades "load" by. Core-safe (no host_agent import). Freshness compares the latest host-level sample against the host_agent plugin's stale window (stale_after_seconds; callers pass agent_stale_after_seconds(app.config)). A host whose agent has gone quiet keeps its last values with fresh=False (#5208). """ from steward.core.metric_series import latest_by_resource if not host_names: return {} now = datetime.now(timezone.utc) stale_after = int(stale_after_seconds) latest = await latest_by_resource( db, "host_agent", list(host_names) + [h + ":/" for h in host_names], set(AGENT_GLANCE_METRICS) | {"disk_used_pct"}) from steward.models.metrics import PluginMetricLatest cores: dict[str, int] = {} for res in (await db.execute( select(PluginMetricLatest.resource_name).where( PluginMetricLatest.source_module == "host_agent", PluginMetricLatest.metric_name == "cpu_pct", PluginMetricLatest.resource_name.like("%:core%")) )).scalars(): host = res.rsplit(":core", 1)[0] cores[host] = cores.get(host, 0) + 1 out: dict[str, dict] = {} for name in host_names: own = {m: r for m, r in latest.get(name, {}).items() if m in AGENT_GLANCE_METRICS} root = latest.get(name + ":/", {}).get("disk_used_pct") if not own and root is None: continue d = {m: r.value for m, r in own.items()} if root is not None: d["disk_root"] = root.value ts = max((r.recorded_at for r in own.values()), default=None) d["last_at"] = ts d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after) d["cores"] = cores.get(name) load = d.get("load_1m") d["load_per_core"] = round(load / d["cores"] * 100) if (load is not None and d["cores"]) else None out[name] = d return out GLANCE_ORDER = {"down": 0, "crit": 1, "stale": 2, "warn": 3, "up": 4, "unknown": 5} def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict]: """The Hosts — Overview rows, worst first (#5251). Each is {host, state, worst, agent}. State is "down" when a linked monitor is down, "stale" when the agent has stopped reporting, else the grade of the reading furthest past its cutoff ("crit" / "warn": CPU, memory, disk / or load per core, against Settings → Thresholds), else "up"; "unknown" for a host with neither monitors nor agent. `worst` is (label, value, grade) or None, from formatting.worst_reading.""" from steward.core.formatting import worst_reading rows = [] for h in hosts: st, a = status.get(h.id), agent.get(h.name) worst = None if a and a.get("fresh"): worst = worst_reading(( ("CPU", a.get("cpu_pct"), "cpu"), ("Memory", a.get("mem_used_pct"), "mem"), ("Disk /", a.get("disk_root"), "disk"), ("Load", a.get("load_per_core"), "load")), thresholds) if st and st["state"] == "down": state = "down" elif a and not a.get("fresh"): state = "stale" elif worst: state = worst[2] elif st or a: state = "up" else: state = "unknown" rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}}) return sorted(rows, key=lambda r: (GLANCE_ORDER[r["state"]], r["host"].name.lower()))