bvandeusenandClaude Opus 5.5
c113cadbdd
Routers: a router in trouble reads 'started 2h ago' or 'all of the last 24 h', from problem_since over the day's history, read for the troubled routers only (a certificate's problem carries its own date). Hosts: a down host reads 'a monitor is down for 3h'; host_glance keeps the down_since host_status already had. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
412 lines
19 KiB
Python
412 lines
19 KiB
Python
"""Host queries shared by the Hosts pages, dashboard widgets and MCP tools.
|
|
|
|
Every function takes an open session and returns plain data; none reads
|
|
Quart's request context, so the web routes and the /mcp tools (which run
|
|
outside Quart's pipeline) call the same code.
|
|
"""
|
|
from __future__ import annotations
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import and_, case, func, select
|
|
|
|
from steward.core.norms import norms_at
|
|
from steward.core.settings import floor_of
|
|
from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor
|
|
from steward.models.hosts import Host
|
|
from steward.models.monitors import Monitor, MonitorResult
|
|
|
|
|
|
# Host-level agent metrics read for the at-a-glance views (plus the root
|
|
# mount's disk_used_pct, read from the "<host>:/" sub-resource).
|
|
AGENT_GLANCE_METRICS = ("cpu_pct", "mem_used_pct", "disk_used_pct", "disk_used_pct_worst", "load_1m")
|
|
|
|
|
|
async def check_name_free(db, name: str, exclude_id: str | None = None) -> None:
|
|
"""ValueError when another host already has this name, or it is blank.
|
|
|
|
Agent metrics, container metrics and the alert rules on them go by the
|
|
host's name, so two hosts sharing one would share one history.
|
|
"""
|
|
if not name:
|
|
raise ValueError("a host needs a name.")
|
|
q = select(Host.id).where(Host.name == name)
|
|
if exclude_id:
|
|
q = q.where(Host.id != exclude_id)
|
|
if (await db.execute(q.limit(1))).first():
|
|
raise ValueError(f"another host is already named {name!r}; pick a different name.")
|
|
|
|
|
|
def agent_stale_after_seconds(config) -> int:
|
|
"""The host_agent plugin's freshness window from app config (default 180s)."""
|
|
return int(((config.get("PLUGINS") or {}).get("host_agent") or {})
|
|
.get("stale_after_seconds", 180))
|
|
|
|
|
|
async def compute_uptime(db) -> dict[str, dict]:
|
|
"""Per-host uptime % for 24h/7d/30d, aggregated over the host's monitors.
|
|
|
|
Returns {host_id: {"24h": float|None, "7d": float|None, "30d": float|None}}.
|
|
"""
|
|
now = datetime.now(timezone.utc)
|
|
cutoff_30d = now - timedelta(days=30)
|
|
cutoff_7d = now - timedelta(days=7)
|
|
cutoff_24h = now - timedelta(hours=24)
|
|
|
|
result = await db.execute(
|
|
select(
|
|
Monitor.host_id,
|
|
func.count(MonitorResult.id).label("total_30d"),
|
|
func.sum(case((MonitorResult.is_up.is_(True), 1), else_=0)).label("up_30d"),
|
|
func.sum(case((MonitorResult.checked_at >= cutoff_7d, 1), else_=0)).label("total_7d"),
|
|
func.sum(case(
|
|
(and_(MonitorResult.checked_at >= cutoff_7d, MonitorResult.is_up.is_(True)), 1),
|
|
else_=0,
|
|
)).label("up_7d"),
|
|
func.sum(case((MonitorResult.checked_at >= cutoff_24h, 1), else_=0)).label("total_24h"),
|
|
func.sum(case(
|
|
(and_(MonitorResult.checked_at >= cutoff_24h, MonitorResult.is_up.is_(True)), 1),
|
|
else_=0,
|
|
)).label("up_24h"),
|
|
)
|
|
.join(Monitor, Monitor.id == MonitorResult.monitor_id)
|
|
.where(Monitor.host_id.isnot(None), MonitorResult.checked_at >= cutoff_30d)
|
|
.group_by(Monitor.host_id)
|
|
)
|
|
|
|
def _pct(up, total):
|
|
return round(float(up) / float(total) * 100, 2) if total else None
|
|
|
|
stats: dict[str, dict] = {}
|
|
for row in result:
|
|
stats[row.host_id] = {
|
|
"24h": _pct(row.up_24h, row.total_24h),
|
|
"7d": _pct(row.up_7d, row.total_7d),
|
|
"30d": _pct(row.up_30d, row.total_30d),
|
|
}
|
|
return stats
|
|
|
|
|
|
async def host_status(db, host_ids: list[str]) -> dict[str, dict]:
|
|
"""Per-host status rollup from the latest result of each linked monitor.
|
|
|
|
{host_id: {"state": "up"|"down"|"pending", "latency_ms": float|None,
|
|
"count": int, "down_since": datetime|None}}. State is down if
|
|
ANY linked monitor's latest result is down (worst-case), up if at least
|
|
one is up, else pending. down_since is when the longest-down of its down
|
|
monitors went down.
|
|
"""
|
|
if not host_ids:
|
|
return {}
|
|
monitors = (await db.execute(
|
|
select(Monitor.id, Monitor.host_id)
|
|
.where(Monitor.host_id.in_(host_ids), Monitor.enabled.is_(True))
|
|
)).all()
|
|
latest = await _last_n_results(db, [m.id for m in monitors], n=1)
|
|
rows = [(latest[mid][-1], host_id, mid) for mid, host_id in monitors if latest.get(mid)]
|
|
out: dict[str, dict] = {}
|
|
down: dict[str, list[str]] = {}
|
|
for res, host_id, mid in rows:
|
|
d = out.setdefault(host_id, {"state": "pending", "latency_ms": None, "count": 0,
|
|
"down_since": None})
|
|
d["count"] += 1
|
|
if not res.is_up:
|
|
d["state"] = "down"
|
|
down.setdefault(host_id, []).append(mid)
|
|
elif d["state"] != "down":
|
|
d["state"] = "up"
|
|
if res.is_up and res.response_ms is not None:
|
|
d["latency_ms"] = (res.response_ms if d["latency_ms"] is None
|
|
else min(d["latency_ms"], res.response_ms))
|
|
if down:
|
|
from steward.monitors.service import down_since
|
|
for host_id, mids in down.items():
|
|
starts = [t for t in [await down_since(db, mid) for mid in mids] if t]
|
|
out[host_id]["down_since"] = min(starts) if starts else None
|
|
return out
|
|
|
|
|
|
async def host_groups(db, host_ids: list[str]) -> dict[str, list[str]]:
|
|
"""{host_id: [group name, …]}, from each host's linked Ansible target.
|
|
Ansible inventory groups are Steward's grouping system (ruling on the
|
|
Hosts system, #5294); a host with no linked target has no groups."""
|
|
from sqlalchemy.orm import selectinload
|
|
from steward.models.ansible_inventory import AnsibleTarget
|
|
if not host_ids:
|
|
return {}
|
|
targets = (await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.host_id.in_(host_ids))
|
|
.options(selectinload(AnsibleTarget.groups)))).scalars().all()
|
|
return {t.host_id: sorted((g.name for g in t.groups), key=str.lower) for t in targets}
|
|
|
|
|
|
async def group_names(db) -> list[str]:
|
|
"""Every inventory group's name, for a group filter."""
|
|
from steward.models.ansible_inventory import AnsibleGroup
|
|
return sorted((await db.execute(select(AnsibleGroup.name))).scalars().all(), key=str.lower)
|
|
|
|
|
|
# The Hosts list's filters and sorts (#5294). NO_GROUP picks hosts in no group.
|
|
HOST_STATES = {"down": "Down", "silent": "Agent silent", "up": "Up", "none": "No monitors"}
|
|
NO_GROUP = "__none__"
|
|
# Each sort and whether it starts descending: the busiest, slowest or least
|
|
# reliable host first; "status" is problems-first, the list's default.
|
|
HOST_SORTS = {"status": False, "name": False, "latency": True, "uptime": False, "cpu": True, "mem": True}
|
|
|
|
|
|
def host_state_keys(status: dict | None, agent: dict | None) -> set[str]:
|
|
"""Which Status filters a host answers to; "Agent silent" can sit beside
|
|
"Down" or "Up", since it is about the agent and they are about monitors."""
|
|
keys = set()
|
|
if status is None:
|
|
keys.add("none")
|
|
elif status["state"] in ("down", "up"):
|
|
keys.add(status["state"])
|
|
if agent and not agent.get("fresh"):
|
|
keys.add("silent")
|
|
return keys
|
|
|
|
|
|
def filter_hosts(hosts, *, q: str = "", group: str = "", state: str = "",
|
|
status: dict, agent: dict, groups: dict) -> list:
|
|
"""Hosts matching a search (name or address, any case), a group (or
|
|
NO_GROUP) and a Status filter; an empty value doesn't filter."""
|
|
q = (q or "").strip().lower()
|
|
out = []
|
|
for h in hosts:
|
|
if q and q not in h.name.lower() and q not in (h.address or "").lower():
|
|
continue
|
|
mine = groups.get(h.id, [])
|
|
if (group == NO_GROUP and mine) or (group and group != NO_GROUP and group not in mine):
|
|
continue
|
|
if state and state not in host_state_keys(status.get(h.id), agent.get(h.name)):
|
|
continue
|
|
out.append(h)
|
|
return out
|
|
|
|
|
|
def sort_hosts(hosts, sort: str, desc: bool, *, status: dict, agent: dict, uptime: dict) -> list:
|
|
"""Hosts in a column's order. Ties keep name order; a host with no value
|
|
for the column goes last whichever way it runs, so "—" never leads."""
|
|
by_name = sorted(hosts, key=lambda h: h.name.lower())
|
|
if sort == "name":
|
|
return list(reversed(by_name)) if desc else by_name
|
|
if sort == "status":
|
|
return sorted(by_name, key=lambda h: host_rank(status.get(h.id), agent.get(h.name)), reverse=desc)
|
|
value = {
|
|
"latency": lambda h: (status.get(h.id) or {}).get("latency_ms"),
|
|
"uptime": lambda h: (uptime.get(h.id) or {}).get("30d"),
|
|
"cpu": lambda h: (agent.get(h.name) or {}).get("cpu_pct"),
|
|
"mem": lambda h: (agent.get(h.name) or {}).get("mem_used_pct"),
|
|
}[sort]
|
|
have = [h for h in by_name if value(h) is not None]
|
|
return sorted(have, key=value, reverse=desc) + [h for h in by_name if value(h) is None]
|
|
|
|
|
|
def host_rank(status: dict | None, agent: dict | None) -> int:
|
|
"""Where a host sorts in the Hosts list: a down monitor first, then an
|
|
agent gone quiet, then monitors not yet checked, then the healthy; a host
|
|
with neither monitors nor agent last."""
|
|
if status and status["state"] == "down":
|
|
return 0
|
|
if agent and not agent.get("fresh"):
|
|
return 1
|
|
if status and status["state"] == "pending":
|
|
return 2
|
|
if status or agent:
|
|
return 3
|
|
return 4
|
|
|
|
|
|
async def host_monitors(db, host_id: str) -> list[dict]:
|
|
"""Display rows for the monitors linked to one host (host hub section)."""
|
|
monitors = (await db.execute(
|
|
select(Monitor).where(Monitor.host_id == host_id).order_by(Monitor.name)
|
|
)).scalars().all()
|
|
ids = [m.id for m in monitors]
|
|
recent = await _last_n_results(db, ids)
|
|
uptime = await _uptime_by_monitor(db, ids)
|
|
data = []
|
|
from steward.monitors.service import down_reason, down_since
|
|
for m in monitors:
|
|
mrows = recent.get(m.id, [])
|
|
latest = mrows[-1] if mrows else None
|
|
state = ("paused" if not m.enabled else "pending" if latest is None
|
|
else "up" if latest.is_up else "down")
|
|
data.append({
|
|
"monitor": m, "latest": latest, "state": state,
|
|
"heartbeat": [{"state": "up" if r.is_up else "down",
|
|
"title": _heartbeat_title(m.type, r)} for r in mrows],
|
|
"uptime_pct": uptime.get(m.id, {}).get("24h"),
|
|
"down_since": await down_since(db, m.id) if state == "down" else None,
|
|
"down_reason": down_reason(latest) if state == "down" else None,
|
|
})
|
|
# Problems first: down, then not yet checked, then up, paused last.
|
|
order = {"down": 0, "pending": 1, "up": 2, "paused": 3}
|
|
return sorted(data, key=lambda d: (order[d["state"]], d["monitor"].name.lower()))
|
|
|
|
|
|
async def agent_overview_by_host(
|
|
db, host_names: list[str], stale_after_seconds: int = 180,
|
|
) -> dict[str, dict]:
|
|
"""Per-host agent glance (cpu/mem/disk/load + freshness) from plugin_metric_latest.
|
|
|
|
{host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root, load_1m,
|
|
cores, load_per_core, last_at, fresh, clock_off_secs, norms}} for each host
|
|
with any agent samples. `norms` is {kind: Norm}, the host's CPU, memory and
|
|
load normals for this hour (host_normals grades by them). Cores are counted from the per-core CPU sub-resources
|
|
("<host>:core<N>"), as on the host page; load_per_core is load_1m as a %
|
|
of them, the figure Settings → Thresholds grades "load" by.
|
|
|
|
Core-safe (no host_agent import). Freshness compares the latest host-level
|
|
sample against the host_agent plugin's stale window (stale_after_seconds;
|
|
callers pass agent_stale_after_seconds(app.config)). A host whose agent has
|
|
gone quiet keeps its last values with fresh=False (#5208).
|
|
"""
|
|
from steward.core.metric_series import latest_by_resource
|
|
if not host_names:
|
|
return {}
|
|
now = datetime.now(timezone.utc)
|
|
stale_after = int(stale_after_seconds)
|
|
latest = await latest_by_resource(
|
|
db, "host_agent", list(host_names) + [h + ":/" for h in host_names],
|
|
set(AGENT_GLANCE_METRICS) | {"disk_used_pct", "inodes_used_pct", "clock_off_secs"})
|
|
from steward.models.metrics import PluginMetricLatest
|
|
cores: dict[str, int] = {}
|
|
for res in (await db.execute(
|
|
select(PluginMetricLatest.resource_name).where(
|
|
PluginMetricLatest.source_module == "host_agent",
|
|
PluginMetricLatest.metric_name == "cpu_pct",
|
|
PluginMetricLatest.resource_name.like("%:core%"))
|
|
)).scalars():
|
|
host = res.rsplit(":core", 1)[0]
|
|
cores[host] = cores.get(host, 0) + 1
|
|
out: dict[str, dict] = {}
|
|
for name in host_names:
|
|
own = {m: r for m, r in latest.get(name, {}).items() if m in AGENT_GLANCE_METRICS}
|
|
root = latest.get(name + ":/", {}).get("disk_used_pct")
|
|
if not own and root is None:
|
|
continue
|
|
d = {m: r.value for m, r in own.items()}
|
|
if root is not None:
|
|
d["disk_root"] = root.value
|
|
inodes = latest.get(name + ":/", {}).get("inodes_used_pct")
|
|
d["inodes_root"] = inodes.value if inodes is not None else None
|
|
ts = max((r.recorded_at for r in own.values()), default=None)
|
|
d["last_at"] = ts
|
|
d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
|
|
clock = latest.get(name, {}).get("clock_off_secs")
|
|
d["clock_off_secs"] = clock.value if clock is not None else None
|
|
d["cores"] = cores.get(name)
|
|
load = d.get("load_1m")
|
|
d["load_per_core"] = round(load / d["cores"] * 100) if (load is not None and d["cores"]) else None
|
|
d["norms"] = {}
|
|
out[name] = d
|
|
learned = await norms_at(db, [("host_agent", h, metric) for h in out
|
|
for metric, _f in HOST_NORMED.values()], now)
|
|
for (_s, host, metric), norm in learned.items():
|
|
out[host]["norms"][_KIND_OF[metric]] = norm
|
|
return out
|
|
|
|
|
|
# The host figures graded against their own normal (#5742): the threshold
|
|
# kind, the metric that learns it and its floor's setting (norms.floor.*).
|
|
# Disk keeps its cutoff: a nearly full disk is a failure (System 182).
|
|
HOST_NORMED = {"cpu": ("cpu_pct", "host_cpu_pct"), "mem": ("mem_used_pct", "host_mem_pct"),
|
|
"load": ("load_pct", "host_load_pct")}
|
|
_KIND_OF = {metric: kind for kind, (metric, _f) in HOST_NORMED.items()}
|
|
|
|
|
|
def fuller(space: float | None, inodes: float | None) -> float | None:
|
|
"""How full a mount is: by space or by inodes, whichever is fuller (#5745)."""
|
|
return max((v for v in (space, inodes) if v is not None), default=None)
|
|
|
|
|
|
def host_normals(norms: dict, thresholds: dict) -> dict:
|
|
"""{kind: (Norm, floor)} for worst_reading and anomalies.graded, from an
|
|
agent glance's `norms`."""
|
|
return {kind: (norm, floor_of(thresholds, HOST_NORMED[kind][1]))
|
|
for kind, norm in (norms or {}).items()}
|
|
|
|
|
|
async def host_norms(db, host_name: str, now: datetime | None = None) -> dict:
|
|
"""{kind: Norm}: one host's CPU, memory and load normals for this hour."""
|
|
learned = await norms_at(db, [("host_agent", host_name, metric)
|
|
for metric, _f in HOST_NORMED.values()],
|
|
now or datetime.now(timezone.utc))
|
|
return {_KIND_OF[metric]: norm for (_s, _h, metric), norm in learned.items()}
|
|
|
|
|
|
def host_grades(values: dict, norms: dict, thresholds: dict) -> dict:
|
|
"""{kind: (status class, why)} for a host's readings ({kind: value}):
|
|
against Settings → Thresholds, except CPU, memory and load within the
|
|
host's own normal, which are "ok" and say what is usual (#5742)."""
|
|
from steward.core.anomalies import graded
|
|
from steward.core.formatting import status_class
|
|
normals = host_normals(norms, thresholds)
|
|
return {kind: (graded(value, kind, thresholds, *normals[kind], "%") if kind in normals
|
|
else (status_class(value, kind, thresholds), None))
|
|
for kind, value in values.items()}
|
|
|
|
|
|
def clock_trouble(agent: dict[str, dict], limit: float) -> dict | None:
|
|
"""Reporting hosts whose clock is more than `limit` seconds from Steward's
|
|
(Settings → Thresholds' clock drift warning, milestone 490),
|
|
worst first: {"hosts": [(name, seconds)], "steward": bool, "typical": s}.
|
|
|
|
`steward` is set when every reporting host (two or more) is off, and by
|
|
about the same amount: then the one clock they all disagree with,
|
|
Steward's own, is the likely culprit. None when every clock is in step."""
|
|
readings = {n: a["clock_off_secs"] for n, a in agent.items()
|
|
if a.get("fresh") and a.get("clock_off_secs") is not None}
|
|
off = {n: v for n, v in readings.items() if v > limit}
|
|
if not off:
|
|
return None
|
|
worst = sorted(off.items(), key=lambda kv: (-kv[1], kv[0]))
|
|
low, high = min(off.values()), max(off.values())
|
|
together = len(readings) >= 2 and len(off) == len(readings) and high - low <= max(5.0, 0.2 * low)
|
|
return {"hosts": worst, "steward": together, "typical": sorted(off.values())[len(off) // 2]}
|
|
|
|
|
|
GLANCE_ORDER = {"down": 0, "crit": 1, "stale": 2, "warn": 3, "up": 4, "unknown": 5}
|
|
|
|
|
|
def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict]:
|
|
"""The Hosts — Overview rows, worst first (#5251).
|
|
|
|
Each is {host, state, worst, agent, down_since}. State is "down" when a linked monitor
|
|
is down, "stale" when the agent has stopped reporting, else the grade of
|
|
the reading furthest past its cutoff ("crit" / "warn": CPU, memory, disk /
|
|
or load per core, against Settings → Thresholds; CPU, memory and load
|
|
within the host's own normal are not past it), else "up"; "unknown" for
|
|
a host with neither monitors nor agent. `worst` is (label, value, grade)
|
|
or None, from formatting.worst_reading; `grades` is host_grades' {kind:
|
|
(grade, why)} for a fresh agent, else {}."""
|
|
from steward.core.formatting import worst_reading
|
|
rows = []
|
|
for h in hosts:
|
|
st, a = status.get(h.id), agent.get(h.name)
|
|
worst, grades = None, {}
|
|
if a and a.get("fresh"):
|
|
grades = host_grades({"cpu": a.get("cpu_pct"), "mem": a.get("mem_used_pct"),
|
|
"disk": fuller(a.get("disk_root"), a.get("inodes_root")),
|
|
"load": a.get("load_per_core")},
|
|
a.get("norms"), thresholds)
|
|
worst = worst_reading((
|
|
("CPU", a.get("cpu_pct"), "cpu"), ("Memory", a.get("mem_used_pct"), "mem"),
|
|
("Disk /", a.get("disk_root"), "disk"), ("Inodes on /", a.get("inodes_root"), "disk"),
|
|
("Load", a.get("load_per_core"), "load")),
|
|
thresholds, host_normals(a.get("norms"), thresholds))
|
|
if st and st["state"] == "down":
|
|
state = "down"
|
|
elif a and not a.get("fresh"):
|
|
state = "stale"
|
|
elif worst:
|
|
state = worst[2]
|
|
elif st or a:
|
|
state = "up"
|
|
else:
|
|
state = "unknown"
|
|
rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}, "grades": grades,
|
|
"down_since": st.get("down_since") if state == "down" else None})
|
|
return sorted(rows, key=lambda r: (GLANCE_ORDER[r["state"]], r["host"].name.lower()))
|