Files
FabledSteward/steward/hosts/service.py
T
bvandeusenandClaude Opus 5.5 c9f5c049db
CI / lint (push) Successful in 3s
CI / integration (push) Successful in 50s
CI / unit (push) Successful in 1m6s
CI / publish (push) Successful in 1m2s
feat(alerts): renames carry their alerts — monitor and host renames move rules, windows and history (#5065)
#5065 "Renames carry their alerts". Alert rules, maintenance-window scopes
and metric history key a thing by (source, name), so a rename used to leave
the rules watching a name nothing reported under, and split the history.

- core/renames.carry_rename moves AlertRule.resource_name,
  MaintenanceWindow.scope_resource, plugin_metrics and plugin_metrics_hourly
  to the new name in the caller's transaction. A host rename also carries
  its parts: host_agent "<host>:…" and docker "<host>/…" (HOST_SOURCES).
  Matching is exact (a prefix compare, not LIKE or a collation range).
  History left under the new name by something deleted is dropped first
  rather than merged in.
- Web monitor edit, MCP update_monitor and web host edit call it; MCP
  answers with `carried` (it used to return a warning naming stranded rules).
- A name another monitor of the same type, or another host, already has is
  refused on create and rename: two would share one history.
- The rename is audited (monitor.updated / host.renamed) with what moved.

Tests: an integration test renames a firing monitor through the web route
and finds its rule still firing and cleared by a sample on the new name,
its window and raw+hourly history moved and an orphan's history dropped; a
host rename moves agent and container parts but not a longer-named host's
or another source's; a guard that every host_agent resource is "<host>" or
"<host>:…".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 14:02:43 -04:00

264 lines
10 KiB
Python

"""Host queries shared by the Hosts pages, dashboard widgets and MCP tools.
Every function takes an open session and returns plain data; none reads
Quart's request context, so the web routes and the /mcp tools (which run
outside Quart's pipeline) call the same code.
"""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from sqlalchemy import and_, case, func, select
from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor
from steward.models.hosts import Host
from steward.models.monitors import Monitor, MonitorResult
# Host-level agent metrics read for the at-a-glance views (plus the root
# mount's disk_used_pct, read from the "<host>:/" sub-resource).
AGENT_GLANCE_METRICS = ("cpu_pct", "mem_used_pct", "disk_used_pct", "disk_used_pct_worst")
async def check_name_free(db, name: str, exclude_id: str | None = None) -> None:
"""ValueError when another host already has this name, or it is blank.
Agent metrics, container metrics and the alert rules on them go by the
host's name, so two hosts sharing one would share one history.
"""
if not name:
raise ValueError("a host needs a name.")
q = select(Host.id).where(Host.name == name)
if exclude_id:
q = q.where(Host.id != exclude_id)
if (await db.execute(q.limit(1))).first():
raise ValueError(f"another host is already named {name!r}; pick a different name.")
def agent_stale_after_seconds(config) -> int:
"""The host_agent plugin's freshness window from app config (default 180s)."""
return int(((config.get("PLUGINS") or {}).get("host_agent") or {})
.get("stale_after_seconds", 180))
async def compute_uptime(db) -> dict[str, dict]:
"""Per-host uptime % for 24h/7d/30d, aggregated over the host's monitors.
Returns {host_id: {"24h": float|None, "7d": float|None, "30d": float|None}}.
"""
now = datetime.now(timezone.utc)
cutoff_30d = now - timedelta(days=30)
cutoff_7d = now - timedelta(days=7)
cutoff_24h = now - timedelta(hours=24)
result = await db.execute(
select(
Monitor.host_id,
func.count(MonitorResult.id).label("total_30d"),
func.sum(case((MonitorResult.is_up.is_(True), 1), else_=0)).label("up_30d"),
func.sum(case((MonitorResult.checked_at >= cutoff_7d, 1), else_=0)).label("total_7d"),
func.sum(case(
(and_(MonitorResult.checked_at >= cutoff_7d, MonitorResult.is_up.is_(True)), 1),
else_=0,
)).label("up_7d"),
func.sum(case((MonitorResult.checked_at >= cutoff_24h, 1), else_=0)).label("total_24h"),
func.sum(case(
(and_(MonitorResult.checked_at >= cutoff_24h, MonitorResult.is_up.is_(True)), 1),
else_=0,
)).label("up_24h"),
)
.join(Monitor, Monitor.id == MonitorResult.monitor_id)
.where(Monitor.host_id.isnot(None), MonitorResult.checked_at >= cutoff_30d)
.group_by(Monitor.host_id)
)
def _pct(up, total):
return round(float(up) / float(total) * 100, 2) if total else None
stats: dict[str, dict] = {}
for row in result:
stats[row.host_id] = {
"24h": _pct(row.up_24h, row.total_24h),
"7d": _pct(row.up_7d, row.total_7d),
"30d": _pct(row.up_30d, row.total_30d),
}
return stats
async def host_status(db, host_ids: list[str]) -> dict[str, dict]:
"""Per-host status rollup from the latest result of each linked monitor.
{host_id: {"state": "up"|"down"|"pending", "latency_ms": float|None,
"count": int}}. State is down if ANY linked monitor's latest
result is down (worst-case), up if at least one is up, else pending.
"""
if not host_ids:
return {}
rn = func.row_number().over(
partition_by=MonitorResult.monitor_id, order_by=MonitorResult.checked_at.desc()
).label("rn")
subq = (
select(MonitorResult.id, rn)
.join(Monitor, Monitor.id == MonitorResult.monitor_id)
.where(Monitor.host_id.in_(host_ids), Monitor.enabled.is_(True))
.subquery()
)
rows = (await db.execute(
select(MonitorResult, Monitor.host_id)
.join(subq, MonitorResult.id == subq.c.id)
.join(Monitor, Monitor.id == MonitorResult.monitor_id)
.where(subq.c.rn == 1)
)).all()
out: dict[str, dict] = {}
for res, host_id in rows:
d = out.setdefault(host_id, {"state": "pending", "latency_ms": None, "count": 0})
d["count"] += 1
if not res.is_up:
d["state"] = "down"
elif d["state"] != "down":
d["state"] = "up"
if res.is_up and res.response_ms is not None:
d["latency_ms"] = (res.response_ms if d["latency_ms"] is None
else min(d["latency_ms"], res.response_ms))
return out
async def host_monitors(db, host_id: str) -> list[dict]:
"""Display rows for the monitors linked to one host (host hub section)."""
monitors = (await db.execute(
select(Monitor).where(Monitor.host_id == host_id).order_by(Monitor.name)
)).scalars().all()
ids = [m.id for m in monitors]
recent = await _last_n_results(db, ids)
uptime = await _uptime_by_monitor(db, ids)
data = []
for m in monitors:
mrows = recent.get(m.id, [])
latest = mrows[-1] if mrows else None
data.append({
"monitor": m, "latest": latest,
"heartbeat": [{"state": "up" if r.is_up else "down",
"title": _heartbeat_title(m.type, r)} for r in mrows],
"uptime_pct": uptime.get(m.id, {}).get("24h"),
})
return data
async def agent_metrics_by_host(db, host_names: list[str]) -> dict[str, dict]:
"""Latest agent cpu/mem per host name from the generic PluginMetric table.
Read directly (PluginMetric is a core model) so the Hosts list can show an
at-a-glance agent column without importing the host_agent plugin.
"""
from steward.models.metrics import PluginMetric
if not host_names:
return {}
wanted = ("cpu_pct", "mem_used_pct")
subq = (
select(
PluginMetric.resource_name, PluginMetric.metric_name,
func.max(PluginMetric.recorded_at).label("m"),
)
.where(
PluginMetric.source_module == "host_agent",
PluginMetric.metric_name.in_(wanted),
PluginMetric.resource_name.in_(host_names),
)
.group_by(PluginMetric.resource_name, PluginMetric.metric_name)
).subquery()
rows = (await db.execute(
select(PluginMetric).join(
subq,
(PluginMetric.resource_name == subq.c.resource_name)
& (PluginMetric.metric_name == subq.c.metric_name)
& (PluginMetric.recorded_at == subq.c.m),
)
)).scalars().all()
out: dict[str, dict] = {}
for r in rows:
out.setdefault(r.resource_name, {})[r.metric_name] = r.value
return out
async def agent_overview_by_host(
db, host_names: list[str], stale_after_seconds: int = 180,
) -> dict[str, dict]:
"""Per-host agent glance (cpu/mem/disk + freshness) from PluginMetric.
{host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root,
last_at, fresh}} for each host with any agent samples.
Core-safe (no host_agent import). Freshness compares the latest host-level
sample against the host_agent plugin's stale window (stale_after_seconds;
callers pass agent_stale_after_seconds(app.config)).
"""
from steward.models.metrics import PluginMetric
if not host_names:
return {}
now = datetime.now(timezone.utc)
stale_after = int(stale_after_seconds)
resources = list(host_names) + [h + ":/" for h in host_names]
subq = (
select(
PluginMetric.resource_name, PluginMetric.metric_name,
func.max(PluginMetric.recorded_at).label("m"),
)
.where(
PluginMetric.source_module == "host_agent",
PluginMetric.resource_name.in_(resources),
PluginMetric.metric_name.in_(AGENT_GLANCE_METRICS),
)
.group_by(PluginMetric.resource_name, PluginMetric.metric_name)
).subquery()
rows = (await db.execute(
select(PluginMetric).join(
subq,
(PluginMetric.resource_name == subq.c.resource_name)
& (PluginMetric.metric_name == subq.c.metric_name)
& (PluginMetric.recorded_at == subq.c.m),
).where(PluginMetric.source_module == "host_agent")
)).scalars().all()
out: dict[str, dict] = {}
for r in rows:
if r.resource_name.endswith(":/") and r.metric_name == "disk_used_pct":
out.setdefault(r.resource_name[:-2], {})["disk_root"] = r.value
elif r.resource_name in host_names:
d = out.setdefault(r.resource_name, {})
d[r.metric_name] = r.value
if d.get("_ts") is None or r.recorded_at > d["_ts"]:
d["_ts"] = r.recorded_at
for d in out.values():
ts = d.pop("_ts", None)
d["last_at"] = ts
d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
return out
async def agent_cpu_sparks_by_host(db, host_names: list[str], hours: int = 1) -> dict[str, str]:
"""{host_name: inline-SVG cpu sparkline} over the last `hours` (core-safe).
A tiny at-a-glance trend in each row — the host-view sparkline, on the widget.
"""
from steward.models.metrics import PluginMetric
from steward.core.status import sparkline_svg
if not host_names:
return {}
cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
rows = (await db.execute(
select(PluginMetric.resource_name, PluginMetric.value, PluginMetric.recorded_at)
.where(
PluginMetric.source_module == "host_agent",
PluginMetric.metric_name == "cpu_pct",
PluginMetric.resource_name.in_(host_names),
PluginMetric.recorded_at >= cutoff,
)
.order_by(PluginMetric.recorded_at)
)).all()
by_host: dict[str, list[float]] = {}
for r in rows:
by_host.setdefault(r.resource_name, []).append(r.value)
# Amber line to read as "CPU"; only draw when there are a couple of points.
return {
name: sparkline_svg(vals[-40:], width=64, height=18, stroke="#c8a840")
for name, vals in by_host.items() if len(vals) >= 2
}