feat(host_agent): Netdata-style fleet overview + per-host detail view
Surfaces the metrics the agent now collects (task #868). Adds a viewer-facing fleet page (/plugins/host_agent/) with per-host cards (CPU/mem/disk/load/temp, stale flag) and a per-host detail page (/plugins/host_agent/<id>/) — the link the fleet widget already pointed at but had no route for. Detail page shows current gauges (CPU, memory incl. swap/cache, load, network rx/tx, disk I/O, temp max, memory/cpu/io PSI), per-core CPU bars, per-mount filesystem bars, and per-interface/disk/sensor breakdowns, plus history charts (utilization %, throughput B/s, load & pressure) over a selectable range. Charts use a linear epoch-ms x-axis so no Chart.js date adapter is needed. Stale state uses the plugin's stale_after_seconds threshold. Repoints the host_resources widget detail link from admin settings to the new fleet page. Completes milestone #68 (task #867). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+174
-28
@@ -11,10 +11,11 @@ import secrets
|
||||
from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for
|
||||
from steward.auth.middleware import require_role
|
||||
from steward.models.users import UserRole
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy import select, func, or_
|
||||
from datetime import timedelta
|
||||
|
||||
from steward.core.settings import public_base_url
|
||||
from steward.core.time_range import parse_range, RANGE_OPTIONS
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.metrics import PluginMetric
|
||||
from .models import HostAgentRegistration
|
||||
@@ -299,35 +300,45 @@ async def agent_source():
|
||||
return Response(body, mimetype="text/x-python")
|
||||
|
||||
|
||||
@host_agent_bp.get("/widget")
|
||||
async def widget_table():
|
||||
"""Fleet-glance table: one row per monitored host, latest metrics."""
|
||||
async with current_app.db_sessionmaker() as session:
|
||||
regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
|
||||
host_ids = [r.host_id for r in regs]
|
||||
hosts = {
|
||||
h.id: h for h in (await session.execute(
|
||||
select(Host).where(Host.id.in_(host_ids))
|
||||
)).scalars().all()
|
||||
} if host_ids else {}
|
||||
def _stale_after_seconds() -> int:
|
||||
return int(
|
||||
current_app.config.get("PLUGINS", {})
|
||||
.get(SOURCE_MODULE, {})
|
||||
.get("stale_after_seconds", 180)
|
||||
)
|
||||
|
||||
subq = (
|
||||
select(
|
||||
PluginMetric.resource_name,
|
||||
PluginMetric.metric_name,
|
||||
func.max(PluginMetric.recorded_at).label("max_ts"),
|
||||
)
|
||||
.where(PluginMetric.source_module == SOURCE_MODULE)
|
||||
.group_by(PluginMetric.resource_name, PluginMetric.metric_name)
|
||||
).subquery()
|
||||
latest_rows = (await session.execute(
|
||||
select(PluginMetric).join(
|
||||
subq,
|
||||
(PluginMetric.resource_name == subq.c.resource_name) &
|
||||
(PluginMetric.metric_name == subq.c.metric_name) &
|
||||
(PluginMetric.recorded_at == subq.c.max_ts),
|
||||
).where(PluginMetric.source_module == SOURCE_MODULE)
|
||||
|
||||
async def _fleet_rows(session) -> list[dict]:
|
||||
"""One row per registered host with its latest host-level metrics + stale flag.
|
||||
|
||||
Only bare host-level resources are used here (sub-resources carry a ':' and
|
||||
are skipped) so the fleet glance stays one line per host.
|
||||
"""
|
||||
regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
|
||||
host_ids = [r.host_id for r in regs]
|
||||
hosts = {
|
||||
h.id: h for h in (await session.execute(
|
||||
select(Host).where(Host.id.in_(host_ids))
|
||||
)).scalars().all()
|
||||
} if host_ids else {}
|
||||
|
||||
subq = (
|
||||
select(
|
||||
PluginMetric.resource_name,
|
||||
PluginMetric.metric_name,
|
||||
func.max(PluginMetric.recorded_at).label("max_ts"),
|
||||
)
|
||||
.where(PluginMetric.source_module == SOURCE_MODULE)
|
||||
.group_by(PluginMetric.resource_name, PluginMetric.metric_name)
|
||||
).subquery()
|
||||
latest_rows = (await session.execute(
|
||||
select(PluginMetric).join(
|
||||
subq,
|
||||
(PluginMetric.resource_name == subq.c.resource_name) &
|
||||
(PluginMetric.metric_name == subq.c.metric_name) &
|
||||
(PluginMetric.recorded_at == subq.c.max_ts),
|
||||
).where(PluginMetric.source_module == SOURCE_MODULE)
|
||||
)).scalars().all()
|
||||
|
||||
latest: dict[str, dict[str, float]] = {}
|
||||
for row in latest_rows:
|
||||
@@ -335,21 +346,46 @@ async def widget_table():
|
||||
continue
|
||||
latest.setdefault(row.resource_name, {})[row.metric_name] = row.value
|
||||
|
||||
stale_after = _stale_after_seconds()
|
||||
now = datetime.now(timezone.utc)
|
||||
rows = []
|
||||
for reg in regs:
|
||||
host = hosts.get(reg.host_id)
|
||||
if host is None:
|
||||
continue
|
||||
m = latest.get(host.name, {})
|
||||
ls = reg.last_seen_at
|
||||
stale = ls is None or (now - ls).total_seconds() > stale_after
|
||||
rows.append({
|
||||
"host": host,
|
||||
"reg": reg,
|
||||
"stale": stale,
|
||||
"cpu_pct": m.get("cpu_pct"),
|
||||
"mem_used_pct": m.get("mem_used_pct"),
|
||||
"disk_worst": m.get("disk_used_pct_worst"),
|
||||
"load_1m": m.get("load_1m"),
|
||||
"temp_max": m.get("temp_c_max"),
|
||||
"net_rx_bps": m.get("net_rx_bps"),
|
||||
"net_tx_bps": m.get("net_tx_bps"),
|
||||
})
|
||||
rows.sort(key=lambda r: r["host"].name.lower())
|
||||
return rows
|
||||
|
||||
|
||||
@host_agent_bp.get("/")
|
||||
@require_role(UserRole.viewer)
|
||||
async def index():
|
||||
"""Fleet overview page — cards per host linking to the detail view."""
|
||||
async with current_app.db_sessionmaker() as session:
|
||||
rows = await _fleet_rows(session)
|
||||
return await render_template("host_list.html", rows=rows)
|
||||
|
||||
|
||||
@host_agent_bp.get("/widget")
|
||||
async def widget_table():
|
||||
"""Fleet-glance dashboard widget: one row per monitored host, latest metrics."""
|
||||
async with current_app.db_sessionmaker() as session:
|
||||
rows = await _fleet_rows(session)
|
||||
return await render_template("widget_table.html", rows=rows)
|
||||
|
||||
|
||||
@@ -381,6 +417,116 @@ async def widget_history():
|
||||
return await render_template("widget_history.html", host=host, series=series, hours=hours)
|
||||
|
||||
|
||||
# Host-level metrics charted on the detail page (sub-resources are shown as
|
||||
# current-value lists, not time series, to keep the page readable).
|
||||
HISTORY_METRICS = (
|
||||
"cpu_pct", "mem_used_pct", "disk_used_pct_worst", "load_1m",
|
||||
"net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
|
||||
"temp_c_max", "psi_mem_some_avg10",
|
||||
)
|
||||
|
||||
|
||||
async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[str, float]]:
|
||||
"""{resource_name: {metric: value}} of the latest sample for a host + sub-resources."""
|
||||
subq = (
|
||||
select(
|
||||
PluginMetric.resource_name,
|
||||
PluginMetric.metric_name,
|
||||
func.max(PluginMetric.recorded_at).label("max_ts"),
|
||||
)
|
||||
.where(PluginMetric.source_module == SOURCE_MODULE)
|
||||
.where(or_(
|
||||
PluginMetric.resource_name == host_name,
|
||||
PluginMetric.resource_name.like(host_name + ":%"),
|
||||
))
|
||||
.group_by(PluginMetric.resource_name, PluginMetric.metric_name)
|
||||
).subquery()
|
||||
rows = (await session.execute(
|
||||
select(PluginMetric).join(
|
||||
subq,
|
||||
(PluginMetric.resource_name == subq.c.resource_name) &
|
||||
(PluginMetric.metric_name == subq.c.metric_name) &
|
||||
(PluginMetric.recorded_at == subq.c.max_ts),
|
||||
).where(PluginMetric.source_module == SOURCE_MODULE)
|
||||
)).scalars().all()
|
||||
out: dict[str, dict[str, float]] = {}
|
||||
for r in rows:
|
||||
out.setdefault(r.resource_name, {})[r.metric_name] = r.value
|
||||
return out
|
||||
|
||||
|
||||
async def _history_for_host(session, host_name: str, since) -> dict[str, list]:
|
||||
"""{metric: [[epoch_ms, value], …]} host-level series since `since`.
|
||||
|
||||
Epoch-ms x values let the charts use a linear axis (no Chart.js date adapter).
|
||||
"""
|
||||
rows = (await session.execute(
|
||||
select(PluginMetric).where(
|
||||
PluginMetric.source_module == SOURCE_MODULE,
|
||||
PluginMetric.resource_name == host_name,
|
||||
PluginMetric.metric_name.in_(HISTORY_METRICS),
|
||||
PluginMetric.recorded_at >= since,
|
||||
).order_by(PluginMetric.recorded_at)
|
||||
)).scalars().all()
|
||||
series: dict[str, list] = {m: [] for m in HISTORY_METRICS}
|
||||
for p in rows:
|
||||
series[p.metric_name].append([int(p.recorded_at.timestamp() * 1000), round(p.value, 2)])
|
||||
return series
|
||||
|
||||
|
||||
@host_agent_bp.get("/<host_id>/")
|
||||
@require_role(UserRole.viewer)
|
||||
async def host_detail(host_id: str):
|
||||
"""Netdata-style per-host detail: current gauges + history charts."""
|
||||
since, range_key = parse_range(request.args.get("range"))
|
||||
async with current_app.db_sessionmaker() as session:
|
||||
host = (await session.execute(
|
||||
select(Host).where(Host.id == host_id))).scalar_one_or_none()
|
||||
if host is None:
|
||||
return _error(404, "not_found")
|
||||
reg = (await session.execute(select(HostAgentRegistration).where(
|
||||
HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
|
||||
latest = await _latest_metrics_for_host(session, host.name)
|
||||
series = await _history_for_host(session, host.name, since)
|
||||
|
||||
hostlvl = latest.get(host.name, {})
|
||||
cores: list = []
|
||||
nets: dict = {}
|
||||
disks_io: dict = {}
|
||||
temps: dict = {}
|
||||
mounts: dict = {}
|
||||
plen = len(host.name) + 1
|
||||
for res, metrics in latest.items():
|
||||
if res == host.name:
|
||||
continue
|
||||
suffix = res[plen:]
|
||||
if suffix.startswith("core"):
|
||||
try:
|
||||
idx = int(suffix[4:])
|
||||
except ValueError:
|
||||
idx = 0
|
||||
cores.append((idx, metrics.get("cpu_pct")))
|
||||
elif suffix.startswith("net:"):
|
||||
nets[suffix[len("net:"):]] = metrics
|
||||
elif suffix.startswith("diskio:"):
|
||||
disks_io[suffix[len("diskio:"):]] = metrics
|
||||
elif suffix.startswith("temp:"):
|
||||
temps[suffix[len("temp:"):]] = metrics.get("temp_c")
|
||||
else:
|
||||
mounts[suffix] = metrics
|
||||
cores.sort(key=lambda c: c[0])
|
||||
|
||||
ls = reg.last_seen_at if reg else None
|
||||
stale = ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()
|
||||
|
||||
return await render_template(
|
||||
"host_detail.html",
|
||||
host=host, reg=reg, stale=stale, hostlvl=hostlvl,
|
||||
cores=cores, nets=nets, disks_io=disks_io, temps=temps, mounts=mounts,
|
||||
series=series, range_key=range_key, range_options=RANGE_OPTIONS,
|
||||
)
|
||||
|
||||
|
||||
def _new_token_pair() -> tuple[str, str]:
|
||||
raw = secrets.token_urlsafe(32)
|
||||
return raw, _hash_token(raw)
|
||||
|
||||
Reference in New Issue
Block a user