From 9e4f1983f81a282fc15e7b4a6bc631ca23ef6c8c Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 20 Jun 2026 12:18:39 -0400 Subject: [PATCH] feat(host_agent): lazy-load full-metrics charts + live-poll current state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The full-metrics page rendered once server-side with the history query inline, so the charts blocked first paint and nothing refreshed without a reload (it reads the latest snapshot the server holds — the agent pushes ~every 30s). Split it into a shell + two HTMX fragments: - Shell (host_detail.html): header + shared time-range toggle + two containers; paints instantly. - Current state (//metrics → _host_metrics.html): identity + gauges + per-core + filesystems + interfaces/disks + temps, from the DISTINCT ON latest query. hx-trigger "load, every 15s" → live numbers at ~the agent cadence. - History charts (//charts → _host_charts.html): the 3 charts + Chart.js, from the date_bin history query. hx-trigger "load, every 60s, rangeChange" so they lazy-load (never block paint), refresh slowly, and follow the range selector. Leak-safe: previous Chart instances are destroyed before re-render. - Range toggle switched from full-reload links to the shared _time_range.html (setTimeRange + rangeChange), so only the fragments refetch. - routes: host_detail (shell) + host_detail_metrics + host_detail_charts, with a _split_host_metrics helper. - tests/test_templates_parse.py now also parses plugin templates (these fragments aren't rendered in the unit lane). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- plugins/host_agent/routes.py | 83 ++++-- .../host_agent/templates/_host_charts.html | 70 +++++ .../host_agent/templates/_host_metrics.html | 182 ++++++++++++ plugins/host_agent/templates/host_detail.html | 272 ++---------------- tests/test_templates_parse.py | 16 +- 5 files changed, 350 insertions(+), 273 deletions(-) create mode 100644 plugins/host_agent/templates/_host_charts.html create mode 100644 plugins/host_agent/templates/_host_metrics.html diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index effb412..61663ef 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -545,30 +545,25 @@ def _downsample(series: list[list], target: int = 120) -> list[list]: return out -@host_agent_bp.get("//") -@require_role(UserRole.viewer) -async def host_detail(host_id: str): - """Netdata-style per-host detail: current gauges + history charts.""" - since, range_key = parse_range(request.args.get("range")) - async with current_app.db_sessionmaker() as session: - host = (await session.execute( - select(Host).where(Host.id == host_id))).scalar_one_or_none() - if host is None: - return _error(404, "not_found") - reg = (await session.execute(select(HostAgentRegistration).where( - HostAgentRegistration.host_id == host_id))).scalar_one_or_none() - latest = await _latest_metrics_for_host(session, host.name) - series = await _history_for_host(session, host.name, since) +# Full-metrics refresh cadences. The agent pushes every ~30s by default, so +# polling the current snapshot at 15s keeps it ≤ one cadence stale; the history +# charts move slowly over a multi-hour/day window, so they refresh less often. +_DETAIL_POLL_SECONDS = 15 +_CHART_POLL_SECONDS = 60 - hostlvl = latest.get(host.name, {}) + +def _split_host_metrics(host_name: str, latest: dict) -> tuple: + """Split a host's latest snapshot into + (hostlvl, cores, nets, disks_io, temps, mounts) for the current-state fragment.""" + hostlvl = latest.get(host_name, {}) cores: list = [] nets: dict = {} disks_io: dict = {} temps: dict = {} mounts: dict = {} - plen = len(host.name) + 1 + plen = len(host_name) + 1 for res, metrics in latest.items(): - if res == host.name: + if res == host_name: continue suffix = res[plen:] if suffix.startswith("core"): @@ -586,18 +581,66 @@ async def host_detail(host_id: str): else: mounts[suffix] = metrics cores.sort(key=lambda c: c[0]) + return hostlvl, cores, nets, disks_io, temps, mounts - ls = reg.last_seen_at if reg else None - stale = ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds() +@host_agent_bp.get("//") +@require_role(UserRole.viewer) +async def host_detail(host_id: str): + """Full-metrics shell: header + range toggle only. The current-state and + history sections stream in via HTMX (host_detail_metrics / host_detail_charts) + so the heavy history query never blocks first paint and the data live-updates.""" + _, range_key = parse_range(request.args.get("range")) + async with current_app.db_sessionmaker() as session: + host = (await session.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return _error(404, "not_found") return await render_template( "host_detail.html", + host=host, current_range=range_key, range_options=RANGE_OPTIONS, + poll_seconds=_DETAIL_POLL_SECONDS, chart_poll_seconds=_CHART_POLL_SECONDS, + ) + + +@host_agent_bp.get("//metrics") +@require_role(UserRole.viewer) +async def host_detail_metrics(host_id: str): + """Current-state fragment (gauges/cores/filesystems/IO/temps) — polled live so + the page shows the latest snapshot the server holds without a full reload.""" + async with current_app.db_sessionmaker() as session: + host = (await session.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return "", 404 + reg = (await session.execute(select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host_id))).scalar_one_or_none() + latest = await _latest_metrics_for_host(session, host.name) + hostlvl, cores, nets, disks_io, temps, mounts = _split_host_metrics(host.name, latest) + ls = reg.last_seen_at if reg else None + stale = ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds() + return await render_template( + "_host_metrics.html", host=host, reg=reg, stale=stale, hostlvl=hostlvl, cores=cores, nets=nets, disks_io=disks_io, temps=temps, mounts=mounts, - series=series, range_key=range_key, range_options=RANGE_OPTIONS, ) +@host_agent_bp.get("//charts") +@require_role(UserRole.viewer) +async def host_detail_charts(host_id: str): + """History-charts fragment — lazy-loaded so the date_bin query never blocks + the page shell; refreshed on a slow cadence and on range change.""" + since, range_key = parse_range(request.args.get("range")) + async with current_app.db_sessionmaker() as session: + host = (await session.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return "", 404 + series = await _history_for_host(session, host.name, since) + return await render_template("_host_charts.html", series=series, range_key=range_key) + + def _new_token_pair() -> tuple[str, str]: raw = secrets.token_urlsafe(32) return raw, _hash_token(raw) diff --git a/plugins/host_agent/templates/_host_charts.html b/plugins/host_agent/templates/_host_charts.html new file mode 100644 index 0000000..96bc142 --- /dev/null +++ b/plugins/host_agent/templates/_host_charts.html @@ -0,0 +1,70 @@ +{# History-charts fragment for the full-metrics page — lazy-loaded via HTMX so + the date_bin history query never blocks the page shell, and refreshed on a + slow cadence / range change. #} +
+
+
Utilization % — last {{ range_key }}
+
+
+
+
Throughput (B/s) — last {{ range_key }}
+
+
+
+
Load & pressure — last {{ range_key }}
+
+
+
+ + diff --git a/plugins/host_agent/templates/_host_metrics.html b/plugins/host_agent/templates/_host_metrics.html new file mode 100644 index 0000000..c06c49b --- /dev/null +++ b/plugins/host_agent/templates/_host_metrics.html @@ -0,0 +1,182 @@ +{# Current-state fragment for the full-metrics page — polled live via HTMX. + Self-contained (defines its own format macros) so each poll re-renders the + latest snapshot the server has. #} +{% macro fmt_bps(v) %} +{%- if v is none -%}— +{%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s +{%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s +{%- else -%}{{ "%.0f"|format(v) }} B/s +{%- endif -%} +{% endmacro %} +{% macro fmt_bytes(v) %} +{%- if v is none -%}— +{%- elif v >= 1125899906842624 -%}{{ "%.1f"|format(v / 1125899906842624) }} PB +{%- elif v >= 1099511627776 -%}{{ "%.1f"|format(v / 1099511627776) }} TB +{%- elif v >= 1073741824 -%}{{ "%.1f"|format(v / 1073741824) }} GB +{%- elif v >= 1048576 -%}{{ "%.0f"|format(v / 1048576) }} MB +{%- else -%}{{ "%.0f"|format(v / 1024) }} KB +{%- endif -%} +{% endmacro %} +{% macro bar(pct) %} +{%- set p = pct if pct is not none else 0 -%} +
+
+
+{% endmacro %} + +{# ── Identity / metadata ─────────────────────────────────────────────────── #} +
+ {% if stale %}stale{% else %}live{% endif %} + Address {{ host.address or "—" }} + {% if reg %} + {% if reg.distro %}OS {{ reg.distro }}{% endif %} + {% if reg.kernel %}Kernel {{ reg.kernel }}{% endif %} + {% if reg.arch %}Arch {{ reg.arch }}{% endif %} + {% if reg.agent_version %}Agent v{{ reg.agent_version }}{% endif %} + {% set up = hostlvl.get('uptime_secs') %} + {% if up %}Uptime {{ (up // 86400)|int }}d {{ ((up % 86400) // 3600)|int }}h{% endif %} + Last seen {{ reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") ~ " UTC" if reg.last_seen_at else "never" }} + {% else %} + No agent registration found for this host. + {% endif %} +
+ +{% if not hostlvl %} +
No metrics reported yet. Once the agent checks in, CPU, memory, network, disk I/O, temperatures, and pressure will appear here.
+{% else %} + +{# ── Current headline gauges ─────────────────────────────────────────────── #} +
+
+
CPU
+ {% set c = hostlvl.get('cpu_pct') %} + {% if c is not none %}{{ "%.0f"|format(c) }}%{% else %}—{% endif %} +
{{ bar(c) }}
+
+
+
Memory
+ {% set mp = hostlvl.get('mem_used_pct') %} + {% if mp is not none %}{{ "%.0f"|format(mp) }}%{% else %}—{% endif %} +
{{ bar(mp) }}
+
+ avail {{ fmt_bytes(hostlvl.get('mem_available_bytes')) }} · cache {{ fmt_bytes(hostlvl.get('mem_cached_bytes')) }} · swap {{ fmt_bytes(hostlvl.get('swap_used_bytes')) }} +
+
+
+
Load
+ {% if hostlvl.get('load_1m') is not none %}{{ "%.2f"|format(hostlvl.get('load_1m')) }}{% else %}—{% endif %} +
+ 5m {% if hostlvl.get('load_5m') is not none %}{{ "%.2f"|format(hostlvl.get('load_5m')) }}{% else %}—{% endif %} · + 15m {% if hostlvl.get('load_15m') is not none %}{{ "%.2f"|format(hostlvl.get('load_15m')) }}{% else %}—{% endif %} +
+
+
+
Network
+
+
↓ {{ fmt_bps(hostlvl.get('net_rx_bps')) }}
+
↑ {{ fmt_bps(hostlvl.get('net_tx_bps')) }}
+
+
+
+
Disk I/O
+
+
rd {{ fmt_bps(hostlvl.get('disk_read_bps')) }}
+
wr {{ fmt_bps(hostlvl.get('disk_write_bps')) }}
+
+
+ {% if hostlvl.get('temp_c_max') is not none %} +
+
Temp (max)
+ {% set t = hostlvl.get('temp_c_max') %} + {{ "%.0f"|format(t) }}°C +
+ {% endif %} + {% if hostlvl.get('psi_mem_some_avg10') is not none %} +
+
Pressure (10s)
+
+
mem {{ "%.1f"|format(hostlvl.get('psi_mem_some_avg10')) }}%
+ {% if hostlvl.get('psi_cpu_some_avg10') is not none %}
cpu {{ "%.1f"|format(hostlvl.get('psi_cpu_some_avg10')) }}%
{% endif %} + {% if hostlvl.get('psi_io_some_avg10') is not none %}
io {{ "%.1f"|format(hostlvl.get('psi_io_some_avg10')) }}%
{% endif %} +
+
+ {% endif %} +
+ +{# ── Per-core CPU ─────────────────────────────────────────────────────────── #} +{% if cores %} +
+
Per-core CPU
+
+ {% for idx, pct in cores %} +
+
+ core {{ idx }}{% if pct is not none %}{{ "%.0f"|format(pct) }}%{% else %}—{% endif %} +
+ {{ bar(pct) }} +
+ {% endfor %} +
+
+{% endif %} + +{# ── Filesystems ──────────────────────────────────────────────────────────── #} +{% if mounts %} +
+
Filesystems
+ {% for mount, m in mounts.items() %} +
+
+ {{ mount }} + {{ fmt_bytes(m.get('disk_used_bytes')) }} / {{ fmt_bytes(m.get('disk_total_bytes')) }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %} +
+ {{ bar(m.get('disk_used_pct')) }} +
+ {% endfor %} +
+{% endif %} + +{# ── Interfaces / disks (short panels, side by side at natural height) ─────── #} +{% if nets or disks_io %} +
+ {% if nets %} +
+
Interfaces
+ {% for iface, m in nets.items() %} +
+ {{ iface }} + ↓ {{ fmt_bps(m.get('net_rx_bps')) }} · ↑ {{ fmt_bps(m.get('net_tx_bps')) }} +
+ {% endfor %} +
+ {% endif %} + {% if disks_io %} +
+
Disks
+ {% for dev, m in disks_io.items() %} +
+ {{ dev }} + rd {{ fmt_bps(m.get('disk_read_bps')) }} · wr {{ fmt_bps(m.get('disk_write_bps')) }} +
+ {% endfor %} +
+ {% endif %} +
+{% endif %} + +{# ── Temperatures (full width; cores flow into a compact multi-column grid) ── #} +{% if temps %} +
+
Temperatures
+
+ {% for label, c in temps.items() %} +
+ {{ label }} + {% if c is not none %}{{ "%.0f"|format(c) }}°C{% else %}—{% endif %} +
+ {% endfor %} +
+
+{% endif %} + +{% endif %} diff --git a/plugins/host_agent/templates/host_detail.html b/plugins/host_agent/templates/host_detail.html index 14c63f3..44b4bb2 100644 --- a/plugins/host_agent/templates/host_detail.html +++ b/plugins/host_agent/templates/host_detail.html @@ -1,262 +1,32 @@ {% extends "base.html" %} {% from "_macros.html" import crumbs %} -{% block title %}{{ host.name }} — Host Agent — Steward{% endblock %} +{% block title %}{{ host.name }} — Metrics — Steward{% endblock %} {% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), (host.name, "/hosts/" ~ host.id), ("Metrics", "")]) }}{% endblock %} - -{% macro fmt_bps(v) %} -{%- if v is none -%}— -{%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s -{%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s -{%- else -%}{{ "%.0f"|format(v) }} B/s -{%- endif -%} -{% endmacro %} -{% macro fmt_bytes(v) %} -{%- if v is none -%}— -{%- elif v >= 1125899906842624 -%}{{ "%.1f"|format(v / 1125899906842624) }} PB -{%- elif v >= 1099511627776 -%}{{ "%.1f"|format(v / 1099511627776) }} TB -{%- elif v >= 1073741824 -%}{{ "%.1f"|format(v / 1073741824) }} GB -{%- elif v >= 1048576 -%}{{ "%.0f"|format(v / 1048576) }} MB -{%- else -%}{{ "%.0f"|format(v / 1024) }} KB -{%- endif -%} -{% endmacro %} -{% macro bar(pct) %} -{%- set p = pct if pct is not none else 0 -%} -
-
-
-{% endmacro %} - {% block content %} +{# Shell only: the current-state and history-chart sections stream in via HTMX + below, so the heavy history query never blocks first paint and the data + refreshes live (≈ the agent's push cadence) without a full reload. #}
-
-

{{ host.name }}

- {% if stale %}stale{% else %}live{% endif %} -
-
- {% for r in range_options %} - {{ r }} - {% endfor %} -
+

{{ host.name }}

+ {% include "_time_range.html" %}
-{# ── Identity / metadata ─────────────────────────────────────────────────── #} -
- Address {{ host.address or "—" }} - {% if reg %} - {% if reg.distro %}OS {{ reg.distro }}{% endif %} - {% if reg.kernel %}Kernel {{ reg.kernel }}{% endif %} - {% if reg.arch %}Arch {{ reg.arch }}{% endif %} - {% if reg.agent_version %}Agent v{{ reg.agent_version }}{% endif %} - {% set up = hostlvl.get('uptime_secs') %} - {% if up %}Uptime {{ (up // 86400)|int }}d {{ ((up % 86400) // 3600)|int }}h{% endif %} - Last seen {{ reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") ~ " UTC" if reg.last_seen_at else "never" }} - {% else %} - No agent registration found for this host. - {% endif %} +{# Current state — lazy + polled live (latest snapshot the server holds). #} +
+
Loading metrics…
-{% if not hostlvl %} -
No metrics reported yet. Once the agent checks in, CPU, memory, network, disk I/O, temperatures, and pressure will appear here.
-{% else %} - -{# ── Current headline gauges ─────────────────────────────────────────────── #} -
-
-
CPU
- {% set c = hostlvl.get('cpu_pct') %} - {% if c is not none %}{{ "%.0f"|format(c) }}%{% else %}—{% endif %} -
{{ bar(c) }}
-
-
-
Memory
- {% set mp = hostlvl.get('mem_used_pct') %} - {% if mp is not none %}{{ "%.0f"|format(mp) }}%{% else %}—{% endif %} -
{{ bar(mp) }}
-
- avail {{ fmt_bytes(hostlvl.get('mem_available_bytes')) }} · cache {{ fmt_bytes(hostlvl.get('mem_cached_bytes')) }} · swap {{ fmt_bytes(hostlvl.get('swap_used_bytes')) }} -
-
-
-
Load
- {% if hostlvl.get('load_1m') is not none %}{{ "%.2f"|format(hostlvl.get('load_1m')) }}{% else %}—{% endif %} -
- 5m {% if hostlvl.get('load_5m') is not none %}{{ "%.2f"|format(hostlvl.get('load_5m')) }}{% else %}—{% endif %} · - 15m {% if hostlvl.get('load_15m') is not none %}{{ "%.2f"|format(hostlvl.get('load_15m')) }}{% else %}—{% endif %} -
-
-
-
Network
-
-
↓ {{ fmt_bps(hostlvl.get('net_rx_bps')) }}
-
↑ {{ fmt_bps(hostlvl.get('net_tx_bps')) }}
-
-
-
-
Disk I/O
-
-
rd {{ fmt_bps(hostlvl.get('disk_read_bps')) }}
-
wr {{ fmt_bps(hostlvl.get('disk_write_bps')) }}
-
-
- {% if hostlvl.get('temp_c_max') is not none %} -
-
Temp (max)
- {% set t = hostlvl.get('temp_c_max') %} - {{ "%.0f"|format(t) }}°C -
- {% endif %} - {% if hostlvl.get('psi_mem_some_avg10') is not none %} -
-
Pressure (10s)
-
-
mem {{ "%.1f"|format(hostlvl.get('psi_mem_some_avg10')) }}%
- {% if hostlvl.get('psi_cpu_some_avg10') is not none %}
cpu {{ "%.1f"|format(hostlvl.get('psi_cpu_some_avg10')) }}%
{% endif %} - {% if hostlvl.get('psi_io_some_avg10') is not none %}
io {{ "%.1f"|format(hostlvl.get('psi_io_some_avg10')) }}%
{% endif %} -
-
- {% endif %} +{# History charts — lazy (never blocks paint), refreshed on a slow cadence and + on range change; range comes from the shared #time-range input. #} +
+
Loading charts…
- -{# ── Per-core CPU ─────────────────────────────────────────────────────────── #} -{% if cores %} -
-
Per-core CPU
-
- {% for idx, pct in cores %} -
-
- core {{ idx }}{% if pct is not none %}{{ "%.0f"|format(pct) }}%{% else %}—{% endif %} -
- {{ bar(pct) }} -
- {% endfor %} -
-
-{% endif %} - -{# ── Filesystems ──────────────────────────────────────────────────────────── #} -{% if mounts %} -
-
Filesystems
- {% for mount, m in mounts.items() %} -
-
- {{ mount }} - {{ fmt_bytes(m.get('disk_used_bytes')) }} / {{ fmt_bytes(m.get('disk_total_bytes')) }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %} -
- {{ bar(m.get('disk_used_pct')) }} -
- {% endfor %} -
-{% endif %} - -{# ── Interfaces / disks (short panels, side by side at natural height) ─────── #} -{% if nets or disks_io %} -
- {% if nets %} -
-
Interfaces
- {% for iface, m in nets.items() %} -
- {{ iface }} - ↓ {{ fmt_bps(m.get('net_rx_bps')) }} · ↑ {{ fmt_bps(m.get('net_tx_bps')) }} -
- {% endfor %} -
- {% endif %} - {% if disks_io %} -
-
Disks
- {% for dev, m in disks_io.items() %} -
- {{ dev }} - rd {{ fmt_bps(m.get('disk_read_bps')) }} · wr {{ fmt_bps(m.get('disk_write_bps')) }} -
- {% endfor %} -
- {% endif %} -
-{% endif %} - -{# ── Temperatures (full width; cores flow into a compact multi-column grid so a - many-core CPU reads wide-and-short instead of one very tall column) ─────── #} -{% if temps %} -
-
Temperatures
-
- {% for label, c in temps.items() %} -
- {{ label }} - {% if c is not none %}{{ "%.0f"|format(c) }}°C{% else %}—{% endif %} -
- {% endfor %} -
-
-{% endif %} - -{# ── History charts ───────────────────────────────────────────────────────── #} -
-
-
Utilization % — last {{ range_key }}
-
-
-
-
Throughput (B/s) — last {{ range_key }}
-
-
-
-
Load & pressure — last {{ range_key }}
-
-
-
- - -{% endif %} {% endblock %} diff --git a/tests/test_templates_parse.py b/tests/test_templates_parse.py index 46e6f25..25dd636 100644 --- a/tests/test_templates_parse.py +++ b/tests/test_templates_parse.py @@ -11,12 +11,24 @@ import jinja2 import steward -_ROOT = pathlib.Path(steward.__file__).parent / "templates" +_REPO = pathlib.Path(steward.__file__).parent.parent +_CORE_TEMPLATES = _REPO / "steward" / "templates" +_PLUGINS = _REPO / "plugins" def test_all_steward_templates_parse(): env = jinja2.Environment() - files = sorted(_ROOT.rglob("*.html")) + files = sorted(_CORE_TEMPLATES.rglob("*.html")) assert files, "no steward templates found" for f in files: env.parse(f.read_text()) # raises TemplateSyntaxError on a bad tag + + +def test_all_plugin_templates_parse(): + # First-party plugins ship templates too (host_agent fragments, docker, …); + # they aren't rendered in the unit lane, so parse them here to catch a bad tag. + env = jinja2.Environment() + files = sorted(_PLUGINS.rglob("templates/**/*.html")) + assert files, "no plugin templates found" + for f in files: + env.parse(f.read_text())