bvandeusenandClaude Opus 5.5
b8f73270e7
The dashboard's host widgets took 18-60 s. Every "latest reading" query (hosts overview, host-agent fleet, host page, vitals, SNMP, MCP get_host) searched plugin_metrics for the newest row per metric. Postgres has no skip scan, so each one read every raw sample in the 7-day window. - plugin_metric_latest: one row per (source, resource, metric). record_metrics upserts it in the same transaction, moving forward only and in sorted key order. Migration 0030 seeds it by stepping the existing index. - Retention drops a latest row once its sample leaves the raw window, so a resource that stopped reporting disappears when its history does, as before. Renames carry it along. - A quiet host keeps its last values, marked stale. A time-bounded query would have shown it as missing. - Vitals sparklines span the 6 h they claim, averaged in SQL to ~120 points, instead of fetching 6 h raw and drawing the last 120 samples. - Dashboard and share-view cells: "Loading…" until the first answer; a 20 s deadline per fetch; "Couldn't load (why)" with Retry now and an automatic retry for cells that don't poll; "as of HH:MM" in the header, and "couldn't refresh · as of" when a refresh fails over shown data. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
97 lines
4.3 KiB
Python
97 lines
4.3 KiB
Python
"""Host-metric read helpers (latest snapshot + bucketed history).
|
|
|
|
Kept separate from routes.py so it imports only the core PluginMetric model — not
|
|
the host_agent ORM models — which lets integration tests import these helpers
|
|
without tripping the plugin-loader's double-registration ("Table already
|
|
defined") guard.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import func, select
|
|
|
|
from steward.core.time_range import bucket_seconds
|
|
from steward.models.metrics import PluginMetric, PluginMetricHourly
|
|
|
|
SOURCE_MODULE = "host_agent"
|
|
|
|
# Host-level metrics charted on the detail page (sub-resources are shown as
|
|
# current-value lists, not time series, to keep the page readable).
|
|
HISTORY_METRICS = (
|
|
"cpu_pct", "mem_used_pct", "disk_used_pct_worst", "load_1m",
|
|
"net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
|
|
"temp_c_max", "psi_mem_some_avg10",
|
|
)
|
|
|
|
|
|
async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[str, float]]:
|
|
"""{resource_name: {metric: value}} — latest sample per (resource, metric) for
|
|
a host + its sub-resources, from plugin_metric_latest (#5208). A host whose
|
|
agent has gone quiet keeps its last values until they age out of the raw
|
|
window; the caller decides staleness from the registration's last_seen_at.
|
|
"""
|
|
from steward.core.metric_series import latest_values
|
|
|
|
out: dict[str, dict[str, float]] = {}
|
|
for r in await latest_values(session, SOURCE_MODULE, host_name, include_children=True):
|
|
out.setdefault(r.resource_name, {})[r.metric_name] = r.value
|
|
return out
|
|
|
|
|
|
async def _history_for_host(session, host_name: str, since, *, raw_days: int = 7) -> dict[str, list]:
|
|
"""{metric: [[epoch_ms, avg_value], …]} host-level series since `since`.
|
|
|
|
Retention rolls raw plugin_metrics older than `raw_days` into hourly averages
|
|
(plugin_metrics_hourly) and deletes the raw rows. So we read the rollup for the
|
|
part of the range older than that boundary and raw (bucket-averaged in SQL,
|
|
capped to ≤1h to match the rollup) for the recent part — never shipping raw
|
|
samples to Python. The two windows don't overlap, so appending hourly-then-raw
|
|
keeps each metric's series time-ordered. Epoch-ms x feeds a linear chart axis.
|
|
"""
|
|
now = datetime.now(timezone.utc)
|
|
raw_cutoff = (now - timedelta(days=raw_days)).replace(minute=0, second=0, microsecond=0)
|
|
series: dict[str, list] = {m: [] for m in HISTORY_METRICS}
|
|
|
|
# ── Older than the raw window: the hourly rollup ──
|
|
if since < raw_cutoff:
|
|
hourly = (await session.execute(
|
|
select(PluginMetricHourly.metric_name, PluginMetricHourly.bucket,
|
|
PluginMetricHourly.value_avg)
|
|
.where(
|
|
PluginMetricHourly.source_module == SOURCE_MODULE,
|
|
PluginMetricHourly.resource_name == host_name,
|
|
PluginMetricHourly.metric_name.in_(HISTORY_METRICS),
|
|
PluginMetricHourly.bucket >= since,
|
|
PluginMetricHourly.bucket < raw_cutoff,
|
|
)
|
|
.order_by(PluginMetricHourly.bucket)
|
|
)).all()
|
|
for metric_name, bucket, avg in hourly:
|
|
series[metric_name].append([int(bucket.timestamp() * 1000), round(float(avg), 2)])
|
|
|
|
# ── Recent part: raw, bucket-averaged in SQL ──
|
|
raw_since = since if since >= raw_cutoff else raw_cutoff
|
|
width_s = bucket_seconds(raw_since, 120)
|
|
if since < raw_cutoff:
|
|
width_s = min(width_s, 3600) # ≤ 1h so it matches the hourly portion
|
|
rbucket = func.date_bin(
|
|
func.make_interval(0, 0, 0, 0, 0, 0, width_s),
|
|
PluginMetric.recorded_at,
|
|
func.to_timestamp(0), # epoch origin
|
|
).label("bucket")
|
|
raw = (await session.execute(
|
|
select(PluginMetric.metric_name, rbucket, func.avg(PluginMetric.value))
|
|
.where(
|
|
PluginMetric.source_module == SOURCE_MODULE,
|
|
PluginMetric.resource_name == host_name,
|
|
PluginMetric.metric_name.in_(HISTORY_METRICS),
|
|
PluginMetric.recorded_at >= raw_since,
|
|
)
|
|
.group_by(PluginMetric.metric_name, rbucket)
|
|
.order_by(rbucket)
|
|
)).all()
|
|
for metric_name, b, avg in raw:
|
|
series[metric_name].append([int(b.timestamp() * 1000), round(float(avg), 2)])
|
|
return series
|