perf(metrics): a latest-value table for "what is it now"; dashboard cells say loading, failed and as-of (#5208)

The dashboard's host widgets took 18-60 s. Every "latest reading" query
(hosts overview, host-agent fleet, host page, vitals, SNMP, MCP get_host)
searched plugin_metrics for the newest row per metric. Postgres has no skip
scan, so each one read every raw sample in the 7-day window.

- plugin_metric_latest: one row per (source, resource, metric).
  record_metrics upserts it in the same transaction, moving forward only and
  in sorted key order. Migration 0030 seeds it by stepping the existing index.
- Retention drops a latest row once its sample leaves the raw window, so a
  resource that stopped reporting disappears when its history does, as before.
  Renames carry it along.
- A quiet host keeps its last values, marked stale. A time-bounded query
  would have shown it as missing.
- Vitals sparklines span the 6 h they claim, averaged in SQL to ~120 points,
  instead of fetching 6 h raw and drawing the last 120 samples.
- Dashboard and share-view cells: "Loading…" until the first answer; a 20 s
  deadline per fetch; "Couldn't load (why)" with Retry now and an automatic
  retry for cells that don't poll; "as of HH:MM" in the header, and
  "couldn't refresh · as of" when a refresh fails over shown data.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/metrics_query.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 from datetime import datetime, timedelta, timezone
 
-from sqlalchemy import func, or_, select
+from sqlalchemy import func, select
 
 from steward.core.time_range import bucket_seconds
 from steward.models.metrics import PluginMetric, PluginMetricHourly
@@ -27,30 +27,14 @@ HISTORY_METRICS = (
 
 async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[str, float]]:
     """{resource_name: {metric: value}} — latest sample per (resource, metric) for
-    a host + its sub-resources.
-
-    DISTINCT ON picks the newest row per group in one index-ordered pass over
-    ix_plugin_metrics_module_resource_metric_recorded, instead of a GROUP-BY-max
-    subquery self-joined back to the table (two passes over the whole history).
+    a host + its sub-resources, from plugin_metric_latest (#5208). A host whose
+    agent has gone quiet keeps its last values until they age out of the raw
+    window; the caller decides staleness from the registration's last_seen_at.
     """
-    rows = (await session.execute(
-        select(PluginMetric)
-        .where(
-            PluginMetric.source_module == SOURCE_MODULE,
-            or_(
-                PluginMetric.resource_name == host_name,
-                PluginMetric.resource_name.like(host_name + ":%"),
-            ),
-        )
-        .distinct(PluginMetric.resource_name, PluginMetric.metric_name)
-        .order_by(
-            PluginMetric.resource_name,
-            PluginMetric.metric_name,
-            PluginMetric.recorded_at.desc(),
-        )
-    )).scalars().all()
+    from steward.core.metric_series import latest_values
+
     out: dict[str, dict[str, float]] = {}
-    for r in rows:
+    for r in await latest_values(session, SOURCE_MODULE, host_name, include_children=True):
         out.setdefault(r.resource_name, {})[r.metric_name] = r.value
     return out
 

plugins/host_agent/routes.py

@@ -13,10 +13,11 @@ import secrets
 from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for, session
 from steward.auth.middleware import public_route, require_role
 from steward.models.users import UserRole
-from sqlalchemy import select, func, or_, and_
+from sqlalchemy import select, or_, and_
 from datetime import timedelta
 
 from steward.core.formatting import worst_reading
+from steward.core.metric_series import latest_by_resource, points as metric_points
 from steward.core.settings import get_setting, public_base_url
 from steward.core.time_range import parse_range, RANGE_OPTIONS
 from steward.models.hosts import Host
@@ -490,37 +491,21 @@ async def _fleet_rows(session) -> list[dict]:
         )).scalars().all()
     } if host_ids else {}
 
-    subq = (
-        select(
-            PluginMetric.resource_name,
-            PluginMetric.metric_name,
-            func.max(PluginMetric.recorded_at).label("max_ts"),
-        )
-        .where(PluginMetric.source_module == SOURCE_MODULE)
-        .group_by(PluginMetric.resource_name, PluginMetric.metric_name)
-    ).subquery()
-    latest_rows = (await session.execute(
-        select(PluginMetric).join(
-            subq,
-            (PluginMetric.resource_name == subq.c.resource_name) &
-            (PluginMetric.metric_name == subq.c.metric_name) &
-            (PluginMetric.recorded_at == subq.c.max_ts),
-        ).where(PluginMetric.source_module == SOURCE_MODULE)
-    )).scalars().all()
-
+    host_names = [h.name for h in hosts.values()]
+    latest_by = await latest_by_resource(
+        session, SOURCE_MODULE, host_names + [n + ":/" for n in host_names])
     latest: dict[str, dict[str, float]] = {}
     root_disk: dict[str, float] = {}  # host_name → root (/) disk %
-    for row in latest_rows:
-        if row.resource_name.endswith(":/") and row.metric_name == "disk_used_pct":
-            root_disk[row.resource_name[:-2]] = row.value
-        if ":" in row.resource_name:
+    for resource, metrics in latest_by.items():
+        if resource.endswith(":/"):
+            if "disk_used_pct" in metrics:
+                root_disk[resource[:-2]] = metrics["disk_used_pct"].value
             continue
-        latest.setdefault(row.resource_name, {})[row.metric_name] = row.value
+        latest[resource] = {m: r.value for m, r in metrics.items()}
 
     # Per-host recent series for the inline sparklines (cpu/mem/load host-level,
     # disk from the root mount), so each metric shows its trend beside the value —
     # the host-panel at-a-glance, on the fleet widget. 1h window keeps it cheap.
-    host_names = [h.name for h in hosts.values()]
     spark_series: dict[str, dict[str, list]] = {
         n: {"cpu": [], "mem": [], "disk": [], "load": []} for n in host_names
     }
@@ -789,37 +774,25 @@ async def host_vitals(host_id: str):
             HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
         latest = await _latest_metrics_for_host(db, host.name) if reg else {}
 
-        # Recent series for the at-a-glance sparklines (cpu/mem/load host-level,
-        # disk from the root mount sub-resource).
+        # The last 6 h for the at-a-glance sparklines (cpu/mem/load host-level,
+        # disk from the root mount), averaged into ~120 points in SQL, so the
+        # line spans the whole window and only what is drawn is fetched.
         series_raw: dict[str, list[float]] = {"cpu": [], "mem": [], "disk": [], "load": []}
         if reg:
             since = datetime.now(timezone.utc) - timedelta(hours=6)
-            spark_rows = (await db.execute(
-                select(PluginMetric).where(
-                    PluginMetric.source_module == SOURCE_MODULE,
-                    PluginMetric.recorded_at >= since,
-                    or_(
-                        and_(PluginMetric.resource_name == host.name,
-                             PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "load_1m"))),
-                        and_(PluginMetric.resource_name == host.name + ":/",
-                             PluginMetric.metric_name == "disk_used_pct"),
-                    ),
-                ).order_by(PluginMetric.recorded_at)
-            )).scalars().all()
-            for r in spark_rows:
-                if r.resource_name == host.name:
-                    key = {"cpu_pct": "cpu", "mem_used_pct": "mem", "load_1m": "load"}.get(r.metric_name)
-                    if key:
-                        series_raw[key].append(r.value)
-                else:
-                    series_raw["disk"].append(r.value)
+            for key, resource, metric in (("cpu", host.name, "cpu_pct"),
+                                          ("mem", host.name, "mem_used_pct"),
+                                          ("load", host.name, "load_1m"),
+                                          ("disk", host.name + ":/", "disk_used_pct")):
+                series_raw[key] = [v for _t, v in await metric_points(
+                    db, SOURCE_MODULE, resource, metric, since, target=120)]
 
     from steward.core.status import sparkline_svg
     hostlvl = latest.get(host.name, {})
     # At-a-glance disk = the root filesystem (what people actually care about),
     # not the "worst" mount which is opaque. Worst stays on the full-metrics page.
     disk_root = (latest.get(host.name + ":/", {}) or {}).get("disk_used_pct")
-    sparks = {k: sparkline_svg(v[-120:]) for k, v in series_raw.items()}
+    sparks = {k: sparkline_svg(v) for k, v in series_raw.items()}
     # Normalize load by core count so it's comparable across hardware (load/core
     # as %). Cores derived from the per-core CPU sub-resources already collected.
     cores = sum(1 for k in latest if k.startswith(host.name + ":core"))