perf(metrics): a latest-value table for "what is it now"; dashboard cells say loading, failed and as-of (#5208)

The dashboard's host widgets took 18-60 s. Every "latest reading" query
(hosts overview, host-agent fleet, host page, vitals, SNMP, MCP get_host)
searched plugin_metrics for the newest row per metric. Postgres has no skip
scan, so each one read every raw sample in the 7-day window.

- plugin_metric_latest: one row per (source, resource, metric).
  record_metrics upserts it in the same transaction, moving forward only and
  in sorted key order. Migration 0030 seeds it by stepping the existing index.
- Retention drops a latest row once its sample leaves the raw window, so a
  resource that stopped reporting disappears when its history does, as before.
  Renames carry it along.
- A quiet host keeps its last values, marked stale. A time-bounded query
  would have shown it as missing.
- Vitals sparklines span the 6 h they claim, averaged in SQL to ~120 points,
  instead of fetching 6 h raw and drawing the last 120 samples.
- Dashboard and share-view cells: "Loading…" until the first answer; a 20 s
  deadline per fetch; "Couldn't load (why)" with Retry now and an automatic
  retry for cells that don't poll; "as of HH:MM" in the header, and
  "couldn't refresh · as of" when a refresh fails over shown data.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/hosts/service.py

@@ -144,92 +144,50 @@ async def host_monitors(db, host_id: str) -> list[dict]:
 
 
 async def agent_metrics_by_host(db, host_names: list[str]) -> dict[str, dict]:
-    """Latest agent cpu/mem per host name from the generic PluginMetric table.
+    """Latest agent cpu/mem per host name, from plugin_metric_latest.
 
-    Read directly (PluginMetric is a core model) so the Hosts list can show an
+    Core-safe (no host_agent import), so the Hosts list can show an
     at-a-glance agent column without importing the host_agent plugin.
     """
-    from steward.models.metrics import PluginMetric
-    if not host_names:
-        return {}
-    wanted = ("cpu_pct", "mem_used_pct")
-    subq = (
-        select(
-            PluginMetric.resource_name, PluginMetric.metric_name,
-            func.max(PluginMetric.recorded_at).label("m"),
-        )
-        .where(
-            PluginMetric.source_module == "host_agent",
-            PluginMetric.metric_name.in_(wanted),
-            PluginMetric.resource_name.in_(host_names),
-        )
-        .group_by(PluginMetric.resource_name, PluginMetric.metric_name)
-    ).subquery()
-    rows = (await db.execute(
-        select(PluginMetric).join(
-            subq,
-            (PluginMetric.resource_name == subq.c.resource_name)
-            & (PluginMetric.metric_name == subq.c.metric_name)
-            & (PluginMetric.recorded_at == subq.c.m),
-        )
-    )).scalars().all()
-    out: dict[str, dict] = {}
-    for r in rows:
-        out.setdefault(r.resource_name, {})[r.metric_name] = r.value
-    return out
+    from steward.core.metric_series import latest_by_resource
+    latest = await latest_by_resource(db, "host_agent", host_names, ("cpu_pct", "mem_used_pct"))
+    return {name: {m: r.value for m, r in metrics.items()} for name, metrics in latest.items()}
 
 
 async def agent_overview_by_host(
     db, host_names: list[str], stale_after_seconds: int = 180,
 ) -> dict[str, dict]:
-    """Per-host agent glance (cpu/mem/disk + freshness) from PluginMetric.
+    """Per-host agent glance (cpu/mem/disk + freshness) from plugin_metric_latest.
 
     {host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root,
                  last_at, fresh}} for each host with any agent samples.
 
     Core-safe (no host_agent import). Freshness compares the latest host-level
     sample against the host_agent plugin's stale window (stale_after_seconds;
-    callers pass agent_stale_after_seconds(app.config)).
+    callers pass agent_stale_after_seconds(app.config)). A host whose agent has
+    gone quiet keeps its last values with fresh=False (#5208).
     """
-    from steward.models.metrics import PluginMetric
+    from steward.core.metric_series import latest_by_resource
     if not host_names:
         return {}
     now = datetime.now(timezone.utc)
     stale_after = int(stale_after_seconds)
-    resources = list(host_names) + [h + ":/" for h in host_names]
-    subq = (
-        select(
-            PluginMetric.resource_name, PluginMetric.metric_name,
-            func.max(PluginMetric.recorded_at).label("m"),
-        )
-        .where(
-            PluginMetric.source_module == "host_agent",
-            PluginMetric.resource_name.in_(resources),
-            PluginMetric.metric_name.in_(AGENT_GLANCE_METRICS),
-        )
-        .group_by(PluginMetric.resource_name, PluginMetric.metric_name)
-    ).subquery()
-    rows = (await db.execute(
-        select(PluginMetric).join(
-            subq,
-            (PluginMetric.resource_name == subq.c.resource_name)
-            & (PluginMetric.metric_name == subq.c.metric_name)
-            & (PluginMetric.recorded_at == subq.c.m),
-        ).where(PluginMetric.source_module == "host_agent")
-    )).scalars().all()
+    latest = await latest_by_resource(
+        db, "host_agent", list(host_names) + [h + ":/" for h in host_names],
+        set(AGENT_GLANCE_METRICS) | {"disk_used_pct"})
     out: dict[str, dict] = {}
-    for r in rows:
-        if r.resource_name.endswith(":/") and r.metric_name == "disk_used_pct":
-            out.setdefault(r.resource_name[:-2], {})["disk_root"] = r.value
-        elif r.resource_name in host_names:
-            d = out.setdefault(r.resource_name, {})
-            d[r.metric_name] = r.value
-            if d.get("_ts") is None or r.recorded_at > d["_ts"]:
-                d["_ts"] = r.recorded_at
-    for d in out.values():
-        ts = d.pop("_ts", None)
+    for name in host_names:
+        own = {m: r for m, r in latest.get(name, {}).items() if m in AGENT_GLANCE_METRICS}
+        root = latest.get(name + ":/", {}).get("disk_used_pct")
+        if not own and root is None:
+            continue
+        d = {m: r.value for m, r in own.items()}
+        if root is not None:
+            d["disk_root"] = root.value
+        ts = max((r.recorded_at for r in own.values()), default=None)
         d["last_at"] = ts
         d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
+        out[name] = d
     return out