perf(metrics): a latest-value table for "what is it now"; dashboard cells say loading, failed and as-of (#5208)

The dashboard's host widgets took 18-60 s. Every "latest reading" query
(hosts overview, host-agent fleet, host page, vitals, SNMP, MCP get_host)
searched plugin_metrics for the newest row per metric. Postgres has no skip
scan, so each one read every raw sample in the 7-day window.

- plugin_metric_latest: one row per (source, resource, metric).
  record_metrics upserts it in the same transaction, moving forward only and
  in sorted key order. Migration 0030 seeds it by stepping the existing index.
- Retention drops a latest row once its sample leaves the raw window, so a
  resource that stopped reporting disappears when its history does, as before.
  Renames carry it along.
- A quiet host keeps its last values, marked stale. A time-bounded query
  would have shown it as missing.
- Vitals sparklines span the 6 h they claim, averaged in SQL to ~120 points,
  instead of fetching 6 h raw and drawing the last 120 samples.
- Dashboard and share-view cells: "Loading…" until the first answer; a 20 s
  deadline per fetch; "Couldn't load (why)" with Retry now and an automatic
  retry for cells that don't poll; "as of HH:MM" in the header, and
  "couldn't refresh · as of" when a refresh fails over shown data.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/alerts.py

@@ -18,7 +18,7 @@ from steward.models.alerts import (
     MaintenanceWindow,
 )
 from steward.core.formatting import human_duration
-from steward.models.metrics import PluginMetric
+from steward.models.metrics import PluginMetric, PluginMetricLatest
 
 if TYPE_CHECKING:
     from quart import Quart
@@ -155,6 +155,34 @@ def _after_commit(session: AsyncSession, fn, *args) -> None:
     info[_AFTER_COMMIT].append((fn, args))
 
 
+LATEST_CHUNK = 1000
+
+
+async def _keep_latest(session: AsyncSession, rows: list[PluginMetric]) -> None:
+    """Move each key's row in plugin_metric_latest to the newest sample in
+    `rows`. A row only moves forward, so a backlog arriving after newer samples
+    leaves it alone. Keys go in sorted order, so two batches sharing keys lock
+    them in the same order and cannot deadlock."""
+    from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+    newest: dict[tuple[str, str, str], PluginMetric] = {}
+    for row in rows:
+        key = (row.source_module, row.resource_name, row.metric_name)
+        if key not in newest or row.recorded_at > newest[key].recorded_at:
+            newest[key] = row
+    values = [{"source_module": k[0], "resource_name": k[1], "metric_name": k[2],
+               "value": newest[k].value, "recorded_at": newest[k].recorded_at}
+              for k in sorted(newest)]
+    # Five parameters a row; chunks keep a statement well under asyncpg's 32767.
+    for i in range(0, len(values), LATEST_CHUNK):
+        stmt = pg_insert(PluginMetricLatest).values(values[i:i + LATEST_CHUNK])
+        await session.execute(stmt.on_conflict_do_update(
+            index_elements=["source_module", "resource_name", "metric_name"],
+            set_={"value": stmt.excluded.value, "recorded_at": stmt.excluded.recorded_at},
+            where=PluginMetricLatest.recorded_at <= stmt.excluded.recorded_at,
+        ))
+
+
 async def record_metrics(session: AsyncSession, rows: list[PluginMetric], phases=None) -> None:
     """Write metric rows and evaluate the alert rules that watch them.
 
@@ -182,6 +210,7 @@ async def record_metrics(session: AsyncSession, rows: list[PluginMetric], phases
             row.recorded_at = now
     session.add_all(rows)
     await session.flush()
+    await _keep_latest(session, rows)
     mark("flush")
 
     by_key: dict[tuple[str, str, str], list[PluginMetric]] = {}

steward/core/metric_series.py

@@ -11,7 +11,7 @@ from datetime import datetime, timedelta, timezone
 from sqlalchemy import func, or_, select, text
 
 from steward.core.time_range import bucket_seconds
-from steward.models.metrics import PluginMetric, PluginMetricHourly
+from steward.models.metrics import PluginMetric, PluginMetricHourly, PluginMetricLatest
 
 # Loose index scans: step through the distinct leading values of the
 # (source_module, resource_name, …) index one probe at a time instead of
@@ -48,26 +48,49 @@ async def resources(db, source_module: str) -> list[str]:
     return [v for (v,) in await db.execute(_RESOURCES_SQL, {"src": source_module})]
 
 
+def _resource_match(resource: str, include_children: bool):
+    match = PluginMetricLatest.resource_name == resource
+    if include_children:
+        like = resource.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + ":%"
+        match = or_(match, PluginMetricLatest.resource_name.like(like, escape="\\"))
+    return match
+
+
 async def latest_values(db, source_module: str, resource: str, *,
-                        include_children: bool = False) -> list[PluginMetric]:
+                        include_children: bool = False) -> list[PluginMetricLatest]:
     """The newest sample of each metric for a resource, sorted by (resource, metric).
 
     include_children also returns the resource's sub-resources, named
-    "<resource>:<part>" (a host's mounts, interfaces, sensors).
+    "<resource>:<part>" (a host's mounts, interfaces, sensors). Read from
+    plugin_metric_latest, so the cost is the resource's metric count, not its
+    history.
     """
-    match = PluginMetric.resource_name == resource
-    if include_children:
-        like = resource.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + ":%"
-        match = or_(match, PluginMetric.resource_name.like(like, escape="\\"))
     return list((await db.execute(
-        select(PluginMetric)
-        .where(PluginMetric.source_module == source_module, match)
-        .distinct(PluginMetric.resource_name, PluginMetric.metric_name)
-        .order_by(PluginMetric.resource_name, PluginMetric.metric_name,
-                  PluginMetric.recorded_at.desc())
+        select(PluginMetricLatest)
+        .where(PluginMetricLatest.source_module == source_module,
+               _resource_match(resource, include_children))
+        .order_by(PluginMetricLatest.resource_name, PluginMetricLatest.metric_name)
     )).scalars())
 
 
+async def latest_by_resource(db, source_module: str, resources, metrics=None,
+                             ) -> dict[str, dict[str, PluginMetricLatest]]:
+    """{resource: {metric: newest sample}} for several resources of a source,
+    optionally only the named metrics. A resource with no samples is absent."""
+    resources = list(resources)
+    if not resources:
+        return {}
+    q = select(PluginMetricLatest).where(
+        PluginMetricLatest.source_module == source_module,
+        PluginMetricLatest.resource_name.in_(resources))
+    if metrics is not None:
+        q = q.where(PluginMetricLatest.metric_name.in_(list(metrics)))
+    out: dict[str, dict[str, PluginMetricLatest]] = {}
+    for r in (await db.execute(q)).scalars():
+        out.setdefault(r.resource_name, {})[r.metric_name] = r
+    return out
+
+
 async def points(db, source_module: str, resource: str, metric: str, since: datetime,
                  *, target: int = 120) -> list[tuple[datetime, float]]:
     """(time, average) points for one metric since `since`, oldest first.

steward/core/metrics_retention.py

@@ -49,7 +49,7 @@ async def rollup_plugin_metrics(
     from sqlalchemy import delete, func, select
     from sqlalchemy.dialects.postgresql import insert as pg_insert
 
-    from steward.models.metrics import PluginMetric, PluginMetricHourly
+    from steward.models.metrics import PluginMetric, PluginMetricHourly, PluginMetricLatest
 
     if now is None:
         now = datetime.now(timezone.utc)
@@ -97,6 +97,11 @@ async def rollup_plugin_metrics(
         await session.execute(
             delete(PluginMetric).where(PluginMetric.recorded_at < raw_cutoff)
         )
+    # A latest value goes with its raw history: a resource that stopped
+    # reporting a raw window ago is no longer "now" anywhere.
+    latest_pruned = (await session.execute(
+        delete(PluginMetricLatest).where(PluginMetricLatest.recorded_at < raw_cutoff)
+    )).rowcount or 0
 
     # ── 2. Prune rolled-up rows beyond the rollup window ──
     rollup_cutoff = now - timedelta(days=rollup_days)
@@ -109,4 +114,5 @@ async def rollup_plugin_metrics(
         "buckets_rolled": rolled,
         "raw_rows_rolled": rolled_rows,
         "rollup_pruned": rollup_pruned,
+        "latest_pruned": latest_pruned,
     }

steward/core/renames.py

@@ -11,7 +11,7 @@ from sqlalchemy import and_, delete, func, or_, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from steward.models.alerts import AlertRule, MaintenanceWindow
-from steward.models.metrics import PluginMetric, PluginMetricHourly
+from steward.models.metrics import PluginMetric, PluginMetricHourly, PluginMetricLatest
 
 # What a host's name keys, and how each source names a part of the host:
 # host_agent "web01:/var", "web01:core3"; docker "web01/nginx".
@@ -47,7 +47,7 @@ async def carry_rename(session: AsyncSession, sources, old: str, new: str) -> di
     if not old or old == new:
         return {}
     sources = tuple(sources)
-    for model in (PluginMetric, PluginMetricHourly):
+    for model in (PluginMetric, PluginMetricHourly, PluginMetricLatest):
         await session.execute(delete(model).where(
             _matches(model.source_module, model.resource_name, sources, new)))
     moved = {}
@@ -58,6 +58,8 @@ async def carry_rename(session: AsyncSession, sources, old: str, new: str) -> di
         ("metric_rows", PluginMetric, PluginMetric.source_module, PluginMetric.resource_name),
         ("hourly_rows", PluginMetricHourly, PluginMetricHourly.source_module,
          PluginMetricHourly.resource_name),
+        ("latest_rows", PluginMetricLatest, PluginMetricLatest.source_module,
+         PluginMetricLatest.resource_name),
     ):
         result = await session.execute(
             update(model).where(_matches(src, res, sources, old))