perf(metrics): a latest-value table for "what is it now"; dashboard cells say loading, failed and as-of (#5208)

The dashboard's host widgets took 18-60 s. Every "latest reading" query
(hosts overview, host-agent fleet, host page, vitals, SNMP, MCP get_host)
searched plugin_metrics for the newest row per metric. Postgres has no skip
scan, so each one read every raw sample in the 7-day window.

- plugin_metric_latest: one row per (source, resource, metric).
  record_metrics upserts it in the same transaction, moving forward only and
  in sorted key order. Migration 0030 seeds it by stepping the existing index.
- Retention drops a latest row once its sample leaves the raw window, so a
  resource that stopped reporting disappears when its history does, as before.
  Renames carry it along.
- A quiet host keeps its last values, marked stale. A time-bounded query
  would have shown it as missing.
- Vitals sparklines span the 6 h they claim, averaged in SQL to ~120 points,
  instead of fetching 6 h raw and drawing the last 120 samples.
- Dashboard and share-view cells: "Loading…" until the first answer; a 20 s
  deadline per fetch; "Couldn't load (why)" with Retry now and an automatic
  retry for cells that don't poll; "as of HH:MM" in the header, and
  "couldn't refresh · as of" when a refresh fails over shown data.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_host_metrics.py

@@ -1,8 +1,9 @@
-"""Integration: host-metrics read paths (DISTINCT ON latest + SQL date_bin history).
+"""Integration: host-metrics read paths (latest values + SQL date_bin history).
 
 Validates the two host_agent query helpers that back the slow host views, against
-a live Postgres: the latest-per-(resource,metric) lookup and the bucket-averaged
-history (which now aggregates in SQL instead of shipping raw rows to Python).
+a live Postgres: the latest-per-(resource,metric) lookup (plugin_metric_latest,
+kept by record_metrics) and the bucket-averaged history (which aggregates in SQL
+instead of shipping raw rows to Python).
 Requires STEWARD_DATABASE_URL.
 """
 from __future__ import annotations
@@ -31,8 +32,9 @@ def app():
 
 
 @_NEEDS_DB
-def test_latest_distinct_on_and_sql_bucketed_history(app):
+def test_latest_values_and_sql_bucketed_history(app):
     from sqlalchemy import text
+    from steward.core.alerts import record_metrics
     from steward.models.metrics import PluginMetric
     from plugins.host_agent.metrics_query import (
         SOURCE_MODULE, _history_for_host, _latest_metrics_for_host,
@@ -62,7 +64,7 @@ def test_latest_distinct_on_and_sql_bucketed_history(app):
                     metric_name="disk_used_pct", value=80.0,
                     recorded_at=now - timedelta(minutes=5),
                 ))
-                s.add_all(rows)
+                await record_metrics(s, rows)
 
             latest = await _latest_metrics_for_host(s, hostname)
             hist = await _history_for_host(s, hostname, now - timedelta(hours=1))
@@ -70,7 +72,7 @@ def test_latest_distinct_on_and_sql_bucketed_history(app):
 
     latest, hist = asyncio.run(_go())
 
-    # DISTINCT ON returns the newest sample per (resource, metric).
+    # The newest sample per (resource, metric).
     assert latest[hostname]["cpu_pct"] == 30.0
     assert latest[hostname + ":/"]["disk_used_pct"] == 80.0
 

tests/integration/test_mcp_tools.py

@@ -44,6 +44,7 @@ def test_diagnostic_tools_over_a_seeded_fleet(app):
     )
     from steward.models.ansible import AnsibleRun, AnsibleRunStatus
     from steward.models.hosts import Host
+    from steward.core.alerts import _keep_latest
     from steward.models.metrics import PluginMetric
     from steward.models.monitors import Monitor, MonitorResult
     from steward.models.users import User, UserRole
@@ -77,16 +78,21 @@ def test_diagnostic_tools_over_a_seeded_fleet(app):
                 s.add(MonitorResult(monitor_id=web_id, checked_at=now - timedelta(minutes=1),
                                     is_up=True, status_code=200, response_ms=40.0,
                                     tls_expires_at=now + timedelta(days=5)))
-                for i, v in enumerate((10.0, 20.0, 95.0)):
-                    s.add(PluginMetric(source_module="host_agent", resource_name=host_name,
-                                       metric_name="cpu_pct", value=v,
-                                       recorded_at=now - timedelta(minutes=40 - i * 20, seconds=30)))
-                s.add(PluginMetric(source_module="host_agent", resource_name=host_name,
-                                   metric_name="mem_used_pct", value=50.0,
-                                   recorded_at=now - timedelta(seconds=30)))
-                s.add(PluginMetric(source_module="host_agent", resource_name=host_name + ":/",
-                                   metric_name="disk_used_pct", value=41.0,
-                                   recorded_at=now - timedelta(seconds=30)))
+                samples = [PluginMetric(source_module="host_agent", resource_name=host_name,
+                                        metric_name="cpu_pct", value=v,
+                                        recorded_at=now - timedelta(minutes=40 - i * 20, seconds=30))
+                           for i, v in enumerate((10.0, 20.0, 95.0))]
+                samples.append(PluginMetric(source_module="host_agent", resource_name=host_name,
+                                            metric_name="mem_used_pct", value=50.0,
+                                            recorded_at=now - timedelta(seconds=30)))
+                samples.append(PluginMetric(source_module="host_agent",
+                                            resource_name=host_name + ":/",
+                                            metric_name="disk_used_pct", value=41.0,
+                                            recorded_at=now - timedelta(seconds=30)))
+                s.add_all(samples)
+                # What record_metrics keeps beside the rows, without evaluating
+                # the rule this seed sets up by hand.
+                await _keep_latest(s, samples)
                 s.add(AlertRule(id=rule_id, name=f"cpu hot {tag}", source_module="host_agent",
                                 resource_name=host_name, metric_name="cpu_pct",
                                 operator=AlertOperator.gt, threshold=90.0,

tests/integration/test_metric_latest.py

@@ -0,0 +1,149 @@
+"""Integration: plugin_metric_latest, the newest sample per metric (#5208).
+
+record_metrics keeps it beside plugin_metrics so "what is it now" reads one row
+per metric instead of every raw sample in the window. Requires
+STEWARD_DATABASE_URL.
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def _sample(resource, metric, value, at):
+    from steward.models.metrics import PluginMetric
+    return PluginMetric(source_module="host_agent", resource_name=resource,
+                        metric_name=metric, value=value, recorded_at=at)
+
+
+async def _latest(s, resource):
+    from sqlalchemy import select
+    from steward.models.metrics import PluginMetricLatest
+    rows = (await s.execute(select(PluginMetricLatest).where(
+        PluginMetricLatest.source_module == "host_agent",
+        PluginMetricLatest.resource_name == resource))).scalars().all()
+    return {r.metric_name: (r.value, r.recorded_at) for r in rows}
+
+
+def test_the_newest_sample_wins_and_a_late_backlog_does_not_move_it_back(app):
+    from steward.core.alerts import record_metrics
+
+    now = datetime.now(timezone.utc).replace(microsecond=0)
+    host = "latest-" + uuid.uuid4().hex[:8]
+
+    async def go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                # One batch holding two samples of a metric: the newer is kept.
+                await record_metrics(s, [
+                    _sample(host, "cpu_pct", 10.0, now - timedelta(minutes=2)),
+                    _sample(host, "cpu_pct", 20.0, now - timedelta(minutes=1)),
+                    _sample(host, "mem_used_pct", 40.0, now - timedelta(minutes=1)),
+                ])
+            after_first = await _latest(s, host)
+            async with s.begin():
+                # An agent catching up posts older samples after newer ones.
+                await record_metrics(s, [
+                    _sample(host, "cpu_pct", 99.0, now - timedelta(minutes=5)),
+                    _sample(host, "mem_used_pct", 41.0, now),
+                ])
+            return after_first, await _latest(s, host)
+
+    first, second = asyncio.run(go())
+    assert first["cpu_pct"] == (20.0, now - timedelta(minutes=1))
+    assert second["cpu_pct"] == (20.0, now - timedelta(minutes=1))    # backlog ignored
+    assert second["mem_used_pct"] == (41.0, now)                      # newer moves it
+
+
+def test_a_quiet_host_keeps_its_last_values_and_reads_stale_not_missing(app):
+    from steward.core.alerts import record_metrics
+    from steward.hosts.service import agent_metrics_by_host, agent_overview_by_host
+
+    now = datetime.now(timezone.utc)
+    quiet, live = ("quiet-" + uuid.uuid4().hex[:8]), ("live-" + uuid.uuid4().hex[:8])
+    two_hours_ago = now - timedelta(hours=2)
+
+    async def go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await record_metrics(s, [
+                    _sample(quiet, "cpu_pct", 12.0, two_hours_ago),
+                    _sample(quiet, "mem_used_pct", 34.0, two_hours_ago),
+                    _sample(quiet + ":/", "disk_used_pct", 56.0, two_hours_ago),
+                    _sample(live, "cpu_pct", 7.0, now - timedelta(seconds=20)),
+                ])
+            overview = await agent_overview_by_host(s, [quiet, live, "never-" + quiet], 180)
+            metrics = await agent_metrics_by_host(s, [quiet, live])
+            return overview, metrics
+
+    overview, metrics = asyncio.run(go())
+    q = overview[quiet]
+    assert (q["cpu_pct"], q["mem_used_pct"], q["disk_root"]) == (12.0, 34.0, 56.0)
+    assert q["fresh"] is False
+    assert abs((q["last_at"] - two_hours_ago).total_seconds()) < 1
+    assert overview[live]["fresh"] is True
+    assert "never-" + quiet not in overview          # no samples at all: absent
+    assert metrics[quiet] == {"cpu_pct": 12.0, "mem_used_pct": 34.0}
+
+
+def test_retention_drops_a_latest_value_with_its_raw_history(app):
+    from steward.core.alerts import record_metrics
+    from steward.core.metrics_retention import rollup_plugin_metrics
+
+    now = datetime.now(timezone.utc)
+    gone, here = ("gone-" + uuid.uuid4().hex[:8]), ("here-" + uuid.uuid4().hex[:8])
+
+    async def go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await record_metrics(s, [
+                    _sample(gone, "cpu_pct", 1.0, now - timedelta(days=10)),
+                    _sample(here, "cpu_pct", 2.0, now - timedelta(hours=1)),
+                ])
+            async with s.begin():
+                counts = await rollup_plugin_metrics(s, raw_days=7, rollup_days=90, now=now)
+            return counts, await _latest(s, gone), await _latest(s, here)
+
+    counts, gone_latest, here_latest = asyncio.run(go())
+    assert counts["latest_pruned"] >= 1
+    assert gone_latest == {}
+    assert here_latest["cpu_pct"][0] == 2.0
+
+
+def test_a_renamed_host_carries_its_latest_values(app):
+    from steward.core.alerts import record_metrics
+    from steward.core.renames import HOST_SOURCES, carry_rename
+
+    now = datetime.now(timezone.utc)
+    old, new = ("old-" + uuid.uuid4().hex[:8]), ("new-" + uuid.uuid4().hex[:8])
+
+    async def go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await record_metrics(s, [
+                    _sample(old, "cpu_pct", 5.0, now),
+                    _sample(old + ":/", "disk_used_pct", 60.0, now),
+                ])
+            async with s.begin():
+                await carry_rename(s, HOST_SOURCES, old, new)
+            return (await _latest(s, old), await _latest(s, new),
+                    await _latest(s, new + ":/"))
+
+    old_latest, new_latest, new_root = asyncio.run(go())
+    assert old_latest == {}
+    assert new_latest["cpu_pct"][0] == 5.0
+    assert new_root["disk_used_pct"][0] == 60.0