feat(metrics): roll plugin_metrics up to hourly to bound storage

plugin_metrics grows by (sources × resources × ~30s cadence); keeping 90d of raw
is a large table. Add a raw→hourly rollup (mirroring the Docker plugin) so only a
short raw window is kept at full resolution, with hourly averages archived longer.

- PluginMetricHourly model + core migration 0024 (plugin_metrics_hourly: avg/max/
  count per source/resource/metric/hour, unique bucket constraint + lookup index).
- steward/core/metrics_retention.rollup_plugin_metrics: date_trunc('hour') agg of
  raw older than the hour-aligned raw window, idempotent pg upsert into hourly,
  delete the rolled raw, prune hourly beyond the rollup window.
- cleanup.py: plugin_metrics is no longer blanket-deleted at data.retention_days;
  _run_metrics_retention drives the rollup with windows read live from settings.
- Settings: metrics.retention.raw_days (7) + rollup_days (90), tunable on the
  Thresholds & Retention page (new "Host metrics retention" card).
- Chart read: _history_for_host merges the hourly rollup (older part of the range)
  with raw date_bin (recent part, capped ≤1h), so 30d charts keep working —
  recent at full resolution, older at hourly. Route passes raw_days from settings.
- Tests: unit (cutoff helpers) + integration (rollup aggregates/prunes; history
  merges hourly + raw) against Postgres.

Speed was already handled by the indexes + SQL aggregation; this is the storage
lever (raw window ~10x smaller).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC

plugins/host_agent/metrics_query.py

@@ -7,10 +7,12 @@ defined") guard.
 """
 from __future__ import annotations
 
+from datetime import datetime, timedelta, timezone
+
 from sqlalchemy import func, or_, select
 
 from steward.core.time_range import bucket_seconds
-from steward.models.metrics import PluginMetric
+from steward.models.metrics import PluginMetric, PluginMetricHourly
 
 SOURCE_MODULE = "host_agent"
 
@@ -53,33 +55,58 @@ async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[st
     return out
 
 
-async def _history_for_host(session, host_name: str, since) -> dict[str, list]:
+async def _history_for_host(session, host_name: str, since, *, raw_days: int = 7) -> dict[str, list]:
     """{metric: [[epoch_ms, avg_value], …]} host-level series since `since`.
 
-    Buckets + averages in SQL (date_bin to ~120 buckets) so we return a readable
-    point count instead of shipping every raw sample to Python and downsampling
-    there — a 30d range was reading hundreds of thousands of rows per load. The
-    bucket width is epoch-aligned so the x axis is stable across refreshes.
-    Epoch-ms x values feed a linear chart axis (no Chart.js date adapter).
+    Retention rolls raw plugin_metrics older than `raw_days` into hourly averages
+    (plugin_metrics_hourly) and deletes the raw rows. So we read the rollup for the
+    part of the range older than that boundary and raw (bucket-averaged in SQL,
+    capped to ≤1h to match the rollup) for the recent part — never shipping raw
+    samples to Python. The two windows don't overlap, so appending hourly-then-raw
+    keeps each metric's series time-ordered. Epoch-ms x feeds a linear chart axis.
     """
-    width_s = bucket_seconds(since, 120)
-    bucket = func.date_bin(
-        func.make_interval(0, 0, 0, 0, 0, 0, width_s),  # width_s seconds
+    now = datetime.now(timezone.utc)
+    raw_cutoff = (now - timedelta(days=raw_days)).replace(minute=0, second=0, microsecond=0)
+    series: dict[str, list] = {m: [] for m in HISTORY_METRICS}
+
+    # ── Older than the raw window: the hourly rollup ──
+    if since < raw_cutoff:
+        hourly = (await session.execute(
+            select(PluginMetricHourly.metric_name, PluginMetricHourly.bucket,
+                   PluginMetricHourly.value_avg)
+            .where(
+                PluginMetricHourly.source_module == SOURCE_MODULE,
+                PluginMetricHourly.resource_name == host_name,
+                PluginMetricHourly.metric_name.in_(HISTORY_METRICS),
+                PluginMetricHourly.bucket >= since,
+                PluginMetricHourly.bucket < raw_cutoff,
+            )
+            .order_by(PluginMetricHourly.bucket)
+        )).all()
+        for metric_name, bucket, avg in hourly:
+            series[metric_name].append([int(bucket.timestamp() * 1000), round(float(avg), 2)])
+
+    # ── Recent part: raw, bucket-averaged in SQL ──
+    raw_since = since if since >= raw_cutoff else raw_cutoff
+    width_s = bucket_seconds(raw_since, 120)
+    if since < raw_cutoff:
+        width_s = min(width_s, 3600)  # ≤ 1h so it matches the hourly portion
+    rbucket = func.date_bin(
+        func.make_interval(0, 0, 0, 0, 0, 0, width_s),
         PluginMetric.recorded_at,
         func.to_timestamp(0),                            # epoch origin
     ).label("bucket")
-    rows = (await session.execute(
-        select(PluginMetric.metric_name, bucket, func.avg(PluginMetric.value))
+    raw = (await session.execute(
+        select(PluginMetric.metric_name, rbucket, func.avg(PluginMetric.value))
         .where(
             PluginMetric.source_module == SOURCE_MODULE,
             PluginMetric.resource_name == host_name,
             PluginMetric.metric_name.in_(HISTORY_METRICS),
-            PluginMetric.recorded_at >= since,
+            PluginMetric.recorded_at >= raw_since,
         )
-        .group_by(PluginMetric.metric_name, bucket)
-        .order_by(bucket)
+        .group_by(PluginMetric.metric_name, rbucket)
+        .order_by(rbucket)
     )).all()
-    series: dict[str, list] = {m: [] for m in HISTORY_METRICS}
-    for metric_name, b, avg in rows:
+    for metric_name, b, avg in raw:
         series[metric_name].append([int(b.timestamp() * 1000), round(float(avg), 2)])
     return series

plugins/host_agent/routes.py

@@ -14,7 +14,7 @@ from steward.models.users import UserRole
 from sqlalchemy import select, func, or_, and_
 from datetime import timedelta
 
-from steward.core.settings import public_base_url
+from steward.core.settings import get_setting, public_base_url
 from steward.core.time_range import parse_range, RANGE_OPTIONS
 from steward.models.hosts import Host
 from steward.models.metrics import PluginMetric
@@ -637,7 +637,8 @@ async def host_detail_charts(host_id: str):
             select(Host).where(Host.id == host_id))).scalar_one_or_none()
         if host is None:
             return "", 404
-        series = await _history_for_host(session, host.name, since)
+        raw_days = int(await get_setting(session, "metrics.retention.raw_days") or 7)
+        series = await _history_for_host(session, host.name, since, raw_days=raw_days)
     return await render_template("_host_charts.html", series=series, range_key=range_key)