feat(metrics): roll plugin_metrics up to hourly to bound storage

plugin_metrics grows by (sources × resources × ~30s cadence); keeping 90d of raw
is a large table. Add a raw→hourly rollup (mirroring the Docker plugin) so only a
short raw window is kept at full resolution, with hourly averages archived longer.

- PluginMetricHourly model + core migration 0024 (plugin_metrics_hourly: avg/max/
  count per source/resource/metric/hour, unique bucket constraint + lookup index).
- steward/core/metrics_retention.rollup_plugin_metrics: date_trunc('hour') agg of
  raw older than the hour-aligned raw window, idempotent pg upsert into hourly,
  delete the rolled raw, prune hourly beyond the rollup window.
- cleanup.py: plugin_metrics is no longer blanket-deleted at data.retention_days;
  _run_metrics_retention drives the rollup with windows read live from settings.
- Settings: metrics.retention.raw_days (7) + rollup_days (90), tunable on the
  Thresholds & Retention page (new "Host metrics retention" card).
- Chart read: _history_for_host merges the hourly rollup (older part of the range)
  with raw date_bin (recent part, capped ≤1h), so 30d charts keep working —
  recent at full resolution, older at hourly. Route passes raw_days from settings.
- Tests: unit (cutoff helpers) + integration (rollup aggregates/prunes; history
  merges hourly + raw) against Postgres.

Speed was already handled by the indexes + SQL aggregation; this is the storage
lever (raw window ~10x smaller).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC

steward/core/cleanup.py

@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
 from sqlalchemy import delete
 
 from steward.models.monitors import MonitorResult
-from steward.models.metrics import PluginMetric
 from steward.models.ansible import AnsibleRun
 
 if TYPE_CHECKING:
@@ -26,7 +25,6 @@ async def run_cleanup(app: "Quart") -> None:
         async with session.begin():
             for model, ts_col in [
                 (MonitorResult, MonitorResult.checked_at),
-                (PluginMetric, PluginMetric.recorded_at),
                 (AnsibleRun, AnsibleRun.started_at),
             ]:
                 result = await session.execute(
@@ -35,9 +33,29 @@ async def run_cleanup(app: "Quart") -> None:
                 if result.rowcount:
                     logger.info(f"Pruned {result.rowcount} rows from {model.__tablename__}")
 
+        # plugin_metrics is NOT blanket-deleted here — it's rolled up to hourly
+        # then pruned, so multi-week host history stays cheap.
+        await _run_metrics_retention(session, now)
         await _run_docker_retention(session, now)
 
 
+async def _run_metrics_retention(session, now: datetime) -> None:
+    """Roll up + prune plugin_metrics (raw → hourly → gone). Windows read fresh
+    from settings each run (rule 25 — UI change takes effect next cleanup, no
+    restart). get_setting's SELECT autobegins, so read inside the begin block."""
+    from steward.core.metrics_retention import rollup_plugin_metrics
+    from steward.core.settings import get_setting
+
+    async with session.begin():
+        raw_days = int(await get_setting(session, "metrics.retention.raw_days") or 7)
+        rollup_days = int(await get_setting(session, "metrics.retention.rollup_days") or 90)
+        counts = await rollup_plugin_metrics(
+            session, raw_days=raw_days, rollup_days=rollup_days, now=now,
+        )
+    if counts and any(counts.values()):
+        logger.info("Metrics retention: %s", counts)
+
+
 async def _run_docker_retention(session, now: datetime) -> None:
     """Drive the docker plugin's rollup + prune via its capability, if loaded.
 

steward/core/metrics_retention.py

@@ -0,0 +1,112 @@
+"""Bound plugin_metrics growth: roll old raw samples up to hourly, prune the rest.
+
+plugin_metrics grows by (sources × resources × sample cadence) — host agents push
+host-level + per-core/mount/iface sub-resources every ~30s, so a fleet accrues
+millions of rows. We keep raw samples for a short window, aggregate everything
+older into hourly averages (plugin_metrics_hourly) and delete the raw rows, then
+prune hourly beyond a longer window. Charts read raw for the recent part of a
+range and hourly for the older part.
+
+Driven by the core cleanup task (steward.core.cleanup). Runs inside the caller's
+open transaction; never opens or commits its own.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+
+def _hour_floor(dt: datetime) -> datetime:
+    """Truncate a datetime down to the start of its hour (drops min/sec/µs)."""
+    return dt.replace(minute=0, second=0, microsecond=0)
+
+
+def _rollup_cutoff(now: datetime, raw_days: int) -> datetime:
+    """Hour-aligned boundary below which raw metrics get rolled up + deleted.
+
+    Aligning to the hour means we only roll up *whole* elapsed hours — a bucket
+    is never split across the keep/roll boundary, so a re-run can't produce a
+    partial-then-complete duplicate for the same hour.
+    """
+    return _hour_floor(now - timedelta(days=raw_days))
+
+
+async def rollup_plugin_metrics(
+    session,
+    *,
+    raw_days: int,
+    rollup_days: int,
+    now: datetime | None = None,
+) -> dict:
+    """Roll up + prune plugin_metrics. Returns a counts dict for logging.
+
+    1. Aggregate plugin_metrics older than the (hour-aligned) raw window into
+       plugin_metrics_hourly (avg/max per source/resource/metric/hour), upserting
+       so a re-run is idempotent, then delete those raw rows.
+    2. Prune rolled-up rows older than the rollup window.
+    """
+    from datetime import timezone
+
+    from sqlalchemy import delete, func, select
+    from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+    from steward.models.metrics import PluginMetric, PluginMetricHourly
+
+    if now is None:
+        now = datetime.now(timezone.utc)
+
+    rolled = rolled_rows = rollup_pruned = 0
+
+    # ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
+    raw_cutoff = _rollup_cutoff(now, raw_days)
+    hour = func.date_trunc("hour", PluginMetric.recorded_at)
+    agg = (
+        select(
+            PluginMetric.source_module,
+            PluginMetric.resource_name,
+            PluginMetric.metric_name,
+            hour.label("bucket"),
+            func.avg(PluginMetric.value).label("value_avg"),
+            func.max(PluginMetric.value).label("value_max"),
+            func.count().label("sample_count"),
+        )
+        .where(PluginMetric.recorded_at < raw_cutoff)
+        .group_by(
+            PluginMetric.source_module, PluginMetric.resource_name,
+            PluginMetric.metric_name, hour,
+        )
+    )
+    for r in (await session.execute(agg)).all():
+        avg_v = float(r.value_avg or 0.0)
+        max_v = float(r.value_max or 0.0)
+        cnt = int(r.sample_count or 0)
+        await session.execute(
+            pg_insert(PluginMetricHourly)
+            .values(
+                source_module=r.source_module, resource_name=r.resource_name,
+                metric_name=r.metric_name, bucket=r.bucket,
+                value_avg=avg_v, value_max=max_v, sample_count=cnt,
+            )
+            .on_conflict_do_update(
+                constraint="uq_plugin_metrics_hourly_bucket",
+                set_={"value_avg": avg_v, "value_max": max_v, "sample_count": cnt},
+            )
+        )
+        rolled += 1
+        rolled_rows += cnt
+    if rolled:
+        await session.execute(
+            delete(PluginMetric).where(PluginMetric.recorded_at < raw_cutoff)
+        )
+
+    # ── 2. Prune rolled-up rows beyond the rollup window ──
+    rollup_cutoff = now - timedelta(days=rollup_days)
+    res = await session.execute(
+        delete(PluginMetricHourly).where(PluginMetricHourly.bucket < rollup_cutoff)
+    )
+    rollup_pruned = res.rowcount or 0
+
+    return {
+        "buckets_rolled": rolled,
+        "raw_rows_rolled": rolled_rows,
+        "rollup_pruned": rollup_pruned,
+    }

steward/core/settings.py

@@ -84,6 +84,10 @@ DEFAULTS: dict[str, Any] = {
     "docker.retention.metrics_raw_days": 7,
     "docker.retention.metrics_rollup_days": 90,
     "docker.retention.events_days": 30,
+    # Host/plugin metrics retention (plugin_metrics): keep a short raw window at
+    # the agent's ~30s cadence, then roll up to hourly averages kept much longer.
+    "metrics.retention.raw_days": 7,
+    "metrics.retention.rollup_days": 90,
     "plugins.index_url": "https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml",
     # Default-enabled plugins. These are the generic, non-vendor-specific
     # bundled plugins (protocols/standards, not a single product) — useful on