feat(metrics): roll plugin_metrics up to hourly to bound storage

plugin_metrics grows by (sources × resources × ~30s cadence); keeping 90d of raw
is a large table. Add a raw→hourly rollup (mirroring the Docker plugin) so only a
short raw window is kept at full resolution, with hourly averages archived longer.

- PluginMetricHourly model + core migration 0024 (plugin_metrics_hourly: avg/max/
  count per source/resource/metric/hour, unique bucket constraint + lookup index).
- steward/core/metrics_retention.rollup_plugin_metrics: date_trunc('hour') agg of
  raw older than the hour-aligned raw window, idempotent pg upsert into hourly,
  delete the rolled raw, prune hourly beyond the rollup window.
- cleanup.py: plugin_metrics is no longer blanket-deleted at data.retention_days;
  _run_metrics_retention drives the rollup with windows read live from settings.
- Settings: metrics.retention.raw_days (7) + rollup_days (90), tunable on the
  Thresholds & Retention page (new "Host metrics retention" card).
- Chart read: _history_for_host merges the hourly rollup (older part of the range)
  with raw date_bin (recent part, capped ≤1h), so 30d charts keep working —
  recent at full resolution, older at hourly. Route passes raw_days from settings.
- Tests: unit (cutoff helpers) + integration (rollup aggregates/prunes; history
  merges hourly + raw) against Postgres.

Speed was already handled by the indexes + SQL aggregation; this is the storage
lever (raw window ~10x smaller).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC

steward/core/cleanup.py

@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
 from sqlalchemy import delete
 
 from steward.models.monitors import MonitorResult
-from steward.models.metrics import PluginMetric
 from steward.models.ansible import AnsibleRun
 
 if TYPE_CHECKING:
@@ -26,7 +25,6 @@ async def run_cleanup(app: "Quart") -> None:
         async with session.begin():
             for model, ts_col in [
                 (MonitorResult, MonitorResult.checked_at),
-                (PluginMetric, PluginMetric.recorded_at),
                 (AnsibleRun, AnsibleRun.started_at),
             ]:
                 result = await session.execute(
@@ -35,9 +33,29 @@ async def run_cleanup(app: "Quart") -> None:
                 if result.rowcount:
                     logger.info(f"Pruned {result.rowcount} rows from {model.__tablename__}")
 
+        # plugin_metrics is NOT blanket-deleted here — it's rolled up to hourly
+        # then pruned, so multi-week host history stays cheap.
+        await _run_metrics_retention(session, now)
         await _run_docker_retention(session, now)
 
 
+async def _run_metrics_retention(session, now: datetime) -> None:
+    """Roll up + prune plugin_metrics (raw → hourly → gone). Windows read fresh
+    from settings each run (rule 25 — UI change takes effect next cleanup, no
+    restart). get_setting's SELECT autobegins, so read inside the begin block."""
+    from steward.core.metrics_retention import rollup_plugin_metrics
+    from steward.core.settings import get_setting
+
+    async with session.begin():
+        raw_days = int(await get_setting(session, "metrics.retention.raw_days") or 7)
+        rollup_days = int(await get_setting(session, "metrics.retention.rollup_days") or 90)
+        counts = await rollup_plugin_metrics(
+            session, raw_days=raw_days, rollup_days=rollup_days, now=now,
+        )
+    if counts and any(counts.values()):
+        logger.info("Metrics retention: %s", counts)
+
+
 async def _run_docker_retention(session, now: datetime) -> None:
     """Drive the docker plugin's rollup + prune via its capability, if loaded.
 

steward/core/metrics_retention.py

@@ -0,0 +1,112 @@
+"""Bound plugin_metrics growth: roll old raw samples up to hourly, prune the rest.
+
+plugin_metrics grows by (sources × resources × sample cadence) — host agents push
+host-level + per-core/mount/iface sub-resources every ~30s, so a fleet accrues
+millions of rows. We keep raw samples for a short window, aggregate everything
+older into hourly averages (plugin_metrics_hourly) and delete the raw rows, then
+prune hourly beyond a longer window. Charts read raw for the recent part of a
+range and hourly for the older part.
+
+Driven by the core cleanup task (steward.core.cleanup). Runs inside the caller's
+open transaction; never opens or commits its own.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+
+def _hour_floor(dt: datetime) -> datetime:
+    """Truncate a datetime down to the start of its hour (drops min/sec/µs)."""
+    return dt.replace(minute=0, second=0, microsecond=0)
+
+
+def _rollup_cutoff(now: datetime, raw_days: int) -> datetime:
+    """Hour-aligned boundary below which raw metrics get rolled up + deleted.
+
+    Aligning to the hour means we only roll up *whole* elapsed hours — a bucket
+    is never split across the keep/roll boundary, so a re-run can't produce a
+    partial-then-complete duplicate for the same hour.
+    """
+    return _hour_floor(now - timedelta(days=raw_days))
+
+
+async def rollup_plugin_metrics(
+    session,
+    *,
+    raw_days: int,
+    rollup_days: int,
+    now: datetime | None = None,
+) -> dict:
+    """Roll up + prune plugin_metrics. Returns a counts dict for logging.
+
+    1. Aggregate plugin_metrics older than the (hour-aligned) raw window into
+       plugin_metrics_hourly (avg/max per source/resource/metric/hour), upserting
+       so a re-run is idempotent, then delete those raw rows.
+    2. Prune rolled-up rows older than the rollup window.
+    """
+    from datetime import timezone
+
+    from sqlalchemy import delete, func, select
+    from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+    from steward.models.metrics import PluginMetric, PluginMetricHourly
+
+    if now is None:
+        now = datetime.now(timezone.utc)
+
+    rolled = rolled_rows = rollup_pruned = 0
+
+    # ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
+    raw_cutoff = _rollup_cutoff(now, raw_days)
+    hour = func.date_trunc("hour", PluginMetric.recorded_at)
+    agg = (
+        select(
+            PluginMetric.source_module,
+            PluginMetric.resource_name,
+            PluginMetric.metric_name,
+            hour.label("bucket"),
+            func.avg(PluginMetric.value).label("value_avg"),
+            func.max(PluginMetric.value).label("value_max"),
+            func.count().label("sample_count"),
+        )
+        .where(PluginMetric.recorded_at < raw_cutoff)
+        .group_by(
+            PluginMetric.source_module, PluginMetric.resource_name,
+            PluginMetric.metric_name, hour,
+        )
+    )
+    for r in (await session.execute(agg)).all():
+        avg_v = float(r.value_avg or 0.0)
+        max_v = float(r.value_max or 0.0)
+        cnt = int(r.sample_count or 0)
+        await session.execute(
+            pg_insert(PluginMetricHourly)
+            .values(
+                source_module=r.source_module, resource_name=r.resource_name,
+                metric_name=r.metric_name, bucket=r.bucket,
+                value_avg=avg_v, value_max=max_v, sample_count=cnt,
+            )
+            .on_conflict_do_update(
+                constraint="uq_plugin_metrics_hourly_bucket",
+                set_={"value_avg": avg_v, "value_max": max_v, "sample_count": cnt},
+            )
+        )
+        rolled += 1
+        rolled_rows += cnt
+    if rolled:
+        await session.execute(
+            delete(PluginMetric).where(PluginMetric.recorded_at < raw_cutoff)
+        )
+
+    # ── 2. Prune rolled-up rows beyond the rollup window ──
+    rollup_cutoff = now - timedelta(days=rollup_days)
+    res = await session.execute(
+        delete(PluginMetricHourly).where(PluginMetricHourly.bucket < rollup_cutoff)
+    )
+    rollup_pruned = res.rowcount or 0
+
+    return {
+        "buckets_rolled": rolled,
+        "raw_rows_rolled": rolled_rows,
+        "rollup_pruned": rollup_pruned,
+    }

steward/core/settings.py

@@ -84,6 +84,10 @@ DEFAULTS: dict[str, Any] = {
     "docker.retention.metrics_raw_days": 7,
     "docker.retention.metrics_rollup_days": 90,
     "docker.retention.events_days": 30,
+    # Host/plugin metrics retention (plugin_metrics): keep a short raw window at
+    # the agent's ~30s cadence, then roll up to hourly averages kept much longer.
+    "metrics.retention.raw_days": 7,
+    "metrics.retention.rollup_days": 90,
     "plugins.index_url": "https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml",
     # Default-enabled plugins. These are the generic, non-vendor-specific
     # bundled plugins (protocols/standards, not a single product) — useful on

steward/migrations/versions/0024_plugin_metrics_hourly.py

@@ -0,0 +1,43 @@
+"""Hourly rollup table for plugin_metrics
+
+Adds plugin_metrics_hourly — the coarse series that retention rolls raw
+plugin_metrics into before pruning them, so multi-day/week host history stays
+cheap to store. One row per (source_module, resource_name, metric_name, hour);
+the unique constraint is the conflict target for the idempotent rollup upsert.
+
+Revision ID: 0024_plugin_metrics_hourly
+Revises: 0023_plugin_metrics_indexes
+Create Date: 2026-06-20
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "0024_plugin_metrics_hourly"
+down_revision: Union[str, None] = "0023_plugin_metrics_indexes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "plugin_metrics_hourly",
+        sa.Column("id", sa.String(length=36), nullable=False),
+        sa.Column("source_module", sa.String(length=64), nullable=False),
+        sa.Column("resource_name", sa.String(length=255), nullable=False),
+        sa.Column("metric_name", sa.String(length=128), nullable=False),
+        sa.Column("bucket", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("value_avg", sa.Float(), nullable=False, server_default="0"),
+        sa.Column("value_max", sa.Float(), nullable=False, server_default="0"),
+        sa.Column("sample_count", sa.Integer(), nullable=False, server_default="0"),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("source_module", "resource_name", "metric_name", "bucket",
+                            name="uq_plugin_metrics_hourly_bucket"),
+    )
+    op.create_index("ix_plugin_metrics_hourly_lookup", "plugin_metrics_hourly",
+                    ["source_module", "resource_name", "metric_name", "bucket"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_plugin_metrics_hourly_lookup", table_name="plugin_metrics_hourly")
+    op.drop_table("plugin_metrics_hourly")

steward/models/metrics.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 import uuid
 from datetime import datetime, timezone
-from sqlalchemy import DateTime, Float, Index, String
+from sqlalchemy import DateTime, Float, Index, Integer, String, UniqueConstraint
 from sqlalchemy.orm import Mapped, mapped_column
 from .base import Base
 
@@ -28,3 +28,30 @@ class PluginMetric(Base):
         Index("ix_plugin_metrics_module_resource_metric_recorded",
               "source_module", "resource_name", "metric_name", "recorded_at"),
     )
+
+
+class PluginMetricHourly(Base):
+    """Hourly rollup of plugin_metrics — the coarse series that retention rolls
+    raw samples into before pruning them, so multi-day/week history stays cheap.
+
+    One row per (source_module, resource_name, metric_name, hour bucket); the
+    unique constraint is the conflict target for the idempotent rollup upsert.
+    Charts read this for the part of a range older than the raw-retention window.
+    """
+    __tablename__ = "plugin_metrics_hourly"
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+    source_module: Mapped[str] = mapped_column(String(64), nullable=False)
+    resource_name: Mapped[str] = mapped_column(String(255), nullable=False)
+    metric_name: Mapped[str] = mapped_column(String(128), nullable=False)
+    bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    value_avg: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
+    value_max: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
+    sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+
+    __table_args__ = (
+        UniqueConstraint("source_module", "resource_name", "metric_name", "bucket",
+                         name="uq_plugin_metrics_hourly_bucket"),
+        Index("ix_plugin_metrics_hourly_lookup",
+              "source_module", "resource_name", "metric_name", "bucket"),
+    )

steward/settings/routes.py

@@ -133,6 +133,8 @@ _RETENTION_FIELDS = [
     ("docker_metrics_raw_days", "docker.retention.metrics_raw_days"),
     ("docker_metrics_rollup_days", "docker.retention.metrics_rollup_days"),
     ("docker_events_days", "docker.retention.events_days"),
+    ("metrics_raw_days", "metrics.retention.raw_days"),
+    ("metrics_rollup_days", "metrics.retention.rollup_days"),
 ]
 
 

steward/templates/settings/thresholds.html

@@ -70,6 +70,21 @@
           "Keep container start/stop/die/health history this long.") }}
 </div>
 
+<div class="card" style="max-width:640px;margin-top:1rem;">
+  <h2 class="section-title" style="margin-bottom:0.5rem;">Host metrics retention</h2>
+  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
+    Bounds how much host-agent metric history is stored (CPU, memory, disk, network,
+    temps, …). Raw per-sample points are kept for the raw window, then rolled up
+    into hourly averages kept for the rollup window. Host charts read raw for the
+    recent part of a range and hourly for older data. Applied by the hourly cleanup.
+  </p>
+
+  {{ days("Raw metrics", "metrics_raw_days", "metrics.retention.raw_days",
+          "Keep per-sample host metrics this long, then roll up to hourly averages.") }}
+  {{ days("Rolled-up metrics", "metrics_rollup_days", "metrics.retention.rollup_days",
+          "Keep the hourly-averaged series this long for multi-week history.") }}
+</div>
+
 <div style="margin-top:1rem;display:flex;align-items:center;gap:1rem;">
   <button type="submit" class="btn">Save</button>
   <span style="font-size:0.82rem;color:var(--text-muted);">Takes effect immediately — no restart.</span>