feat(metrics): roll plugin_metrics up to hourly to bound storage
plugin_metrics grows by (sources × resources × ~30s cadence); keeping 90d of raw
is a large table. Add a raw→hourly rollup (mirroring the Docker plugin) so only a
short raw window is kept at full resolution, with hourly averages archived longer.
- PluginMetricHourly model + core migration 0024 (plugin_metrics_hourly: avg/max/
count per source/resource/metric/hour, unique bucket constraint + lookup index).
- steward/core/metrics_retention.rollup_plugin_metrics: date_trunc('hour') agg of
raw older than the hour-aligned raw window, idempotent pg upsert into hourly,
delete the rolled raw, prune hourly beyond the rollup window.
- cleanup.py: plugin_metrics is no longer blanket-deleted at data.retention_days;
_run_metrics_retention drives the rollup with windows read live from settings.
- Settings: metrics.retention.raw_days (7) + rollup_days (90), tunable on the
Thresholds & Retention page (new "Host metrics retention" card).
- Chart read: _history_for_host merges the hourly rollup (older part of the range)
with raw date_bin (recent part, capped ≤1h), so 30d charts keep working —
recent at full resolution, older at hourly. Route passes raw_days from settings.
- Tests: unit (cutoff helpers) + integration (rollup aggregates/prunes; history
merges hourly + raw) against Postgres.
Speed was already handled by the indexes + SQL aggregation; this is the storage
lever (raw window ~10x smaller).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
"""Hourly rollup table for plugin_metrics
|
||||
|
||||
Adds plugin_metrics_hourly — the coarse series that retention rolls raw
|
||||
plugin_metrics into before pruning them, so multi-day/week host history stays
|
||||
cheap to store. One row per (source_module, resource_name, metric_name, hour);
|
||||
the unique constraint is the conflict target for the idempotent rollup upsert.
|
||||
|
||||
Revision ID: 0024_plugin_metrics_hourly
|
||||
Revises: 0023_plugin_metrics_indexes
|
||||
Create Date: 2026-06-20
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0024_plugin_metrics_hourly"
|
||||
down_revision: Union[str, None] = "0023_plugin_metrics_indexes"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"plugin_metrics_hourly",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("source_module", sa.String(length=64), nullable=False),
|
||||
sa.Column("resource_name", sa.String(length=255), nullable=False),
|
||||
sa.Column("metric_name", sa.String(length=128), nullable=False),
|
||||
sa.Column("bucket", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("value_avg", sa.Float(), nullable=False, server_default="0"),
|
||||
sa.Column("value_max", sa.Float(), nullable=False, server_default="0"),
|
||||
sa.Column("sample_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("source_module", "resource_name", "metric_name", "bucket",
|
||||
name="uq_plugin_metrics_hourly_bucket"),
|
||||
)
|
||||
op.create_index("ix_plugin_metrics_hourly_lookup", "plugin_metrics_hourly",
|
||||
["source_module", "resource_name", "metric_name", "bucket"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_plugin_metrics_hourly_lookup", table_name="plugin_metrics_hourly")
|
||||
op.drop_table("plugin_metrics_hourly")
|
||||
Reference in New Issue
Block a user