bvandeusen
8af297670e
plugin_metrics grows by (sources × resources × ~30s cadence); keeping 90d of raw
is a large table. Add a raw→hourly rollup (mirroring the Docker plugin) so only a
short raw window is kept at full resolution, with hourly averages archived longer.
- PluginMetricHourly model + core migration 0024 (plugin_metrics_hourly: avg/max/
count per source/resource/metric/hour, unique bucket constraint + lookup index).
- steward/core/metrics_retention.rollup_plugin_metrics: date_trunc('hour') agg of
raw older than the hour-aligned raw window, idempotent pg upsert into hourly,
delete the rolled raw, prune hourly beyond the rollup window.
- cleanup.py: plugin_metrics is no longer blanket-deleted at data.retention_days;
_run_metrics_retention drives the rollup with windows read live from settings.
- Settings: metrics.retention.raw_days (7) + rollup_days (90), tunable on the
Thresholds & Retention page (new "Host metrics retention" card).
- Chart read: _history_for_host merges the hourly rollup (older part of the range)
with raw date_bin (recent part, capped ≤1h), so 30d charts keep working —
recent at full resolution, older at hourly. Route passes raw_days from settings.
- Tests: unit (cutoff helpers) + integration (rollup aggregates/prunes; history
merges hourly + raw) against Postgres.
Speed was already handled by the indexes + SQL aggregation; this is the storage
lever (raw window ~10x smaller).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC
58 lines
2.8 KiB
Python
58 lines
2.8 KiB
Python
from __future__ import annotations
|
||
import uuid
|
||
from datetime import datetime, timezone
|
||
from sqlalchemy import DateTime, Float, Index, Integer, String, UniqueConstraint
|
||
from sqlalchemy.orm import Mapped, mapped_column
|
||
from .base import Base
|
||
|
||
|
||
class PluginMetric(Base):
|
||
__tablename__ = "plugin_metrics"
|
||
|
||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
||
source_module: Mapped[str] = mapped_column(String(64), nullable=False)
|
||
resource_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||
metric_name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||
value: Mapped[float] = mapped_column(Float, nullable=False)
|
||
recorded_at: Mapped[datetime] = mapped_column(
|
||
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
|
||
)
|
||
|
||
# This time-series table grows by (sources × resources × sample cadence); every
|
||
# host-detail / full-metrics / dashboard-widget read filters by
|
||
# (source_module, resource_name) over a recorded_at range. Without these it's a
|
||
# full sequential scan on every load.
|
||
__table_args__ = (
|
||
Index("ix_plugin_metrics_module_resource_recorded",
|
||
"source_module", "resource_name", "recorded_at"),
|
||
Index("ix_plugin_metrics_module_resource_metric_recorded",
|
||
"source_module", "resource_name", "metric_name", "recorded_at"),
|
||
)
|
||
|
||
|
||
class PluginMetricHourly(Base):
|
||
"""Hourly rollup of plugin_metrics — the coarse series that retention rolls
|
||
raw samples into before pruning them, so multi-day/week history stays cheap.
|
||
|
||
One row per (source_module, resource_name, metric_name, hour bucket); the
|
||
unique constraint is the conflict target for the idempotent rollup upsert.
|
||
Charts read this for the part of a range older than the raw-retention window.
|
||
"""
|
||
__tablename__ = "plugin_metrics_hourly"
|
||
|
||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
||
source_module: Mapped[str] = mapped_column(String(64), nullable=False)
|
||
resource_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||
metric_name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||
bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||
value_avg: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||
value_max: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||
sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||
|
||
__table_args__ = (
|
||
UniqueConstraint("source_module", "resource_name", "metric_name", "bucket",
|
||
name="uq_plugin_metrics_hourly_bucket"),
|
||
Index("ix_plugin_metrics_hourly_lookup",
|
||
"source_module", "resource_name", "metric_name", "bucket"),
|
||
)
|