bvandeusen
8af297670e
plugin_metrics grows by (sources × resources × ~30s cadence); keeping 90d of raw
is a large table. Add a raw→hourly rollup (mirroring the Docker plugin) so only a
short raw window is kept at full resolution, with hourly averages archived longer.
- PluginMetricHourly model + core migration 0024 (plugin_metrics_hourly: avg/max/
count per source/resource/metric/hour, unique bucket constraint + lookup index).
- steward/core/metrics_retention.rollup_plugin_metrics: date_trunc('hour') agg of
raw older than the hour-aligned raw window, idempotent pg upsert into hourly,
delete the rolled raw, prune hourly beyond the rollup window.
- cleanup.py: plugin_metrics is no longer blanket-deleted at data.retention_days;
_run_metrics_retention drives the rollup with windows read live from settings.
- Settings: metrics.retention.raw_days (7) + rollup_days (90), tunable on the
Thresholds & Retention page (new "Host metrics retention" card).
- Chart read: _history_for_host merges the hourly rollup (older part of the range)
with raw date_bin (recent part, capped ≤1h), so 30d charts keep working —
recent at full resolution, older at hourly. Route passes raw_days from settings.
- Tests: unit (cutoff helpers) + integration (rollup aggregates/prunes; history
merges hourly + raw) against Postgres.
Speed was already handled by the indexes + SQL aggregation; this is the storage
lever (raw window ~10x smaller).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC
86 lines
3.7 KiB
Python
86 lines
3.7 KiB
Python
from __future__ import annotations
|
|
import logging
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import delete
|
|
|
|
from steward.models.monitors import MonitorResult
|
|
from steward.models.ansible import AnsibleRun
|
|
|
|
if TYPE_CHECKING:
|
|
from quart import Quart
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def run_cleanup(app: "Quart") -> None:
|
|
"""Delete rows older than DATA_RETENTION_DAYS from time-series tables, then
|
|
run Docker-specific rollup + retention (delegated to the docker plugin)."""
|
|
retention_days: int = app.config.get("DATA_RETENTION_DAYS", 90)
|
|
now = datetime.now(timezone.utc)
|
|
cutoff = now - timedelta(days=retention_days)
|
|
|
|
async with app.db_sessionmaker() as session:
|
|
async with session.begin():
|
|
for model, ts_col in [
|
|
(MonitorResult, MonitorResult.checked_at),
|
|
(AnsibleRun, AnsibleRun.started_at),
|
|
]:
|
|
result = await session.execute(
|
|
delete(model).where(ts_col < cutoff)
|
|
)
|
|
if result.rowcount:
|
|
logger.info(f"Pruned {result.rowcount} rows from {model.__tablename__}")
|
|
|
|
# plugin_metrics is NOT blanket-deleted here — it's rolled up to hourly
|
|
# then pruned, so multi-week host history stays cheap.
|
|
await _run_metrics_retention(session, now)
|
|
await _run_docker_retention(session, now)
|
|
|
|
|
|
async def _run_metrics_retention(session, now: datetime) -> None:
|
|
"""Roll up + prune plugin_metrics (raw → hourly → gone). Windows read fresh
|
|
from settings each run (rule 25 — UI change takes effect next cleanup, no
|
|
restart). get_setting's SELECT autobegins, so read inside the begin block."""
|
|
from steward.core.metrics_retention import rollup_plugin_metrics
|
|
from steward.core.settings import get_setting
|
|
|
|
async with session.begin():
|
|
raw_days = int(await get_setting(session, "metrics.retention.raw_days") or 7)
|
|
rollup_days = int(await get_setting(session, "metrics.retention.rollup_days") or 90)
|
|
counts = await rollup_plugin_metrics(
|
|
session, raw_days=raw_days, rollup_days=rollup_days, now=now,
|
|
)
|
|
if counts and any(counts.values()):
|
|
logger.info("Metrics retention: %s", counts)
|
|
|
|
|
|
async def _run_docker_retention(session, now: datetime) -> None:
|
|
"""Drive the docker plugin's rollup + prune via its capability, if loaded.
|
|
|
|
Windows are read fresh from settings each run (rule 25 — a change in the
|
|
Settings UI takes effect on the next hourly cleanup, no restart). Kept in its
|
|
own transaction so a docker-side failure can't roll back the generic prune
|
|
above. No-op when the docker plugin is disabled (capability absent).
|
|
"""
|
|
from steward.core.capabilities import has_capability, invoke_capability
|
|
if not has_capability("docker.run_retention"):
|
|
return
|
|
from steward.core.settings import get_setting
|
|
from steward.models.users import UserRole
|
|
|
|
# Reads + rollup/prune share one transaction — get_setting's SELECT would
|
|
# otherwise autobegin one, making a later session.begin() raise.
|
|
async with session.begin():
|
|
raw_days = int(await get_setting(session, "docker.retention.metrics_raw_days") or 7)
|
|
rollup_days = int(await get_setting(session, "docker.retention.metrics_rollup_days") or 90)
|
|
events_days = int(await get_setting(session, "docker.retention.events_days") or 30)
|
|
counts = await invoke_capability(
|
|
"docker.run_retention", UserRole.viewer, session,
|
|
events_days=events_days, metrics_raw_days=raw_days,
|
|
metrics_rollup_days=rollup_days, now=now,
|
|
)
|
|
if counts and any(counts.values()):
|
|
logger.info("Docker retention: %s", counts)
|