feat(metrics): roll plugin_metrics up to hourly to bound storage
plugin_metrics grows by (sources × resources × ~30s cadence); keeping 90d of raw
is a large table. Add a raw→hourly rollup (mirroring the Docker plugin) so only a
short raw window is kept at full resolution, with hourly averages archived longer.
- PluginMetricHourly model + core migration 0024 (plugin_metrics_hourly: avg/max/
count per source/resource/metric/hour, unique bucket constraint + lookup index).
- steward/core/metrics_retention.rollup_plugin_metrics: date_trunc('hour') agg of
raw older than the hour-aligned raw window, idempotent pg upsert into hourly,
delete the rolled raw, prune hourly beyond the rollup window.
- cleanup.py: plugin_metrics is no longer blanket-deleted at data.retention_days;
_run_metrics_retention drives the rollup with windows read live from settings.
- Settings: metrics.retention.raw_days (7) + rollup_days (90), tunable on the
Thresholds & Retention page (new "Host metrics retention" card).
- Chart read: _history_for_host merges the hourly rollup (older part of the range)
with raw date_bin (recent part, capped ≤1h), so 30d charts keep working —
recent at full resolution, older at hourly. Route passes raw_days from settings.
- Tests: unit (cutoff helpers) + integration (rollup aggregates/prunes; history
merges hourly + raw) against Postgres.
Speed was already handled by the indexes + SQL aggregation; this is the storage
lever (raw window ~10x smaller).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC
This commit is contained in:
@@ -7,10 +7,12 @@ defined") guard.
|
|||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
from sqlalchemy import func, or_, select
|
from sqlalchemy import func, or_, select
|
||||||
|
|
||||||
from steward.core.time_range import bucket_seconds
|
from steward.core.time_range import bucket_seconds
|
||||||
from steward.models.metrics import PluginMetric
|
from steward.models.metrics import PluginMetric, PluginMetricHourly
|
||||||
|
|
||||||
SOURCE_MODULE = "host_agent"
|
SOURCE_MODULE = "host_agent"
|
||||||
|
|
||||||
@@ -53,33 +55,58 @@ async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[st
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
async def _history_for_host(session, host_name: str, since) -> dict[str, list]:
|
async def _history_for_host(session, host_name: str, since, *, raw_days: int = 7) -> dict[str, list]:
|
||||||
"""{metric: [[epoch_ms, avg_value], …]} host-level series since `since`.
|
"""{metric: [[epoch_ms, avg_value], …]} host-level series since `since`.
|
||||||
|
|
||||||
Buckets + averages in SQL (date_bin to ~120 buckets) so we return a readable
|
Retention rolls raw plugin_metrics older than `raw_days` into hourly averages
|
||||||
point count instead of shipping every raw sample to Python and downsampling
|
(plugin_metrics_hourly) and deletes the raw rows. So we read the rollup for the
|
||||||
there — a 30d range was reading hundreds of thousands of rows per load. The
|
part of the range older than that boundary and raw (bucket-averaged in SQL,
|
||||||
bucket width is epoch-aligned so the x axis is stable across refreshes.
|
capped to ≤1h to match the rollup) for the recent part — never shipping raw
|
||||||
Epoch-ms x values feed a linear chart axis (no Chart.js date adapter).
|
samples to Python. The two windows don't overlap, so appending hourly-then-raw
|
||||||
|
keeps each metric's series time-ordered. Epoch-ms x feeds a linear chart axis.
|
||||||
"""
|
"""
|
||||||
width_s = bucket_seconds(since, 120)
|
now = datetime.now(timezone.utc)
|
||||||
bucket = func.date_bin(
|
raw_cutoff = (now - timedelta(days=raw_days)).replace(minute=0, second=0, microsecond=0)
|
||||||
func.make_interval(0, 0, 0, 0, 0, 0, width_s), # width_s seconds
|
series: dict[str, list] = {m: [] for m in HISTORY_METRICS}
|
||||||
|
|
||||||
|
# ── Older than the raw window: the hourly rollup ──
|
||||||
|
if since < raw_cutoff:
|
||||||
|
hourly = (await session.execute(
|
||||||
|
select(PluginMetricHourly.metric_name, PluginMetricHourly.bucket,
|
||||||
|
PluginMetricHourly.value_avg)
|
||||||
|
.where(
|
||||||
|
PluginMetricHourly.source_module == SOURCE_MODULE,
|
||||||
|
PluginMetricHourly.resource_name == host_name,
|
||||||
|
PluginMetricHourly.metric_name.in_(HISTORY_METRICS),
|
||||||
|
PluginMetricHourly.bucket >= since,
|
||||||
|
PluginMetricHourly.bucket < raw_cutoff,
|
||||||
|
)
|
||||||
|
.order_by(PluginMetricHourly.bucket)
|
||||||
|
)).all()
|
||||||
|
for metric_name, bucket, avg in hourly:
|
||||||
|
series[metric_name].append([int(bucket.timestamp() * 1000), round(float(avg), 2)])
|
||||||
|
|
||||||
|
# ── Recent part: raw, bucket-averaged in SQL ──
|
||||||
|
raw_since = since if since >= raw_cutoff else raw_cutoff
|
||||||
|
width_s = bucket_seconds(raw_since, 120)
|
||||||
|
if since < raw_cutoff:
|
||||||
|
width_s = min(width_s, 3600) # ≤ 1h so it matches the hourly portion
|
||||||
|
rbucket = func.date_bin(
|
||||||
|
func.make_interval(0, 0, 0, 0, 0, 0, width_s),
|
||||||
PluginMetric.recorded_at,
|
PluginMetric.recorded_at,
|
||||||
func.to_timestamp(0), # epoch origin
|
func.to_timestamp(0), # epoch origin
|
||||||
).label("bucket")
|
).label("bucket")
|
||||||
rows = (await session.execute(
|
raw = (await session.execute(
|
||||||
select(PluginMetric.metric_name, bucket, func.avg(PluginMetric.value))
|
select(PluginMetric.metric_name, rbucket, func.avg(PluginMetric.value))
|
||||||
.where(
|
.where(
|
||||||
PluginMetric.source_module == SOURCE_MODULE,
|
PluginMetric.source_module == SOURCE_MODULE,
|
||||||
PluginMetric.resource_name == host_name,
|
PluginMetric.resource_name == host_name,
|
||||||
PluginMetric.metric_name.in_(HISTORY_METRICS),
|
PluginMetric.metric_name.in_(HISTORY_METRICS),
|
||||||
PluginMetric.recorded_at >= since,
|
PluginMetric.recorded_at >= raw_since,
|
||||||
)
|
)
|
||||||
.group_by(PluginMetric.metric_name, bucket)
|
.group_by(PluginMetric.metric_name, rbucket)
|
||||||
.order_by(bucket)
|
.order_by(rbucket)
|
||||||
)).all()
|
)).all()
|
||||||
series: dict[str, list] = {m: [] for m in HISTORY_METRICS}
|
for metric_name, b, avg in raw:
|
||||||
for metric_name, b, avg in rows:
|
|
||||||
series[metric_name].append([int(b.timestamp() * 1000), round(float(avg), 2)])
|
series[metric_name].append([int(b.timestamp() * 1000), round(float(avg), 2)])
|
||||||
return series
|
return series
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from steward.models.users import UserRole
|
|||||||
from sqlalchemy import select, func, or_, and_
|
from sqlalchemy import select, func, or_, and_
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
|
|
||||||
from steward.core.settings import public_base_url
|
from steward.core.settings import get_setting, public_base_url
|
||||||
from steward.core.time_range import parse_range, RANGE_OPTIONS
|
from steward.core.time_range import parse_range, RANGE_OPTIONS
|
||||||
from steward.models.hosts import Host
|
from steward.models.hosts import Host
|
||||||
from steward.models.metrics import PluginMetric
|
from steward.models.metrics import PluginMetric
|
||||||
@@ -637,7 +637,8 @@ async def host_detail_charts(host_id: str):
|
|||||||
select(Host).where(Host.id == host_id))).scalar_one_or_none()
|
select(Host).where(Host.id == host_id))).scalar_one_or_none()
|
||||||
if host is None:
|
if host is None:
|
||||||
return "", 404
|
return "", 404
|
||||||
series = await _history_for_host(session, host.name, since)
|
raw_days = int(await get_setting(session, "metrics.retention.raw_days") or 7)
|
||||||
|
series = await _history_for_host(session, host.name, since, raw_days=raw_days)
|
||||||
return await render_template("_host_charts.html", series=series, range_key=range_key)
|
return await render_template("_host_charts.html", series=series, range_key=range_key)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+20
-2
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
|
|||||||
from sqlalchemy import delete
|
from sqlalchemy import delete
|
||||||
|
|
||||||
from steward.models.monitors import MonitorResult
|
from steward.models.monitors import MonitorResult
|
||||||
from steward.models.metrics import PluginMetric
|
|
||||||
from steward.models.ansible import AnsibleRun
|
from steward.models.ansible import AnsibleRun
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
@@ -26,7 +25,6 @@ async def run_cleanup(app: "Quart") -> None:
|
|||||||
async with session.begin():
|
async with session.begin():
|
||||||
for model, ts_col in [
|
for model, ts_col in [
|
||||||
(MonitorResult, MonitorResult.checked_at),
|
(MonitorResult, MonitorResult.checked_at),
|
||||||
(PluginMetric, PluginMetric.recorded_at),
|
|
||||||
(AnsibleRun, AnsibleRun.started_at),
|
(AnsibleRun, AnsibleRun.started_at),
|
||||||
]:
|
]:
|
||||||
result = await session.execute(
|
result = await session.execute(
|
||||||
@@ -35,9 +33,29 @@ async def run_cleanup(app: "Quart") -> None:
|
|||||||
if result.rowcount:
|
if result.rowcount:
|
||||||
logger.info(f"Pruned {result.rowcount} rows from {model.__tablename__}")
|
logger.info(f"Pruned {result.rowcount} rows from {model.__tablename__}")
|
||||||
|
|
||||||
|
# plugin_metrics is NOT blanket-deleted here — it's rolled up to hourly
|
||||||
|
# then pruned, so multi-week host history stays cheap.
|
||||||
|
await _run_metrics_retention(session, now)
|
||||||
await _run_docker_retention(session, now)
|
await _run_docker_retention(session, now)
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_metrics_retention(session, now: datetime) -> None:
|
||||||
|
"""Roll up + prune plugin_metrics (raw → hourly → gone). Windows read fresh
|
||||||
|
from settings each run (rule 25 — UI change takes effect next cleanup, no
|
||||||
|
restart). get_setting's SELECT autobegins, so read inside the begin block."""
|
||||||
|
from steward.core.metrics_retention import rollup_plugin_metrics
|
||||||
|
from steward.core.settings import get_setting
|
||||||
|
|
||||||
|
async with session.begin():
|
||||||
|
raw_days = int(await get_setting(session, "metrics.retention.raw_days") or 7)
|
||||||
|
rollup_days = int(await get_setting(session, "metrics.retention.rollup_days") or 90)
|
||||||
|
counts = await rollup_plugin_metrics(
|
||||||
|
session, raw_days=raw_days, rollup_days=rollup_days, now=now,
|
||||||
|
)
|
||||||
|
if counts and any(counts.values()):
|
||||||
|
logger.info("Metrics retention: %s", counts)
|
||||||
|
|
||||||
|
|
||||||
async def _run_docker_retention(session, now: datetime) -> None:
|
async def _run_docker_retention(session, now: datetime) -> None:
|
||||||
"""Drive the docker plugin's rollup + prune via its capability, if loaded.
|
"""Drive the docker plugin's rollup + prune via its capability, if loaded.
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,112 @@
|
|||||||
|
"""Bound plugin_metrics growth: roll old raw samples up to hourly, prune the rest.
|
||||||
|
|
||||||
|
plugin_metrics grows by (sources × resources × sample cadence) — host agents push
|
||||||
|
host-level + per-core/mount/iface sub-resources every ~30s, so a fleet accrues
|
||||||
|
millions of rows. We keep raw samples for a short window, aggregate everything
|
||||||
|
older into hourly averages (plugin_metrics_hourly) and delete the raw rows, then
|
||||||
|
prune hourly beyond a longer window. Charts read raw for the recent part of a
|
||||||
|
range and hourly for the older part.
|
||||||
|
|
||||||
|
Driven by the core cleanup task (steward.core.cleanup). Runs inside the caller's
|
||||||
|
open transaction; never opens or commits its own.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
|
|
||||||
|
def _hour_floor(dt: datetime) -> datetime:
|
||||||
|
"""Truncate a datetime down to the start of its hour (drops min/sec/µs)."""
|
||||||
|
return dt.replace(minute=0, second=0, microsecond=0)
|
||||||
|
|
||||||
|
|
||||||
|
def _rollup_cutoff(now: datetime, raw_days: int) -> datetime:
|
||||||
|
"""Hour-aligned boundary below which raw metrics get rolled up + deleted.
|
||||||
|
|
||||||
|
Aligning to the hour means we only roll up *whole* elapsed hours — a bucket
|
||||||
|
is never split across the keep/roll boundary, so a re-run can't produce a
|
||||||
|
partial-then-complete duplicate for the same hour.
|
||||||
|
"""
|
||||||
|
return _hour_floor(now - timedelta(days=raw_days))
|
||||||
|
|
||||||
|
|
||||||
|
async def rollup_plugin_metrics(
|
||||||
|
session,
|
||||||
|
*,
|
||||||
|
raw_days: int,
|
||||||
|
rollup_days: int,
|
||||||
|
now: datetime | None = None,
|
||||||
|
) -> dict:
|
||||||
|
"""Roll up + prune plugin_metrics. Returns a counts dict for logging.
|
||||||
|
|
||||||
|
1. Aggregate plugin_metrics older than the (hour-aligned) raw window into
|
||||||
|
plugin_metrics_hourly (avg/max per source/resource/metric/hour), upserting
|
||||||
|
so a re-run is idempotent, then delete those raw rows.
|
||||||
|
2. Prune rolled-up rows older than the rollup window.
|
||||||
|
"""
|
||||||
|
from datetime import timezone
|
||||||
|
|
||||||
|
from sqlalchemy import delete, func, select
|
||||||
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
|
|
||||||
|
from steward.models.metrics import PluginMetric, PluginMetricHourly
|
||||||
|
|
||||||
|
if now is None:
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
rolled = rolled_rows = rollup_pruned = 0
|
||||||
|
|
||||||
|
# ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
|
||||||
|
raw_cutoff = _rollup_cutoff(now, raw_days)
|
||||||
|
hour = func.date_trunc("hour", PluginMetric.recorded_at)
|
||||||
|
agg = (
|
||||||
|
select(
|
||||||
|
PluginMetric.source_module,
|
||||||
|
PluginMetric.resource_name,
|
||||||
|
PluginMetric.metric_name,
|
||||||
|
hour.label("bucket"),
|
||||||
|
func.avg(PluginMetric.value).label("value_avg"),
|
||||||
|
func.max(PluginMetric.value).label("value_max"),
|
||||||
|
func.count().label("sample_count"),
|
||||||
|
)
|
||||||
|
.where(PluginMetric.recorded_at < raw_cutoff)
|
||||||
|
.group_by(
|
||||||
|
PluginMetric.source_module, PluginMetric.resource_name,
|
||||||
|
PluginMetric.metric_name, hour,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for r in (await session.execute(agg)).all():
|
||||||
|
avg_v = float(r.value_avg or 0.0)
|
||||||
|
max_v = float(r.value_max or 0.0)
|
||||||
|
cnt = int(r.sample_count or 0)
|
||||||
|
await session.execute(
|
||||||
|
pg_insert(PluginMetricHourly)
|
||||||
|
.values(
|
||||||
|
source_module=r.source_module, resource_name=r.resource_name,
|
||||||
|
metric_name=r.metric_name, bucket=r.bucket,
|
||||||
|
value_avg=avg_v, value_max=max_v, sample_count=cnt,
|
||||||
|
)
|
||||||
|
.on_conflict_do_update(
|
||||||
|
constraint="uq_plugin_metrics_hourly_bucket",
|
||||||
|
set_={"value_avg": avg_v, "value_max": max_v, "sample_count": cnt},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
rolled += 1
|
||||||
|
rolled_rows += cnt
|
||||||
|
if rolled:
|
||||||
|
await session.execute(
|
||||||
|
delete(PluginMetric).where(PluginMetric.recorded_at < raw_cutoff)
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── 2. Prune rolled-up rows beyond the rollup window ──
|
||||||
|
rollup_cutoff = now - timedelta(days=rollup_days)
|
||||||
|
res = await session.execute(
|
||||||
|
delete(PluginMetricHourly).where(PluginMetricHourly.bucket < rollup_cutoff)
|
||||||
|
)
|
||||||
|
rollup_pruned = res.rowcount or 0
|
||||||
|
|
||||||
|
return {
|
||||||
|
"buckets_rolled": rolled,
|
||||||
|
"raw_rows_rolled": rolled_rows,
|
||||||
|
"rollup_pruned": rollup_pruned,
|
||||||
|
}
|
||||||
@@ -84,6 +84,10 @@ DEFAULTS: dict[str, Any] = {
|
|||||||
"docker.retention.metrics_raw_days": 7,
|
"docker.retention.metrics_raw_days": 7,
|
||||||
"docker.retention.metrics_rollup_days": 90,
|
"docker.retention.metrics_rollup_days": 90,
|
||||||
"docker.retention.events_days": 30,
|
"docker.retention.events_days": 30,
|
||||||
|
# Host/plugin metrics retention (plugin_metrics): keep a short raw window at
|
||||||
|
# the agent's ~30s cadence, then roll up to hourly averages kept much longer.
|
||||||
|
"metrics.retention.raw_days": 7,
|
||||||
|
"metrics.retention.rollup_days": 90,
|
||||||
"plugins.index_url": "https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml",
|
"plugins.index_url": "https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml",
|
||||||
# Default-enabled plugins. These are the generic, non-vendor-specific
|
# Default-enabled plugins. These are the generic, non-vendor-specific
|
||||||
# bundled plugins (protocols/standards, not a single product) — useful on
|
# bundled plugins (protocols/standards, not a single product) — useful on
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
"""Hourly rollup table for plugin_metrics
|
||||||
|
|
||||||
|
Adds plugin_metrics_hourly — the coarse series that retention rolls raw
|
||||||
|
plugin_metrics into before pruning them, so multi-day/week host history stays
|
||||||
|
cheap to store. One row per (source_module, resource_name, metric_name, hour);
|
||||||
|
the unique constraint is the conflict target for the idempotent rollup upsert.
|
||||||
|
|
||||||
|
Revision ID: 0024_plugin_metrics_hourly
|
||||||
|
Revises: 0023_plugin_metrics_indexes
|
||||||
|
Create Date: 2026-06-20
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
revision: str = "0024_plugin_metrics_hourly"
|
||||||
|
down_revision: Union[str, None] = "0023_plugin_metrics_indexes"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"plugin_metrics_hourly",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("source_module", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("resource_name", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("metric_name", sa.String(length=128), nullable=False),
|
||||||
|
sa.Column("bucket", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("value_avg", sa.Float(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("value_max", sa.Float(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("sample_count", sa.Integer(), nullable=False, server_default="0"),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
sa.UniqueConstraint("source_module", "resource_name", "metric_name", "bucket",
|
||||||
|
name="uq_plugin_metrics_hourly_bucket"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_plugin_metrics_hourly_lookup", "plugin_metrics_hourly",
|
||||||
|
["source_module", "resource_name", "metric_name", "bucket"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_plugin_metrics_hourly_lookup", table_name="plugin_metrics_hourly")
|
||||||
|
op.drop_table("plugin_metrics_hourly")
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
import uuid
|
import uuid
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from sqlalchemy import DateTime, Float, Index, String
|
from sqlalchemy import DateTime, Float, Index, Integer, String, UniqueConstraint
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
from .base import Base
|
from .base import Base
|
||||||
|
|
||||||
@@ -28,3 +28,30 @@ class PluginMetric(Base):
|
|||||||
Index("ix_plugin_metrics_module_resource_metric_recorded",
|
Index("ix_plugin_metrics_module_resource_metric_recorded",
|
||||||
"source_module", "resource_name", "metric_name", "recorded_at"),
|
"source_module", "resource_name", "metric_name", "recorded_at"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PluginMetricHourly(Base):
|
||||||
|
"""Hourly rollup of plugin_metrics — the coarse series that retention rolls
|
||||||
|
raw samples into before pruning them, so multi-day/week history stays cheap.
|
||||||
|
|
||||||
|
One row per (source_module, resource_name, metric_name, hour bucket); the
|
||||||
|
unique constraint is the conflict target for the idempotent rollup upsert.
|
||||||
|
Charts read this for the part of a range older than the raw-retention window.
|
||||||
|
"""
|
||||||
|
__tablename__ = "plugin_metrics_hourly"
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||||
|
source_module: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
resource_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
metric_name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
value_avg: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||||
|
value_max: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||||
|
sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("source_module", "resource_name", "metric_name", "bucket",
|
||||||
|
name="uq_plugin_metrics_hourly_bucket"),
|
||||||
|
Index("ix_plugin_metrics_hourly_lookup",
|
||||||
|
"source_module", "resource_name", "metric_name", "bucket"),
|
||||||
|
)
|
||||||
|
|||||||
@@ -133,6 +133,8 @@ _RETENTION_FIELDS = [
|
|||||||
("docker_metrics_raw_days", "docker.retention.metrics_raw_days"),
|
("docker_metrics_raw_days", "docker.retention.metrics_raw_days"),
|
||||||
("docker_metrics_rollup_days", "docker.retention.metrics_rollup_days"),
|
("docker_metrics_rollup_days", "docker.retention.metrics_rollup_days"),
|
||||||
("docker_events_days", "docker.retention.events_days"),
|
("docker_events_days", "docker.retention.events_days"),
|
||||||
|
("metrics_raw_days", "metrics.retention.raw_days"),
|
||||||
|
("metrics_rollup_days", "metrics.retention.rollup_days"),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -70,6 +70,21 @@
|
|||||||
"Keep container start/stop/die/health history this long.") }}
|
"Keep container start/stop/die/health history this long.") }}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="card" style="max-width:640px;margin-top:1rem;">
|
||||||
|
<h2 class="section-title" style="margin-bottom:0.5rem;">Host metrics retention</h2>
|
||||||
|
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
|
||||||
|
Bounds how much host-agent metric history is stored (CPU, memory, disk, network,
|
||||||
|
temps, …). Raw per-sample points are kept for the raw window, then rolled up
|
||||||
|
into hourly averages kept for the rollup window. Host charts read raw for the
|
||||||
|
recent part of a range and hourly for older data. Applied by the hourly cleanup.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
{{ days("Raw metrics", "metrics_raw_days", "metrics.retention.raw_days",
|
||||||
|
"Keep per-sample host metrics this long, then roll up to hourly averages.") }}
|
||||||
|
{{ days("Rolled-up metrics", "metrics_rollup_days", "metrics.retention.rollup_days",
|
||||||
|
"Keep the hourly-averaged series this long for multi-week history.") }}
|
||||||
|
</div>
|
||||||
|
|
||||||
<div style="margin-top:1rem;display:flex;align-items:center;gap:1rem;">
|
<div style="margin-top:1rem;display:flex;align-items:center;gap:1rem;">
|
||||||
<button type="submit" class="btn">Save</button>
|
<button type="submit" class="btn">Save</button>
|
||||||
<span style="font-size:0.82rem;color:var(--text-muted);">Takes effect immediately — no restart.</span>
|
<span style="font-size:0.82rem;color:var(--text-muted);">Takes effect immediately — no restart.</span>
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
"""Unit tests for the plugin_metrics rollup cutoff helpers (no DB)."""
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from steward.core.metrics_retention import _hour_floor, _rollup_cutoff
|
||||||
|
|
||||||
|
|
||||||
|
def test_hour_floor_drops_sub_hour():
|
||||||
|
dt = datetime(2026, 6, 20, 15, 42, 9, 123456, tzinfo=timezone.utc)
|
||||||
|
assert _hour_floor(dt) == datetime(2026, 6, 20, 15, 0, 0, 0, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def test_rollup_cutoff_is_hour_aligned_and_offset():
|
||||||
|
now = datetime(2026, 6, 20, 15, 42, 9, tzinfo=timezone.utc)
|
||||||
|
cutoff = _rollup_cutoff(now, 7)
|
||||||
|
assert (cutoff.minute, cutoff.second, cutoff.microsecond) == (0, 0, 0)
|
||||||
|
# 7 whole days back, then floored to the hour.
|
||||||
|
assert cutoff == datetime(2026, 6, 13, 15, 0, 0, 0, tzinfo=timezone.utc)
|
||||||
@@ -78,3 +78,84 @@ def test_latest_distinct_on_and_sql_bucketed_history(app):
|
|||||||
cpu = hist["cpu_pct"]
|
cpu = hist["cpu_pct"]
|
||||||
assert cpu, "expected bucketed cpu_pct history"
|
assert cpu, "expected bucketed cpu_pct history"
|
||||||
assert all(10.0 <= v <= 30.0 for _, v in cpu)
|
assert all(10.0 <= v <= 30.0 for _, v in cpu)
|
||||||
|
|
||||||
|
|
||||||
|
@_NEEDS_DB
|
||||||
|
def test_rollup_aggregates_old_raw_and_prunes(app):
|
||||||
|
from sqlalchemy import text
|
||||||
|
from steward.core.metrics_retention import rollup_plugin_metrics
|
||||||
|
from steward.models.metrics import PluginMetric
|
||||||
|
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
hostname = "rollup-host-" + uuid.uuid4().hex[:8]
|
||||||
|
|
||||||
|
async def _go():
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
async with s.begin():
|
||||||
|
for tbl in ("plugin_metrics", "plugin_metrics_hourly"):
|
||||||
|
await s.execute(
|
||||||
|
text(f"DELETE FROM {tbl} WHERE resource_name LIKE :p"),
|
||||||
|
{"p": hostname + "%"},
|
||||||
|
)
|
||||||
|
old = (now - timedelta(days=10)).replace(minute=5, second=0, microsecond=0)
|
||||||
|
s.add_all([
|
||||||
|
PluginMetric(source_module="host_agent", resource_name=hostname,
|
||||||
|
metric_name="cpu_pct", value=10.0, recorded_at=old),
|
||||||
|
PluginMetric(source_module="host_agent", resource_name=hostname,
|
||||||
|
metric_name="cpu_pct", value=20.0,
|
||||||
|
recorded_at=old.replace(minute=35)), # same hour bucket
|
||||||
|
PluginMetric(source_module="host_agent", resource_name=hostname,
|
||||||
|
metric_name="cpu_pct", value=99.0,
|
||||||
|
recorded_at=now - timedelta(hours=1)), # recent → kept raw
|
||||||
|
])
|
||||||
|
async with s.begin():
|
||||||
|
counts = await rollup_plugin_metrics(s, raw_days=7, rollup_days=90, now=now)
|
||||||
|
hrly = (await s.execute(text(
|
||||||
|
"SELECT value_avg, sample_count FROM plugin_metrics_hourly "
|
||||||
|
"WHERE resource_name = :h AND metric_name = 'cpu_pct'"), {"h": hostname})).all()
|
||||||
|
raw_left = (await s.execute(text(
|
||||||
|
"SELECT count(*) FROM plugin_metrics WHERE resource_name = :h"), {"h": hostname})).scalar()
|
||||||
|
return counts, hrly, raw_left
|
||||||
|
|
||||||
|
counts, hrly, raw_left = asyncio.run(_go())
|
||||||
|
assert counts["buckets_rolled"] == 1
|
||||||
|
assert counts["raw_rows_rolled"] == 2
|
||||||
|
assert len(hrly) == 1
|
||||||
|
assert abs(float(hrly[0][0]) - 15.0) < 0.001 # avg(10, 20)
|
||||||
|
assert hrly[0][1] == 2
|
||||||
|
assert raw_left == 1 # only the recent sample remains
|
||||||
|
|
||||||
|
|
||||||
|
@_NEEDS_DB
|
||||||
|
def test_history_merges_hourly_and_raw(app):
|
||||||
|
from sqlalchemy import text
|
||||||
|
from steward.models.metrics import PluginMetric, PluginMetricHourly
|
||||||
|
from plugins.host_agent.metrics_query import _history_for_host
|
||||||
|
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
hostname = "merge-host-" + uuid.uuid4().hex[:8]
|
||||||
|
|
||||||
|
async def _go():
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
async with s.begin():
|
||||||
|
for tbl in ("plugin_metrics", "plugin_metrics_hourly"):
|
||||||
|
await s.execute(
|
||||||
|
text(f"DELETE FROM {tbl} WHERE resource_name LIKE :p"),
|
||||||
|
{"p": hostname + "%"},
|
||||||
|
)
|
||||||
|
old_bucket = (now - timedelta(days=10)).replace(minute=0, second=0, microsecond=0)
|
||||||
|
s.add(PluginMetricHourly(
|
||||||
|
source_module="host_agent", resource_name=hostname, metric_name="cpu_pct",
|
||||||
|
bucket=old_bucket, value_avg=42.0, value_max=50.0, sample_count=120))
|
||||||
|
s.add(PluginMetric(
|
||||||
|
source_module="host_agent", resource_name=hostname, metric_name="cpu_pct",
|
||||||
|
value=77.0, recorded_at=now - timedelta(hours=1)))
|
||||||
|
# 30d range with a 7d raw window → spans the rollup boundary.
|
||||||
|
return await _history_for_host(s, hostname, now - timedelta(days=30), raw_days=7)
|
||||||
|
|
||||||
|
hist = asyncio.run(_go())
|
||||||
|
cpu = hist["cpu_pct"]
|
||||||
|
vals = [v for _, v in cpu]
|
||||||
|
assert 42.0 in vals, "expected the rolled-up hourly point"
|
||||||
|
assert 77.0 in vals, "expected the recent raw point"
|
||||||
|
assert cpu == sorted(cpu, key=lambda p: p[0]), "series must be time-ordered"
|
||||||
|
|||||||
Reference in New Issue
Block a user