feat(metrics): roll plugin_metrics up to hourly to bound storage
plugin_metrics grows by (sources × resources × ~30s cadence); keeping 90d of raw
is a large table. Add a raw→hourly rollup (mirroring the Docker plugin) so only a
short raw window is kept at full resolution, with hourly averages archived longer.
- PluginMetricHourly model + core migration 0024 (plugin_metrics_hourly: avg/max/
count per source/resource/metric/hour, unique bucket constraint + lookup index).
- steward/core/metrics_retention.rollup_plugin_metrics: date_trunc('hour') agg of
raw older than the hour-aligned raw window, idempotent pg upsert into hourly,
delete the rolled raw, prune hourly beyond the rollup window.
- cleanup.py: plugin_metrics is no longer blanket-deleted at data.retention_days;
_run_metrics_retention drives the rollup with windows read live from settings.
- Settings: metrics.retention.raw_days (7) + rollup_days (90), tunable on the
Thresholds & Retention page (new "Host metrics retention" card).
- Chart read: _history_for_host merges the hourly rollup (older part of the range)
with raw date_bin (recent part, capped ≤1h), so 30d charts keep working —
recent at full resolution, older at hourly. Route passes raw_days from settings.
- Tests: unit (cutoff helpers) + integration (rollup aggregates/prunes; history
merges hourly + raw) against Postgres.
Speed was already handled by the indexes + SQL aggregation; this is the storage
lever (raw window ~10x smaller).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC
This commit is contained in:
@@ -7,10 +7,12 @@ defined") guard.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import func, or_, select
|
||||
|
||||
from steward.core.time_range import bucket_seconds
|
||||
from steward.models.metrics import PluginMetric
|
||||
from steward.models.metrics import PluginMetric, PluginMetricHourly
|
||||
|
||||
SOURCE_MODULE = "host_agent"
|
||||
|
||||
@@ -53,33 +55,58 @@ async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[st
|
||||
return out
|
||||
|
||||
|
||||
async def _history_for_host(session, host_name: str, since) -> dict[str, list]:
|
||||
async def _history_for_host(session, host_name: str, since, *, raw_days: int = 7) -> dict[str, list]:
|
||||
"""{metric: [[epoch_ms, avg_value], …]} host-level series since `since`.
|
||||
|
||||
Buckets + averages in SQL (date_bin to ~120 buckets) so we return a readable
|
||||
point count instead of shipping every raw sample to Python and downsampling
|
||||
there — a 30d range was reading hundreds of thousands of rows per load. The
|
||||
bucket width is epoch-aligned so the x axis is stable across refreshes.
|
||||
Epoch-ms x values feed a linear chart axis (no Chart.js date adapter).
|
||||
Retention rolls raw plugin_metrics older than `raw_days` into hourly averages
|
||||
(plugin_metrics_hourly) and deletes the raw rows. So we read the rollup for the
|
||||
part of the range older than that boundary and raw (bucket-averaged in SQL,
|
||||
capped to ≤1h to match the rollup) for the recent part — never shipping raw
|
||||
samples to Python. The two windows don't overlap, so appending hourly-then-raw
|
||||
keeps each metric's series time-ordered. Epoch-ms x feeds a linear chart axis.
|
||||
"""
|
||||
width_s = bucket_seconds(since, 120)
|
||||
bucket = func.date_bin(
|
||||
func.make_interval(0, 0, 0, 0, 0, 0, width_s), # width_s seconds
|
||||
now = datetime.now(timezone.utc)
|
||||
raw_cutoff = (now - timedelta(days=raw_days)).replace(minute=0, second=0, microsecond=0)
|
||||
series: dict[str, list] = {m: [] for m in HISTORY_METRICS}
|
||||
|
||||
# ── Older than the raw window: the hourly rollup ──
|
||||
if since < raw_cutoff:
|
||||
hourly = (await session.execute(
|
||||
select(PluginMetricHourly.metric_name, PluginMetricHourly.bucket,
|
||||
PluginMetricHourly.value_avg)
|
||||
.where(
|
||||
PluginMetricHourly.source_module == SOURCE_MODULE,
|
||||
PluginMetricHourly.resource_name == host_name,
|
||||
PluginMetricHourly.metric_name.in_(HISTORY_METRICS),
|
||||
PluginMetricHourly.bucket >= since,
|
||||
PluginMetricHourly.bucket < raw_cutoff,
|
||||
)
|
||||
.order_by(PluginMetricHourly.bucket)
|
||||
)).all()
|
||||
for metric_name, bucket, avg in hourly:
|
||||
series[metric_name].append([int(bucket.timestamp() * 1000), round(float(avg), 2)])
|
||||
|
||||
# ── Recent part: raw, bucket-averaged in SQL ──
|
||||
raw_since = since if since >= raw_cutoff else raw_cutoff
|
||||
width_s = bucket_seconds(raw_since, 120)
|
||||
if since < raw_cutoff:
|
||||
width_s = min(width_s, 3600) # ≤ 1h so it matches the hourly portion
|
||||
rbucket = func.date_bin(
|
||||
func.make_interval(0, 0, 0, 0, 0, 0, width_s),
|
||||
PluginMetric.recorded_at,
|
||||
func.to_timestamp(0), # epoch origin
|
||||
).label("bucket")
|
||||
rows = (await session.execute(
|
||||
select(PluginMetric.metric_name, bucket, func.avg(PluginMetric.value))
|
||||
raw = (await session.execute(
|
||||
select(PluginMetric.metric_name, rbucket, func.avg(PluginMetric.value))
|
||||
.where(
|
||||
PluginMetric.source_module == SOURCE_MODULE,
|
||||
PluginMetric.resource_name == host_name,
|
||||
PluginMetric.metric_name.in_(HISTORY_METRICS),
|
||||
PluginMetric.recorded_at >= since,
|
||||
PluginMetric.recorded_at >= raw_since,
|
||||
)
|
||||
.group_by(PluginMetric.metric_name, bucket)
|
||||
.order_by(bucket)
|
||||
.group_by(PluginMetric.metric_name, rbucket)
|
||||
.order_by(rbucket)
|
||||
)).all()
|
||||
series: dict[str, list] = {m: [] for m in HISTORY_METRICS}
|
||||
for metric_name, b, avg in rows:
|
||||
for metric_name, b, avg in raw:
|
||||
series[metric_name].append([int(b.timestamp() * 1000), round(float(avg), 2)])
|
||||
return series
|
||||
|
||||
@@ -14,7 +14,7 @@ from steward.models.users import UserRole
|
||||
from sqlalchemy import select, func, or_, and_
|
||||
from datetime import timedelta
|
||||
|
||||
from steward.core.settings import public_base_url
|
||||
from steward.core.settings import get_setting, public_base_url
|
||||
from steward.core.time_range import parse_range, RANGE_OPTIONS
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.metrics import PluginMetric
|
||||
@@ -637,7 +637,8 @@ async def host_detail_charts(host_id: str):
|
||||
select(Host).where(Host.id == host_id))).scalar_one_or_none()
|
||||
if host is None:
|
||||
return "", 404
|
||||
series = await _history_for_host(session, host.name, since)
|
||||
raw_days = int(await get_setting(session, "metrics.retention.raw_days") or 7)
|
||||
series = await _history_for_host(session, host.name, since, raw_days=raw_days)
|
||||
return await render_template("_host_charts.html", series=series, range_key=range_key)
|
||||
|
||||
|
||||
|
||||
+20
-2
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
|
||||
from sqlalchemy import delete
|
||||
|
||||
from steward.models.monitors import MonitorResult
|
||||
from steward.models.metrics import PluginMetric
|
||||
from steward.models.ansible import AnsibleRun
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -26,7 +25,6 @@ async def run_cleanup(app: "Quart") -> None:
|
||||
async with session.begin():
|
||||
for model, ts_col in [
|
||||
(MonitorResult, MonitorResult.checked_at),
|
||||
(PluginMetric, PluginMetric.recorded_at),
|
||||
(AnsibleRun, AnsibleRun.started_at),
|
||||
]:
|
||||
result = await session.execute(
|
||||
@@ -35,9 +33,29 @@ async def run_cleanup(app: "Quart") -> None:
|
||||
if result.rowcount:
|
||||
logger.info(f"Pruned {result.rowcount} rows from {model.__tablename__}")
|
||||
|
||||
# plugin_metrics is NOT blanket-deleted here — it's rolled up to hourly
|
||||
# then pruned, so multi-week host history stays cheap.
|
||||
await _run_metrics_retention(session, now)
|
||||
await _run_docker_retention(session, now)
|
||||
|
||||
|
||||
async def _run_metrics_retention(session, now: datetime) -> None:
|
||||
"""Roll up + prune plugin_metrics (raw → hourly → gone). Windows read fresh
|
||||
from settings each run (rule 25 — UI change takes effect next cleanup, no
|
||||
restart). get_setting's SELECT autobegins, so read inside the begin block."""
|
||||
from steward.core.metrics_retention import rollup_plugin_metrics
|
||||
from steward.core.settings import get_setting
|
||||
|
||||
async with session.begin():
|
||||
raw_days = int(await get_setting(session, "metrics.retention.raw_days") or 7)
|
||||
rollup_days = int(await get_setting(session, "metrics.retention.rollup_days") or 90)
|
||||
counts = await rollup_plugin_metrics(
|
||||
session, raw_days=raw_days, rollup_days=rollup_days, now=now,
|
||||
)
|
||||
if counts and any(counts.values()):
|
||||
logger.info("Metrics retention: %s", counts)
|
||||
|
||||
|
||||
async def _run_docker_retention(session, now: datetime) -> None:
|
||||
"""Drive the docker plugin's rollup + prune via its capability, if loaded.
|
||||
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Bound plugin_metrics growth: roll old raw samples up to hourly, prune the rest.
|
||||
|
||||
plugin_metrics grows by (sources × resources × sample cadence) — host agents push
|
||||
host-level + per-core/mount/iface sub-resources every ~30s, so a fleet accrues
|
||||
millions of rows. We keep raw samples for a short window, aggregate everything
|
||||
older into hourly averages (plugin_metrics_hourly) and delete the raw rows, then
|
||||
prune hourly beyond a longer window. Charts read raw for the recent part of a
|
||||
range and hourly for the older part.
|
||||
|
||||
Driven by the core cleanup task (steward.core.cleanup). Runs inside the caller's
|
||||
open transaction; never opens or commits its own.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
|
||||
def _hour_floor(dt: datetime) -> datetime:
|
||||
"""Truncate a datetime down to the start of its hour (drops min/sec/µs)."""
|
||||
return dt.replace(minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
def _rollup_cutoff(now: datetime, raw_days: int) -> datetime:
|
||||
"""Hour-aligned boundary below which raw metrics get rolled up + deleted.
|
||||
|
||||
Aligning to the hour means we only roll up *whole* elapsed hours — a bucket
|
||||
is never split across the keep/roll boundary, so a re-run can't produce a
|
||||
partial-then-complete duplicate for the same hour.
|
||||
"""
|
||||
return _hour_floor(now - timedelta(days=raw_days))
|
||||
|
||||
|
||||
async def rollup_plugin_metrics(
|
||||
session,
|
||||
*,
|
||||
raw_days: int,
|
||||
rollup_days: int,
|
||||
now: datetime | None = None,
|
||||
) -> dict:
|
||||
"""Roll up + prune plugin_metrics. Returns a counts dict for logging.
|
||||
|
||||
1. Aggregate plugin_metrics older than the (hour-aligned) raw window into
|
||||
plugin_metrics_hourly (avg/max per source/resource/metric/hour), upserting
|
||||
so a re-run is idempotent, then delete those raw rows.
|
||||
2. Prune rolled-up rows older than the rollup window.
|
||||
"""
|
||||
from datetime import timezone
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from steward.models.metrics import PluginMetric, PluginMetricHourly
|
||||
|
||||
if now is None:
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
rolled = rolled_rows = rollup_pruned = 0
|
||||
|
||||
# ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
|
||||
raw_cutoff = _rollup_cutoff(now, raw_days)
|
||||
hour = func.date_trunc("hour", PluginMetric.recorded_at)
|
||||
agg = (
|
||||
select(
|
||||
PluginMetric.source_module,
|
||||
PluginMetric.resource_name,
|
||||
PluginMetric.metric_name,
|
||||
hour.label("bucket"),
|
||||
func.avg(PluginMetric.value).label("value_avg"),
|
||||
func.max(PluginMetric.value).label("value_max"),
|
||||
func.count().label("sample_count"),
|
||||
)
|
||||
.where(PluginMetric.recorded_at < raw_cutoff)
|
||||
.group_by(
|
||||
PluginMetric.source_module, PluginMetric.resource_name,
|
||||
PluginMetric.metric_name, hour,
|
||||
)
|
||||
)
|
||||
for r in (await session.execute(agg)).all():
|
||||
avg_v = float(r.value_avg or 0.0)
|
||||
max_v = float(r.value_max or 0.0)
|
||||
cnt = int(r.sample_count or 0)
|
||||
await session.execute(
|
||||
pg_insert(PluginMetricHourly)
|
||||
.values(
|
||||
source_module=r.source_module, resource_name=r.resource_name,
|
||||
metric_name=r.metric_name, bucket=r.bucket,
|
||||
value_avg=avg_v, value_max=max_v, sample_count=cnt,
|
||||
)
|
||||
.on_conflict_do_update(
|
||||
constraint="uq_plugin_metrics_hourly_bucket",
|
||||
set_={"value_avg": avg_v, "value_max": max_v, "sample_count": cnt},
|
||||
)
|
||||
)
|
||||
rolled += 1
|
||||
rolled_rows += cnt
|
||||
if rolled:
|
||||
await session.execute(
|
||||
delete(PluginMetric).where(PluginMetric.recorded_at < raw_cutoff)
|
||||
)
|
||||
|
||||
# ── 2. Prune rolled-up rows beyond the rollup window ──
|
||||
rollup_cutoff = now - timedelta(days=rollup_days)
|
||||
res = await session.execute(
|
||||
delete(PluginMetricHourly).where(PluginMetricHourly.bucket < rollup_cutoff)
|
||||
)
|
||||
rollup_pruned = res.rowcount or 0
|
||||
|
||||
return {
|
||||
"buckets_rolled": rolled,
|
||||
"raw_rows_rolled": rolled_rows,
|
||||
"rollup_pruned": rollup_pruned,
|
||||
}
|
||||
@@ -84,6 +84,10 @@ DEFAULTS: dict[str, Any] = {
|
||||
"docker.retention.metrics_raw_days": 7,
|
||||
"docker.retention.metrics_rollup_days": 90,
|
||||
"docker.retention.events_days": 30,
|
||||
# Host/plugin metrics retention (plugin_metrics): keep a short raw window at
|
||||
# the agent's ~30s cadence, then roll up to hourly averages kept much longer.
|
||||
"metrics.retention.raw_days": 7,
|
||||
"metrics.retention.rollup_days": 90,
|
||||
"plugins.index_url": "https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml",
|
||||
# Default-enabled plugins. These are the generic, non-vendor-specific
|
||||
# bundled plugins (protocols/standards, not a single product) — useful on
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Hourly rollup table for plugin_metrics
|
||||
|
||||
Adds plugin_metrics_hourly — the coarse series that retention rolls raw
|
||||
plugin_metrics into before pruning them, so multi-day/week host history stays
|
||||
cheap to store. One row per (source_module, resource_name, metric_name, hour);
|
||||
the unique constraint is the conflict target for the idempotent rollup upsert.
|
||||
|
||||
Revision ID: 0024_plugin_metrics_hourly
|
||||
Revises: 0023_plugin_metrics_indexes
|
||||
Create Date: 2026-06-20
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0024_plugin_metrics_hourly"
|
||||
down_revision: Union[str, None] = "0023_plugin_metrics_indexes"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"plugin_metrics_hourly",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("source_module", sa.String(length=64), nullable=False),
|
||||
sa.Column("resource_name", sa.String(length=255), nullable=False),
|
||||
sa.Column("metric_name", sa.String(length=128), nullable=False),
|
||||
sa.Column("bucket", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("value_avg", sa.Float(), nullable=False, server_default="0"),
|
||||
sa.Column("value_max", sa.Float(), nullable=False, server_default="0"),
|
||||
sa.Column("sample_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("source_module", "resource_name", "metric_name", "bucket",
|
||||
name="uq_plugin_metrics_hourly_bucket"),
|
||||
)
|
||||
op.create_index("ix_plugin_metrics_hourly_lookup", "plugin_metrics_hourly",
|
||||
["source_module", "resource_name", "metric_name", "bucket"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_plugin_metrics_hourly_lookup", table_name="plugin_metrics_hourly")
|
||||
op.drop_table("plugin_metrics_hourly")
|
||||
@@ -1,7 +1,7 @@
|
||||
from __future__ import annotations
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from sqlalchemy import DateTime, Float, Index, String
|
||||
from sqlalchemy import DateTime, Float, Index, Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from .base import Base
|
||||
|
||||
@@ -28,3 +28,30 @@ class PluginMetric(Base):
|
||||
Index("ix_plugin_metrics_module_resource_metric_recorded",
|
||||
"source_module", "resource_name", "metric_name", "recorded_at"),
|
||||
)
|
||||
|
||||
|
||||
class PluginMetricHourly(Base):
|
||||
"""Hourly rollup of plugin_metrics — the coarse series that retention rolls
|
||||
raw samples into before pruning them, so multi-day/week history stays cheap.
|
||||
|
||||
One row per (source_module, resource_name, metric_name, hour bucket); the
|
||||
unique constraint is the conflict target for the idempotent rollup upsert.
|
||||
Charts read this for the part of a range older than the raw-retention window.
|
||||
"""
|
||||
__tablename__ = "plugin_metrics_hourly"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
source_module: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
resource_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
metric_name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
value_avg: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||
value_max: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||
sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_module", "resource_name", "metric_name", "bucket",
|
||||
name="uq_plugin_metrics_hourly_bucket"),
|
||||
Index("ix_plugin_metrics_hourly_lookup",
|
||||
"source_module", "resource_name", "metric_name", "bucket"),
|
||||
)
|
||||
|
||||
@@ -133,6 +133,8 @@ _RETENTION_FIELDS = [
|
||||
("docker_metrics_raw_days", "docker.retention.metrics_raw_days"),
|
||||
("docker_metrics_rollup_days", "docker.retention.metrics_rollup_days"),
|
||||
("docker_events_days", "docker.retention.events_days"),
|
||||
("metrics_raw_days", "metrics.retention.raw_days"),
|
||||
("metrics_rollup_days", "metrics.retention.rollup_days"),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -70,6 +70,21 @@
|
||||
"Keep container start/stop/die/health history this long.") }}
|
||||
</div>
|
||||
|
||||
<div class="card" style="max-width:640px;margin-top:1rem;">
|
||||
<h2 class="section-title" style="margin-bottom:0.5rem;">Host metrics retention</h2>
|
||||
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
|
||||
Bounds how much host-agent metric history is stored (CPU, memory, disk, network,
|
||||
temps, …). Raw per-sample points are kept for the raw window, then rolled up
|
||||
into hourly averages kept for the rollup window. Host charts read raw for the
|
||||
recent part of a range and hourly for older data. Applied by the hourly cleanup.
|
||||
</p>
|
||||
|
||||
{{ days("Raw metrics", "metrics_raw_days", "metrics.retention.raw_days",
|
||||
"Keep per-sample host metrics this long, then roll up to hourly averages.") }}
|
||||
{{ days("Rolled-up metrics", "metrics_rollup_days", "metrics.retention.rollup_days",
|
||||
"Keep the hourly-averaged series this long for multi-week history.") }}
|
||||
</div>
|
||||
|
||||
<div style="margin-top:1rem;display:flex;align-items:center;gap:1rem;">
|
||||
<button type="submit" class="btn">Save</button>
|
||||
<span style="font-size:0.82rem;color:var(--text-muted);">Takes effect immediately — no restart.</span>
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Unit tests for the plugin_metrics rollup cutoff helpers (no DB)."""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from steward.core.metrics_retention import _hour_floor, _rollup_cutoff
|
||||
|
||||
|
||||
def test_hour_floor_drops_sub_hour():
|
||||
dt = datetime(2026, 6, 20, 15, 42, 9, 123456, tzinfo=timezone.utc)
|
||||
assert _hour_floor(dt) == datetime(2026, 6, 20, 15, 0, 0, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def test_rollup_cutoff_is_hour_aligned_and_offset():
|
||||
now = datetime(2026, 6, 20, 15, 42, 9, tzinfo=timezone.utc)
|
||||
cutoff = _rollup_cutoff(now, 7)
|
||||
assert (cutoff.minute, cutoff.second, cutoff.microsecond) == (0, 0, 0)
|
||||
# 7 whole days back, then floored to the hour.
|
||||
assert cutoff == datetime(2026, 6, 13, 15, 0, 0, 0, tzinfo=timezone.utc)
|
||||
@@ -78,3 +78,84 @@ def test_latest_distinct_on_and_sql_bucketed_history(app):
|
||||
cpu = hist["cpu_pct"]
|
||||
assert cpu, "expected bucketed cpu_pct history"
|
||||
assert all(10.0 <= v <= 30.0 for _, v in cpu)
|
||||
|
||||
|
||||
@_NEEDS_DB
|
||||
def test_rollup_aggregates_old_raw_and_prunes(app):
|
||||
from sqlalchemy import text
|
||||
from steward.core.metrics_retention import rollup_plugin_metrics
|
||||
from steward.models.metrics import PluginMetric
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
hostname = "rollup-host-" + uuid.uuid4().hex[:8]
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
for tbl in ("plugin_metrics", "plugin_metrics_hourly"):
|
||||
await s.execute(
|
||||
text(f"DELETE FROM {tbl} WHERE resource_name LIKE :p"),
|
||||
{"p": hostname + "%"},
|
||||
)
|
||||
old = (now - timedelta(days=10)).replace(minute=5, second=0, microsecond=0)
|
||||
s.add_all([
|
||||
PluginMetric(source_module="host_agent", resource_name=hostname,
|
||||
metric_name="cpu_pct", value=10.0, recorded_at=old),
|
||||
PluginMetric(source_module="host_agent", resource_name=hostname,
|
||||
metric_name="cpu_pct", value=20.0,
|
||||
recorded_at=old.replace(minute=35)), # same hour bucket
|
||||
PluginMetric(source_module="host_agent", resource_name=hostname,
|
||||
metric_name="cpu_pct", value=99.0,
|
||||
recorded_at=now - timedelta(hours=1)), # recent → kept raw
|
||||
])
|
||||
async with s.begin():
|
||||
counts = await rollup_plugin_metrics(s, raw_days=7, rollup_days=90, now=now)
|
||||
hrly = (await s.execute(text(
|
||||
"SELECT value_avg, sample_count FROM plugin_metrics_hourly "
|
||||
"WHERE resource_name = :h AND metric_name = 'cpu_pct'"), {"h": hostname})).all()
|
||||
raw_left = (await s.execute(text(
|
||||
"SELECT count(*) FROM plugin_metrics WHERE resource_name = :h"), {"h": hostname})).scalar()
|
||||
return counts, hrly, raw_left
|
||||
|
||||
counts, hrly, raw_left = asyncio.run(_go())
|
||||
assert counts["buckets_rolled"] == 1
|
||||
assert counts["raw_rows_rolled"] == 2
|
||||
assert len(hrly) == 1
|
||||
assert abs(float(hrly[0][0]) - 15.0) < 0.001 # avg(10, 20)
|
||||
assert hrly[0][1] == 2
|
||||
assert raw_left == 1 # only the recent sample remains
|
||||
|
||||
|
||||
@_NEEDS_DB
|
||||
def test_history_merges_hourly_and_raw(app):
|
||||
from sqlalchemy import text
|
||||
from steward.models.metrics import PluginMetric, PluginMetricHourly
|
||||
from plugins.host_agent.metrics_query import _history_for_host
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
hostname = "merge-host-" + uuid.uuid4().hex[:8]
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
for tbl in ("plugin_metrics", "plugin_metrics_hourly"):
|
||||
await s.execute(
|
||||
text(f"DELETE FROM {tbl} WHERE resource_name LIKE :p"),
|
||||
{"p": hostname + "%"},
|
||||
)
|
||||
old_bucket = (now - timedelta(days=10)).replace(minute=0, second=0, microsecond=0)
|
||||
s.add(PluginMetricHourly(
|
||||
source_module="host_agent", resource_name=hostname, metric_name="cpu_pct",
|
||||
bucket=old_bucket, value_avg=42.0, value_max=50.0, sample_count=120))
|
||||
s.add(PluginMetric(
|
||||
source_module="host_agent", resource_name=hostname, metric_name="cpu_pct",
|
||||
value=77.0, recorded_at=now - timedelta(hours=1)))
|
||||
# 30d range with a 7d raw window → spans the rollup boundary.
|
||||
return await _history_for_host(s, hostname, now - timedelta(days=30), raw_days=7)
|
||||
|
||||
hist = asyncio.run(_go())
|
||||
cpu = hist["cpu_pct"]
|
||||
vals = [v for _, v in cpu]
|
||||
assert 42.0 in vals, "expected the rolled-up hourly point"
|
||||
assert 77.0 in vals, "expected the recent raw point"
|
||||
assert cpu == sorted(cpu, key=lambda p: p[0]), "series must be time-ordered"
|
||||
|
||||
Reference in New Issue
Block a user