perf(host_agent): aggregate metric history in SQL; DISTINCT ON for latest
Follow-on to the plugin_metrics indexes. plugin_metrics is already retention- bounded (core cleanup prunes > data.retention_days, default 90d) and charts top out at 30d, so the cost wasn't growth — it was the read path shipping raw rows to Python. - _history_for_host: bucket + average in SQL via date_bin (epoch-aligned, ~120 buckets) instead of fetching every raw sample (a 30d range was hundreds of thousands of rows) and downsampling in Python. Uses the new (source_module, resource_name, recorded_at) index. - _latest_metrics_for_host: DISTINCT ON (resource_name, metric_name) ORDER BY recorded_at DESC — newest row per group in one index-ordered pass, replacing the GROUP-BY-max subquery self-joined back to the whole history. - Integration test validates both against Postgres. Deliberately not a materialized raw→hourly rollup: these query-side changes deliver the speed; a rollup would additionally cut storage and remains a future option if scale demands it. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC
This commit is contained in:
@@ -0,0 +1,80 @@
|
||||
"""Integration: host-metrics read paths (DISTINCT ON latest + SQL date_bin history).
|
||||
|
||||
Validates the two host_agent query helpers that back the slow host views, against
|
||||
a live Postgres: the latest-per-(resource,metric) lookup and the bucket-averaged
|
||||
history (which now aggregates in SQL instead of shipping raw rows to Python).
|
||||
Requires STEWARD_DATABASE_URL.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
_NEEDS_DB = pytest.mark.skipif(
|
||||
not os.environ.get("STEWARD_DATABASE_URL"),
|
||||
reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
if not os.environ.get("STEWARD_DATABASE_URL"):
|
||||
pytest.skip("needs Postgres")
|
||||
from steward.app import create_app
|
||||
return create_app(testing=False)
|
||||
|
||||
|
||||
@_NEEDS_DB
|
||||
def test_latest_distinct_on_and_sql_bucketed_history(app):
|
||||
from sqlalchemy import text
|
||||
from steward.models.metrics import PluginMetric
|
||||
from plugins.host_agent.routes import (
|
||||
SOURCE_MODULE, _history_for_host, _latest_metrics_for_host,
|
||||
)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
hostname = "metrics-host-" + uuid.uuid4().hex[:8]
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await s.execute(
|
||||
text("DELETE FROM plugin_metrics WHERE resource_name LIKE :p"),
|
||||
{"p": hostname + "%"},
|
||||
)
|
||||
rows = []
|
||||
# Host-level CPU samples, oldest → newest (10, 20, 30).
|
||||
for i, val in enumerate([10.0, 20.0, 30.0]):
|
||||
rows.append(PluginMetric(
|
||||
source_module=SOURCE_MODULE, resource_name=hostname,
|
||||
metric_name="cpu_pct", value=val,
|
||||
recorded_at=now - timedelta(minutes=30 - i * 10),
|
||||
))
|
||||
# A sub-resource (root mount) to exercise the host:% match.
|
||||
rows.append(PluginMetric(
|
||||
source_module=SOURCE_MODULE, resource_name=hostname + ":/",
|
||||
metric_name="disk_used_pct", value=80.0,
|
||||
recorded_at=now - timedelta(minutes=5),
|
||||
))
|
||||
s.add_all(rows)
|
||||
|
||||
latest = await _latest_metrics_for_host(s, hostname)
|
||||
hist = await _history_for_host(s, hostname, now - timedelta(hours=1))
|
||||
return latest, hist
|
||||
|
||||
latest, hist = asyncio.run(_go())
|
||||
|
||||
# DISTINCT ON returns the newest sample per (resource, metric).
|
||||
assert latest[hostname]["cpu_pct"] == 30.0
|
||||
assert latest[hostname + ":/"]["disk_used_pct"] == 80.0
|
||||
|
||||
# SQL date_bin aggregation returns cpu_pct buckets; averages stay in range.
|
||||
cpu = hist["cpu_pct"]
|
||||
assert cpu, "expected bucketed cpu_pct history"
|
||||
assert all(10.0 <= v <= 30.0 for _, v in cpu)
|
||||
Reference in New Issue
Block a user