bvandeusenandClaude Opus 5.5
2ade4ebc46
Rollups exist only for hours older than the raw window (7 days by default), so the first deploy read an empty week and judged nothing. refresh now averages the raw samples by the hour and adds any rolled-up hours in the window. Sensors are named from plugin_metric_latest, so each series is read through its index instead of a pattern over every host_agent row. An integration test covers both halves against Postgres. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
214 lines
9.6 KiB
Python
214 lines
9.6 KiB
Python
"""Integration: host-metrics read paths (latest values + SQL date_bin history).
|
|
|
|
Validates the two host_agent query helpers that back the slow host views, against
|
|
a live Postgres: the latest-per-(resource,metric) lookup (plugin_metric_latest,
|
|
kept by record_metrics) and the bucket-averaged history (which aggregates in SQL
|
|
instead of shipping raw rows to Python).
|
|
Requires STEWARD_DATABASE_URL.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
_NEEDS_DB = pytest.mark.skipif(
|
|
not os.environ.get("STEWARD_DATABASE_URL"),
|
|
reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)",
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_latest_values_and_sql_bucketed_history(app):
|
|
from sqlalchemy import text
|
|
from steward.core.alerts import record_metrics
|
|
from steward.models.metrics import PluginMetric
|
|
from plugins.host_agent.metrics_query import (
|
|
SOURCE_MODULE, _history_for_host, _latest_metrics_for_host,
|
|
)
|
|
|
|
now = datetime.now(timezone.utc)
|
|
hostname = "metrics-host-" + uuid.uuid4().hex[:8]
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await s.execute(
|
|
text("DELETE FROM plugin_metrics WHERE resource_name LIKE :p"),
|
|
{"p": hostname + "%"},
|
|
)
|
|
rows = []
|
|
# Host-level CPU samples, oldest → newest (10, 20, 30).
|
|
for i, val in enumerate([10.0, 20.0, 30.0]):
|
|
rows.append(PluginMetric(
|
|
source_module=SOURCE_MODULE, resource_name=hostname,
|
|
metric_name="cpu_pct", value=val,
|
|
recorded_at=now - timedelta(minutes=30 - i * 10),
|
|
))
|
|
# A sub-resource (root mount) to exercise the host:% match.
|
|
rows.append(PluginMetric(
|
|
source_module=SOURCE_MODULE, resource_name=hostname + ":/",
|
|
metric_name="disk_used_pct", value=80.0,
|
|
recorded_at=now - timedelta(minutes=5),
|
|
))
|
|
await record_metrics(s, rows)
|
|
|
|
latest = await _latest_metrics_for_host(s, hostname)
|
|
hist = await _history_for_host(s, hostname, now - timedelta(hours=1))
|
|
return latest, hist
|
|
|
|
latest, hist = asyncio.run(_go())
|
|
|
|
# The newest sample per (resource, metric).
|
|
assert latest[hostname]["cpu_pct"] == 30.0
|
|
assert latest[hostname + ":/"]["disk_used_pct"] == 80.0
|
|
|
|
# SQL date_bin aggregation returns cpu_pct buckets; averages stay in range.
|
|
cpu = hist["cpu_pct"]
|
|
assert cpu, "expected bucketed cpu_pct history"
|
|
assert all(10.0 <= v <= 30.0 for _, v in cpu)
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_rollup_aggregates_old_raw_and_prunes(app):
|
|
from sqlalchemy import text
|
|
from steward.core.metrics_retention import rollup_plugin_metrics
|
|
from steward.models.metrics import PluginMetric
|
|
|
|
now = datetime.now(timezone.utc)
|
|
hostname = "rollup-host-" + uuid.uuid4().hex[:8]
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
for tbl in ("plugin_metrics", "plugin_metrics_hourly"):
|
|
await s.execute(
|
|
text(f"DELETE FROM {tbl} WHERE resource_name LIKE :p"),
|
|
{"p": hostname + "%"},
|
|
)
|
|
old = (now - timedelta(days=10)).replace(minute=5, second=0, microsecond=0)
|
|
s.add_all([
|
|
PluginMetric(source_module="host_agent", resource_name=hostname,
|
|
metric_name="cpu_pct", value=10.0, recorded_at=old),
|
|
PluginMetric(source_module="host_agent", resource_name=hostname,
|
|
metric_name="cpu_pct", value=20.0,
|
|
recorded_at=old.replace(minute=35)), # same hour bucket
|
|
PluginMetric(source_module="host_agent", resource_name=hostname,
|
|
metric_name="cpu_pct", value=99.0,
|
|
recorded_at=now - timedelta(hours=1)), # recent → kept raw
|
|
])
|
|
async with s.begin():
|
|
counts = await rollup_plugin_metrics(s, raw_days=7, rollup_days=90, now=now)
|
|
hrly = (await s.execute(text(
|
|
"SELECT value_avg, sample_count FROM plugin_metrics_hourly "
|
|
"WHERE resource_name = :h AND metric_name = 'cpu_pct'"), {"h": hostname})).all()
|
|
raw_left = (await s.execute(text(
|
|
"SELECT count(*) FROM plugin_metrics WHERE resource_name = :h"), {"h": hostname})).scalar()
|
|
return counts, hrly, raw_left
|
|
|
|
counts, hrly, raw_left = asyncio.run(_go())
|
|
# The counts span the whole table, which other tests share (docker
|
|
# snapshots are recorded at their own, older times, #5752): this host's
|
|
# bucket is checked exactly below.
|
|
assert counts["buckets_rolled"] >= 1
|
|
assert counts["raw_rows_rolled"] >= 2
|
|
assert len(hrly) == 1
|
|
assert abs(float(hrly[0][0]) - 15.0) < 0.001 # avg(10, 20)
|
|
assert hrly[0][1] == 2
|
|
assert raw_left == 1 # only the recent sample remains
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_history_merges_hourly_and_raw(app):
|
|
from sqlalchemy import text
|
|
from steward.models.metrics import PluginMetric, PluginMetricHourly
|
|
from plugins.host_agent.metrics_query import _history_for_host
|
|
|
|
now = datetime.now(timezone.utc)
|
|
hostname = "merge-host-" + uuid.uuid4().hex[:8]
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
for tbl in ("plugin_metrics", "plugin_metrics_hourly"):
|
|
await s.execute(
|
|
text(f"DELETE FROM {tbl} WHERE resource_name LIKE :p"),
|
|
{"p": hostname + "%"},
|
|
)
|
|
old_bucket = (now - timedelta(days=10)).replace(minute=0, second=0, microsecond=0)
|
|
s.add(PluginMetricHourly(
|
|
source_module="host_agent", resource_name=hostname, metric_name="cpu_pct",
|
|
bucket=old_bucket, value_avg=42.0, value_max=50.0, sample_count=120))
|
|
s.add(PluginMetric(
|
|
source_module="host_agent", resource_name=hostname, metric_name="cpu_pct",
|
|
value=77.0, recorded_at=now - timedelta(hours=1)))
|
|
# 30d range with a 7d raw window → spans the rollup boundary.
|
|
return await _history_for_host(s, hostname, now - timedelta(days=30), raw_days=7)
|
|
|
|
hist = asyncio.run(_go())
|
|
cpu = hist["cpu_pct"]
|
|
vals = [v for _, v in cpu]
|
|
assert 42.0 in vals, "expected the rolled-up hourly point"
|
|
assert 77.0 in vals, "expected the recent raw point"
|
|
assert cpu == sorted(cpu, key=lambda p: p[0]), "series must be time-ordered"
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_sensors_are_judged_from_raw_hours_and_rollups_together(app):
|
|
"""The live instance keeps raw samples for 7 days and rolls up only what
|
|
is older, so a week of hourly rollups alone is empty: the first deploy of
|
|
#5783 judged nothing. refresh reads both, by sensor name."""
|
|
import math
|
|
|
|
from sqlalchemy import text
|
|
from steward.models.metrics import PluginMetric, PluginMetricHourly, PluginMetricLatest
|
|
from plugins.host_agent import sensors
|
|
|
|
now = datetime.now(timezone.utc).replace(minute=30, second=0, microsecond=0)
|
|
host = "sensor-host-" + uuid.uuid4().hex[:8]
|
|
|
|
def cpu(i):
|
|
return 65 + 15 * math.sin(i / 24 * 2 * math.pi)
|
|
|
|
series = {"Package id 0": cpu, "SYSTIN": lambda i: 114 - 0.1 * (cpu(i) - 65),
|
|
"CPUTIN": lambda i: 40 + 0.3 * (cpu(i) - 65)}
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
for tbl in ("plugin_metrics", "plugin_metrics_hourly", "plugin_metric_latest"):
|
|
await s.execute(text(f"DELETE FROM {tbl} WHERE resource_name LIKE :p"), {"p": host + "%"})
|
|
for label, f in series.items():
|
|
res = f"{host}:temp:{label}"
|
|
for i in range(100): # 100 h back; the oldest 30 rolled up
|
|
at = now - timedelta(hours=100 - i)
|
|
if i < 30:
|
|
s.add(PluginMetricHourly(source_module="host_agent", resource_name=res,
|
|
metric_name="temp_c", bucket=at.replace(minute=0),
|
|
value_avg=f(i), value_max=f(i), sample_count=1))
|
|
else: # two samples an hour, averaged by refresh
|
|
for m, d in ((5, -0.2), (35, 0.2)):
|
|
s.add(PluginMetric(source_module="host_agent", resource_name=res,
|
|
metric_name="temp_c", value=f(i) + d,
|
|
recorded_at=at.replace(minute=m)))
|
|
s.add(PluginMetricLatest(source_module="host_agent", resource_name=res,
|
|
metric_name="temp_c", value=f(99), recorded_at=now))
|
|
return await sensors.refresh(s, now)
|
|
|
|
got = asyncio.run(_go())
|
|
assert list(got[host]) == ["SYSTIN"]
|
|
assert got[host]["SYSTIN"].startswith("falls as the CPU warms (r -1.00 over 100 h")
|