bvandeusenandClaude Opus 5.5
ec13f3797d
On the live instance the hourly refresh took 3 min 8 s, averaging a week of raw temp_c samples each time. The hourly averages are now kept in memory. The first refresh after startup reads the whole window. Each later one reads raw samples from the start of the hour the last read ended in, so the hour that was still filling is read again whole. merge_hours folds the new rows in and drops hours older than the window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
224 lines
10 KiB
Python
224 lines
10 KiB
Python
"""Integration: host-metrics read paths (latest values + SQL date_bin history).
|
|
|
|
Validates the two host_agent query helpers that back the slow host views, against
|
|
a live Postgres: the latest-per-(resource,metric) lookup (plugin_metric_latest,
|
|
kept by record_metrics) and the bucket-averaged history (which aggregates in SQL
|
|
instead of shipping raw rows to Python).
|
|
Requires STEWARD_DATABASE_URL.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
_NEEDS_DB = pytest.mark.skipif(
|
|
not os.environ.get("STEWARD_DATABASE_URL"),
|
|
reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)",
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_latest_values_and_sql_bucketed_history(app):
|
|
from sqlalchemy import text
|
|
from steward.core.alerts import record_metrics
|
|
from steward.models.metrics import PluginMetric
|
|
from plugins.host_agent.metrics_query import (
|
|
SOURCE_MODULE, _history_for_host, _latest_metrics_for_host,
|
|
)
|
|
|
|
now = datetime.now(timezone.utc)
|
|
hostname = "metrics-host-" + uuid.uuid4().hex[:8]
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await s.execute(
|
|
text("DELETE FROM plugin_metrics WHERE resource_name LIKE :p"),
|
|
{"p": hostname + "%"},
|
|
)
|
|
rows = []
|
|
# Host-level CPU samples, oldest → newest (10, 20, 30).
|
|
for i, val in enumerate([10.0, 20.0, 30.0]):
|
|
rows.append(PluginMetric(
|
|
source_module=SOURCE_MODULE, resource_name=hostname,
|
|
metric_name="cpu_pct", value=val,
|
|
recorded_at=now - timedelta(minutes=30 - i * 10),
|
|
))
|
|
# A sub-resource (root mount) to exercise the host:% match.
|
|
rows.append(PluginMetric(
|
|
source_module=SOURCE_MODULE, resource_name=hostname + ":/",
|
|
metric_name="disk_used_pct", value=80.0,
|
|
recorded_at=now - timedelta(minutes=5),
|
|
))
|
|
await record_metrics(s, rows)
|
|
|
|
latest = await _latest_metrics_for_host(s, hostname)
|
|
hist = await _history_for_host(s, hostname, now - timedelta(hours=1))
|
|
return latest, hist
|
|
|
|
latest, hist = asyncio.run(_go())
|
|
|
|
# The newest sample per (resource, metric).
|
|
assert latest[hostname]["cpu_pct"] == 30.0
|
|
assert latest[hostname + ":/"]["disk_used_pct"] == 80.0
|
|
|
|
# SQL date_bin aggregation returns cpu_pct buckets; averages stay in range.
|
|
cpu = hist["cpu_pct"]
|
|
assert cpu, "expected bucketed cpu_pct history"
|
|
assert all(10.0 <= v <= 30.0 for _, v in cpu)
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_rollup_aggregates_old_raw_and_prunes(app):
|
|
from sqlalchemy import text
|
|
from steward.core.metrics_retention import rollup_plugin_metrics
|
|
from steward.models.metrics import PluginMetric
|
|
|
|
now = datetime.now(timezone.utc)
|
|
hostname = "rollup-host-" + uuid.uuid4().hex[:8]
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
for tbl in ("plugin_metrics", "plugin_metrics_hourly"):
|
|
await s.execute(
|
|
text(f"DELETE FROM {tbl} WHERE resource_name LIKE :p"),
|
|
{"p": hostname + "%"},
|
|
)
|
|
old = (now - timedelta(days=10)).replace(minute=5, second=0, microsecond=0)
|
|
s.add_all([
|
|
PluginMetric(source_module="host_agent", resource_name=hostname,
|
|
metric_name="cpu_pct", value=10.0, recorded_at=old),
|
|
PluginMetric(source_module="host_agent", resource_name=hostname,
|
|
metric_name="cpu_pct", value=20.0,
|
|
recorded_at=old.replace(minute=35)), # same hour bucket
|
|
PluginMetric(source_module="host_agent", resource_name=hostname,
|
|
metric_name="cpu_pct", value=99.0,
|
|
recorded_at=now - timedelta(hours=1)), # recent → kept raw
|
|
])
|
|
async with s.begin():
|
|
counts = await rollup_plugin_metrics(s, raw_days=7, rollup_days=90, now=now)
|
|
hrly = (await s.execute(text(
|
|
"SELECT value_avg, sample_count FROM plugin_metrics_hourly "
|
|
"WHERE resource_name = :h AND metric_name = 'cpu_pct'"), {"h": hostname})).all()
|
|
raw_left = (await s.execute(text(
|
|
"SELECT count(*) FROM plugin_metrics WHERE resource_name = :h"), {"h": hostname})).scalar()
|
|
return counts, hrly, raw_left
|
|
|
|
counts, hrly, raw_left = asyncio.run(_go())
|
|
# The counts span the whole table, which other tests share (docker
|
|
# snapshots are recorded at their own, older times, #5752): this host's
|
|
# bucket is checked exactly below.
|
|
assert counts["buckets_rolled"] >= 1
|
|
assert counts["raw_rows_rolled"] >= 2
|
|
assert len(hrly) == 1
|
|
assert abs(float(hrly[0][0]) - 15.0) < 0.001 # avg(10, 20)
|
|
assert hrly[0][1] == 2
|
|
assert raw_left == 1 # only the recent sample remains
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_history_merges_hourly_and_raw(app):
|
|
from sqlalchemy import text
|
|
from steward.models.metrics import PluginMetric, PluginMetricHourly
|
|
from plugins.host_agent.metrics_query import _history_for_host
|
|
|
|
now = datetime.now(timezone.utc)
|
|
hostname = "merge-host-" + uuid.uuid4().hex[:8]
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
for tbl in ("plugin_metrics", "plugin_metrics_hourly"):
|
|
await s.execute(
|
|
text(f"DELETE FROM {tbl} WHERE resource_name LIKE :p"),
|
|
{"p": hostname + "%"},
|
|
)
|
|
old_bucket = (now - timedelta(days=10)).replace(minute=0, second=0, microsecond=0)
|
|
s.add(PluginMetricHourly(
|
|
source_module="host_agent", resource_name=hostname, metric_name="cpu_pct",
|
|
bucket=old_bucket, value_avg=42.0, value_max=50.0, sample_count=120))
|
|
s.add(PluginMetric(
|
|
source_module="host_agent", resource_name=hostname, metric_name="cpu_pct",
|
|
value=77.0, recorded_at=now - timedelta(hours=1)))
|
|
# 30d range with a 7d raw window → spans the rollup boundary.
|
|
return await _history_for_host(s, hostname, now - timedelta(days=30), raw_days=7)
|
|
|
|
hist = asyncio.run(_go())
|
|
cpu = hist["cpu_pct"]
|
|
vals = [v for _, v in cpu]
|
|
assert 42.0 in vals, "expected the rolled-up hourly point"
|
|
assert 77.0 in vals, "expected the recent raw point"
|
|
assert cpu == sorted(cpu, key=lambda p: p[0]), "series must be time-ordered"
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_sensors_are_judged_from_raw_hours_and_rollups_together(app):
|
|
"""The live instance keeps raw samples for 7 days and rolls up only what
|
|
is older, so a week of hourly rollups alone is empty: the first deploy of
|
|
#5783 judged nothing. refresh reads both, by sensor name."""
|
|
import math
|
|
|
|
from sqlalchemy import text
|
|
from steward.models.metrics import PluginMetric, PluginMetricHourly, PluginMetricLatest
|
|
from plugins.host_agent import sensors
|
|
|
|
now = datetime.now(timezone.utc).replace(minute=30, second=0, microsecond=0)
|
|
host = "sensor-host-" + uuid.uuid4().hex[:8]
|
|
|
|
def cpu(i):
|
|
return 65 + 15 * math.sin(i / 24 * 2 * math.pi)
|
|
|
|
series = {"Package id 0": cpu, "SYSTIN": lambda i: 114 - 0.1 * (cpu(i) - 65),
|
|
"CPUTIN": lambda i: 40 + 0.3 * (cpu(i) - 65)}
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
for tbl in ("plugin_metrics", "plugin_metrics_hourly", "plugin_metric_latest"):
|
|
await s.execute(text(f"DELETE FROM {tbl} WHERE resource_name LIKE :p"), {"p": host + "%"})
|
|
for label, f in series.items():
|
|
res = f"{host}:temp:{label}"
|
|
for i in range(100): # 100 h back; the oldest 30 rolled up
|
|
at = now - timedelta(hours=100 - i)
|
|
if i < 30:
|
|
s.add(PluginMetricHourly(source_module="host_agent", resource_name=res,
|
|
metric_name="temp_c", bucket=at.replace(minute=0),
|
|
value_avg=f(i), value_max=f(i), sample_count=1))
|
|
else: # two samples an hour, averaged by refresh
|
|
for m, d in ((5, -0.2), (35, 0.2)):
|
|
s.add(PluginMetric(source_module="host_agent", resource_name=res,
|
|
metric_name="temp_c", value=f(i) + d,
|
|
recorded_at=at.replace(minute=m)))
|
|
s.add(PluginMetricLatest(source_module="host_agent", resource_name=res,
|
|
metric_name="temp_c", value=f(99), recorded_at=now))
|
|
return await sensors.refresh(s, now)
|
|
|
|
sensors._hours.clear()
|
|
sensors._read_from = None
|
|
got = asyncio.run(_go())
|
|
assert list(got[host]) == ["SYSTIN"]
|
|
assert got[host]["SYSTIN"].startswith("falls as the CPU warms (r -1.00 over 100 h")
|
|
|
|
async def _again(): # the next hour reads only the hour it ended in
|
|
async with app.db_sessionmaker() as s:
|
|
return await sensors.refresh(s, now + timedelta(hours=1))
|
|
|
|
assert sensors._read_from == now.replace(minute=0)
|
|
again = asyncio.run(_again())
|
|
assert again[host]["SYSTIN"].startswith("falls as the CPU warms (r -1.00 over 100 h")
|