bvandeusenandClaude Opus 5.5
9a11f5788f
The second asyncio.run got a pooled connection bound to the first loop. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
222 lines
10 KiB
Python
222 lines
10 KiB
Python
"""Integration: host-metrics read paths (latest values + SQL date_bin history).
|
|
|
|
Validates the two host_agent query helpers that back the slow host views, against
|
|
a live Postgres: the latest-per-(resource,metric) lookup (plugin_metric_latest,
|
|
kept by record_metrics) and the bucket-averaged history (which aggregates in SQL
|
|
instead of shipping raw rows to Python).
|
|
Requires STEWARD_DATABASE_URL.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
_NEEDS_DB = pytest.mark.skipif(
|
|
not os.environ.get("STEWARD_DATABASE_URL"),
|
|
reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)",
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_latest_values_and_sql_bucketed_history(app):
|
|
from sqlalchemy import text
|
|
from steward.core.alerts import record_metrics
|
|
from steward.models.metrics import PluginMetric
|
|
from plugins.host_agent.metrics_query import (
|
|
SOURCE_MODULE, _history_for_host, _latest_metrics_for_host,
|
|
)
|
|
|
|
now = datetime.now(timezone.utc)
|
|
hostname = "metrics-host-" + uuid.uuid4().hex[:8]
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await s.execute(
|
|
text("DELETE FROM plugin_metrics WHERE resource_name LIKE :p"),
|
|
{"p": hostname + "%"},
|
|
)
|
|
rows = []
|
|
# Host-level CPU samples, oldest → newest (10, 20, 30).
|
|
for i, val in enumerate([10.0, 20.0, 30.0]):
|
|
rows.append(PluginMetric(
|
|
source_module=SOURCE_MODULE, resource_name=hostname,
|
|
metric_name="cpu_pct", value=val,
|
|
recorded_at=now - timedelta(minutes=30 - i * 10),
|
|
))
|
|
# A sub-resource (root mount) to exercise the host:% match.
|
|
rows.append(PluginMetric(
|
|
source_module=SOURCE_MODULE, resource_name=hostname + ":/",
|
|
metric_name="disk_used_pct", value=80.0,
|
|
recorded_at=now - timedelta(minutes=5),
|
|
))
|
|
await record_metrics(s, rows)
|
|
|
|
latest = await _latest_metrics_for_host(s, hostname)
|
|
hist = await _history_for_host(s, hostname, now - timedelta(hours=1))
|
|
return latest, hist
|
|
|
|
latest, hist = asyncio.run(_go())
|
|
|
|
# The newest sample per (resource, metric).
|
|
assert latest[hostname]["cpu_pct"] == 30.0
|
|
assert latest[hostname + ":/"]["disk_used_pct"] == 80.0
|
|
|
|
# SQL date_bin aggregation returns cpu_pct buckets; averages stay in range.
|
|
cpu = hist["cpu_pct"]
|
|
assert cpu, "expected bucketed cpu_pct history"
|
|
assert all(10.0 <= v <= 30.0 for _, v in cpu)
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_rollup_aggregates_old_raw_and_prunes(app):
|
|
from sqlalchemy import text
|
|
from steward.core.metrics_retention import rollup_plugin_metrics
|
|
from steward.models.metrics import PluginMetric
|
|
|
|
now = datetime.now(timezone.utc)
|
|
hostname = "rollup-host-" + uuid.uuid4().hex[:8]
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
for tbl in ("plugin_metrics", "plugin_metrics_hourly"):
|
|
await s.execute(
|
|
text(f"DELETE FROM {tbl} WHERE resource_name LIKE :p"),
|
|
{"p": hostname + "%"},
|
|
)
|
|
old = (now - timedelta(days=10)).replace(minute=5, second=0, microsecond=0)
|
|
s.add_all([
|
|
PluginMetric(source_module="host_agent", resource_name=hostname,
|
|
metric_name="cpu_pct", value=10.0, recorded_at=old),
|
|
PluginMetric(source_module="host_agent", resource_name=hostname,
|
|
metric_name="cpu_pct", value=20.0,
|
|
recorded_at=old.replace(minute=35)), # same hour bucket
|
|
PluginMetric(source_module="host_agent", resource_name=hostname,
|
|
metric_name="cpu_pct", value=99.0,
|
|
recorded_at=now - timedelta(hours=1)), # recent → kept raw
|
|
])
|
|
async with s.begin():
|
|
counts = await rollup_plugin_metrics(s, raw_days=7, rollup_days=90, now=now)
|
|
hrly = (await s.execute(text(
|
|
"SELECT value_avg, sample_count FROM plugin_metrics_hourly "
|
|
"WHERE resource_name = :h AND metric_name = 'cpu_pct'"), {"h": hostname})).all()
|
|
raw_left = (await s.execute(text(
|
|
"SELECT count(*) FROM plugin_metrics WHERE resource_name = :h"), {"h": hostname})).scalar()
|
|
return counts, hrly, raw_left
|
|
|
|
counts, hrly, raw_left = asyncio.run(_go())
|
|
# The counts span the whole table, which other tests share (docker
|
|
# snapshots are recorded at their own, older times, #5752): this host's
|
|
# bucket is checked exactly below.
|
|
assert counts["buckets_rolled"] >= 1
|
|
assert counts["raw_rows_rolled"] >= 2
|
|
assert len(hrly) == 1
|
|
assert abs(float(hrly[0][0]) - 15.0) < 0.001 # avg(10, 20)
|
|
assert hrly[0][1] == 2
|
|
assert raw_left == 1 # only the recent sample remains
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_history_merges_hourly_and_raw(app):
|
|
from sqlalchemy import text
|
|
from steward.models.metrics import PluginMetric, PluginMetricHourly
|
|
from plugins.host_agent.metrics_query import _history_for_host
|
|
|
|
now = datetime.now(timezone.utc)
|
|
hostname = "merge-host-" + uuid.uuid4().hex[:8]
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
for tbl in ("plugin_metrics", "plugin_metrics_hourly"):
|
|
await s.execute(
|
|
text(f"DELETE FROM {tbl} WHERE resource_name LIKE :p"),
|
|
{"p": hostname + "%"},
|
|
)
|
|
old_bucket = (now - timedelta(days=10)).replace(minute=0, second=0, microsecond=0)
|
|
s.add(PluginMetricHourly(
|
|
source_module="host_agent", resource_name=hostname, metric_name="cpu_pct",
|
|
bucket=old_bucket, value_avg=42.0, value_max=50.0, sample_count=120))
|
|
s.add(PluginMetric(
|
|
source_module="host_agent", resource_name=hostname, metric_name="cpu_pct",
|
|
value=77.0, recorded_at=now - timedelta(hours=1)))
|
|
# 30d range with a 7d raw window → spans the rollup boundary.
|
|
return await _history_for_host(s, hostname, now - timedelta(days=30), raw_days=7)
|
|
|
|
hist = asyncio.run(_go())
|
|
cpu = hist["cpu_pct"]
|
|
vals = [v for _, v in cpu]
|
|
assert 42.0 in vals, "expected the rolled-up hourly point"
|
|
assert 77.0 in vals, "expected the recent raw point"
|
|
assert cpu == sorted(cpu, key=lambda p: p[0]), "series must be time-ordered"
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_sensors_are_judged_from_raw_hours_and_rollups_together(app):
|
|
"""The live instance keeps raw samples for 7 days and rolls up only what
|
|
is older, so a week of hourly rollups alone is empty: the first deploy of
|
|
#5783 judged nothing. refresh reads both, by sensor name."""
|
|
import math
|
|
|
|
from sqlalchemy import text
|
|
from steward.models.metrics import PluginMetric, PluginMetricHourly, PluginMetricLatest
|
|
from plugins.host_agent import sensors
|
|
|
|
now = datetime.now(timezone.utc).replace(minute=30, second=0, microsecond=0)
|
|
host = "sensor-host-" + uuid.uuid4().hex[:8]
|
|
|
|
def cpu(i):
|
|
return 65 + 15 * math.sin(i / 24 * 2 * math.pi)
|
|
|
|
series = {"Package id 0": cpu, "SYSTIN": lambda i: 114 - 0.1 * (cpu(i) - 65),
|
|
"CPUTIN": lambda i: 40 + 0.3 * (cpu(i) - 65)}
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
for tbl in ("plugin_metrics", "plugin_metrics_hourly", "plugin_metric_latest"):
|
|
await s.execute(text(f"DELETE FROM {tbl} WHERE resource_name LIKE :p"), {"p": host + "%"})
|
|
for label, f in series.items():
|
|
res = f"{host}:temp:{label}"
|
|
for i in range(100): # 100 h back; the oldest 30 rolled up
|
|
at = now - timedelta(hours=100 - i)
|
|
if i < 30:
|
|
s.add(PluginMetricHourly(source_module="host_agent", resource_name=res,
|
|
metric_name="temp_c", bucket=at.replace(minute=0),
|
|
value_avg=f(i), value_max=f(i), sample_count=1))
|
|
else: # two samples an hour, averaged by refresh
|
|
for m, d in ((5, -0.2), (35, 0.2)):
|
|
s.add(PluginMetric(source_module="host_agent", resource_name=res,
|
|
metric_name="temp_c", value=f(i) + d,
|
|
recorded_at=at.replace(minute=m)))
|
|
s.add(PluginMetricLatest(source_module="host_agent", resource_name=res,
|
|
metric_name="temp_c", value=f(99), recorded_at=now))
|
|
first = await sensors.refresh(s, now)
|
|
read_from = sensors._read_from
|
|
# The next hour reads only the hour the last read ended in; one
|
|
# event loop, since the engine's pooled connections belong to it.
|
|
return first, read_from, await sensors.refresh(s, now + timedelta(hours=1))
|
|
|
|
sensors._hours.clear()
|
|
sensors._read_from = None
|
|
got, read_from, again = asyncio.run(_go())
|
|
assert list(got[host]) == ["SYSTIN"]
|
|
assert got[host]["SYSTIN"].startswith("falls as the CPU warms (r -1.00 over 100 h")
|
|
assert read_from == now.replace(minute=0)
|
|
assert again[host]["SYSTIN"].startswith("falls as the CPU warms (r -1.00 over 100 h")
|