bvandeusenandClaude Opus 5.5
4c84bc8560
A metric registered with learn_normal() gets, for every resource reporting it, a usual hold (the 90th percentile across past days of each hour's p95) and a usual peak for each slot: - an hour of the day once 5 days of history exist; - an hour of the week once history reaches back 3 weeks, so weekly jobs are learned too. Each slot takes in the hours either side, so a job that runs a little late stays normal. One SQL statement relearns everything hourly into metric_norms, from the rollup plus the raw window not rolled up yet; norms_at() reads the slot a moment falls in. Nothing is registered yet: Traefik (#5741) and hosts (#5742) opt in. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
107 lines
4.0 KiB
Python
107 lines
4.0 KiB
Python
"""Integration: Steward learns each metric's normal per hour (#5739).
|
||
|
||
The operator's case (System 182): a nightly backup that runs hot at 02:00
|
||
must read as normal at 02:00, and the same level at 14:00 must not, so the
|
||
normal has to be per hour and built from what each hour held, not its peak.
|
||
Requires STEWARD_DATABASE_URL.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import os
|
||
import uuid
|
||
from datetime import datetime, timedelta, timezone
|
||
|
||
import pytest
|
||
|
||
pytestmark = pytest.mark.integration
|
||
|
||
|
||
@pytest.fixture
|
||
def app():
|
||
if not os.environ.get("STEWARD_DATABASE_URL"):
|
||
pytest.skip("needs Postgres")
|
||
from steward.app import create_app
|
||
return create_app(testing=False)
|
||
|
||
|
||
@pytest.fixture
|
||
def learned():
|
||
from steward.core import norms
|
||
norms.learn_normal("normtest", "cpu_pct", 5)
|
||
yield
|
||
norms.LEARNED.pop(("normtest", "cpu_pct"), None)
|
||
|
||
|
||
# Monday 00:00 UTC, so week slots are easy to read.
|
||
NOW = datetime(2026, 10, 12, 0, 0, tzinfo=timezone.utc)
|
||
|
||
|
||
def _hours(res: str, days: int, level):
|
||
"""An hourly rollup row for each of the last `days` days' hours, held at
|
||
level(day, hour) -> (p95, max)."""
|
||
from steward.models.metrics import PluginMetricHourly
|
||
rows = []
|
||
for d in range(1, days + 1):
|
||
for h in range(24):
|
||
bucket = NOW - timedelta(days=d) + timedelta(hours=h)
|
||
p95, peak = level(bucket, h)
|
||
rows.append(PluginMetricHourly(source_module="normtest", resource_name=res,
|
||
metric_name="cpu_pct", bucket=bucket,
|
||
value_avg=p95, value_max=peak, value_p95=p95,
|
||
sample_count=60))
|
||
return rows
|
||
|
||
|
||
def _learn(app, rows):
|
||
from steward.core.norms import norms_at, rebuild_norms
|
||
|
||
async def _go():
|
||
async with app.db_sessionmaker() as s:
|
||
async with s.begin():
|
||
s.add_all(rows)
|
||
async with s.begin():
|
||
await rebuild_norms(s, NOW)
|
||
res = rows[0].resource_name
|
||
out = {}
|
||
for at in (NOW.replace(hour=2), NOW.replace(hour=3), NOW.replace(hour=14),
|
||
NOW + timedelta(days=1, hours=2)):
|
||
got = await norms_at(s, [("normtest", res, "cpu_pct")], at)
|
||
out[at] = got.get(("normtest", res, "cpu_pct"))
|
||
return out
|
||
|
||
return asyncio.run(_go())
|
||
|
||
|
||
def test_a_nightly_backup_is_normal_at_its_hour_and_not_at_another(app, learned):
|
||
res = f"night-{uuid.uuid4().hex[:8]}"
|
||
# Every night 02:00–03:00 holds 90%; the rest of the day holds 10% and
|
||
# spikes briefly to 95% (a high max, a low p95).
|
||
got = _learn(app, _hours(res, 7, lambda b, h: (90.0, 95.0) if h == 2 else (10.0, 95.0)))
|
||
at2, at3, at14 = (got[NOW.replace(hour=h)] for h in (2, 3, 14))
|
||
assert at2.kind == "day" and at2.days_seen == 7
|
||
assert at2.usual_hold == 90.0
|
||
# 03:00 takes in 02:00 as a neighbour: a backup that runs late stays normal.
|
||
assert at3.usual_hold == 90.0
|
||
# 14:00 holds 10: the spikes raised its peak, never its hold.
|
||
assert at14.usual_hold == 10.0 and at14.usual_peak == 95.0
|
||
|
||
|
||
def test_too_little_history_learns_no_normal(app, learned):
|
||
res = f"young-{uuid.uuid4().hex[:8]}"
|
||
got = _learn(app, _hours(res, 4, lambda b, h: (10.0, 10.0)))
|
||
assert all(n is None for n in got.values())
|
||
|
||
|
||
def test_three_weeks_of_history_learns_the_hour_of_the_week(app, learned):
|
||
res = f"weekly-{uuid.uuid4().hex[:8]}"
|
||
# A weekly job: Tuesdays 02:00 hold 80; every other hour holds 5.
|
||
got = _learn(app, _hours(res, 22, lambda b, h: (80.0, 80.0) if b.weekday() == 1 and h == 2
|
||
else (5.0, 5.0)))
|
||
tuesday_2 = got[NOW + timedelta(days=1, hours=2)]
|
||
monday_2 = got[NOW.replace(hour=2)]
|
||
assert tuesday_2.kind == "week" and tuesday_2.slot == 26 and tuesday_2.usual_hold == 80.0
|
||
# Monday 02:00 is far from Tuesday's job: on an hour-of-day normal it
|
||
# would have been learned as the job's level.
|
||
assert monday_2.usual_hold == 5.0
|