"""Integration: Steward learns each metric's normal per hour (#5739). The operator's case (System 182): a nightly backup that runs hot at 02:00 must read as normal at 02:00, and the same level at 14:00 must not, so the normal has to be per hour and built from what each hour held, not its peak. Requires STEWARD_DATABASE_URL. """ from __future__ import annotations import asyncio import os import uuid from datetime import datetime, timedelta, timezone import pytest pytestmark = pytest.mark.integration @pytest.fixture def app(): if not os.environ.get("STEWARD_DATABASE_URL"): pytest.skip("needs Postgres") from steward.app import create_app return create_app(testing=False) @pytest.fixture def learned(): from steward.core import norms norms.learn_normal("normtest", "cpu_pct", 5) yield norms.LEARNED.pop(("normtest", "cpu_pct"), None) # Monday 00:00 UTC, so week slots are easy to read. NOW = datetime(2026, 10, 12, 0, 0, tzinfo=timezone.utc) def _hours(res: str, days: int, level): """An hourly rollup row for each of the last `days` days' hours, held at level(day, hour) -> (p95, max).""" from steward.models.metrics import PluginMetricHourly rows = [] for d in range(1, days + 1): for h in range(24): bucket = NOW - timedelta(days=d) + timedelta(hours=h) p95, peak = level(bucket, h) rows.append(PluginMetricHourly(source_module="normtest", resource_name=res, metric_name="cpu_pct", bucket=bucket, value_avg=p95, value_max=peak, value_p95=p95, sample_count=60)) return rows def _learn(app, rows): from steward.core.norms import norms_at, rebuild_norms async def _go(): async with app.db_sessionmaker() as s: async with s.begin(): s.add_all(rows) async with s.begin(): await rebuild_norms(s, NOW) res = rows[0].resource_name out = {} for at in (NOW.replace(hour=2), NOW.replace(hour=3), NOW.replace(hour=14), NOW + timedelta(days=1, hours=2)): got = await norms_at(s, [("normtest", res, "cpu_pct")], at) out[at] = got.get(("normtest", res, "cpu_pct")) return out return asyncio.run(_go()) def test_a_nightly_backup_is_normal_at_its_hour_and_not_at_another(app, learned): res = f"night-{uuid.uuid4().hex[:8]}" # Every night 02:00–03:00 holds 90%; the rest of the day holds 10% and # spikes briefly to 95% (a high max, a low p95). got = _learn(app, _hours(res, 7, lambda b, h: (90.0, 95.0) if h == 2 else (10.0, 95.0))) at2, at3, at14 = (got[NOW.replace(hour=h)] for h in (2, 3, 14)) assert at2.kind == "day" and at2.days_seen == 7 assert at2.usual_hold == 90.0 # 03:00 takes in 02:00 as a neighbour: a backup that runs late stays normal. assert at3.usual_hold == 90.0 # 14:00 holds 10: the spikes raised its peak, never its hold. assert at14.usual_hold == 10.0 and at14.usual_peak == 95.0 def test_too_little_history_learns_no_normal(app, learned): res = f"young-{uuid.uuid4().hex[:8]}" got = _learn(app, _hours(res, 4, lambda b, h: (10.0, 10.0))) assert all(n is None for n in got.values()) def test_three_weeks_of_history_learns_the_hour_of_the_week(app, learned): res = f"weekly-{uuid.uuid4().hex[:8]}" # A weekly job: Tuesdays 02:00 hold 80; every other hour holds 5. got = _learn(app, _hours(res, 22, lambda b, h: (80.0, 80.0) if b.weekday() == 1 and h == 2 else (5.0, 5.0))) tuesday_2 = got[NOW + timedelta(days=1, hours=2)] monday_2 = got[NOW.replace(hour=2)] assert tuesday_2.kind == "week" and tuesday_2.slot == 26 and tuesday_2.usual_hold == 80.0 # Monday 02:00 is far from Tuesday's job: on an hour-of-day normal it # would have been learned as the job's level. assert monday_2.usual_hold == 5.0