feat(norms): Steward learns each metric's normal per hour of day, then week (#5739)

A metric registered with learn_normal() gets, for every resource reporting
it, a usual hold (the 90th percentile across past days of each hour's p95)
and a usual peak for each slot:
- an hour of the day once 5 days of history exist;
- an hour of the week once history reaches back 3 weeks, so weekly jobs are
  learned too.
Each slot takes in the hours either side, so a job that runs a little late
stays normal. One SQL statement relearns everything hourly into
metric_norms, from the rollup plus the raw window not rolled up yet;
norms_at() reads the slot a moment falls in. Nothing is registered yet:
Traefik (#5741) and hosts (#5742) opt in.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_norms.py

@@ -0,0 +1,27 @@
+"""Each metric's normal, by time slot (#5739): the slot arithmetic and the
+registry. Learning itself runs in SQL; tests/integration/test_norms.py
+covers it against Postgres."""
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+from steward.core import norms
+
+
+def test_a_day_slot_is_the_utc_hour():
+    at = datetime(2026, 10, 14, 2, 30, tzinfo=timezone(timedelta(hours=-5)))  # 07:30 UTC
+    assert norms.slot_of(at, "day") == 7
+
+
+def test_a_week_slot_counts_hours_from_monday_utc():
+    monday = datetime(2026, 10, 12, 0, 10, tzinfo=timezone.utc)
+    assert norms.slot_of(monday, "week") == 0
+    assert norms.slot_of(monday + timedelta(days=2, hours=2), "week") == 50   # Wed 02:10
+    assert norms.slot_of(monday + timedelta(days=6, hours=23), "week") == 167
+
+
+def test_a_learned_metric_is_registered_once_with_its_floor():
+    norms.learn_normal("test", "x_pct", 2)
+    norms.learn_normal("test", "x_pct", 3)
+    assert norms.LEARNED[("test", "x_pct")] == norms.Learned("test", "x_pct", 3.0)
+    del norms.LEARNED[("test", "x_pct")]

tests/integration/test_norms.py

@@ -0,0 +1,106 @@
+"""Integration: Steward learns each metric's normal per hour (#5739).
+
+The operator's case (System 182): a nightly backup that runs hot at 02:00
+must read as normal at 02:00, and the same level at 14:00 must not, so the
+normal has to be per hour and built from what each hour held, not its peak.
+Requires STEWARD_DATABASE_URL.
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+@pytest.fixture
+def learned():
+    from steward.core import norms
+    norms.learn_normal("normtest", "cpu_pct", 5)
+    yield
+    norms.LEARNED.pop(("normtest", "cpu_pct"), None)
+
+
+# Monday 00:00 UTC, so week slots are easy to read.
+NOW = datetime(2026, 10, 12, 0, 0, tzinfo=timezone.utc)
+
+
+def _hours(res: str, days: int, level):
+    """An hourly rollup row for each of the last `days` days' hours, held at
+    level(day, hour) -> (p95, max)."""
+    from steward.models.metrics import PluginMetricHourly
+    rows = []
+    for d in range(1, days + 1):
+        for h in range(24):
+            bucket = NOW - timedelta(days=d) + timedelta(hours=h)
+            p95, peak = level(bucket, h)
+            rows.append(PluginMetricHourly(source_module="normtest", resource_name=res,
+                                           metric_name="cpu_pct", bucket=bucket,
+                                           value_avg=p95, value_max=peak, value_p95=p95,
+                                           sample_count=60))
+    return rows
+
+
+def _learn(app, rows):
+    from steward.core.norms import norms_at, rebuild_norms
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add_all(rows)
+            async with s.begin():
+                await rebuild_norms(s, NOW)
+            res = rows[0].resource_name
+            out = {}
+            for at in (NOW.replace(hour=2), NOW.replace(hour=3), NOW.replace(hour=14),
+                       NOW + timedelta(days=1, hours=2)):
+                got = await norms_at(s, [("normtest", res, "cpu_pct")], at)
+                out[at] = got.get(("normtest", res, "cpu_pct"))
+            return out
+
+    return asyncio.run(_go())
+
+
+def test_a_nightly_backup_is_normal_at_its_hour_and_not_at_another(app, learned):
+    res = f"night-{uuid.uuid4().hex[:8]}"
+    # Every night 02:00–03:00 holds 90%; the rest of the day holds 10% and
+    # spikes briefly to 95% (a high max, a low p95).
+    got = _learn(app, _hours(res, 7, lambda b, h: (90.0, 95.0) if h == 2 else (10.0, 95.0)))
+    at2, at3, at14 = (got[NOW.replace(hour=h)] for h in (2, 3, 14))
+    assert at2.kind == "day" and at2.days_seen == 7
+    assert at2.usual_hold == 90.0
+    # 03:00 takes in 02:00 as a neighbour: a backup that runs late stays normal.
+    assert at3.usual_hold == 90.0
+    # 14:00 holds 10: the spikes raised its peak, never its hold.
+    assert at14.usual_hold == 10.0 and at14.usual_peak == 95.0
+
+
+def test_too_little_history_learns_no_normal(app, learned):
+    res = f"young-{uuid.uuid4().hex[:8]}"
+    got = _learn(app, _hours(res, 4, lambda b, h: (10.0, 10.0)))
+    assert all(n is None for n in got.values())
+
+
+def test_three_weeks_of_history_learns_the_hour_of_the_week(app, learned):
+    res = f"weekly-{uuid.uuid4().hex[:8]}"
+    # A weekly job: Tuesdays 02:00 hold 80; every other hour holds 5.
+    got = _learn(app, _hours(res, 22, lambda b, h: (80.0, 80.0) if b.weekday() == 1 and h == 2
+                                                    else (5.0, 5.0)))
+    tuesday_2 = got[NOW + timedelta(days=1, hours=2)]
+    monday_2 = got[NOW.replace(hour=2)]
+    assert tuesday_2.kind == "week" and tuesday_2.slot == 26 and tuesday_2.usual_hold == 80.0
+    # Monday 02:00 is far from Tuesday's job: on an hour-of-day normal it
+    # would have been learned as the job's level.
+    assert monday_2.usual_hold == 5.0