feat(traefik): routers are graded against their own normal (#5741)

#5741 "Traefik routers are graded against their own normal", milestone 513.

- A router's 4xx share and p95 learn their normal (learn_normal in the
  plugin's setup). Once a router has 5 days of history, a reading over the
  fixed cutoff but inside its normal is healthy, and the healthy table says
  it is usual for that router. One out of its normal is a warning that says
  what is usual, and a metric anomaly.
- The 5xx share is a failure: fixed cutoff always, whatever the history or
  traffic (System 182's ruling).
- A 4xx share over fewer requests than Settings -> Thresholds' new "Fewest
  requests to judge 4xx" (30 per 15 minutes) isn't graded on the page, and
  isn't judged for anomalies.
- Core: learn_normal(gate=(metric, least)) judges a key only while its
  resource's gate metric averages at least `least`; `least` may be a
  function, so it follows a setting. learn_normal(since=...) learns only
  from history after a moment: Traefik latency learns from #5755's
  per-scrape figures only.
- The Traefik page lists the routers' metric anomalies of the last 30 days
  (_anomaly_card.html). The dashboard widget keeps its why to problems.
- Thresholds and docs say when the cutoffs give way to a normal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_anomalies.py

@@ -271,3 +271,65 @@ def test_cleanup_prunes_reviewed_anomalies_and_never_one_to_review(app, src):
             )).scalars()}
 
     assert asyncio.run(_go()) == {kept}
+
+
+def test_a_gated_metric_is_judged_only_while_its_gate_is_open(app, src):
+    """A router's 4xx share means nothing over a handful of requests (#5741):
+    learned with gate=("req", 1.0), it is judged only while the resource's
+    own "req" averages at least 1 over the window."""
+    from steward.core import norms
+    from steward.core.anomalies import WINDOW
+    from steward.models.metrics import PluginMetric, PluginMetricLatest
+    norms.learn_normal(src, METRIC, 5, gate=("req", 1.0))
+    res = _res("gated")
+    quiet, busy = NOW, NOW + timedelta(hours=2)
+
+    async def _req(s, until, value):
+        n = int(WINDOW.total_seconds() // 60)
+        s.add_all([PluginMetric(source_module=src, resource_name=res, metric_name="req",
+                                value=value, recorded_at=until - timedelta(minutes=m)) for m in range(n)])
+        await s.merge(PluginMetricLatest(source_module=src, resource_name=res, metric_name="req",
+                                         value=value, recorded_at=until))
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await _req(s, quiet, 0.2)
+                await _req(s, busy, 5.0)
+        return await _steps(app, src, res, [(quiet, 60.0), (busy, 60.0)])
+
+    rows, spans = asyncio.run(_go())
+    (a,) = rows                                   # only the busy window opened one
+    assert spans[a.id][0].started_at == busy - WINDOW
+
+
+def test_a_metric_learns_only_from_its_since(app, src):
+    """A metric whose meaning changed (Traefik latency, #5755) learns only
+    from history after `since`: seven days held 90, but only the last three
+    are after it, under the five a normal needs."""
+    from steward.core import norms
+    from steward.core.norms import norms_at, rebuild_norms
+    from steward.models.metrics import PluginMetricHourly
+    res = _res("since")
+    rows = [PluginMetricHourly(source_module=src, resource_name=res, metric_name=METRIC,
+                               bucket=NOW.replace(hour=0) - timedelta(days=d) + timedelta(hours=h),
+                               value_avg=90.0, value_max=90.0, value_p95=90.0, sample_count=60)
+            for d in range(1, 8) for h in range(24)]
+
+    async def _go(since):
+        norms.learn_normal(src, METRIC, 5, since=since)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await rebuild_norms(s, NOW)
+            got = await norms_at(s, [(src, res, METRIC)], NOW.replace(hour=14))
+            return got.get((src, res, METRIC))
+
+    async def _both():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add_all(rows)
+        return await _go(None), await _go(NOW.replace(hour=0) - timedelta(days=3))
+
+    learned, held_back = asyncio.run(_both())
+    assert learned is not None and learned.usual_hold == 90.0
+    assert held_back is None

tests/plugins/traefik/test_normals.py

@@ -0,0 +1,85 @@
+"""A router's 4xx share and p95 are graded against its own normal once it has
+one (#5741, System 182): a router over the fixed cutoff but inside its normal
+is healthy and says so; one out of its normal is a warning that says what is
+usual. The 5xx share keeps its fixed cutoff, and a 4xx share over too few
+requests isn't graded."""
+from types import SimpleNamespace as NS
+
+from plugins.traefik.routes import STATE_WINDOW_WORDS, grade_routers, router_state
+from steward.core.norms import Norm
+from steward.core.settings import to_thresholds_cfg
+from tests.plugins.traefik.test_traefik_page import _latest
+
+TH = to_thresholds_cfg({})          # 4xx warn over 1%, 5xx crit over 0.1%, latency crit 200 ms
+
+
+def _norm(hold, peak=None):
+    return Norm(kind="day", slot=14, usual_hold=hold, usual_peak=peak or hold * 1.5, days_seen=7)
+
+
+def _window(e4=0.0, e5=0.0, p95=20.0, requests=500.0):
+    return NS(error_rate_4xx_pct=e4, error_rate_5xx_pct=e5, latency_p95_ms=p95, requests=requests)
+
+
+def test_an_error_pages_router_answering_4xx_all_day_reads_healthy():
+    state, why = router_state(_window(e4=62.0), TH, STATE_WINDOW_WORDS,
+                              {"error_rate_4xx_pct": _norm(60.0)})
+    assert state == "ok"
+    assert why == "4xx 62.0% is usual 60.0% at this hour"
+
+
+def test_a_router_that_flapped_on_the_cutoff_stays_healthy_inside_its_normal():
+    """authentik-like: a login wall's 401s sit around 1–3%, either side of the
+    1% cutoff, so it came and went from the problem list every few polls."""
+    norm = {"error_rate_4xx_pct": _norm(2.8)}
+    for e4 in (0.6, 1.4, 2.9, 1.1, 3.2, 0.9):
+        assert router_state(_window(e4=e4), TH, STATE_WINDOW_WORDS, norm)[0] == "ok", e4
+
+
+def test_out_of_its_normal_is_a_warning_that_says_what_is_usual():
+    state, why = router_state(_window(e4=12.0), TH, STATE_WINDOW_WORDS,
+                              {"error_rate_4xx_pct": _norm(2.8)})
+    assert state == "warn"
+    assert why == "12.0% of responses are 4xx over the last 15 min, usual 2.8% at this hour"
+
+
+def test_a_departure_smaller_than_the_floor_is_no_news():
+    # 1.25 × 1.2% is 1.5%, but 2.0% is only 0.8 points over: under the 2-point floor.
+    assert router_state(_window(e4=2.0), TH, "", {"error_rate_4xx_pct": _norm(1.2)})[0] == "ok"
+
+
+def test_latency_is_graded_against_its_normal_too():
+    slow_usually = {"latency_p95_ms": _norm(900.0)}
+    assert router_state(_window(p95=950.0), TH, "", slow_usually) == ("ok", "p95 950 ms is usual 900 ms at this hour")
+    state, why = router_state(_window(p95=3000.0), TH, "", slow_usually)
+    assert state == "warn" and why == "p95 latency is 3000 ms, usual 900 ms at this hour"
+
+
+def test_without_a_normal_the_fixed_cutoffs_still_apply():
+    assert router_state(_window(e4=5.0), TH, "")[0] == "warn"
+    assert router_state(_window(p95=450.0), TH, "")[0] == "warn"
+
+
+def test_5xx_keeps_its_cutoff_whatever_the_history_or_traffic():
+    usual_5xx = {"error_rate_4xx_pct": _norm(50.0)}
+    assert router_state(_window(e5=40.0), TH, "", usual_5xx)[0] == "crit"
+    assert router_state(_window(e5=40.0, requests=4.0), TH, "")[0] == "crit"
+
+
+def test_a_4xx_share_over_a_handful_of_requests_is_not_graded():
+    assert router_state(_window(e4=33.0, requests=3.0), TH, "") == ("ok", None)
+    assert router_state(_window(e4=33.0, requests=300.0), TH, "")[0] == "warn"
+
+
+def test_grade_routers_hands_each_router_its_own_normals():
+    routers = [{"name": "errors@docker", "latest": _latest(e4=60.0)},
+               {"name": "app@docker", "latest": _latest(e4=60.0)}]
+    window = {"errors@docker": _window(e4=60.0), "app@docker": _window(e4=60.0)}
+    grade_routers(routers, window, TH, {"errors@docker": {"error_rate_4xx_pct": _norm(58.0)}})
+    assert [r["state"] for r in routers] == ["ok", "warn"]
+
+
+def test_the_fewest_requests_to_judge_is_the_thresholds_setting():
+    fewer = to_thresholds_cfg({"traefik.threshold.min_requests": 5})
+    assert router_state(_window(e4=33.0, requests=10.0), TH, "")[0] == "ok"        # default 30
+    assert router_state(_window(e4=33.0, requests=10.0), fewer, "")[0] == "warn"