feat(traefik): routers are graded against their own normal (#5741)

#5741 "Traefik routers are graded against their own normal", milestone 513.

- A router's 4xx share and p95 learn their normal (learn_normal in the
  plugin's setup). Once a router has 5 days of history, a reading over the
  fixed cutoff but inside its normal is healthy, and the healthy table says
  it is usual for that router. One out of its normal is a warning that says
  what is usual, and a metric anomaly.
- The 5xx share is a failure: fixed cutoff always, whatever the history or
  traffic (System 182's ruling).
- A 4xx share over fewer requests than Settings -> Thresholds' new "Fewest
  requests to judge 4xx" (30 per 15 minutes) isn't graded on the page, and
  isn't judged for anomalies.
- Core: learn_normal(gate=(metric, least)) judges a key only while its
  resource's gate metric averages at least `least`; `least` may be a
  function, so it follows a setting. learn_normal(since=...) learns only
  from history after a moment: Traefik latency learns from #5755's
  per-scrape figures only.
- The Traefik page lists the routers' metric anomalies of the last 30 days
  (_anomaly_card.html). The dashboard widget keeps its why to problems.
- Thresholds and docs say when the cutoffs give way to a normal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/norms.py

@@ -31,10 +31,11 @@ absorbed on its own, and a lapsed anomaly is never learned.
 """
 from __future__ import annotations
 
+from collections.abc import Callable
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 
-from sqlalchemy import ARRAY, String, bindparam, delete, select, text, tuple_
+from sqlalchemy import ARRAY, DateTime, String, bindparam, delete, select, text, tuple_
 
 from steward.models.norms import MetricNorm
 
@@ -50,19 +51,31 @@ USUAL = 0.9             # "usual" is this percentile across past days: one odd d
 class Learned:
     """A metric that learns a normal. `floor` is the smallest departure,
     in the metric's own unit, worth calling an anomaly (#5740), so that
-    0.1% → 0.3% of a share or 2 ms → 4 ms of latency is never news."""
+    0.1% → 0.3% of a share or 2 ms → 4 ms of latency is never news.
+
+    `gate` is (metric, least): the resource is judged only while that metric
+    of its own averages at least `least` over the window, as a router's
+    error share means nothing over a handful of requests (#5741). `least`
+    may be a function, read at each pass, for a minimum that is a setting. `since`:
+    history before it is not learned from, for a metric whose meaning
+    changed then."""
     source_module: str
     metric_name: str
     floor: float
+    gate: tuple[str, float | Callable[[], float]] | None = None
+    since: datetime | None = None
 
 
 LEARNED: dict[tuple[str, str], Learned] = {}
 
 
-def learn_normal(source_module: str, metric_name: str, floor: float) -> None:
+def learn_normal(source_module: str, metric_name: str, floor: float, *,
+                 gate: tuple[str, float | Callable[[], float]] | None = None,
+                 since: datetime | None = None) -> None:
     """Have Steward learn this metric's normal for every resource reporting
     it. A plugin calls this from setup(), as it calls prune_by_age."""
-    LEARNED[(source_module, metric_name)] = Learned(source_module, metric_name, float(floor))
+    LEARNED[(source_module, metric_name)] = Learned(source_module, metric_name, float(floor),
+                                                    gate, since)
 
 
 @dataclass(frozen=True)
@@ -89,17 +102,21 @@ def slot_of(at: datetime, kind: str) -> int:
 # each slot's usual hold and peak across past days, every hour counted in its
 # own slot and its neighbours'.
 _REBUILD = text("""
-WITH all_hours AS (
-    SELECT source_module AS s, resource_name AS r, metric_name AS m, bucket AS b,
-           COALESCE(value_p95, value_avg) AS hold, value_max AS peak
-    FROM plugin_metrics_hourly
-    WHERE bucket >= :week_since AND source_module || '/' || metric_name = ANY(:keys)
+WITH learned AS (
+    SELECT key, since FROM unnest(CAST(:keys AS text[]), CAST(:starts AS timestamptz[])) AS k(key, since)
+),
+all_hours AS (
+    SELECT h.source_module AS s, h.resource_name AS r, h.metric_name AS m, h.bucket AS b,
+           COALESCE(h.value_p95, h.value_avg) AS hold, h.value_max AS peak
+    FROM plugin_metrics_hourly h
+    JOIN learned k ON k.key = h.source_module || '/' || h.metric_name
+    WHERE h.bucket >= k.since
     UNION ALL
-    SELECT source_module, resource_name, metric_name, date_trunc('hour', recorded_at),
-           percentile_cont(0.95) WITHIN GROUP (ORDER BY value), max(value)
-    FROM plugin_metrics
-    WHERE recorded_at >= :week_since AND recorded_at < :this_hour
-      AND source_module || '/' || metric_name = ANY(:keys)
+    SELECT p.source_module, p.resource_name, p.metric_name, date_trunc('hour', p.recorded_at),
+           percentile_cont(0.95) WITHIN GROUP (ORDER BY p.value), max(p.value)
+    FROM plugin_metrics p
+    JOIN learned k ON k.key = p.source_module || '/' || p.metric_name
+    WHERE p.recorded_at >= k.since AND p.recorded_at < :this_hour
     GROUP BY 1, 2, 3, 4
 ),
 hours AS (
@@ -139,7 +156,8 @@ SELECT s, r, m, (slot + n.off + slots) % slots, kind,
        days, :now
 FROM slotted CROSS JOIN generate_series(-CAST(:neighbours AS int), CAST(:neighbours AS int)) AS n(off)
 GROUP BY s, r, m, (slot + n.off + slots) % slots, kind, days
-""").bindparams(bindparam("keys", type_=ARRAY(String)))
+""").bindparams(bindparam("keys", type_=ARRAY(String)),
+               bindparam("starts", type_=ARRAY(DateTime(timezone=True))))
 
 
 async def rebuild_norms(session, now: datetime | None = None) -> int:
@@ -147,12 +165,13 @@ async def rebuild_norms(session, now: datetime | None = None) -> int:
     inside the caller's transaction. Returns how many slot rows it wrote."""
     now = now or datetime.now(timezone.utc)
     await session.execute(delete(MetricNorm))
-    keys = [f"{s}/{m}" for s, m in LEARNED]
-    if not keys:
+    if not LEARNED:
         return 0
+    week_since = now - WEEK_WINDOW
     result = await session.execute(_REBUILD, {
-        "keys": keys,
-        "week_since": now - WEEK_WINDOW,
+        "keys": [f"{s}/{m}" for s, m in LEARNED],
+        # Each key learns from WEEK_WINDOW back, or from its own `since`.
+        "starts": [max(week_since, k.since) if k.since else week_since for k in LEARNED.values()],
         "day_since": now - DAY_WINDOW,
         "this_hour": now.replace(minute=0, second=0, microsecond=0),
         "week_ready": now - timedelta(days=WEEK_MIN_DAYS),