feat(traefik): routers are graded against their own normal (#5741)

#5741 "Traefik routers are graded against their own normal", milestone 513.

- A router's 4xx share and p95 learn their normal (learn_normal in the
  plugin's setup). Once a router has 5 days of history, a reading over the
  fixed cutoff but inside its normal is healthy, and the healthy table says
  it is usual for that router. One out of its normal is a warning that says
  what is usual, and a metric anomaly.
- The 5xx share is a failure: fixed cutoff always, whatever the history or
  traffic (System 182's ruling).
- A 4xx share over fewer requests than Settings -> Thresholds' new "Fewest
  requests to judge 4xx" (30 per 15 minutes) isn't graded on the page, and
  isn't judged for anomalies.
- Core: learn_normal(gate=(metric, least)) judges a key only while its
  resource's gate metric averages at least `least`; `least` may be a
  function, so it follows a setting. learn_normal(since=...) learns only
  from history after a moment: Traefik latency learns from #5755's
  per-scrape figures only.
- The Traefik page lists the routers' metric anomalies of the last 30 days
  (_anomaly_card.html). The dashboard widget keeps its why to problems.
- Thresholds and docs say when the cutoffs give way to a normal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/__init__.py

@@ -20,6 +20,19 @@ def setup(app: "Quart") -> None:
     prune_by_age(TraefikStatusCode.scraped_at)
     prune_by_age(TraefikGlobalStat.scraped_at)
     prune_by_age(TraefikAccessSummary.period_end)
+    # Each router's 4xx share and p95 learn their normal (#5741), judged only
+    # while the router answers the Thresholds minimum of requests a window
+    # (read at each pass, so a saved change applies at once). Latency is
+    # learned from the scrape-by-scrape figures only: before #5755 it was a
+    # percentile of every response since Traefik started.
+    from datetime import datetime, timezone
+    from steward.core.norms import learn_normal
+    from .routes import FLOOR_4XX, FLOOR_P95, STATE_WINDOW, min_requests
+    gate = ("request_rate",
+            lambda: min_requests(app.config.get("THRESHOLDS", {})) / STATE_WINDOW.total_seconds())
+    learn_normal("traefik", "error_rate_4xx_pct", FLOOR_4XX, gate=gate)
+    learn_normal("traefik", "latency_p95_ms", FLOOR_P95, gate=gate,
+                 since=datetime(2026, 10, 14, 18, 47, tzinfo=timezone.utc))
 
 
 def get_scheduled_tasks() -> list:

plugins/traefik/routes.py

@@ -11,7 +11,9 @@ import time
 from steward.auth.middleware import require_role
 from steward.models.users import UserRole
 from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample, time_bucket
+from steward.core import anomalies as anom
 from steward.core.formatting import cert_left, status_class
+from steward.core.norms import norms_at
 from steward.models.metrics import PluginMetricLatest
 from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
 from .scheduler import router_domains
@@ -52,21 +54,67 @@ def error_cutoffs(thresholds: dict) -> tuple[float, float]:
             float(thresholds.get("error_4xx", {}).get("warn", 1.0)))
 
 
-def router_state(reading, thresholds: dict, over: str = "") -> tuple[str, str | None]:
+# Each router's 4xx share and p95 are judged against its own normal once it
+# has one (#5741, System 182): an error-pages service answers 4xx all day and
+# that is its normal, not a problem. Departing by less than the floor is never
+# news. A share over fewer than min_requests() in the window is not graded: one
+# 404 of three requests is 33%. The 5xx share is a failure and keeps its
+# fixed cutoff whatever the history or traffic (the operator's ruling).
+FLOOR_4XX = 2.0           # percentage points
+FLOOR_P95 = 50.0          # ms
+NORMED = ("error_rate_4xx_pct", "latency_p95_ms")
+
+
+def min_requests(thresholds: dict) -> float:
+    """Fewest requests over the STATE_WINDOW for a 4xx share to be graded
+    (Settings → Thresholds, traefik.threshold.min_requests)."""
+    return float(thresholds.get("router_requests", {}).get("warn", 30))
+
+
+def _against(value: float, norm, floor: float, unit: str) -> tuple[bool, str]:
+    """(departs, what is usual) for a reading over its fixed cutoff."""
+    if norm is None:
+        return True, ""
+    usual = f"usual {norm.usual_hold:.1f}{unit}" if unit == "%" else f"usual {norm.usual_hold:.0f}{unit}"
+    return anom.departs(value, norm, floor), f"{usual} at this hour"
+
+
+def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = None) -> tuple[str, str | None]:
     """(grade, why) for a router's reading (recent_window's, or one scrape's):
     crit while it serves 5xx past its cutoff, warn on a p95 past the latency
     critical cutoff or a 4xx share past its cutoff (all Settings → Thresholds),
-    otherwise ok with nothing to say. `over` names the span the reading
-    covers, for the why."""
+    otherwise ok. `norms` ({metric: Norm}) are the router's normals for this
+    hour: a p95 or 4xx share past its cutoff but within its normal is ok, and
+    `why` then says it is usual. `over` names the span the reading covers."""
+    norms = norms or {}
     error_5xx, error_4xx = error_cutoffs(thresholds)
     if (reading.error_rate_5xx_pct or 0) > error_5xx:
         return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}"
+    usual = []
     p95 = reading.latency_p95_ms
     if p95 is not None and status_class(p95, "latency", thresholds) == "crit":
-        return "warn", f"p95 latency is {p95:.0f} ms{over}"
-    if (reading.error_rate_4xx_pct or 0) > error_4xx:
-        return "warn", f"{reading.error_rate_4xx_pct:.1f}% of responses are 4xx{over}"
-    return "ok", None
+        out, said = _against(p95, norms.get("latency_p95_ms"), FLOOR_P95, " ms")
+        if out:
+            return "warn", f"p95 latency is {p95:.0f} ms{over}" + (f", {said}" if said else "")
+        usual.append(f"p95 {p95:.0f} ms is {said}")
+    requests = getattr(reading, "requests", None)
+    e4 = reading.error_rate_4xx_pct or 0
+    if e4 > error_4xx and (requests is None or requests >= min_requests(thresholds)):
+        out, said = _against(e4, norms.get("error_rate_4xx_pct"), FLOOR_4XX, "%")
+        if out:
+            return "warn", f"{e4:.1f}% of responses are 4xx{over}" + (f", {said}" if said else "")
+        usual.append(f"4xx {e4:.1f}% is {said}")
+    return "ok", ("; ".join(usual) or None)
+
+
+async def router_norms(db, names: list[str], now: datetime) -> dict[str, dict]:
+    """{router: {metric: Norm}}: each router's learned normals for this hour.
+    A router without enough history yet is absent, and graded on the cutoffs."""
+    got = await norms_at(db, [("traefik", n, m) for n in names for m in NORMED], now)
+    out: dict[str, dict] = {}
+    for (_s, router, metric), norm in got.items():
+        out.setdefault(router, {})[metric] = norm
+    return out
 
 
 # A router is graded on its last STATE_WINDOW of traffic, not on one scrape:
@@ -81,7 +129,8 @@ STATE_WINDOW_WORDS = " over the last 15 min"
 async def recent_window(db, names: list[str], now: datetime) -> dict[str, SimpleNamespace]:
     """{router: one reading of its last STATE_WINDOW}: the 4xx and 5xx shares
     weighted by each scrape's traffic (an idle scrape's 100% counts for
-    little), p95 averaged. A router with no rows in the window is absent."""
+    little), p95 averaged, and about how many requests it answered. A router
+    with no rows in the window is absent."""
     if not names:
         return {}
     rate = TraefikMetric.request_rate
@@ -94,6 +143,7 @@ async def recent_window(db, names: list[str], now: datetime) -> dict[str, Simple
             func.avg(TraefikMetric.error_rate_4xx_pct).label("e4"),
             func.avg(TraefikMetric.error_rate_5xx_pct).label("e5"),
             func.avg(TraefikMetric.latency_p95_ms).label("p95"),
+            func.avg(rate).label("rate"),
         )
         .where(TraefikMetric.router_name.in_(names))
         .where(TraefikMetric.scraped_at >= now - STATE_WINDOW)
@@ -106,6 +156,7 @@ async def recent_window(db, names: list[str], now: datetime) -> dict[str, Simple
             error_rate_4xx_pct=r.e4w / req if req > 0 else r.e4,
             error_rate_5xx_pct=r.e5w / req if req > 0 else r.e5,
             latency_p95_ms=r.p95,
+            requests=(r.rate or 0.0) * STATE_WINDOW.total_seconds(),
         )
     return out
 
@@ -129,13 +180,15 @@ async def status_codes(db, names: list[str], since: datetime) -> dict[str, list[
     return out
 
 
-def grade_routers(routers: list[dict], window: dict, thresholds: dict) -> None:
+def grade_routers(routers: list[dict], window: dict, thresholds: dict, norms: dict | None = None) -> None:
     """Set each router's state and why from its recent window, or from its
-    latest scrape when the window holds nothing."""
+    latest scrape when the window holds nothing, against its normals
+    (router_norms') where it has them."""
+    norms = norms or {}
     for r in routers:
-        reading = window.get(r["name"])
-        r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS) if reading
-                                else router_state(r["latest"], thresholds))
+        reading, own = window.get(r["name"]), norms.get(r["name"])
+        r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS, own) if reading
+                                else router_state(r["latest"], thresholds, norms=own))
 
 
 # A router missing from this many scrapes in a row is no longer reported:
@@ -291,8 +344,12 @@ async def rows():
         names = [r["name"] for r in current]
         history = await router_history(db, names, since, bucket_seconds(since))
         phases.mark("history")
-        grade_routers(current, await recent_window(db, names, now), thresholds)
+        grade_routers(current, await recent_window(db, names, now), thresholds,
+                      await router_norms(db, names, now))
         phases.mark("window")
+        # The routers' metric anomalies, last 30 days (#5741).
+        anomalies = await anom.views(db, await anom.list_anomalies(
+            db, source="traefik", since=now - timedelta(days=30), limit=20), links=False)
         for r in current:
             points = history[r["name"]]
             r["since"], r["since_before"] = (problem_since(points, thresholds) if r["state"] != "ok"
@@ -364,6 +421,9 @@ async def rows():
         certs=certs,
         old_certs=old_certs,
         range_key=range_key,
+        anomalies=anomalies,
+        back="/plugins/traefik/",
+        base_resource=None,
     )
     phases.mark("render")
     phases.report("rows", f"routers={len(current)} range={range_key}")
@@ -388,6 +448,7 @@ async def widget():
         phases.mark("latest")
         current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
         window = await recent_window(db, [r["name"] for r in current], now)
+        norms = await router_norms(db, [r["name"] for r in current], now)
         phases.mark("window")
         certs = (await db.execute(select(TraefikCert))).scalars().all()
         phases.mark("certs")
@@ -396,7 +457,7 @@ async def widget():
     served, _old = current_certs(certs, scrape)
     domains = router_domains()
     by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
-    grade_routers(current, window, thresholds)
+    grade_routers(current, window, thresholds, norms)
     for r in current:
         r["cert"] = by_router.get(r["name"])
         if r["cert"]:

plugins/traefik/templates/traefik/rows.html

@@ -1,7 +1,8 @@
 {# traefik/rows.html — HTMX fragment for the Traefik page: the verdict (swapped
    into #traefik-verdict above the request chart, #5710), routers that need
-   attention (routes.router_state), the proxy's own figures, the healthy
-   routers folded into one table, then certificates by soonest expiry. #}
+   attention (routes.router_state), their metric anomalies, the proxy's own
+   figures, the healthy routers folded into one table (one over a cutoff but
+   within its normal says so, #5741), then certificates by soonest expiry. #}
 {% from "_macros.html" import stat_tile, status_badge %}
 
 {# ── Verdict: what needs a look and since when, then the counts ─────────── #}
@@ -77,6 +78,10 @@
 </div>
 {% endif %}
 
+{# A router's 4xx share or p95 out of its own normal (#5741), and the last
+   30 days' anomalies, to review here. #}
+{% include "_anomaly_card.html" %}
+
 {{ proxy_figures() }}
 
 {# ── Healthy routers, folded ─────────────────────────────────────────────── #}
@@ -89,7 +94,7 @@
     <tbody>
     {% for r in healthy %}
     <tr>
-      <td style="word-break:break-all;">{{ r.name }}</td>
+      <td style="word-break:break-all;">{{ r.name }}{% if r.why %}<div class="field-hint">{{ r.why }}</div>{% endif %}</td>
       <td>{{ "%.2f"|format(r.latest.request_rate) }}</td>
       <td>{{ r.latest.response_bytes_rate|rate }}</td>
       {{ latency_cell(r.latest.latency_p95_ms) }}

plugins/traefik/templates/traefik/widget.html

@@ -24,7 +24,7 @@
     <td class="w-name" title="{{ r.name }}">
       {{- status_dot(r.state) -}}
       {{ r.name.split('@')[0] }}
-      {%- if r.why %}<span class="w-sub tone-{{ r.state }}">{{ r.why }}</span>{% endif %}
+      {%- if r.why and r.state != "ok" %}<span class="w-sub tone-{{ r.state }}">{{ r.why }}</span>{% endif %}
     </td>
     <td>{% if m.request_rate is none %}—{% else %}{{ "%.2f"|format(m.request_rate) }}{% endif %}</td>
     <td class="w-wide-only">{% if m.response_bytes_rate is none %}—{% else %}{{ m.response_bytes_rate|rate }}{% endif %}</td>