feat(traefik): routers are graded against their own normal (#5741)

#5741 "Traefik routers are graded against their own normal", milestone 513.

- A router's 4xx share and p95 learn their normal (learn_normal in the
  plugin's setup). Once a router has 5 days of history, a reading over the
  fixed cutoff but inside its normal is healthy, and the healthy table says
  it is usual for that router. One out of its normal is a warning that says
  what is usual, and a metric anomaly.
- The 5xx share is a failure: fixed cutoff always, whatever the history or
  traffic (System 182's ruling).
- A 4xx share over fewer requests than Settings -> Thresholds' new "Fewest
  requests to judge 4xx" (30 per 15 minutes) isn't graded on the page, and
  isn't judged for anomalies.
- Core: learn_normal(gate=(metric, least)) judges a key only while its
  resource's gate metric averages at least `least`; `least` may be a
  function, so it follows a setting. learn_normal(since=...) learns only
  from history after a moment: Traefik latency learns from #5755's
  per-scrape figures only.
- The Traefik page lists the routers' metric anomalies of the last 30 days
  (_anomaly_card.html). The dashboard widget keeps its why to problems.
- Thresholds and docs say when the cutoffs give way to a normal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/plugins/traefik.md

@@ -89,6 +89,8 @@ The full Traefik page at `/plugins/traefik/` reads top to bottom, for the time r
 
 A router is graded on its last 15 minutes against Settings → Thresholds. When a problem started is read from the router's history over the range.
 
+Once a router has 5 days of history, its 4xx share and p95 latency are judged against its own normal for that hour (see [Normals and anomalies](../architecture.md)) instead of the fixed cutoffs. One over the cutoff but within its normal is healthy, and the healthy table says it is usual for that router (an error-pages service that answers 4xx all day, a login wall's steady 401s). One out of its normal is a warning that says what is usual, and a metric anomaly to review; the Traefik page lists the routers' anomalies of the last 30 days. A 4xx share over fewer requests in the 15 minutes than Settings → Thresholds allows (30 by default) isn't judged at all. The 5xx share is a failure and always keeps its fixed cutoff. Latency is learned only from #5755's per-scrape figures, so its normal arrives 5 days after that release.
+
 ---
 
 ## File Locations

plugins/traefik/__init__.py

@@ -20,6 +20,19 @@ def setup(app: "Quart") -> None:
     prune_by_age(TraefikStatusCode.scraped_at)
     prune_by_age(TraefikGlobalStat.scraped_at)
     prune_by_age(TraefikAccessSummary.period_end)
+    # Each router's 4xx share and p95 learn their normal (#5741), judged only
+    # while the router answers the Thresholds minimum of requests a window
+    # (read at each pass, so a saved change applies at once). Latency is
+    # learned from the scrape-by-scrape figures only: before #5755 it was a
+    # percentile of every response since Traefik started.
+    from datetime import datetime, timezone
+    from steward.core.norms import learn_normal
+    from .routes import FLOOR_4XX, FLOOR_P95, STATE_WINDOW, min_requests
+    gate = ("request_rate",
+            lambda: min_requests(app.config.get("THRESHOLDS", {})) / STATE_WINDOW.total_seconds())
+    learn_normal("traefik", "error_rate_4xx_pct", FLOOR_4XX, gate=gate)
+    learn_normal("traefik", "latency_p95_ms", FLOOR_P95, gate=gate,
+                 since=datetime(2026, 10, 14, 18, 47, tzinfo=timezone.utc))
 
 
 def get_scheduled_tasks() -> list:

plugins/traefik/routes.py

@@ -11,7 +11,9 @@ import time
 from steward.auth.middleware import require_role
 from steward.models.users import UserRole
 from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample, time_bucket
+from steward.core import anomalies as anom
 from steward.core.formatting import cert_left, status_class
+from steward.core.norms import norms_at
 from steward.models.metrics import PluginMetricLatest
 from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
 from .scheduler import router_domains
@@ -52,21 +54,67 @@ def error_cutoffs(thresholds: dict) -> tuple[float, float]:
             float(thresholds.get("error_4xx", {}).get("warn", 1.0)))
 
 
-def router_state(reading, thresholds: dict, over: str = "") -> tuple[str, str | None]:
+# Each router's 4xx share and p95 are judged against its own normal once it
+# has one (#5741, System 182): an error-pages service answers 4xx all day and
+# that is its normal, not a problem. Departing by less than the floor is never
+# news. A share over fewer than min_requests() in the window is not graded: one
+# 404 of three requests is 33%. The 5xx share is a failure and keeps its
+# fixed cutoff whatever the history or traffic (the operator's ruling).
+FLOOR_4XX = 2.0           # percentage points
+FLOOR_P95 = 50.0          # ms
+NORMED = ("error_rate_4xx_pct", "latency_p95_ms")
+
+
+def min_requests(thresholds: dict) -> float:
+    """Fewest requests over the STATE_WINDOW for a 4xx share to be graded
+    (Settings → Thresholds, traefik.threshold.min_requests)."""
+    return float(thresholds.get("router_requests", {}).get("warn", 30))
+
+
+def _against(value: float, norm, floor: float, unit: str) -> tuple[bool, str]:
+    """(departs, what is usual) for a reading over its fixed cutoff."""
+    if norm is None:
+        return True, ""
+    usual = f"usual {norm.usual_hold:.1f}{unit}" if unit == "%" else f"usual {norm.usual_hold:.0f}{unit}"
+    return anom.departs(value, norm, floor), f"{usual} at this hour"
+
+
+def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = None) -> tuple[str, str | None]:
     """(grade, why) for a router's reading (recent_window's, or one scrape's):
     crit while it serves 5xx past its cutoff, warn on a p95 past the latency
     critical cutoff or a 4xx share past its cutoff (all Settings → Thresholds),
-    otherwise ok with nothing to say. `over` names the span the reading
-    covers, for the why."""
+    otherwise ok. `norms` ({metric: Norm}) are the router's normals for this
+    hour: a p95 or 4xx share past its cutoff but within its normal is ok, and
+    `why` then says it is usual. `over` names the span the reading covers."""
+    norms = norms or {}
     error_5xx, error_4xx = error_cutoffs(thresholds)
     if (reading.error_rate_5xx_pct or 0) > error_5xx:
         return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}"
+    usual = []
     p95 = reading.latency_p95_ms
     if p95 is not None and status_class(p95, "latency", thresholds) == "crit":
-        return "warn", f"p95 latency is {p95:.0f} ms{over}"
-    if (reading.error_rate_4xx_pct or 0) > error_4xx:
-        return "warn", f"{reading.error_rate_4xx_pct:.1f}% of responses are 4xx{over}"
-    return "ok", None
+        out, said = _against(p95, norms.get("latency_p95_ms"), FLOOR_P95, " ms")
+        if out:
+            return "warn", f"p95 latency is {p95:.0f} ms{over}" + (f", {said}" if said else "")
+        usual.append(f"p95 {p95:.0f} ms is {said}")
+    requests = getattr(reading, "requests", None)
+    e4 = reading.error_rate_4xx_pct or 0
+    if e4 > error_4xx and (requests is None or requests >= min_requests(thresholds)):
+        out, said = _against(e4, norms.get("error_rate_4xx_pct"), FLOOR_4XX, "%")
+        if out:
+            return "warn", f"{e4:.1f}% of responses are 4xx{over}" + (f", {said}" if said else "")
+        usual.append(f"4xx {e4:.1f}% is {said}")
+    return "ok", ("; ".join(usual) or None)
+
+
+async def router_norms(db, names: list[str], now: datetime) -> dict[str, dict]:
+    """{router: {metric: Norm}}: each router's learned normals for this hour.
+    A router without enough history yet is absent, and graded on the cutoffs."""
+    got = await norms_at(db, [("traefik", n, m) for n in names for m in NORMED], now)
+    out: dict[str, dict] = {}
+    for (_s, router, metric), norm in got.items():
+        out.setdefault(router, {})[metric] = norm
+    return out
 
 
 # A router is graded on its last STATE_WINDOW of traffic, not on one scrape:
@@ -81,7 +129,8 @@ STATE_WINDOW_WORDS = " over the last 15 min"
 async def recent_window(db, names: list[str], now: datetime) -> dict[str, SimpleNamespace]:
     """{router: one reading of its last STATE_WINDOW}: the 4xx and 5xx shares
     weighted by each scrape's traffic (an idle scrape's 100% counts for
-    little), p95 averaged. A router with no rows in the window is absent."""
+    little), p95 averaged, and about how many requests it answered. A router
+    with no rows in the window is absent."""
     if not names:
         return {}
     rate = TraefikMetric.request_rate
@@ -94,6 +143,7 @@ async def recent_window(db, names: list[str], now: datetime) -> dict[str, Simple
             func.avg(TraefikMetric.error_rate_4xx_pct).label("e4"),
             func.avg(TraefikMetric.error_rate_5xx_pct).label("e5"),
             func.avg(TraefikMetric.latency_p95_ms).label("p95"),
+            func.avg(rate).label("rate"),
         )
         .where(TraefikMetric.router_name.in_(names))
         .where(TraefikMetric.scraped_at >= now - STATE_WINDOW)
@@ -106,6 +156,7 @@ async def recent_window(db, names: list[str], now: datetime) -> dict[str, Simple
             error_rate_4xx_pct=r.e4w / req if req > 0 else r.e4,
             error_rate_5xx_pct=r.e5w / req if req > 0 else r.e5,
             latency_p95_ms=r.p95,
+            requests=(r.rate or 0.0) * STATE_WINDOW.total_seconds(),
         )
     return out
 
@@ -129,13 +180,15 @@ async def status_codes(db, names: list[str], since: datetime) -> dict[str, list[
     return out
 
 
-def grade_routers(routers: list[dict], window: dict, thresholds: dict) -> None:
+def grade_routers(routers: list[dict], window: dict, thresholds: dict, norms: dict | None = None) -> None:
     """Set each router's state and why from its recent window, or from its
-    latest scrape when the window holds nothing."""
+    latest scrape when the window holds nothing, against its normals
+    (router_norms') where it has them."""
+    norms = norms or {}
     for r in routers:
-        reading = window.get(r["name"])
-        r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS) if reading
-                                else router_state(r["latest"], thresholds))
+        reading, own = window.get(r["name"]), norms.get(r["name"])
+        r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS, own) if reading
+                                else router_state(r["latest"], thresholds, norms=own))
 
 
 # A router missing from this many scrapes in a row is no longer reported:
@@ -291,8 +344,12 @@ async def rows():
         names = [r["name"] for r in current]
         history = await router_history(db, names, since, bucket_seconds(since))
         phases.mark("history")
-        grade_routers(current, await recent_window(db, names, now), thresholds)
+        grade_routers(current, await recent_window(db, names, now), thresholds,
+                      await router_norms(db, names, now))
         phases.mark("window")
+        # The routers' metric anomalies, last 30 days (#5741).
+        anomalies = await anom.views(db, await anom.list_anomalies(
+            db, source="traefik", since=now - timedelta(days=30), limit=20), links=False)
         for r in current:
             points = history[r["name"]]
             r["since"], r["since_before"] = (problem_since(points, thresholds) if r["state"] != "ok"
@@ -364,6 +421,9 @@ async def rows():
         certs=certs,
         old_certs=old_certs,
         range_key=range_key,
+        anomalies=anomalies,
+        back="/plugins/traefik/",
+        base_resource=None,
     )
     phases.mark("render")
     phases.report("rows", f"routers={len(current)} range={range_key}")
@@ -388,6 +448,7 @@ async def widget():
         phases.mark("latest")
         current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
         window = await recent_window(db, [r["name"] for r in current], now)
+        norms = await router_norms(db, [r["name"] for r in current], now)
         phases.mark("window")
         certs = (await db.execute(select(TraefikCert))).scalars().all()
         phases.mark("certs")
@@ -396,7 +457,7 @@ async def widget():
     served, _old = current_certs(certs, scrape)
     domains = router_domains()
     by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
-    grade_routers(current, window, thresholds)
+    grade_routers(current, window, thresholds, norms)
     for r in current:
         r["cert"] = by_router.get(r["name"])
         if r["cert"]:

plugins/traefik/templates/traefik/rows.html

@@ -1,7 +1,8 @@
 {# traefik/rows.html — HTMX fragment for the Traefik page: the verdict (swapped
    into #traefik-verdict above the request chart, #5710), routers that need
-   attention (routes.router_state), the proxy's own figures, the healthy
-   routers folded into one table, then certificates by soonest expiry. #}
+   attention (routes.router_state), their metric anomalies, the proxy's own
+   figures, the healthy routers folded into one table (one over a cutoff but
+   within its normal says so, #5741), then certificates by soonest expiry. #}
 {% from "_macros.html" import stat_tile, status_badge %}
 
 {# ── Verdict: what needs a look and since when, then the counts ─────────── #}
@@ -77,6 +78,10 @@
 </div>
 {% endif %}
 
+{# A router's 4xx share or p95 out of its own normal (#5741), and the last
+   30 days' anomalies, to review here. #}
+{% include "_anomaly_card.html" %}
+
 {{ proxy_figures() }}
 
 {# ── Healthy routers, folded ─────────────────────────────────────────────── #}
@@ -89,7 +94,7 @@
     <tbody>
     {% for r in healthy %}
     <tr>
-      <td style="word-break:break-all;">{{ r.name }}</td>
+      <td style="word-break:break-all;">{{ r.name }}{% if r.why %}<div class="field-hint">{{ r.why }}</div>{% endif %}</td>
       <td>{{ "%.2f"|format(r.latest.request_rate) }}</td>
       <td>{{ r.latest.response_bytes_rate|rate }}</td>
       {{ latency_cell(r.latest.latency_p95_ms) }}

plugins/traefik/templates/traefik/widget.html

@@ -24,7 +24,7 @@
     <td class="w-name" title="{{ r.name }}">
       {{- status_dot(r.state) -}}
       {{ r.name.split('@')[0] }}
-      {%- if r.why %}<span class="w-sub tone-{{ r.state }}">{{ r.why }}</span>{% endif %}
+      {%- if r.why and r.state != "ok" %}<span class="w-sub tone-{{ r.state }}">{{ r.why }}</span>{% endif %}
     </td>
     <td>{% if m.request_rate is none %}—{% else %}{{ "%.2f"|format(m.request_rate) }}{% endif %}</td>
     <td class="w-wide-only">{% if m.response_bytes_rate is none %}—{% else %}{{ m.response_bytes_rate|rate }}{% endif %}</td>

steward/core/anomalies.py

@@ -36,6 +36,15 @@ REVIEWS = (TO_REVIEW, NORMAL, REAL, LAPSED)
 MARKS = (NORMAL, REAL)
 
 
+def _gate_open(learned, key: tuple[str, str, str], reported: dict) -> bool:
+    """Whether `key` may be judged: ungated, or its resource's gate metric
+    averaged at least the gate's least over the window."""
+    if learned.gate is None:
+        return True
+    metric, least = learned.gate
+    return reported.get((key[0], key[1], metric), 0.0) >= (least() if callable(least) else least)
+
+
 def departs(level: float, norm: Norm, floor: float) -> bool:
     """Whether `level` is out of `norm`: above its usual hold by DEPART times
     and by at least `floor`, so a 0.1% → 0.3% share is never news."""
@@ -60,8 +69,10 @@ WHERE l.source_module || '/' || l.metric_name = ANY(:keys)
 
 
 async def levels_now(session, now: datetime) -> dict[tuple[str, str, str], float]:
-    """{(source, resource, metric): its average over the last WINDOW}."""
-    keys = [f"{s}/{m}" for s, m in LEARNED]
+    """{(source, resource, metric): its average over the last WINDOW}, for
+    every learned metric and every metric one is gated on."""
+    keys = sorted({f"{s}/{m}" for s, m in LEARNED}
+                  | {f"{k.source_module}/{k.gate[0]}" for k in LEARNED.values() if k.gate})
     if not keys:
         return {}
     rows = await session.execute(_LEVELS, {"keys": keys, "since": now - WINDOW, "now": now})
@@ -72,7 +83,9 @@ async def detect(session, now: datetime | None = None) -> dict:
     """One pass: open, extend and close occurrences. Runs inside the caller's
     transaction. Returns {"opened", "closed"} counts."""
     now = now or datetime.now(timezone.utc)
-    levels = await levels_now(session, now)
+    reported = await levels_now(session, now)
+    levels = {key: level for key, level in reported.items()
+              if (key[0], key[2]) in LEARNED and _gate_open(LEARNED[(key[0], key[2])], key, reported)}
     norms = await norms_at(session, list(levels), now)
     open_now = {
         (a.source_module, a.resource_name, a.metric_name): (span, a)
@@ -97,8 +110,8 @@ async def detect(session, now: datetime | None = None) -> dict:
         elif out:
             await _occur(session, key, level, norm, now)
             opened += 1
-    # An open occurrence whose key stopped reporting, or lost its normal,
-    # can't be judged any more: it ends now.
+    # An open occurrence whose key stopped reporting, lost its normal or fell
+    # below its gate can't be judged any more: it ends now.
     for span, anomaly in open_now.values():
         _close(anomaly, span, now)
         closed += 1

steward/core/norms.py

@@ -31,10 +31,11 @@ absorbed on its own, and a lapsed anomaly is never learned.
 """
 from __future__ import annotations
 
+from collections.abc import Callable
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 
-from sqlalchemy import ARRAY, String, bindparam, delete, select, text, tuple_
+from sqlalchemy import ARRAY, DateTime, String, bindparam, delete, select, text, tuple_
 
 from steward.models.norms import MetricNorm
 
@@ -50,19 +51,31 @@ USUAL = 0.9             # "usual" is this percentile across past days: one odd d
 class Learned:
     """A metric that learns a normal. `floor` is the smallest departure,
     in the metric's own unit, worth calling an anomaly (#5740), so that
-    0.1% → 0.3% of a share or 2 ms → 4 ms of latency is never news."""
+    0.1% → 0.3% of a share or 2 ms → 4 ms of latency is never news.
+
+    `gate` is (metric, least): the resource is judged only while that metric
+    of its own averages at least `least` over the window, as a router's
+    error share means nothing over a handful of requests (#5741). `least`
+    may be a function, read at each pass, for a minimum that is a setting. `since`:
+    history before it is not learned from, for a metric whose meaning
+    changed then."""
     source_module: str
     metric_name: str
     floor: float
+    gate: tuple[str, float | Callable[[], float]] | None = None
+    since: datetime | None = None
 
 
 LEARNED: dict[tuple[str, str], Learned] = {}
 
 
-def learn_normal(source_module: str, metric_name: str, floor: float) -> None:
+def learn_normal(source_module: str, metric_name: str, floor: float, *,
+                 gate: tuple[str, float | Callable[[], float]] | None = None,
+                 since: datetime | None = None) -> None:
     """Have Steward learn this metric's normal for every resource reporting
     it. A plugin calls this from setup(), as it calls prune_by_age."""
-    LEARNED[(source_module, metric_name)] = Learned(source_module, metric_name, float(floor))
+    LEARNED[(source_module, metric_name)] = Learned(source_module, metric_name, float(floor),
+                                                    gate, since)
 
 
 @dataclass(frozen=True)
@@ -89,17 +102,21 @@ def slot_of(at: datetime, kind: str) -> int:
 # each slot's usual hold and peak across past days, every hour counted in its
 # own slot and its neighbours'.
 _REBUILD = text("""
-WITH all_hours AS (
-    SELECT source_module AS s, resource_name AS r, metric_name AS m, bucket AS b,
-           COALESCE(value_p95, value_avg) AS hold, value_max AS peak
-    FROM plugin_metrics_hourly
-    WHERE bucket >= :week_since AND source_module || '/' || metric_name = ANY(:keys)
+WITH learned AS (
+    SELECT key, since FROM unnest(CAST(:keys AS text[]), CAST(:starts AS timestamptz[])) AS k(key, since)
+),
+all_hours AS (
+    SELECT h.source_module AS s, h.resource_name AS r, h.metric_name AS m, h.bucket AS b,
+           COALESCE(h.value_p95, h.value_avg) AS hold, h.value_max AS peak
+    FROM plugin_metrics_hourly h
+    JOIN learned k ON k.key = h.source_module || '/' || h.metric_name
+    WHERE h.bucket >= k.since
     UNION ALL
-    SELECT source_module, resource_name, metric_name, date_trunc('hour', recorded_at),
-           percentile_cont(0.95) WITHIN GROUP (ORDER BY value), max(value)
-    FROM plugin_metrics
-    WHERE recorded_at >= :week_since AND recorded_at < :this_hour
-      AND source_module || '/' || metric_name = ANY(:keys)
+    SELECT p.source_module, p.resource_name, p.metric_name, date_trunc('hour', p.recorded_at),
+           percentile_cont(0.95) WITHIN GROUP (ORDER BY p.value), max(p.value)
+    FROM plugin_metrics p
+    JOIN learned k ON k.key = p.source_module || '/' || p.metric_name
+    WHERE p.recorded_at >= k.since AND p.recorded_at < :this_hour
     GROUP BY 1, 2, 3, 4
 ),
 hours AS (
@@ -139,7 +156,8 @@ SELECT s, r, m, (slot + n.off + slots) % slots, kind,
        days, :now
 FROM slotted CROSS JOIN generate_series(-CAST(:neighbours AS int), CAST(:neighbours AS int)) AS n(off)
 GROUP BY s, r, m, (slot + n.off + slots) % slots, kind, days
-""").bindparams(bindparam("keys", type_=ARRAY(String)))
+""").bindparams(bindparam("keys", type_=ARRAY(String)),
+               bindparam("starts", type_=ARRAY(DateTime(timezone=True))))
 
 
 async def rebuild_norms(session, now: datetime | None = None) -> int:
@@ -147,12 +165,13 @@ async def rebuild_norms(session, now: datetime | None = None) -> int:
     inside the caller's transaction. Returns how many slot rows it wrote."""
     now = now or datetime.now(timezone.utc)
     await session.execute(delete(MetricNorm))
-    keys = [f"{s}/{m}" for s, m in LEARNED]
-    if not keys:
+    if not LEARNED:
         return 0
+    week_since = now - WEEK_WINDOW
     result = await session.execute(_REBUILD, {
-        "keys": keys,
-        "week_since": now - WEEK_WINDOW,
+        "keys": [f"{s}/{m}" for s, m in LEARNED],
+        # Each key learns from WEEK_WINDOW back, or from its own `since`.
+        "starts": [max(week_since, k.since) if k.since else week_since for k in LEARNED.values()],
         "day_since": now - DAY_WINDOW,
         "this_hour": now.replace(minute=0, second=0, microsecond=0),
         "week_ready": now - timedelta(days=WEEK_MIN_DAYS),

steward/core/settings.py

@@ -97,6 +97,9 @@ DEFAULTS: dict[str, Any] = {
     # than this share of 4xx is worth a look (#5380).
     "traefik.threshold.error_5xx_pct": 0.1,
     "traefik.threshold.error_4xx_pct": 1.0,
+    # A router's 4xx share over fewer requests than this in 15 minutes isn't
+    # graded, on the page or for anomalies: one 404 of three is 33% (#5741).
+    "traefik.threshold.min_requests": 30,
     # Degraded/critical cutoffs for metric coloring (warn=amber, crit=red).
     # cpu/mem/disk/load are percentages (load is load-per-core %); temp in °C.
     # uptime is "higher is better" so its warn/crit are floors. Ping latency
@@ -534,6 +537,9 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict:
                       "crit": g("traefik.threshold.error_5xx_pct", 0.1), "dir": "high"},
         "error_4xx": {"warn": g("traefik.threshold.error_4xx_pct", 1.0),
                       "crit": g("traefik.threshold.error_4xx_pct", 1.0), "dir": "high"},
+        # Fewer requests than this in 15 minutes and a router's 4xx share isn't graded.
+        "router_requests": {"warn": g("traefik.threshold.min_requests", 30),
+                            "crit": g("traefik.threshold.min_requests", 30), "dir": "low"},
     }
 
 

steward/settings/routes.py

@@ -144,6 +144,7 @@ _THRESHOLD_FIELDS = [
     ("crash_loop_deaths", "thresholds.crash_loop_deaths", False),
     ("error_5xx_pct", "traefik.threshold.error_5xx_pct", True),
     ("error_4xx_pct", "traefik.threshold.error_4xx_pct", True),
+    ("min_requests", "traefik.threshold.min_requests", False),
 ]
 
 # Docker time-series retention windows (days). Read fresh by the cleanup task,

steward/templates/settings/thresholds.html

@@ -40,7 +40,7 @@
   {{ pair("Disk usage",   "disk_warn", "thresholds.disk_warn", "disk_crit", "thresholds.disk_crit", "%", "Filesystem usage.") }}
   {{ pair("Load",         "load_warn", "thresholds.load_warn", "load_crit", "thresholds.load_crit", "% per core", "Load average normalized by core count (100% = one full core per core).") }}
   {{ pair("Temperature",  "temp_warn", "thresholds.temp_warn", "temp_crit", "thresholds.temp_crit", "°C", "Hottest reported sensor per host.") }}
-  {{ pair("Response time", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "How long a monitor's check takes to answer. Also grades Traefik routers' p95 and the UniFi WAN round trip.") }}
+  {{ pair("Response time", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "How long a monitor's check takes to answer. Also grades Traefik routers' p95 (until a router has its own normal) and the UniFi WAN round trip.") }}
   {{ pair("Uptime / SLA", "uptime_warn", "thresholds.uptime_warn", "uptime_crit", "thresholds.uptime_crit", "%", "Lower is worse — amber below warn, red below crit.", step="0.1") }}
   {{ pair("TLS certificate expiry", "tls_warn", "thresholds.tls_warn_days", "tls_crit", "thresholds.tls_crit_days", "days left", "HTTPS monitors whose certificate expires within these many days are flagged. Lower is worse.") }}
   {{ pair("Clock drift", "clock_warn", "thresholds.clock_warn_s", "clock_crit", "thresholds.clock_crit_s", "seconds", "How far a host's clock may be from Steward's, either way, before its page and the Hosts list call it out.") }}
@@ -69,13 +69,21 @@
     <label for="error_5xx_pct">Server errors (5xx) <span style="color:var(--text-muted);font-size:0.8rem;">(% of responses)</span></label>
     <input type="number" id="error_5xx_pct" name="error_5xx_pct" min="0" max="100" step="0.01"
            value="{{ settings['traefik.threshold.error_5xx_pct'] }}" style="max-width:110px;margin-top:0.25rem;">
-    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">Above this share, a router is failing (red).</div>
+    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">Above this share, a router is failing (red), whatever its history.</div>
   </div>
   <div class="form-group" style="margin-bottom:0.25rem;">
     <label for="error_4xx_pct">Client errors (4xx) <span style="color:var(--text-muted);font-size:0.8rem;">(% of responses)</span></label>
     <input type="number" id="error_4xx_pct" name="error_4xx_pct" min="0" max="100" step="0.1"
            value="{{ settings['traefik.threshold.error_4xx_pct'] }}" style="max-width:110px;margin-top:0.25rem;">
-    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">Above this share, a router is worth a look (amber): missing routes, failed sign-ins.</div>
+    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">Above this share, a router is worth a look (amber): missing routes, failed sign-ins.
+      Once a router has 5 days of history it is judged against its own normal for the hour instead, as is its p95,
+      so one that always answers some 4xx stops being listed.</div>
+  </div>
+  <div class="form-group" style="margin:1.1rem 0 0.25rem;">
+    <label for="min_requests">Fewest requests to judge 4xx <span style="color:var(--text-muted);font-size:0.8rem;">(in 15 minutes)</span></label>
+    <input type="number" id="min_requests" name="min_requests" min="0" step="1"
+           value="{{ settings['traefik.threshold.min_requests'] }}" style="max-width:110px;margin-top:0.25rem;">
+    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">Below this, a router's 4xx share isn't judged, on its page or as an anomaly: one 404 of three requests is 33%. 5xx is always judged.</div>
   </div>
 </div>
 {% endif %}

tests/integration/test_anomalies.py

@@ -271,3 +271,65 @@ def test_cleanup_prunes_reviewed_anomalies_and_never_one_to_review(app, src):
             )).scalars()}
 
     assert asyncio.run(_go()) == {kept}
+
+
+def test_a_gated_metric_is_judged_only_while_its_gate_is_open(app, src):
+    """A router's 4xx share means nothing over a handful of requests (#5741):
+    learned with gate=("req", 1.0), it is judged only while the resource's
+    own "req" averages at least 1 over the window."""
+    from steward.core import norms
+    from steward.core.anomalies import WINDOW
+    from steward.models.metrics import PluginMetric, PluginMetricLatest
+    norms.learn_normal(src, METRIC, 5, gate=("req", 1.0))
+    res = _res("gated")
+    quiet, busy = NOW, NOW + timedelta(hours=2)
+
+    async def _req(s, until, value):
+        n = int(WINDOW.total_seconds() // 60)
+        s.add_all([PluginMetric(source_module=src, resource_name=res, metric_name="req",
+                                value=value, recorded_at=until - timedelta(minutes=m)) for m in range(n)])
+        await s.merge(PluginMetricLatest(source_module=src, resource_name=res, metric_name="req",
+                                         value=value, recorded_at=until))
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await _req(s, quiet, 0.2)
+                await _req(s, busy, 5.0)
+        return await _steps(app, src, res, [(quiet, 60.0), (busy, 60.0)])
+
+    rows, spans = asyncio.run(_go())
+    (a,) = rows                                   # only the busy window opened one
+    assert spans[a.id][0].started_at == busy - WINDOW
+
+
+def test_a_metric_learns_only_from_its_since(app, src):
+    """A metric whose meaning changed (Traefik latency, #5755) learns only
+    from history after `since`: seven days held 90, but only the last three
+    are after it, under the five a normal needs."""
+    from steward.core import norms
+    from steward.core.norms import norms_at, rebuild_norms
+    from steward.models.metrics import PluginMetricHourly
+    res = _res("since")
+    rows = [PluginMetricHourly(source_module=src, resource_name=res, metric_name=METRIC,
+                               bucket=NOW.replace(hour=0) - timedelta(days=d) + timedelta(hours=h),
+                               value_avg=90.0, value_max=90.0, value_p95=90.0, sample_count=60)
+            for d in range(1, 8) for h in range(24)]
+
+    async def _go(since):
+        norms.learn_normal(src, METRIC, 5, since=since)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await rebuild_norms(s, NOW)
+            got = await norms_at(s, [(src, res, METRIC)], NOW.replace(hour=14))
+            return got.get((src, res, METRIC))
+
+    async def _both():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add_all(rows)
+        return await _go(None), await _go(NOW.replace(hour=0) - timedelta(days=3))
+
+    learned, held_back = asyncio.run(_both())
+    assert learned is not None and learned.usual_hold == 90.0
+    assert held_back is None

tests/plugins/traefik/test_normals.py

@@ -0,0 +1,85 @@
+"""A router's 4xx share and p95 are graded against its own normal once it has
+one (#5741, System 182): a router over the fixed cutoff but inside its normal
+is healthy and says so; one out of its normal is a warning that says what is
+usual. The 5xx share keeps its fixed cutoff, and a 4xx share over too few
+requests isn't graded."""
+from types import SimpleNamespace as NS
+
+from plugins.traefik.routes import STATE_WINDOW_WORDS, grade_routers, router_state
+from steward.core.norms import Norm
+from steward.core.settings import to_thresholds_cfg
+from tests.plugins.traefik.test_traefik_page import _latest
+
+TH = to_thresholds_cfg({})          # 4xx warn over 1%, 5xx crit over 0.1%, latency crit 200 ms
+
+
+def _norm(hold, peak=None):
+    return Norm(kind="day", slot=14, usual_hold=hold, usual_peak=peak or hold * 1.5, days_seen=7)
+
+
+def _window(e4=0.0, e5=0.0, p95=20.0, requests=500.0):
+    return NS(error_rate_4xx_pct=e4, error_rate_5xx_pct=e5, latency_p95_ms=p95, requests=requests)
+
+
+def test_an_error_pages_router_answering_4xx_all_day_reads_healthy():
+    state, why = router_state(_window(e4=62.0), TH, STATE_WINDOW_WORDS,
+                              {"error_rate_4xx_pct": _norm(60.0)})
+    assert state == "ok"
+    assert why == "4xx 62.0% is usual 60.0% at this hour"
+
+
+def test_a_router_that_flapped_on_the_cutoff_stays_healthy_inside_its_normal():
+    """authentik-like: a login wall's 401s sit around 1–3%, either side of the
+    1% cutoff, so it came and went from the problem list every few polls."""
+    norm = {"error_rate_4xx_pct": _norm(2.8)}
+    for e4 in (0.6, 1.4, 2.9, 1.1, 3.2, 0.9):
+        assert router_state(_window(e4=e4), TH, STATE_WINDOW_WORDS, norm)[0] == "ok", e4
+
+
+def test_out_of_its_normal_is_a_warning_that_says_what_is_usual():
+    state, why = router_state(_window(e4=12.0), TH, STATE_WINDOW_WORDS,
+                              {"error_rate_4xx_pct": _norm(2.8)})
+    assert state == "warn"
+    assert why == "12.0% of responses are 4xx over the last 15 min, usual 2.8% at this hour"
+
+
+def test_a_departure_smaller_than_the_floor_is_no_news():
+    # 1.25 × 1.2% is 1.5%, but 2.0% is only 0.8 points over: under the 2-point floor.
+    assert router_state(_window(e4=2.0), TH, "", {"error_rate_4xx_pct": _norm(1.2)})[0] == "ok"
+
+
+def test_latency_is_graded_against_its_normal_too():
+    slow_usually = {"latency_p95_ms": _norm(900.0)}
+    assert router_state(_window(p95=950.0), TH, "", slow_usually) == ("ok", "p95 950 ms is usual 900 ms at this hour")
+    state, why = router_state(_window(p95=3000.0), TH, "", slow_usually)
+    assert state == "warn" and why == "p95 latency is 3000 ms, usual 900 ms at this hour"
+
+
+def test_without_a_normal_the_fixed_cutoffs_still_apply():
+    assert router_state(_window(e4=5.0), TH, "")[0] == "warn"
+    assert router_state(_window(p95=450.0), TH, "")[0] == "warn"
+
+
+def test_5xx_keeps_its_cutoff_whatever_the_history_or_traffic():
+    usual_5xx = {"error_rate_4xx_pct": _norm(50.0)}
+    assert router_state(_window(e5=40.0), TH, "", usual_5xx)[0] == "crit"
+    assert router_state(_window(e5=40.0, requests=4.0), TH, "")[0] == "crit"
+
+
+def test_a_4xx_share_over_a_handful_of_requests_is_not_graded():
+    assert router_state(_window(e4=33.0, requests=3.0), TH, "") == ("ok", None)
+    assert router_state(_window(e4=33.0, requests=300.0), TH, "")[0] == "warn"
+
+
+def test_grade_routers_hands_each_router_its_own_normals():
+    routers = [{"name": "errors@docker", "latest": _latest(e4=60.0)},
+               {"name": "app@docker", "latest": _latest(e4=60.0)}]
+    window = {"errors@docker": _window(e4=60.0), "app@docker": _window(e4=60.0)}
+    grade_routers(routers, window, TH, {"errors@docker": {"error_rate_4xx_pct": _norm(58.0)}})
+    assert [r["state"] for r in routers] == ["ok", "warn"]
+
+
+def test_the_fewest_requests_to_judge_is_the_thresholds_setting():
+    fewer = to_thresholds_cfg({"traefik.threshold.min_requests": 5})
+    assert router_state(_window(e4=33.0, requests=10.0), TH, "")[0] == "ok"        # default 30
+    assert router_state(_window(e4=33.0, requests=10.0), fewer, "")[0] == "warn"