feat(traefik): a 4xx share is worth a look only once it is a lot of 4xx (#6105)

Three 404s on a quiet router read 7.5% and were raised as an anomaly; most of
the 4xx anomalies on review stood for 3 to 7 errors in 15 minutes. A new
Settings → Thresholds minimum, traefik.threshold.min_4xx (default 20, more
than one a minute held for the window), applies to both the router's grade
on its page and to anomaly detection, as min_5xx does for 5xx.

learn_normal takes `fewest`: for a share gated on a per-second rate, the
fewest events (rate × window × share) it must stand for to be judged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/__init__.py

@@ -22,15 +22,17 @@ def setup(app: "Quart") -> None:
     prune_by_age(TraefikAccessSummary.period_end)
     # Each router's 4xx share and p95 learn their normal (#5741), judged only
     # while the router answers the Thresholds minimum of requests a window
-    # (read at each pass, so a saved change applies at once). Latency is
+    # (read at each pass, so a saved change applies at once), and the 4xx
+    # share only once it stands for the minimum of 4xx (#6105). Latency is
     # learned from the scrape-by-scrape figures only: before #5755 it was a
     # percentile of every response since Traefik started.
     from datetime import datetime, timezone
     from steward.core.norms import floor_setting, learn_normal
-    from .routes import STATE_WINDOW, min_requests
+    from .routes import STATE_WINDOW, min_4xx, min_requests
     gate = ("request_rate",
             lambda: min_requests(app.config.get("THRESHOLDS", {})) / STATE_WINDOW.total_seconds())
-    learn_normal("traefik", "error_rate_4xx_pct", floor_setting(app, "router_4xx_pct"), gate=gate)
+    learn_normal("traefik", "error_rate_4xx_pct", floor_setting(app, "router_4xx_pct"), gate=gate,
+                 fewest=lambda: min_4xx(app.config.get("THRESHOLDS", {})))
     learn_normal("traefik", "latency_p95_ms", floor_setting(app, "router_p95_ms"), gate=gate,
                  since=datetime(2026, 10, 14, 18, 47, tzinfo=timezone.utc))
 

plugins/traefik/mcp_tools.py

@@ -117,7 +117,8 @@ async def list_routers(state: str = "", limit: int = 100) -> dict:
     out = bounded(rows, clamp_limit(limit, 100, 500), key="routers",
                   more="Pass a larger `limit`, or a `state`.")
     out["cutoffs"] = {"error_5xx_pct": error_5xx, "error_4xx_pct": error_4xx,
-                      "min_requests_15m": page.min_requests(thresholds)}
+                      "min_requests_15m": page.min_requests(thresholds),
+                      "min_5xx_15m": page.min_5xx(thresholds), "min_4xx_15m": page.min_4xx(thresholds)}
     out["gone"] = [{"router": r["name"], "last_scraped_at": iso(r["latest"].scraped_at)} for r in gone[:20]]
     return out
 

plugins/traefik/routes.py

@@ -65,6 +65,8 @@ def error_cutoffs(thresholds: dict) -> tuple[float, float]:
 # keeps its fixed cutoff whatever the history; it shares the 4xx share's
 # minimum of traffic (the operator, #5859), and is failing only once
 # min_5xx() errors were answered: one 502 of 70 requests is 1.4% (#5865).
+# A 4xx share is worth a look only once min_4xx() were answered: three 404s
+# on a quiet router read 7%, and some 4xx are part of any day (#6105).
 NORMED = ("error_rate_4xx_pct", "latency_p95_ms")
 
 
@@ -80,6 +82,13 @@ def min_5xx(thresholds: dict) -> float:
     return float(thresholds.get("router_5xx", {}).get("warn", 3))
 
 
+def min_4xx(thresholds: dict) -> float:
+    """Fewest 4xx answered over the STATE_WINDOW for a 4xx share to be worth a
+    look, on the page or as an anomaly (Settings → Thresholds,
+    traefik.threshold.min_4xx)."""
+    return float(thresholds.get("router_4xx", {}).get("warn", 20))
+
+
 def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = None) -> tuple[str, str | None]:
     """(grade, why) for a router's reading (recent_window's, or one scrape's):
     crit while it serves 5xx past its cutoff, warn on a p95 past the latency
@@ -87,8 +96,9 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None =
     otherwise ok. `norms` ({metric: Norm}) are the router's normals for this
     hour: a p95 or 4xx share past its cutoff but within its normal is ok, and
     `why` then says it is usual. `over` names the span the reading covers.
-    Neither share is graded over fewer than min_requests() requests, and a
-    5xx share is failing only once about min_5xx() errors were answered."""
+    Neither share is graded over fewer than min_requests() requests, a 5xx
+    share is failing only once about min_5xx() errors were answered, and a
+    4xx share is worth a look only once about min_4xx() were."""
     norms = norms or {}
     error_5xx, error_4xx = error_cutoffs(thresholds)
     requests = getattr(reading, "requests", None)
@@ -106,7 +116,8 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None =
             return "warn", f"p95 latency is {p95:.0f} ms{over}" + (f", {said}" if said else "")
         usual.append(f"p95 {p95:.0f} ms is {said}")
     e4 = reading.error_rate_4xx_pct or 0
-    if enough and e4 > error_4xx:
+    errors4 = None if requests is None else round(requests * e4 / 100.0)
+    if enough and e4 > error_4xx and (errors4 is None or errors4 >= min_4xx(thresholds)):
         out, said = anom.against(e4, norms.get("error_rate_4xx_pct"),
                                   floor_of(thresholds, "router_4xx_pct"), "%")
         if out: