feat(traefik): a 4xx share is worth a look only once it is a lot of 4xx (#6105)

Three 404s on a quiet router read 7.5% and were raised as an anomaly; most of
the 4xx anomalies on review stood for 3 to 7 errors in 15 minutes. A new
Settings → Thresholds minimum, traefik.threshold.min_4xx (default 20, more
than one a minute held for the window), applies to both the router's grade
on its page and to anomaly detection, as min_5xx does for 5xx.

learn_normal takes `fewest`: for a share gated on a per-second rate, the
fewest events (rate × window × share) it must stand for to be judged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/__init__.py

@@ -22,15 +22,17 @@ def setup(app: "Quart") -> None:
     prune_by_age(TraefikAccessSummary.period_end)
     # Each router's 4xx share and p95 learn their normal (#5741), judged only
     # while the router answers the Thresholds minimum of requests a window
-    # (read at each pass, so a saved change applies at once). Latency is
+    # (read at each pass, so a saved change applies at once), and the 4xx
+    # share only once it stands for the minimum of 4xx (#6105). Latency is
     # learned from the scrape-by-scrape figures only: before #5755 it was a
     # percentile of every response since Traefik started.
     from datetime import datetime, timezone
     from steward.core.norms import floor_setting, learn_normal
-    from .routes import STATE_WINDOW, min_requests
+    from .routes import STATE_WINDOW, min_4xx, min_requests
     gate = ("request_rate",
             lambda: min_requests(app.config.get("THRESHOLDS", {})) / STATE_WINDOW.total_seconds())
-    learn_normal("traefik", "error_rate_4xx_pct", floor_setting(app, "router_4xx_pct"), gate=gate)
+    learn_normal("traefik", "error_rate_4xx_pct", floor_setting(app, "router_4xx_pct"), gate=gate,
+                 fewest=lambda: min_4xx(app.config.get("THRESHOLDS", {})))
     learn_normal("traefik", "latency_p95_ms", floor_setting(app, "router_p95_ms"), gate=gate,
                  since=datetime(2026, 10, 14, 18, 47, tzinfo=timezone.utc))
 

plugins/traefik/mcp_tools.py

@@ -117,7 +117,8 @@ async def list_routers(state: str = "", limit: int = 100) -> dict:
     out = bounded(rows, clamp_limit(limit, 100, 500), key="routers",
                   more="Pass a larger `limit`, or a `state`.")
     out["cutoffs"] = {"error_5xx_pct": error_5xx, "error_4xx_pct": error_4xx,
-                      "min_requests_15m": page.min_requests(thresholds)}
+                      "min_requests_15m": page.min_requests(thresholds),
+                      "min_5xx_15m": page.min_5xx(thresholds), "min_4xx_15m": page.min_4xx(thresholds)}
     out["gone"] = [{"router": r["name"], "last_scraped_at": iso(r["latest"].scraped_at)} for r in gone[:20]]
     return out
 

plugins/traefik/routes.py

@@ -65,6 +65,8 @@ def error_cutoffs(thresholds: dict) -> tuple[float, float]:
 # keeps its fixed cutoff whatever the history; it shares the 4xx share's
 # minimum of traffic (the operator, #5859), and is failing only once
 # min_5xx() errors were answered: one 502 of 70 requests is 1.4% (#5865).
+# A 4xx share is worth a look only once min_4xx() were answered: three 404s
+# on a quiet router read 7%, and some 4xx are part of any day (#6105).
 NORMED = ("error_rate_4xx_pct", "latency_p95_ms")
 
 
@@ -80,6 +82,13 @@ def min_5xx(thresholds: dict) -> float:
     return float(thresholds.get("router_5xx", {}).get("warn", 3))
 
 
+def min_4xx(thresholds: dict) -> float:
+    """Fewest 4xx answered over the STATE_WINDOW for a 4xx share to be worth a
+    look, on the page or as an anomaly (Settings → Thresholds,
+    traefik.threshold.min_4xx)."""
+    return float(thresholds.get("router_4xx", {}).get("warn", 20))
+
+
 def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = None) -> tuple[str, str | None]:
     """(grade, why) for a router's reading (recent_window's, or one scrape's):
     crit while it serves 5xx past its cutoff, warn on a p95 past the latency
@@ -87,8 +96,9 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None =
     otherwise ok. `norms` ({metric: Norm}) are the router's normals for this
     hour: a p95 or 4xx share past its cutoff but within its normal is ok, and
     `why` then says it is usual. `over` names the span the reading covers.
-    Neither share is graded over fewer than min_requests() requests, and a
-    5xx share is failing only once about min_5xx() errors were answered."""
+    Neither share is graded over fewer than min_requests() requests, a 5xx
+    share is failing only once about min_5xx() errors were answered, and a
+    4xx share is worth a look only once about min_4xx() were."""
     norms = norms or {}
     error_5xx, error_4xx = error_cutoffs(thresholds)
     requests = getattr(reading, "requests", None)
@@ -106,7 +116,8 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None =
             return "warn", f"p95 latency is {p95:.0f} ms{over}" + (f", {said}" if said else "")
         usual.append(f"p95 {p95:.0f} ms is {said}")
     e4 = reading.error_rate_4xx_pct or 0
-    if enough and e4 > error_4xx:
+    errors4 = None if requests is None else round(requests * e4 / 100.0)
+    if enough and e4 > error_4xx and (errors4 is None or errors4 >= min_4xx(thresholds)):
         out, said = anom.against(e4, norms.get("error_rate_4xx_pct"),
                                   floor_of(thresholds, "router_4xx_pct"), "%")
         if out:

steward/core/anomalies.py

@@ -38,11 +38,18 @@ MARKS = (NORMAL, REAL)
 
 def _gate_open(learned, key: tuple[str, str, str], reported: dict) -> bool:
     """Whether `key` may be judged: ungated, or its resource's gate metric
-    averaged at least the gate's least over the window."""
+    averaged at least the gate's least over the window, and the share stands
+    for at least `fewest` events in it."""
     if learned.gate is None:
         return True
     metric, least = learned.gate
-    return reported.get((key[0], key[1], metric), 0.0) >= (least() if callable(least) else least)
+    rate = reported.get((key[0], key[1], metric), 0.0)
+    if rate < (least() if callable(least) else least):
+        return False
+    if learned.fewest is None:
+        return True
+    fewest = learned.fewest() if callable(learned.fewest) else learned.fewest
+    return rate * WINDOW.total_seconds() * reported.get(key, 0.0) / 100.0 >= fewest
 
 
 def departs(level: float, norm: Norm, floor: float) -> bool:

steward/core/norms.py

@@ -57,7 +57,11 @@ class Learned:
     `gate` is (metric, least): the resource is judged only while that metric
     of its own averages at least `least` over the window, as a router's
     error share means nothing over a handful of requests (#5741). `least`
-    may be a function, read at each pass, for a minimum that is a setting. `since`:
+    may be a function, read at each pass, for a minimum that is a setting.
+    `fewest`, for a share (%) gated on a per-second rate: the fewest events
+    the share stands for over the window (rate × window × share / 100), as
+    three 404s are 7% of a quiet router and no news (#6105); a number or a
+    function, like `least`. `since`:
     history before it is not learned from, for a metric whose meaning
     changed then. `parts=False` learns only whole resources, not their parts
     ("<host>:core3"): a host's CPU has a normal, each of its cores doesn't."""
@@ -67,6 +71,7 @@ class Learned:
     gate: tuple[str, float | Callable[[], float]] | None = None
     since: datetime | None = None
     parts: bool = True
+    fewest: float | Callable[[], float] | None = None
 
     @property
     def least_departure(self) -> float:
@@ -83,11 +88,13 @@ LEARNED: dict[tuple[str, str], Learned] = {}
 
 def learn_normal(source_module: str, metric_name: str, floor: float | Callable[[], float], *,
                  gate: tuple[str, float | Callable[[], float]] | None = None,
-                 since: datetime | None = None, parts: bool = True) -> None:
+                 since: datetime | None = None, parts: bool = True,
+                 fewest: float | Callable[[], float] | None = None) -> None:
     """Have Steward learn this metric's normal for every resource reporting
     it. A plugin calls this from setup(), as it calls prune_by_age."""
     LEARNED[(source_module, metric_name)] = Learned(
-        source_module, metric_name, floor if callable(floor) else float(floor), gate, since, parts)
+        source_module, metric_name, floor if callable(floor) else float(floor), gate, since, parts,
+        fewest)
 
 
 def floor_setting(app, name: str) -> Callable[[], float]:

steward/core/settings.py

@@ -103,6 +103,12 @@ DEFAULTS: dict[str, Any] = {
     # ...and a 5xx share past its cutoff is failing only once this many 5xx
     # were answered in those 15 minutes: one 502 of 70 is 1.4% (#5865).
     "traefik.threshold.min_5xx": 3,
+    # ...and a 4xx share past its cutoff, or departing its normal, is worth a
+    # look only once this many 4xx were answered: some 404s and failed
+    # sign-ins are expected all day; it is a lot of them that needs acting on.
+    # 20 is more than one a minute held for the window; the blips it drops
+    # were 3 to 7 errors (#6105).
+    "traefik.threshold.min_4xx": 20,
     # The smallest departure from a metric's normal worth calling an anomaly
     # (#5742), on top of being a quarter over its usual hold: a share's floor
     # is in percentage points, a response time's in ms. 5 points is the
@@ -560,6 +566,9 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict:
         # Fewer 5xx than this in 15 minutes and a router isn't failing, whatever the share.
         "router_5xx": {"warn": g("traefik.threshold.min_5xx", 3),
                        "crit": g("traefik.threshold.min_5xx", 3), "dir": "low"},
+        # Fewer 4xx than this in 15 minutes and a router's 4xx share isn't news.
+        "router_4xx": {"warn": g("traefik.threshold.min_4xx", 20),
+                       "crit": g("traefik.threshold.min_4xx", 20), "dir": "low"},
         # Not a cutoff: each normal's floor (norms.floor.*), read by floor_of().
         "floors": {name: float(g(f"norms.floor.{name}", DEFAULTS[f"norms.floor.{name}"]))
                    for name in NORM_FLOORS},

steward/mcp/tools/anomalies.py

@@ -312,7 +312,9 @@ async def get_metric_normal(source_module: str, resource: str, metric: str) -> d
         "learned": bool(slots), "kind": kind,
         "floor": rnd(floor, 3), "floor_setting": learned.floor_name,
         "only_while": ({"metric": learned.gate[0],
-                        "at_least": rnd(learned.gate[1]() if callable(learned.gate[1]) else learned.gate[1], 3)}
+                        "at_least": rnd(learned.gate[1]() if callable(learned.gate[1]) else learned.gate[1], 3),
+                        "fewest_events": (None if learned.fewest is None else
+                                          rnd(learned.fewest() if callable(learned.fewest) else learned.fewest, 3))}
                        if learned.gate else None),
         "now": ({"slot": current.slot, "label": slot_label(current.kind, current.slot),
                  "usual_hold": rnd(current.usual_hold, 3), "usual_peak": rnd(current.usual_peak, 3),

steward/settings/routes.py

@@ -147,6 +147,7 @@ _THRESHOLD_FIELDS = [
     ("error_4xx_pct", "traefik.threshold.error_4xx_pct", True),
     ("min_requests", "traefik.threshold.min_requests", False),
     ("min_5xx", "traefik.threshold.min_5xx", False),
+    ("min_4xx", "traefik.threshold.min_4xx", False),
     # Each normal's floor (#5742), named by its own setting key.
     *[(f"norms.floor.{name}", f"norms.floor.{name}", True) for name in NORM_FLOORS],
 ]

steward/templates/settings/thresholds.html

@@ -92,6 +92,12 @@
            value="{{ settings['traefik.threshold.min_5xx'] }}" style="max-width:110px;margin-top:0.25rem;">
     <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">A router past the 5xx share above is failing only once it has answered this many 5xx: one 502 among 70 requests is 1.4%, but it is one error.</div>
   </div>
+  <div class="form-group" style="margin:1.1rem 0 0.25rem;">
+    <label for="min_4xx">Fewest client errors worth a look <span style="color:var(--text-muted);font-size:0.8rem;">(4xx in 15 minutes)</span></label>
+    <input type="number" id="min_4xx" name="min_4xx" min="1" step="1"
+           value="{{ settings['traefik.threshold.min_4xx'] }}" style="max-width:110px;margin-top:0.25rem;">
+    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">A router's 4xx share is flagged, on its page or as an anomaly, only once it has answered this many 4xx: a few 404s and failed sign-ins are part of a normal day; a lot of them is what needs acting on.</div>
+  </div>
 </div>
 {% endif %}
 

tests/core/test_anomalies.py

@@ -9,7 +9,7 @@ from zoneinfo import ZoneInfo
 from quart import render_template, session
 
 from steward.core import mail
-from steward.core.anomalies import DEPART, AnomalyView, departs
+from steward.core.anomalies import DEPART, WINDOW, AnomalyView, _gate_open, departs
 from steward.core.norms import Norm
 from tests.test_style_tokens import hex_colours
 
@@ -146,3 +146,15 @@ def test_a_week_without_anomalies_leaves_the_section_out():
     quiet = {**REPORT, "anomalies": {"seen": [], "lapsed": [], "to_review": 0}}
     _, text, html = mail.report_email({}, quiet, to)
     assert "anomal" not in (text + html).lower()
+
+
+def test_a_share_gated_on_a_rate_is_judged_only_once_it_stands_for_enough_events():
+    """#6105: three 404s on a quiet router read 7%. With `fewest`, the share
+    is judged only once rate × window × share reaches it."""
+    learned = SimpleNamespace(gate=("req", lambda: 30 / WINDOW.total_seconds()), fewest=lambda: 20)
+    key = ("traefik", "r", "e4")
+    per_s = 100 / WINDOW.total_seconds()                               # 100 requests in the window
+    assert not _gate_open(learned, key, {key: 7.5, ("traefik", "r", "req"): per_s})          # 7.5 errors
+    assert _gate_open(learned, key, {key: 25.0, ("traefik", "r", "req"): per_s})             # 25
+    assert not _gate_open(learned, key, {key: 90.0, ("traefik", "r", "req"): 10 / WINDOW.total_seconds()})
+    assert _gate_open(SimpleNamespace(gate=("req", 0.0), fewest=None), key, {key: 1.0})

tests/plugins/traefik/test_normals.py

@@ -91,6 +91,14 @@ def test_grade_routers_hands_each_router_its_own_normals():
 
 
 def test_the_fewest_requests_to_judge_is_the_thresholds_setting():
-    fewer = to_thresholds_cfg({"traefik.threshold.min_requests": 5})
+    fewer = to_thresholds_cfg({"traefik.threshold.min_requests": 5, "traefik.threshold.min_4xx": 1})
     assert router_state(_window(e4=33.0, requests=10.0), TH, "")[0] == "ok"        # default 30
     assert router_state(_window(e4=33.0, requests=10.0), fewer, "")[0] == "warn"
+
+
+def test_a_4xx_share_is_worth_a_look_only_once_enough_4xx_were_answered():
+    # librarian@swarm on 2026-10-19: 7.5% of about 100 requests, so 7 errors (#6105)
+    assert router_state(_window(e4=7.5, requests=100.0), TH, "") == ("ok", None)
+    assert router_state(_window(e4=7.5, requests=300.0), TH, "")[0] == "warn"     # 22 errors
+    fewer = to_thresholds_cfg({"traefik.threshold.min_4xx": 5})
+    assert router_state(_window(e4=7.5, requests=100.0), fewer, "")[0] == "warn"