feat(traefik): a 4xx share is worth a look only once it is a lot of 4xx (#6105)

Three 404s on a quiet router read 7.5% and were raised as an anomaly; most of
the 4xx anomalies on review stood for 3 to 7 errors in 15 minutes. A new
Settings → Thresholds minimum, traefik.threshold.min_4xx (default 20, more
than one a minute held for the window), applies to both the router's grade
on its page and to anomaly detection, as min_5xx does for 5xx.

learn_normal takes `fewest`: for a share gated on a per-second rate, the
fewest events (rate × window × share) it must stand for to be judged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/anomalies.py

@@ -38,11 +38,18 @@ MARKS = (NORMAL, REAL)
 
 def _gate_open(learned, key: tuple[str, str, str], reported: dict) -> bool:
     """Whether `key` may be judged: ungated, or its resource's gate metric
-    averaged at least the gate's least over the window."""
+    averaged at least the gate's least over the window, and the share stands
+    for at least `fewest` events in it."""
     if learned.gate is None:
         return True
     metric, least = learned.gate
-    return reported.get((key[0], key[1], metric), 0.0) >= (least() if callable(least) else least)
+    rate = reported.get((key[0], key[1], metric), 0.0)
+    if rate < (least() if callable(least) else least):
+        return False
+    if learned.fewest is None:
+        return True
+    fewest = learned.fewest() if callable(learned.fewest) else learned.fewest
+    return rate * WINDOW.total_seconds() * reported.get(key, 0.0) / 100.0 >= fewest
 
 
 def departs(level: float, norm: Norm, floor: float) -> bool:

steward/core/norms.py

@@ -57,7 +57,11 @@ class Learned:
     `gate` is (metric, least): the resource is judged only while that metric
     of its own averages at least `least` over the window, as a router's
     error share means nothing over a handful of requests (#5741). `least`
-    may be a function, read at each pass, for a minimum that is a setting. `since`:
+    may be a function, read at each pass, for a minimum that is a setting.
+    `fewest`, for a share (%) gated on a per-second rate: the fewest events
+    the share stands for over the window (rate × window × share / 100), as
+    three 404s are 7% of a quiet router and no news (#6105); a number or a
+    function, like `least`. `since`:
     history before it is not learned from, for a metric whose meaning
     changed then. `parts=False` learns only whole resources, not their parts
     ("<host>:core3"): a host's CPU has a normal, each of its cores doesn't."""
@@ -67,6 +71,7 @@ class Learned:
     gate: tuple[str, float | Callable[[], float]] | None = None
     since: datetime | None = None
     parts: bool = True
+    fewest: float | Callable[[], float] | None = None
 
     @property
     def least_departure(self) -> float:
@@ -83,11 +88,13 @@ LEARNED: dict[tuple[str, str], Learned] = {}
 
 def learn_normal(source_module: str, metric_name: str, floor: float | Callable[[], float], *,
                  gate: tuple[str, float | Callable[[], float]] | None = None,
-                 since: datetime | None = None, parts: bool = True) -> None:
+                 since: datetime | None = None, parts: bool = True,
+                 fewest: float | Callable[[], float] | None = None) -> None:
     """Have Steward learn this metric's normal for every resource reporting
     it. A plugin calls this from setup(), as it calls prune_by_age."""
     LEARNED[(source_module, metric_name)] = Learned(
-        source_module, metric_name, floor if callable(floor) else float(floor), gate, since, parts)
+        source_module, metric_name, floor if callable(floor) else float(floor), gate, since, parts,
+        fewest)
 
 
 def floor_setting(app, name: str) -> Callable[[], float]:

steward/core/settings.py

@@ -103,6 +103,12 @@ DEFAULTS: dict[str, Any] = {
     # ...and a 5xx share past its cutoff is failing only once this many 5xx
     # were answered in those 15 minutes: one 502 of 70 is 1.4% (#5865).
     "traefik.threshold.min_5xx": 3,
+    # ...and a 4xx share past its cutoff, or departing its normal, is worth a
+    # look only once this many 4xx were answered: some 404s and failed
+    # sign-ins are expected all day; it is a lot of them that needs acting on.
+    # 20 is more than one a minute held for the window; the blips it drops
+    # were 3 to 7 errors (#6105).
+    "traefik.threshold.min_4xx": 20,
     # The smallest departure from a metric's normal worth calling an anomaly
     # (#5742), on top of being a quarter over its usual hold: a share's floor
     # is in percentage points, a response time's in ms. 5 points is the
@@ -560,6 +566,9 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict:
         # Fewer 5xx than this in 15 minutes and a router isn't failing, whatever the share.
         "router_5xx": {"warn": g("traefik.threshold.min_5xx", 3),
                        "crit": g("traefik.threshold.min_5xx", 3), "dir": "low"},
+        # Fewer 4xx than this in 15 minutes and a router's 4xx share isn't news.
+        "router_4xx": {"warn": g("traefik.threshold.min_4xx", 20),
+                       "crit": g("traefik.threshold.min_4xx", 20), "dir": "low"},
         # Not a cutoff: each normal's floor (norms.floor.*), read by floor_of().
         "floors": {name: float(g(f"norms.floor.{name}", DEFAULTS[f"norms.floor.{name}"]))
                    for name in NORM_FLOORS},