feat(traefik): a 4xx share is worth a look only once it is a lot of 4xx (#6105)

Three 404s on a quiet router read 7.5% and were raised as an anomaly; most of
the 4xx anomalies on review stood for 3 to 7 errors in 15 minutes. A new
Settings → Thresholds minimum, traefik.threshold.min_4xx (default 20, more
than one a minute held for the window), applies to both the router's grade
on its page and to anomaly detection, as min_5xx does for 5xx.

learn_normal takes `fewest`: for a share gated on a per-second rate, the
fewest events (rate × window × share) it must stand for to be judged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/anomalies.py

@@ -38,11 +38,18 @@ MARKS = (NORMAL, REAL)
 
 def _gate_open(learned, key: tuple[str, str, str], reported: dict) -> bool:
     """Whether `key` may be judged: ungated, or its resource's gate metric
-    averaged at least the gate's least over the window."""
+    averaged at least the gate's least over the window, and the share stands
+    for at least `fewest` events in it."""
     if learned.gate is None:
         return True
     metric, least = learned.gate
-    return reported.get((key[0], key[1], metric), 0.0) >= (least() if callable(least) else least)
+    rate = reported.get((key[0], key[1], metric), 0.0)
+    if rate < (least() if callable(least) else least):
+        return False
+    if learned.fewest is None:
+        return True
+    fewest = learned.fewest() if callable(learned.fewest) else learned.fewest
+    return rate * WINDOW.total_seconds() * reported.get(key, 0.0) / 100.0 >= fewest
 
 
 def departs(level: float, norm: Norm, floor: float) -> bool:

steward/core/norms.py

@@ -57,7 +57,11 @@ class Learned:
     `gate` is (metric, least): the resource is judged only while that metric
     of its own averages at least `least` over the window, as a router's
     error share means nothing over a handful of requests (#5741). `least`
-    may be a function, read at each pass, for a minimum that is a setting. `since`:
+    may be a function, read at each pass, for a minimum that is a setting.
+    `fewest`, for a share (%) gated on a per-second rate: the fewest events
+    the share stands for over the window (rate × window × share / 100), as
+    three 404s are 7% of a quiet router and no news (#6105); a number or a
+    function, like `least`. `since`:
     history before it is not learned from, for a metric whose meaning
     changed then. `parts=False` learns only whole resources, not their parts
     ("<host>:core3"): a host's CPU has a normal, each of its cores doesn't."""
@@ -67,6 +71,7 @@ class Learned:
     gate: tuple[str, float | Callable[[], float]] | None = None
     since: datetime | None = None
     parts: bool = True
+    fewest: float | Callable[[], float] | None = None
 
     @property
     def least_departure(self) -> float:
@@ -83,11 +88,13 @@ LEARNED: dict[tuple[str, str], Learned] = {}
 
 def learn_normal(source_module: str, metric_name: str, floor: float | Callable[[], float], *,
                  gate: tuple[str, float | Callable[[], float]] | None = None,
-                 since: datetime | None = None, parts: bool = True) -> None:
+                 since: datetime | None = None, parts: bool = True,
+                 fewest: float | Callable[[], float] | None = None) -> None:
     """Have Steward learn this metric's normal for every resource reporting
     it. A plugin calls this from setup(), as it calls prune_by_age."""
     LEARNED[(source_module, metric_name)] = Learned(
-        source_module, metric_name, floor if callable(floor) else float(floor), gate, since, parts)
+        source_module, metric_name, floor if callable(floor) else float(floor), gate, since, parts,
+        fewest)
 
 
 def floor_setting(app, name: str) -> Callable[[], float]:

steward/core/settings.py

@@ -103,6 +103,12 @@ DEFAULTS: dict[str, Any] = {
     # ...and a 5xx share past its cutoff is failing only once this many 5xx
     # were answered in those 15 minutes: one 502 of 70 is 1.4% (#5865).
     "traefik.threshold.min_5xx": 3,
+    # ...and a 4xx share past its cutoff, or departing its normal, is worth a
+    # look only once this many 4xx were answered: some 404s and failed
+    # sign-ins are expected all day; it is a lot of them that needs acting on.
+    # 20 is more than one a minute held for the window; the blips it drops
+    # were 3 to 7 errors (#6105).
+    "traefik.threshold.min_4xx": 20,
     # The smallest departure from a metric's normal worth calling an anomaly
     # (#5742), on top of being a quarter over its usual hold: a share's floor
     # is in percentage points, a response time's in ms. 5 points is the
@@ -560,6 +566,9 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict:
         # Fewer 5xx than this in 15 minutes and a router isn't failing, whatever the share.
         "router_5xx": {"warn": g("traefik.threshold.min_5xx", 3),
                        "crit": g("traefik.threshold.min_5xx", 3), "dir": "low"},
+        # Fewer 4xx than this in 15 minutes and a router's 4xx share isn't news.
+        "router_4xx": {"warn": g("traefik.threshold.min_4xx", 20),
+                       "crit": g("traefik.threshold.min_4xx", 20), "dir": "low"},
         # Not a cutoff: each normal's floor (norms.floor.*), read by floor_of().
         "floors": {name: float(g(f"norms.floor.{name}", DEFAULTS[f"norms.floor.{name}"]))
                    for name in NORM_FLOORS},

steward/mcp/tools/anomalies.py

@@ -312,7 +312,9 @@ async def get_metric_normal(source_module: str, resource: str, metric: str) -> d
         "learned": bool(slots), "kind": kind,
         "floor": rnd(floor, 3), "floor_setting": learned.floor_name,
         "only_while": ({"metric": learned.gate[0],
-                        "at_least": rnd(learned.gate[1]() if callable(learned.gate[1]) else learned.gate[1], 3)}
+                        "at_least": rnd(learned.gate[1]() if callable(learned.gate[1]) else learned.gate[1], 3),
+                        "fewest_events": (None if learned.fewest is None else
+                                          rnd(learned.fewest() if callable(learned.fewest) else learned.fewest, 3))}
                        if learned.gate else None),
         "now": ({"slot": current.slot, "label": slot_label(current.kind, current.slot),
                  "usual_hold": rnd(current.usual_hold, 3), "usual_peak": rnd(current.usual_peak, 3),

steward/settings/routes.py

@@ -147,6 +147,7 @@ _THRESHOLD_FIELDS = [
     ("error_4xx_pct", "traefik.threshold.error_4xx_pct", True),
     ("min_requests", "traefik.threshold.min_requests", False),
     ("min_5xx", "traefik.threshold.min_5xx", False),
+    ("min_4xx", "traefik.threshold.min_4xx", False),
     # Each normal's floor (#5742), named by its own setting key.
     *[(f"norms.floor.{name}", f"norms.floor.{name}", True) for name in NORM_FLOORS],
 ]

steward/templates/settings/thresholds.html

@@ -92,6 +92,12 @@
            value="{{ settings['traefik.threshold.min_5xx'] }}" style="max-width:110px;margin-top:0.25rem;">
     <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">A router past the 5xx share above is failing only once it has answered this many 5xx: one 502 among 70 requests is 1.4%, but it is one error.</div>
   </div>
+  <div class="form-group" style="margin:1.1rem 0 0.25rem;">
+    <label for="min_4xx">Fewest client errors worth a look <span style="color:var(--text-muted);font-size:0.8rem;">(4xx in 15 minutes)</span></label>
+    <input type="number" id="min_4xx" name="min_4xx" min="1" step="1"
+           value="{{ settings['traefik.threshold.min_4xx'] }}" style="max-width:110px;margin-top:0.25rem;">
+    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">A router's 4xx share is flagged, on its page or as an anomaly, only once it has answered this many 4xx: a few 404s and failed sign-ins are part of a normal day; a lot of them is what needs acting on.</div>
+  </div>
 </div>
 {% endif %}