fix(traefik): a router fails on 5xx only once it answered a few server errors (#5865)

One 502 among about 70 requests read steward@swarm critical at 1.4%:
past the 30-request minimum and far past the 0.1% cutoff, yet one
error. A new Settings → Thresholds field, "Fewest server errors to
fail" (traefik.threshold.min_5xx, default 3), must also be reached in
the 15-minute window; the 0.1% cutoff still catches a busy router.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/routes.py

@@ -63,7 +63,8 @@ def error_cutoffs(thresholds: dict) -> tuple[float, float]:
 # fewer than min_requests() in the window is not graded: one 404 of three
 # requests is 33%, two 500s of sixteen 12%. The 5xx share is a failure and
 # keeps its fixed cutoff whatever the history; it shares the 4xx share's
-# minimum of traffic (the operator, #5859).
+# minimum of traffic (the operator, #5859), and is failing only once
+# min_5xx() errors were answered: one 502 of 70 requests is 1.4% (#5865).
 NORMED = ("error_rate_4xx_pct", "latency_p95_ms")
 
 
@@ -73,6 +74,12 @@ def min_requests(thresholds: dict) -> float:
     return float(thresholds.get("router_requests", {}).get("warn", 30))
 
 
+def min_5xx(thresholds: dict) -> float:
+    """Fewest 5xx answered over the STATE_WINDOW for a 5xx share past its
+    cutoff to be failing (Settings → Thresholds, traefik.threshold.min_5xx)."""
+    return float(thresholds.get("router_5xx", {}).get("warn", 3))
+
+
 def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = None) -> tuple[str, str | None]:
     """(grade, why) for a router's reading (recent_window's, or one scrape's):
     crit while it serves 5xx past its cutoff, warn on a p95 past the latency
@@ -80,12 +87,15 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None =
     otherwise ok. `norms` ({metric: Norm}) are the router's normals for this
     hour: a p95 or 4xx share past its cutoff but within its normal is ok, and
     `why` then says it is usual. `over` names the span the reading covers.
-    Neither share is graded over fewer than min_requests() requests."""
+    Neither share is graded over fewer than min_requests() requests, and a
+    5xx share is failing only once about min_5xx() errors were answered."""
     norms = norms or {}
     error_5xx, error_4xx = error_cutoffs(thresholds)
     requests = getattr(reading, "requests", None)
     enough = requests is None or requests >= min_requests(thresholds)
-    if enough and (reading.error_rate_5xx_pct or 0) > error_5xx:
+    e5 = reading.error_rate_5xx_pct or 0
+    errors = None if requests is None else round(requests * e5 / 100.0)
+    if enough and e5 > error_5xx and (errors is None or errors >= min_5xx(thresholds)):
         return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}"
     usual = []
     p95 = reading.latency_p95_ms