fix(traefik): a router fails on 5xx only once it answered a few server errors (#5865)

One 502 among about 70 requests read steward@swarm critical at 1.4%:
past the 30-request minimum and far past the 0.1% cutoff, yet one
error. A new Settings → Thresholds field, "Fewest server errors to
fail" (traefik.threshold.min_5xx, default 3), must also be reached in
the 15-minute window; the 0.1% cutoff still catches a busy router.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/plugins/traefik.md

@@ -107,7 +107,7 @@ The full Traefik page at `/plugins/traefik/` reads top to bottom, for the time r
 
 A router is graded on its last 15 minutes against Settings → Thresholds. When a problem started is read from the router's history over the range.
 
-Once a router has 5 days of history, its 4xx share and p95 latency are judged against its own normal for that hour (see [Normals and anomalies](../architecture.md)) instead of the fixed cutoffs. One over the cutoff but within its normal is healthy, and the healthy table says it is usual for that router (an error-pages service that answers 4xx all day, a login wall's steady 401s). One out of its normal is a warning that says what is usual, and a metric anomaly to review; the Traefik page lists the routers' anomalies of the last 30 days. How far out counts is set on Settings → Thresholds → Normals (2 points of 4xx share, 50 ms of p95 by default). A 4xx or 5xx share over fewer requests in the 15 minutes than Settings → Thresholds allows (30 by default) isn't judged at all: two 500s of sixteen requests would otherwise read 12% and critical. The 5xx share is a failure and keeps its fixed cutoff whatever the router's history. Latency is learned only from #5755's per-scrape figures, so its normal arrives 5 days after that release.
+Once a router has 5 days of history, its 4xx share and p95 latency are judged against its own normal for that hour (see [Normals and anomalies](../architecture.md)) instead of the fixed cutoffs. One over the cutoff but within its normal is healthy, and the healthy table says it is usual for that router (an error-pages service that answers 4xx all day, a login wall's steady 401s). One out of its normal is a warning that says what is usual, and a metric anomaly to review; the Traefik page lists the routers' anomalies of the last 30 days. How far out counts is set on Settings → Thresholds → Normals (2 points of 4xx share, 50 ms of p95 by default). A 4xx or 5xx share over fewer requests in the 15 minutes than Settings → Thresholds allows (30 by default) isn't judged at all: two 500s of sixteen requests would otherwise read 12% and critical. A 5xx share past its cutoff is failing only once the router answered at least 3 server errors in those 15 minutes (Settings → Thresholds, "Fewest server errors to fail"): one 502 among 70 requests is 1.4%, but it is one error. The 5xx share is a failure and keeps its fixed cutoff whatever the router's history. Latency is learned only from #5755's per-scrape figures, so its normal arrives 5 days after that release.
 
 ---
 

plugins/traefik/routes.py

@@ -63,7 +63,8 @@ def error_cutoffs(thresholds: dict) -> tuple[float, float]:
 # fewer than min_requests() in the window is not graded: one 404 of three
 # requests is 33%, two 500s of sixteen 12%. The 5xx share is a failure and
 # keeps its fixed cutoff whatever the history; it shares the 4xx share's
-# minimum of traffic (the operator, #5859).
+# minimum of traffic (the operator, #5859), and is failing only once
+# min_5xx() errors were answered: one 502 of 70 requests is 1.4% (#5865).
 NORMED = ("error_rate_4xx_pct", "latency_p95_ms")
 
 
@@ -73,6 +74,12 @@ def min_requests(thresholds: dict) -> float:
     return float(thresholds.get("router_requests", {}).get("warn", 30))
 
 
+def min_5xx(thresholds: dict) -> float:
+    """Fewest 5xx answered over the STATE_WINDOW for a 5xx share past its
+    cutoff to be failing (Settings → Thresholds, traefik.threshold.min_5xx)."""
+    return float(thresholds.get("router_5xx", {}).get("warn", 3))
+
+
 def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = None) -> tuple[str, str | None]:
     """(grade, why) for a router's reading (recent_window's, or one scrape's):
     crit while it serves 5xx past its cutoff, warn on a p95 past the latency
@@ -80,12 +87,15 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None =
     otherwise ok. `norms` ({metric: Norm}) are the router's normals for this
     hour: a p95 or 4xx share past its cutoff but within its normal is ok, and
     `why` then says it is usual. `over` names the span the reading covers.
-    Neither share is graded over fewer than min_requests() requests."""
+    Neither share is graded over fewer than min_requests() requests, and a
+    5xx share is failing only once about min_5xx() errors were answered."""
     norms = norms or {}
     error_5xx, error_4xx = error_cutoffs(thresholds)
     requests = getattr(reading, "requests", None)
     enough = requests is None or requests >= min_requests(thresholds)
-    if enough and (reading.error_rate_5xx_pct or 0) > error_5xx:
+    e5 = reading.error_rate_5xx_pct or 0
+    errors = None if requests is None else round(requests * e5 / 100.0)
+    if enough and e5 > error_5xx and (errors is None or errors >= min_5xx(thresholds)):
         return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}"
     usual = []
     p95 = reading.latency_p95_ms

steward/core/settings.py

@@ -100,6 +100,9 @@ DEFAULTS: dict[str, Any] = {
     # A router's 4xx share over fewer requests than this in 15 minutes isn't
     # graded, on the page or for anomalies: one 404 of three is 33% (#5741).
     "traefik.threshold.min_requests": 30,
+    # ...and a 5xx share past its cutoff is failing only once this many 5xx
+    # were answered in those 15 minutes: one 502 of 70 is 1.4% (#5865).
+    "traefik.threshold.min_5xx": 3,
     # The smallest departure from a metric's normal worth calling an anomaly
     # (#5742), on top of being a quarter over its usual hold: a share's floor
     # is in percentage points, a response time's in ms. 5 points is the
@@ -553,6 +556,9 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict:
         # Fewer requests than this in 15 minutes and a router's 4xx share isn't graded.
         "router_requests": {"warn": g("traefik.threshold.min_requests", 30),
                             "crit": g("traefik.threshold.min_requests", 30), "dir": "low"},
+        # Fewer 5xx than this in 15 minutes and a router isn't failing, whatever the share.
+        "router_5xx": {"warn": g("traefik.threshold.min_5xx", 3),
+                       "crit": g("traefik.threshold.min_5xx", 3), "dir": "low"},
         # Not a cutoff: each normal's floor (norms.floor.*), read by floor_of().
         "floors": {name: float(g(f"norms.floor.{name}", DEFAULTS[f"norms.floor.{name}"]))
                    for name in NORM_FLOORS},

steward/settings/routes.py

@@ -146,6 +146,7 @@ _THRESHOLD_FIELDS = [
     ("error_5xx_pct", "traefik.threshold.error_5xx_pct", True),
     ("error_4xx_pct", "traefik.threshold.error_4xx_pct", True),
     ("min_requests", "traefik.threshold.min_requests", False),
+    ("min_5xx", "traefik.threshold.min_5xx", False),
     # Each normal's floor (#5742), named by its own setting key.
     *[(f"norms.floor.{name}", f"norms.floor.{name}", True) for name in NORM_FLOORS],
 ]

steward/templates/settings/thresholds.html

@@ -86,6 +86,12 @@
     <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">Below this, a router's 4xx and 5xx shares aren't judged, on its page or as an anomaly: one 404 of three requests is 33%, two 500s of sixteen 12%.
       A quiet router that fails every request is left to its uptime monitor until it has this many.</div>
   </div>
+  <div class="form-group" style="margin:1.1rem 0 0.25rem;">
+    <label for="min_5xx">Fewest server errors to fail <span style="color:var(--text-muted);font-size:0.8rem;">(5xx in 15 minutes)</span></label>
+    <input type="number" id="min_5xx" name="min_5xx" min="1" step="1"
+           value="{{ settings['traefik.threshold.min_5xx'] }}" style="max-width:110px;margin-top:0.25rem;">
+    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">A router past the 5xx share above is failing only once it has answered this many 5xx: one 502 among 70 requests is 1.4%, but it is one error.</div>
+  </div>
 </div>
 {% endif %}
 

tests/plugins/traefik/test_normals.py

@@ -74,6 +74,14 @@ def test_a_share_over_a_handful_of_requests_is_not_graded():
     assert router_state(_window(e5=12.5, requests=30.0), TH, "")[0] == "crit"
 
 
+def test_a_5xx_share_fails_only_once_enough_errors_were_answered():
+    # steward@swarm on 2026-10-15: one 502 among about 70 requests (#5865)
+    assert router_state(_window(e5=1.43, requests=70.0), TH, "") == ("ok", None)
+    assert router_state(_window(e5=4.29, requests=70.0), TH, "")[0] == "crit"      # 3 errors
+    stricter = to_thresholds_cfg({"traefik.threshold.min_5xx": 1})
+    assert router_state(_window(e5=1.43, requests=70.0), stricter, "")[0] == "crit"
+
+
 def test_grade_routers_hands_each_router_its_own_normals():
     routers = [{"name": "errors@docker", "latest": _latest(e4=60.0)},
                {"name": "app@docker", "latest": _latest(e4=60.0)}]