fix(traefik): a router's 5xx share is judged only over enough requests, like its 4xx share (#5859)
Two 500s of about sixteen requests graded roundtable@swarm critical at 9.1%. The Settings → Thresholds minimum of requests in 15 minutes (30 by default) now guards both shares; the 5xx share still keeps its fixed cutoff whatever the router's history. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -4,7 +4,7 @@ Registered through register_mcp_tools() in __init__.py, so they exist only
|
||||
while the plugin is enabled. Routers are graded exactly as the Traefik page
|
||||
grades them (routes.py): over the last 15 minutes, the 5xx share against its
|
||||
fixed cutoff, the 4xx share and p95 against the router's own normal where it
|
||||
has one.
|
||||
has one; neither share over fewer requests than Settings → Thresholds asks.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -61,14 +61,15 @@ def error_cutoffs(thresholds: dict) -> tuple[float, float]:
|
||||
# that is its normal, not a problem. Departing by less than the floor
|
||||
# (Settings → Thresholds, norms.floor.router_*) is never news. A share over
|
||||
# fewer than min_requests() in the window is not graded: one 404 of three
|
||||
# requests is 33%. The 5xx share is a failure and keeps its fixed cutoff
|
||||
# whatever the history or traffic (the operator's ruling).
|
||||
# requests is 33%, two 500s of sixteen 12%. The 5xx share is a failure and
|
||||
# keeps its fixed cutoff whatever the history; it shares the 4xx share's
|
||||
# minimum of traffic (the operator, #5859).
|
||||
NORMED = ("error_rate_4xx_pct", "latency_p95_ms")
|
||||
|
||||
|
||||
def min_requests(thresholds: dict) -> float:
|
||||
"""Fewest requests over the STATE_WINDOW for a 4xx share to be graded
|
||||
(Settings → Thresholds, traefik.threshold.min_requests)."""
|
||||
"""Fewest requests over the STATE_WINDOW for a 4xx or 5xx share to be
|
||||
graded (Settings → Thresholds, traefik.threshold.min_requests)."""
|
||||
return float(thresholds.get("router_requests", {}).get("warn", 30))
|
||||
|
||||
|
||||
@@ -78,10 +79,13 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None =
|
||||
critical cutoff or a 4xx share past its cutoff (all Settings → Thresholds),
|
||||
otherwise ok. `norms` ({metric: Norm}) are the router's normals for this
|
||||
hour: a p95 or 4xx share past its cutoff but within its normal is ok, and
|
||||
`why` then says it is usual. `over` names the span the reading covers."""
|
||||
`why` then says it is usual. `over` names the span the reading covers.
|
||||
Neither share is graded over fewer than min_requests() requests."""
|
||||
norms = norms or {}
|
||||
error_5xx, error_4xx = error_cutoffs(thresholds)
|
||||
if (reading.error_rate_5xx_pct or 0) > error_5xx:
|
||||
requests = getattr(reading, "requests", None)
|
||||
enough = requests is None or requests >= min_requests(thresholds)
|
||||
if enough and (reading.error_rate_5xx_pct or 0) > error_5xx:
|
||||
return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}"
|
||||
usual = []
|
||||
p95 = reading.latency_p95_ms
|
||||
@@ -91,9 +95,8 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None =
|
||||
if out:
|
||||
return "warn", f"p95 latency is {p95:.0f} ms{over}" + (f", {said}" if said else "")
|
||||
usual.append(f"p95 {p95:.0f} ms is {said}")
|
||||
requests = getattr(reading, "requests", None)
|
||||
e4 = reading.error_rate_4xx_pct or 0
|
||||
if e4 > error_4xx and (requests is None or requests >= min_requests(thresholds)):
|
||||
if enough and e4 > error_4xx:
|
||||
out, said = anom.against(e4, norms.get("error_rate_4xx_pct"),
|
||||
floor_of(thresholds, "router_4xx_pct"), "%")
|
||||
if out:
|
||||
|
||||
Reference in New Issue
Block a user