From a24b9da69dc5b1ccb331621ef52c5c4cabc5edd8 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 15 Oct 2026 15:49:51 -0400 Subject: [PATCH] fix(traefik): a router fails on 5xx only once it answered a few server errors (#5865) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One 502 among about 70 requests read steward@swarm critical at 1.4%: past the 30-request minimum and far past the 0.1% cutoff, yet one error. A new Settings → Thresholds field, "Fewest server errors to fail" (traefik.threshold.min_5xx, default 3), must also be reached in the 15-minute window; the 0.1% cutoff still catches a busy router. Co-Authored-By: Claude Opus 5.5 --- docs/plugins/traefik.md | 2 +- plugins/traefik/routes.py | 16 +++++++++++++--- steward/core/settings.py | 6 ++++++ steward/settings/routes.py | 1 + steward/templates/settings/thresholds.html | 6 ++++++ tests/plugins/traefik/test_normals.py | 8 ++++++++ 6 files changed, 35 insertions(+), 4 deletions(-) diff --git a/docs/plugins/traefik.md b/docs/plugins/traefik.md index cee323e..0cb8f3e 100644 --- a/docs/plugins/traefik.md +++ b/docs/plugins/traefik.md @@ -107,7 +107,7 @@ The full Traefik page at `/plugins/traefik/` reads top to bottom, for the time r A router is graded on its last 15 minutes against Settings → Thresholds. When a problem started is read from the router's history over the range. -Once a router has 5 days of history, its 4xx share and p95 latency are judged against its own normal for that hour (see [Normals and anomalies](../architecture.md)) instead of the fixed cutoffs. One over the cutoff but within its normal is healthy, and the healthy table says it is usual for that router (an error-pages service that answers 4xx all day, a login wall's steady 401s). One out of its normal is a warning that says what is usual, and a metric anomaly to review; the Traefik page lists the routers' anomalies of the last 30 days. How far out counts is set on Settings → Thresholds → Normals (2 points of 4xx share, 50 ms of p95 by default). A 4xx or 5xx share over fewer requests in the 15 minutes than Settings → Thresholds allows (30 by default) isn't judged at all: two 500s of sixteen requests would otherwise read 12% and critical. The 5xx share is a failure and keeps its fixed cutoff whatever the router's history. Latency is learned only from #5755's per-scrape figures, so its normal arrives 5 days after that release. +Once a router has 5 days of history, its 4xx share and p95 latency are judged against its own normal for that hour (see [Normals and anomalies](../architecture.md)) instead of the fixed cutoffs. One over the cutoff but within its normal is healthy, and the healthy table says it is usual for that router (an error-pages service that answers 4xx all day, a login wall's steady 401s). One out of its normal is a warning that says what is usual, and a metric anomaly to review; the Traefik page lists the routers' anomalies of the last 30 days. How far out counts is set on Settings → Thresholds → Normals (2 points of 4xx share, 50 ms of p95 by default). A 4xx or 5xx share over fewer requests in the 15 minutes than Settings → Thresholds allows (30 by default) isn't judged at all: two 500s of sixteen requests would otherwise read 12% and critical. A 5xx share past its cutoff is failing only once the router answered at least 3 server errors in those 15 minutes (Settings → Thresholds, "Fewest server errors to fail"): one 502 among 70 requests is 1.4%, but it is one error. The 5xx share is a failure and keeps its fixed cutoff whatever the router's history. Latency is learned only from #5755's per-scrape figures, so its normal arrives 5 days after that release. --- diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py index 03385bd..bee60c4 100644 --- a/plugins/traefik/routes.py +++ b/plugins/traefik/routes.py @@ -63,7 +63,8 @@ def error_cutoffs(thresholds: dict) -> tuple[float, float]: # fewer than min_requests() in the window is not graded: one 404 of three # requests is 33%, two 500s of sixteen 12%. The 5xx share is a failure and # keeps its fixed cutoff whatever the history; it shares the 4xx share's -# minimum of traffic (the operator, #5859). +# minimum of traffic (the operator, #5859), and is failing only once +# min_5xx() errors were answered: one 502 of 70 requests is 1.4% (#5865). NORMED = ("error_rate_4xx_pct", "latency_p95_ms") @@ -73,6 +74,12 @@ def min_requests(thresholds: dict) -> float: return float(thresholds.get("router_requests", {}).get("warn", 30)) +def min_5xx(thresholds: dict) -> float: + """Fewest 5xx answered over the STATE_WINDOW for a 5xx share past its + cutoff to be failing (Settings → Thresholds, traefik.threshold.min_5xx).""" + return float(thresholds.get("router_5xx", {}).get("warn", 3)) + + def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = None) -> tuple[str, str | None]: """(grade, why) for a router's reading (recent_window's, or one scrape's): crit while it serves 5xx past its cutoff, warn on a p95 past the latency @@ -80,12 +87,15 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = otherwise ok. `norms` ({metric: Norm}) are the router's normals for this hour: a p95 or 4xx share past its cutoff but within its normal is ok, and `why` then says it is usual. `over` names the span the reading covers. - Neither share is graded over fewer than min_requests() requests.""" + Neither share is graded over fewer than min_requests() requests, and a + 5xx share is failing only once about min_5xx() errors were answered.""" norms = norms or {} error_5xx, error_4xx = error_cutoffs(thresholds) requests = getattr(reading, "requests", None) enough = requests is None or requests >= min_requests(thresholds) - if enough and (reading.error_rate_5xx_pct or 0) > error_5xx: + e5 = reading.error_rate_5xx_pct or 0 + errors = None if requests is None else round(requests * e5 / 100.0) + if enough and e5 > error_5xx and (errors is None or errors >= min_5xx(thresholds)): return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}" usual = [] p95 = reading.latency_p95_ms diff --git a/steward/core/settings.py b/steward/core/settings.py index 4b57745..c89bff5 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -100,6 +100,9 @@ DEFAULTS: dict[str, Any] = { # A router's 4xx share over fewer requests than this in 15 minutes isn't # graded, on the page or for anomalies: one 404 of three is 33% (#5741). "traefik.threshold.min_requests": 30, + # ...and a 5xx share past its cutoff is failing only once this many 5xx + # were answered in those 15 minutes: one 502 of 70 is 1.4% (#5865). + "traefik.threshold.min_5xx": 3, # The smallest departure from a metric's normal worth calling an anomaly # (#5742), on top of being a quarter over its usual hold: a share's floor # is in percentage points, a response time's in ms. 5 points is the @@ -553,6 +556,9 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict: # Fewer requests than this in 15 minutes and a router's 4xx share isn't graded. "router_requests": {"warn": g("traefik.threshold.min_requests", 30), "crit": g("traefik.threshold.min_requests", 30), "dir": "low"}, + # Fewer 5xx than this in 15 minutes and a router isn't failing, whatever the share. + "router_5xx": {"warn": g("traefik.threshold.min_5xx", 3), + "crit": g("traefik.threshold.min_5xx", 3), "dir": "low"}, # Not a cutoff: each normal's floor (norms.floor.*), read by floor_of(). "floors": {name: float(g(f"norms.floor.{name}", DEFAULTS[f"norms.floor.{name}"])) for name in NORM_FLOORS}, diff --git a/steward/settings/routes.py b/steward/settings/routes.py index b216444..e8a5ce0 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -146,6 +146,7 @@ _THRESHOLD_FIELDS = [ ("error_5xx_pct", "traefik.threshold.error_5xx_pct", True), ("error_4xx_pct", "traefik.threshold.error_4xx_pct", True), ("min_requests", "traefik.threshold.min_requests", False), + ("min_5xx", "traefik.threshold.min_5xx", False), # Each normal's floor (#5742), named by its own setting key. *[(f"norms.floor.{name}", f"norms.floor.{name}", True) for name in NORM_FLOORS], ] diff --git a/steward/templates/settings/thresholds.html b/steward/templates/settings/thresholds.html index 1d5470a..635ed54 100644 --- a/steward/templates/settings/thresholds.html +++ b/steward/templates/settings/thresholds.html @@ -86,6 +86,12 @@
Below this, a router's 4xx and 5xx shares aren't judged, on its page or as an anomaly: one 404 of three requests is 33%, two 500s of sixteen 12%. A quiet router that fails every request is left to its uptime monitor until it has this many.
+
+ + +
A router past the 5xx share above is failing only once it has answered this many 5xx: one 502 among 70 requests is 1.4%, but it is one error.
+
{% endif %} diff --git a/tests/plugins/traefik/test_normals.py b/tests/plugins/traefik/test_normals.py index f7023ee..95f0705 100644 --- a/tests/plugins/traefik/test_normals.py +++ b/tests/plugins/traefik/test_normals.py @@ -74,6 +74,14 @@ def test_a_share_over_a_handful_of_requests_is_not_graded(): assert router_state(_window(e5=12.5, requests=30.0), TH, "")[0] == "crit" +def test_a_5xx_share_fails_only_once_enough_errors_were_answered(): + # steward@swarm on 2026-10-15: one 502 among about 70 requests (#5865) + assert router_state(_window(e5=1.43, requests=70.0), TH, "") == ("ok", None) + assert router_state(_window(e5=4.29, requests=70.0), TH, "")[0] == "crit" # 3 errors + stricter = to_thresholds_cfg({"traefik.threshold.min_5xx": 1}) + assert router_state(_window(e5=1.43, requests=70.0), stricter, "")[0] == "crit" + + def test_grade_routers_hands_each_router_its_own_normals(): routers = [{"name": "errors@docker", "latest": _latest(e4=60.0)}, {"name": "app@docker", "latest": _latest(e4=60.0)}]