From 80674603bc6ab3d89a83f426ea11352396a514c6 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 15 Oct 2026 15:35:57 -0400 Subject: [PATCH] fix(traefik): a router's 5xx share is judged only over enough requests, like its 4xx share (#5859) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two 500s of about sixteen requests graded roundtable@swarm critical at 9.1%. The Settings → Thresholds minimum of requests in 15 minutes (30 by default) now guards both shares; the 5xx share still keeps its fixed cutoff whatever the router's history. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 2 +- docs/plugins/traefik.md | 2 +- plugins/traefik/mcp_tools.py | 2 +- plugins/traefik/routes.py | 19 +++++++++++-------- steward/templates/settings/thresholds.html | 5 +++-- tests/plugins/traefik/test_normals.py | 13 ++++++++----- 6 files changed, 25 insertions(+), 18 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 18946c5..85e10cb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -123,7 +123,7 @@ What learns a normal, and the floor each departure must clear (Settings → Thre | Monitor response time | `` / `response_ms` | 100 ms | | Router 4xx share, p95 | `traefik` / `error_rate_4xx_pct`, `latency_p95_ms` | 2 points, 50 ms | -Host metrics learn with `parts=False`: the host learns, its cores (`:core`) don't. Disk space, a monitor that is down, an agent gone quiet and a router's 5xx share are failures and keep their fixed cutoffs. +Host metrics learn with `parts=False`: the host learns, its cores (`:core`) don't. Disk space, a monitor that is down, an agent gone quiet and a router's 5xx share are failures and keep their fixed cutoffs (a router's shares are judged only over enough requests). Pages grade by the normal through `anomalies.graded()` (hosts' `host_grades` and `worst_reading(normals=)`, `docker.service.usage(thresholds=, norms=)`, `monitors.service.latency_grade`): a reading past its cutoff but within its normal reads healthy and says what is usual at this hour. The host, monitor, container and Traefik pages list their anomalies with `_anomaly_card.html`. diff --git a/docs/plugins/traefik.md b/docs/plugins/traefik.md index 7fc33d1..cee323e 100644 --- a/docs/plugins/traefik.md +++ b/docs/plugins/traefik.md @@ -107,7 +107,7 @@ The full Traefik page at `/plugins/traefik/` reads top to bottom, for the time r A router is graded on its last 15 minutes against Settings → Thresholds. When a problem started is read from the router's history over the range. -Once a router has 5 days of history, its 4xx share and p95 latency are judged against its own normal for that hour (see [Normals and anomalies](../architecture.md)) instead of the fixed cutoffs. One over the cutoff but within its normal is healthy, and the healthy table says it is usual for that router (an error-pages service that answers 4xx all day, a login wall's steady 401s). One out of its normal is a warning that says what is usual, and a metric anomaly to review; the Traefik page lists the routers' anomalies of the last 30 days. How far out counts is set on Settings → Thresholds → Normals (2 points of 4xx share, 50 ms of p95 by default). A 4xx share over fewer requests in the 15 minutes than Settings → Thresholds allows (30 by default) isn't judged at all. The 5xx share is a failure and always keeps its fixed cutoff. Latency is learned only from #5755's per-scrape figures, so its normal arrives 5 days after that release. +Once a router has 5 days of history, its 4xx share and p95 latency are judged against its own normal for that hour (see [Normals and anomalies](../architecture.md)) instead of the fixed cutoffs. One over the cutoff but within its normal is healthy, and the healthy table says it is usual for that router (an error-pages service that answers 4xx all day, a login wall's steady 401s). One out of its normal is a warning that says what is usual, and a metric anomaly to review; the Traefik page lists the routers' anomalies of the last 30 days. How far out counts is set on Settings → Thresholds → Normals (2 points of 4xx share, 50 ms of p95 by default). A 4xx or 5xx share over fewer requests in the 15 minutes than Settings → Thresholds allows (30 by default) isn't judged at all: two 500s of sixteen requests would otherwise read 12% and critical. The 5xx share is a failure and keeps its fixed cutoff whatever the router's history. Latency is learned only from #5755's per-scrape figures, so its normal arrives 5 days after that release. --- diff --git a/plugins/traefik/mcp_tools.py b/plugins/traefik/mcp_tools.py index bf1f3ca..1ed5ec2 100644 --- a/plugins/traefik/mcp_tools.py +++ b/plugins/traefik/mcp_tools.py @@ -4,7 +4,7 @@ Registered through register_mcp_tools() in __init__.py, so they exist only while the plugin is enabled. Routers are graded exactly as the Traefik page grades them (routes.py): over the last 15 minutes, the 5xx share against its fixed cutoff, the 4xx share and p95 against the router's own normal where it -has one. +has one; neither share over fewer requests than Settings → Thresholds asks. """ from __future__ import annotations diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py index 2d034de..03385bd 100644 --- a/plugins/traefik/routes.py +++ b/plugins/traefik/routes.py @@ -61,14 +61,15 @@ def error_cutoffs(thresholds: dict) -> tuple[float, float]: # that is its normal, not a problem. Departing by less than the floor # (Settings → Thresholds, norms.floor.router_*) is never news. A share over # fewer than min_requests() in the window is not graded: one 404 of three -# requests is 33%. The 5xx share is a failure and keeps its fixed cutoff -# whatever the history or traffic (the operator's ruling). +# requests is 33%, two 500s of sixteen 12%. The 5xx share is a failure and +# keeps its fixed cutoff whatever the history; it shares the 4xx share's +# minimum of traffic (the operator, #5859). NORMED = ("error_rate_4xx_pct", "latency_p95_ms") def min_requests(thresholds: dict) -> float: - """Fewest requests over the STATE_WINDOW for a 4xx share to be graded - (Settings → Thresholds, traefik.threshold.min_requests).""" + """Fewest requests over the STATE_WINDOW for a 4xx or 5xx share to be + graded (Settings → Thresholds, traefik.threshold.min_requests).""" return float(thresholds.get("router_requests", {}).get("warn", 30)) @@ -78,10 +79,13 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = critical cutoff or a 4xx share past its cutoff (all Settings → Thresholds), otherwise ok. `norms` ({metric: Norm}) are the router's normals for this hour: a p95 or 4xx share past its cutoff but within its normal is ok, and - `why` then says it is usual. `over` names the span the reading covers.""" + `why` then says it is usual. `over` names the span the reading covers. + Neither share is graded over fewer than min_requests() requests.""" norms = norms or {} error_5xx, error_4xx = error_cutoffs(thresholds) - if (reading.error_rate_5xx_pct or 0) > error_5xx: + requests = getattr(reading, "requests", None) + enough = requests is None or requests >= min_requests(thresholds) + if enough and (reading.error_rate_5xx_pct or 0) > error_5xx: return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}" usual = [] p95 = reading.latency_p95_ms @@ -91,9 +95,8 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = if out: return "warn", f"p95 latency is {p95:.0f} ms{over}" + (f", {said}" if said else "") usual.append(f"p95 {p95:.0f} ms is {said}") - requests = getattr(reading, "requests", None) e4 = reading.error_rate_4xx_pct or 0 - if e4 > error_4xx and (requests is None or requests >= min_requests(thresholds)): + if enough and e4 > error_4xx: out, said = anom.against(e4, norms.get("error_rate_4xx_pct"), floor_of(thresholds, "router_4xx_pct"), "%") if out: diff --git a/steward/templates/settings/thresholds.html b/steward/templates/settings/thresholds.html index d80be1c..1d5470a 100644 --- a/steward/templates/settings/thresholds.html +++ b/steward/templates/settings/thresholds.html @@ -80,10 +80,11 @@ so one that always answers some 4xx stops being listed.
- + -
Below this, a router's 4xx share isn't judged, on its page or as an anomaly: one 404 of three requests is 33%. 5xx is always judged.
+
Below this, a router's 4xx and 5xx shares aren't judged, on its page or as an anomaly: one 404 of three requests is 33%, two 500s of sixteen 12%. + A quiet router that fails every request is left to its uptime monitor until it has this many.
{% endif %} diff --git a/tests/plugins/traefik/test_normals.py b/tests/plugins/traefik/test_normals.py index c8b98f4..f7023ee 100644 --- a/tests/plugins/traefik/test_normals.py +++ b/tests/plugins/traefik/test_normals.py @@ -1,8 +1,8 @@ """A router's 4xx share and p95 are graded against its own normal once it has one (#5741, System 182): a router over the fixed cutoff but inside its normal is healthy and says so; one out of its normal is a warning that says what is -usual. The 5xx share keeps its fixed cutoff, and a 4xx share over too few -requests isn't graded.""" +usual. The 5xx share keeps its fixed cutoff, and neither share over too few +requests is graded.""" from types import SimpleNamespace as NS from plugins.traefik.routes import STATE_WINDOW_WORDS, grade_routers, router_state @@ -60,15 +60,18 @@ def test_without_a_normal_the_fixed_cutoffs_still_apply(): assert router_state(_window(p95=450.0), TH, "")[0] == "warn" -def test_5xx_keeps_its_cutoff_whatever_the_history_or_traffic(): +def test_5xx_keeps_its_cutoff_whatever_the_history(): usual_5xx = {"error_rate_4xx_pct": _norm(50.0)} assert router_state(_window(e5=40.0), TH, "", usual_5xx)[0] == "crit" - assert router_state(_window(e5=40.0, requests=4.0), TH, "")[0] == "crit" + assert router_state(_window(e5=40.0, requests=300.0), TH, "", usual_5xx)[0] == "crit" -def test_a_4xx_share_over_a_handful_of_requests_is_not_graded(): +def test_a_share_over_a_handful_of_requests_is_not_graded(): assert router_state(_window(e4=33.0, requests=3.0), TH, "") == ("ok", None) assert router_state(_window(e4=33.0, requests=300.0), TH, "")[0] == "warn" + # roundtable@swarm on 2026-10-15: two 500s of about sixteen requests (#5859) + assert router_state(_window(e5=12.5, requests=16.0), TH, "") == ("ok", None) + assert router_state(_window(e5=12.5, requests=30.0), TH, "")[0] == "crit" def test_grade_routers_hands_each_router_its_own_normals():