bvandeusenandClaude Opus 5.5
a24b9da69d
One 502 among about 70 requests read steward@swarm critical at 1.4%: past the 30-request minimum and far past the 0.1% cutoff, yet one error. A new Settings → Thresholds field, "Fewest server errors to fail" (traefik.threshold.min_5xx, default 3), must also be reached in the 15-minute window; the 0.1% cutoff still catches a busy router. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
97 lines
4.7 KiB
Python
97 lines
4.7 KiB
Python
"""A router's 4xx share and p95 are graded against its own normal once it has
|
||
one (#5741, System 182): a router over the fixed cutoff but inside its normal
|
||
is healthy and says so; one out of its normal is a warning that says what is
|
||
usual. The 5xx share keeps its fixed cutoff, and neither share over too few
|
||
requests is graded."""
|
||
from types import SimpleNamespace as NS
|
||
|
||
from plugins.traefik.routes import STATE_WINDOW_WORDS, grade_routers, router_state
|
||
from steward.core.norms import Norm
|
||
from steward.core.settings import to_thresholds_cfg
|
||
from tests.plugins.traefik.test_traefik_page import _latest
|
||
|
||
TH = to_thresholds_cfg({}) # 4xx warn over 1%, 5xx crit over 0.1%, latency crit 200 ms
|
||
|
||
|
||
def _norm(hold, peak=None):
|
||
return Norm(kind="day", slot=14, usual_hold=hold, usual_peak=peak or hold * 1.5, days_seen=7)
|
||
|
||
|
||
def _window(e4=0.0, e5=0.0, p95=20.0, requests=500.0):
|
||
return NS(error_rate_4xx_pct=e4, error_rate_5xx_pct=e5, latency_p95_ms=p95, requests=requests)
|
||
|
||
|
||
def test_an_error_pages_router_answering_4xx_all_day_reads_healthy():
|
||
state, why = router_state(_window(e4=62.0), TH, STATE_WINDOW_WORDS,
|
||
{"error_rate_4xx_pct": _norm(60.0)})
|
||
assert state == "ok"
|
||
assert why == "4xx 62.0% is usual 60.0% at this hour"
|
||
|
||
|
||
def test_a_router_that_flapped_on_the_cutoff_stays_healthy_inside_its_normal():
|
||
"""authentik-like: a login wall's 401s sit around 1–3%, either side of the
|
||
1% cutoff, so it came and went from the problem list every few polls."""
|
||
norm = {"error_rate_4xx_pct": _norm(2.8)}
|
||
for e4 in (0.6, 1.4, 2.9, 1.1, 3.2, 0.9):
|
||
assert router_state(_window(e4=e4), TH, STATE_WINDOW_WORDS, norm)[0] == "ok", e4
|
||
|
||
|
||
def test_out_of_its_normal_is_a_warning_that_says_what_is_usual():
|
||
state, why = router_state(_window(e4=12.0), TH, STATE_WINDOW_WORDS,
|
||
{"error_rate_4xx_pct": _norm(2.8)})
|
||
assert state == "warn"
|
||
assert why == "12.0% of responses are 4xx over the last 15 min, usual 2.8% at this hour"
|
||
|
||
|
||
def test_a_departure_smaller_than_the_floor_is_no_news():
|
||
# 1.25 × 1.2% is 1.5%, but 2.0% is only 0.8 points over: under the 2-point floor.
|
||
assert router_state(_window(e4=2.0), TH, "", {"error_rate_4xx_pct": _norm(1.2)})[0] == "ok"
|
||
|
||
|
||
def test_latency_is_graded_against_its_normal_too():
|
||
slow_usually = {"latency_p95_ms": _norm(900.0)}
|
||
assert router_state(_window(p95=950.0), TH, "", slow_usually) == ("ok", "p95 950 ms is usual 900 ms at this hour")
|
||
state, why = router_state(_window(p95=3000.0), TH, "", slow_usually)
|
||
assert state == "warn" and why == "p95 latency is 3000 ms, usual 900 ms at this hour"
|
||
|
||
|
||
def test_without_a_normal_the_fixed_cutoffs_still_apply():
|
||
assert router_state(_window(e4=5.0), TH, "")[0] == "warn"
|
||
assert router_state(_window(p95=450.0), TH, "")[0] == "warn"
|
||
|
||
|
||
def test_5xx_keeps_its_cutoff_whatever_the_history():
|
||
usual_5xx = {"error_rate_4xx_pct": _norm(50.0)}
|
||
assert router_state(_window(e5=40.0), TH, "", usual_5xx)[0] == "crit"
|
||
assert router_state(_window(e5=40.0, requests=300.0), TH, "", usual_5xx)[0] == "crit"
|
||
|
||
|
||
def test_a_share_over_a_handful_of_requests_is_not_graded():
|
||
assert router_state(_window(e4=33.0, requests=3.0), TH, "") == ("ok", None)
|
||
assert router_state(_window(e4=33.0, requests=300.0), TH, "")[0] == "warn"
|
||
# roundtable@swarm on 2026-10-15: two 500s of about sixteen requests (#5859)
|
||
assert router_state(_window(e5=12.5, requests=16.0), TH, "") == ("ok", None)
|
||
assert router_state(_window(e5=12.5, requests=30.0), TH, "")[0] == "crit"
|
||
|
||
|
||
def test_a_5xx_share_fails_only_once_enough_errors_were_answered():
|
||
# steward@swarm on 2026-10-15: one 502 among about 70 requests (#5865)
|
||
assert router_state(_window(e5=1.43, requests=70.0), TH, "") == ("ok", None)
|
||
assert router_state(_window(e5=4.29, requests=70.0), TH, "")[0] == "crit" # 3 errors
|
||
stricter = to_thresholds_cfg({"traefik.threshold.min_5xx": 1})
|
||
assert router_state(_window(e5=1.43, requests=70.0), stricter, "")[0] == "crit"
|
||
|
||
|
||
def test_grade_routers_hands_each_router_its_own_normals():
|
||
routers = [{"name": "errors@docker", "latest": _latest(e4=60.0)},
|
||
{"name": "app@docker", "latest": _latest(e4=60.0)}]
|
||
window = {"errors@docker": _window(e4=60.0), "app@docker": _window(e4=60.0)}
|
||
grade_routers(routers, window, TH, {"errors@docker": {"error_rate_4xx_pct": _norm(58.0)}})
|
||
assert [r["state"] for r in routers] == ["ok", "warn"]
|
||
|
||
|
||
def test_the_fewest_requests_to_judge_is_the_thresholds_setting():
|
||
fewer = to_thresholds_cfg({"traefik.threshold.min_requests": 5})
|
||
assert router_state(_window(e4=33.0, requests=10.0), TH, "")[0] == "ok" # default 30
|
||
assert router_state(_window(e4=33.0, requests=10.0), fewer, "")[0] == "warn"
|