"""A router's 4xx share and p95 are graded against its own normal once it has one (#5741, System 182): a router over the fixed cutoff but inside its normal is healthy and says so; one out of its normal is a warning that says what is usual. The 5xx share keeps its fixed cutoff, and neither share over too few requests is graded.""" from types import SimpleNamespace as NS from plugins.traefik.routes import STATE_WINDOW_WORDS, grade_routers, router_state from steward.core.norms import Norm from steward.core.settings import to_thresholds_cfg from tests.plugins.traefik.test_traefik_page import _latest TH = to_thresholds_cfg({}) # 4xx warn over 1%, 5xx crit over 0.1%, latency crit 200 ms def _norm(hold, peak=None): return Norm(kind="day", slot=14, usual_hold=hold, usual_peak=peak or hold * 1.5, days_seen=7) def _window(e4=0.0, e5=0.0, p95=20.0, requests=500.0): return NS(error_rate_4xx_pct=e4, error_rate_5xx_pct=e5, latency_p95_ms=p95, requests=requests) def test_an_error_pages_router_answering_4xx_all_day_reads_healthy(): state, why = router_state(_window(e4=62.0), TH, STATE_WINDOW_WORDS, {"error_rate_4xx_pct": _norm(60.0)}) assert state == "ok" assert why == "4xx 62.0% is usual 60.0% at this hour" def test_a_router_that_flapped_on_the_cutoff_stays_healthy_inside_its_normal(): """authentik-like: a login wall's 401s sit around 1–3%, either side of the 1% cutoff, so it came and went from the problem list every few polls.""" norm = {"error_rate_4xx_pct": _norm(2.8)} for e4 in (0.6, 1.4, 2.9, 1.1, 3.2, 0.9): assert router_state(_window(e4=e4), TH, STATE_WINDOW_WORDS, norm)[0] == "ok", e4 def test_out_of_its_normal_is_a_warning_that_says_what_is_usual(): state, why = router_state(_window(e4=12.0), TH, STATE_WINDOW_WORDS, {"error_rate_4xx_pct": _norm(2.8)}) assert state == "warn" assert why == "12.0% of responses are 4xx over the last 15 min, usual 2.8% at this hour" def test_a_departure_smaller_than_the_floor_is_no_news(): # 1.25 × 1.2% is 1.5%, but 2.0% is only 0.8 points over: under the 2-point floor. assert router_state(_window(e4=2.0), TH, "", {"error_rate_4xx_pct": _norm(1.2)})[0] == "ok" def test_latency_is_graded_against_its_normal_too(): slow_usually = {"latency_p95_ms": _norm(900.0)} assert router_state(_window(p95=950.0), TH, "", slow_usually) == ("ok", "p95 950 ms is usual 900 ms at this hour") state, why = router_state(_window(p95=3000.0), TH, "", slow_usually) assert state == "warn" and why == "p95 latency is 3000 ms, usual 900 ms at this hour" def test_without_a_normal_the_fixed_cutoffs_still_apply(): assert router_state(_window(e4=5.0), TH, "")[0] == "warn" assert router_state(_window(p95=450.0), TH, "")[0] == "warn" def test_5xx_keeps_its_cutoff_whatever_the_history(): usual_5xx = {"error_rate_4xx_pct": _norm(50.0)} assert router_state(_window(e5=40.0), TH, "", usual_5xx)[0] == "crit" assert router_state(_window(e5=40.0, requests=300.0), TH, "", usual_5xx)[0] == "crit" def test_a_share_over_a_handful_of_requests_is_not_graded(): assert router_state(_window(e4=33.0, requests=3.0), TH, "") == ("ok", None) assert router_state(_window(e4=33.0, requests=300.0), TH, "")[0] == "warn" # roundtable@swarm on 2026-10-15: two 500s of about sixteen requests (#5859) assert router_state(_window(e5=12.5, requests=16.0), TH, "") == ("ok", None) assert router_state(_window(e5=12.5, requests=30.0), TH, "")[0] == "crit" def test_a_5xx_share_fails_only_once_enough_errors_were_answered(): # steward@swarm on 2026-10-15: one 502 among about 70 requests (#5865) assert router_state(_window(e5=1.43, requests=70.0), TH, "") == ("ok", None) assert router_state(_window(e5=4.29, requests=70.0), TH, "")[0] == "crit" # 3 errors stricter = to_thresholds_cfg({"traefik.threshold.min_5xx": 1}) assert router_state(_window(e5=1.43, requests=70.0), stricter, "")[0] == "crit" def test_grade_routers_hands_each_router_its_own_normals(): routers = [{"name": "errors@docker", "latest": _latest(e4=60.0)}, {"name": "app@docker", "latest": _latest(e4=60.0)}] window = {"errors@docker": _window(e4=60.0), "app@docker": _window(e4=60.0)} grade_routers(routers, window, TH, {"errors@docker": {"error_rate_4xx_pct": _norm(58.0)}}) assert [r["state"] for r in routers] == ["ok", "warn"] def test_the_fewest_requests_to_judge_is_the_thresholds_setting(): fewer = to_thresholds_cfg({"traefik.threshold.min_requests": 5, "traefik.threshold.min_4xx": 1}) assert router_state(_window(e4=33.0, requests=10.0), TH, "")[0] == "ok" # default 30 assert router_state(_window(e4=33.0, requests=10.0), fewer, "")[0] == "warn" def test_a_4xx_share_is_worth_a_look_only_once_enough_4xx_were_answered(): # librarian@swarm on 2026-10-19: 7.5% of about 100 requests, so 7 errors (#6105) assert router_state(_window(e4=7.5, requests=100.0), TH, "") == ("ok", None) assert router_state(_window(e4=7.5, requests=300.0), TH, "")[0] == "warn" # 22 errors fewer = to_thresholds_cfg({"traefik.threshold.min_4xx": 5}) assert router_state(_window(e4=7.5, requests=100.0), fewer, "")[0] == "warn"