"""A router's 4xx share and p95 are graded against its own normal once it has one (#5741, System 182): a router over the fixed cutoff but inside its normal is healthy and says so; one out of its normal is a warning that says what is usual. The 5xx share keeps its fixed cutoff, and a 4xx share over too few requests isn't graded.""" from types import SimpleNamespace as NS from plugins.traefik.routes import STATE_WINDOW_WORDS, grade_routers, router_state from steward.core.norms import Norm from steward.core.settings import to_thresholds_cfg from tests.plugins.traefik.test_traefik_page import _latest TH = to_thresholds_cfg({}) # 4xx warn over 1%, 5xx crit over 0.1%, latency crit 200 ms def _norm(hold, peak=None): return Norm(kind="day", slot=14, usual_hold=hold, usual_peak=peak or hold * 1.5, days_seen=7) def _window(e4=0.0, e5=0.0, p95=20.0, requests=500.0): return NS(error_rate_4xx_pct=e4, error_rate_5xx_pct=e5, latency_p95_ms=p95, requests=requests) def test_an_error_pages_router_answering_4xx_all_day_reads_healthy(): state, why = router_state(_window(e4=62.0), TH, STATE_WINDOW_WORDS, {"error_rate_4xx_pct": _norm(60.0)}) assert state == "ok" assert why == "4xx 62.0% is usual 60.0% at this hour" def test_a_router_that_flapped_on_the_cutoff_stays_healthy_inside_its_normal(): """authentik-like: a login wall's 401s sit around 1–3%, either side of the 1% cutoff, so it came and went from the problem list every few polls.""" norm = {"error_rate_4xx_pct": _norm(2.8)} for e4 in (0.6, 1.4, 2.9, 1.1, 3.2, 0.9): assert router_state(_window(e4=e4), TH, STATE_WINDOW_WORDS, norm)[0] == "ok", e4 def test_out_of_its_normal_is_a_warning_that_says_what_is_usual(): state, why = router_state(_window(e4=12.0), TH, STATE_WINDOW_WORDS, {"error_rate_4xx_pct": _norm(2.8)}) assert state == "warn" assert why == "12.0% of responses are 4xx over the last 15 min, usual 2.8% at this hour" def test_a_departure_smaller_than_the_floor_is_no_news(): # 1.25 × 1.2% is 1.5%, but 2.0% is only 0.8 points over: under the 2-point floor. assert router_state(_window(e4=2.0), TH, "", {"error_rate_4xx_pct": _norm(1.2)})[0] == "ok" def test_latency_is_graded_against_its_normal_too(): slow_usually = {"latency_p95_ms": _norm(900.0)} assert router_state(_window(p95=950.0), TH, "", slow_usually) == ("ok", "p95 950 ms is usual 900 ms at this hour") state, why = router_state(_window(p95=3000.0), TH, "", slow_usually) assert state == "warn" and why == "p95 latency is 3000 ms, usual 900 ms at this hour" def test_without_a_normal_the_fixed_cutoffs_still_apply(): assert router_state(_window(e4=5.0), TH, "")[0] == "warn" assert router_state(_window(p95=450.0), TH, "")[0] == "warn" def test_5xx_keeps_its_cutoff_whatever_the_history_or_traffic(): usual_5xx = {"error_rate_4xx_pct": _norm(50.0)} assert router_state(_window(e5=40.0), TH, "", usual_5xx)[0] == "crit" assert router_state(_window(e5=40.0, requests=4.0), TH, "")[0] == "crit" def test_a_4xx_share_over_a_handful_of_requests_is_not_graded(): assert router_state(_window(e4=33.0, requests=3.0), TH, "") == ("ok", None) assert router_state(_window(e4=33.0, requests=300.0), TH, "")[0] == "warn" def test_grade_routers_hands_each_router_its_own_normals(): routers = [{"name": "errors@docker", "latest": _latest(e4=60.0)}, {"name": "app@docker", "latest": _latest(e4=60.0)}] window = {"errors@docker": _window(e4=60.0), "app@docker": _window(e4=60.0)} grade_routers(routers, window, TH, {"errors@docker": {"error_rate_4xx_pct": _norm(58.0)}}) assert [r["state"] for r in routers] == ["ok", "warn"] def test_the_fewest_requests_to_judge_is_the_thresholds_setting(): fewer = to_thresholds_cfg({"traefik.threshold.min_requests": 5}) assert router_state(_window(e4=33.0, requests=10.0), TH, "")[0] == "ok" # default 30 assert router_state(_window(e4=33.0, requests=10.0), fewer, "")[0] == "warn"