bvandeusenandClaude Opus 5.5
9f14620271
#5741 "Traefik routers are graded against their own normal", milestone 513. - A router's 4xx share and p95 learn their normal (learn_normal in the plugin's setup). Once a router has 5 days of history, a reading over the fixed cutoff but inside its normal is healthy, and the healthy table says it is usual for that router. One out of its normal is a warning that says what is usual, and a metric anomaly. - The 5xx share is a failure: fixed cutoff always, whatever the history or traffic (System 182's ruling). - A 4xx share over fewer requests than Settings -> Thresholds' new "Fewest requests to judge 4xx" (30 per 15 minutes) isn't graded on the page, and isn't judged for anomalies. - Core: learn_normal(gate=(metric, least)) judges a key only while its resource's gate metric averages at least `least`; `least` may be a function, so it follows a setting. learn_normal(since=...) learns only from history after a moment: Traefik latency learns from #5755's per-scrape figures only. - The Traefik page lists the routers' metric anomalies of the last 30 days (_anomaly_card.html). The dashboard widget keeps its why to problems. - Thresholds and docs say when the cutoffs give way to a normal. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
86 lines
4.0 KiB
Python
86 lines
4.0 KiB
Python
"""A router's 4xx share and p95 are graded against its own normal once it has
|
||
one (#5741, System 182): a router over the fixed cutoff but inside its normal
|
||
is healthy and says so; one out of its normal is a warning that says what is
|
||
usual. The 5xx share keeps its fixed cutoff, and a 4xx share over too few
|
||
requests isn't graded."""
|
||
from types import SimpleNamespace as NS
|
||
|
||
from plugins.traefik.routes import STATE_WINDOW_WORDS, grade_routers, router_state
|
||
from steward.core.norms import Norm
|
||
from steward.core.settings import to_thresholds_cfg
|
||
from tests.plugins.traefik.test_traefik_page import _latest
|
||
|
||
TH = to_thresholds_cfg({}) # 4xx warn over 1%, 5xx crit over 0.1%, latency crit 200 ms
|
||
|
||
|
||
def _norm(hold, peak=None):
|
||
return Norm(kind="day", slot=14, usual_hold=hold, usual_peak=peak or hold * 1.5, days_seen=7)
|
||
|
||
|
||
def _window(e4=0.0, e5=0.0, p95=20.0, requests=500.0):
|
||
return NS(error_rate_4xx_pct=e4, error_rate_5xx_pct=e5, latency_p95_ms=p95, requests=requests)
|
||
|
||
|
||
def test_an_error_pages_router_answering_4xx_all_day_reads_healthy():
|
||
state, why = router_state(_window(e4=62.0), TH, STATE_WINDOW_WORDS,
|
||
{"error_rate_4xx_pct": _norm(60.0)})
|
||
assert state == "ok"
|
||
assert why == "4xx 62.0% is usual 60.0% at this hour"
|
||
|
||
|
||
def test_a_router_that_flapped_on_the_cutoff_stays_healthy_inside_its_normal():
|
||
"""authentik-like: a login wall's 401s sit around 1–3%, either side of the
|
||
1% cutoff, so it came and went from the problem list every few polls."""
|
||
norm = {"error_rate_4xx_pct": _norm(2.8)}
|
||
for e4 in (0.6, 1.4, 2.9, 1.1, 3.2, 0.9):
|
||
assert router_state(_window(e4=e4), TH, STATE_WINDOW_WORDS, norm)[0] == "ok", e4
|
||
|
||
|
||
def test_out_of_its_normal_is_a_warning_that_says_what_is_usual():
|
||
state, why = router_state(_window(e4=12.0), TH, STATE_WINDOW_WORDS,
|
||
{"error_rate_4xx_pct": _norm(2.8)})
|
||
assert state == "warn"
|
||
assert why == "12.0% of responses are 4xx over the last 15 min, usual 2.8% at this hour"
|
||
|
||
|
||
def test_a_departure_smaller_than_the_floor_is_no_news():
|
||
# 1.25 × 1.2% is 1.5%, but 2.0% is only 0.8 points over: under the 2-point floor.
|
||
assert router_state(_window(e4=2.0), TH, "", {"error_rate_4xx_pct": _norm(1.2)})[0] == "ok"
|
||
|
||
|
||
def test_latency_is_graded_against_its_normal_too():
|
||
slow_usually = {"latency_p95_ms": _norm(900.0)}
|
||
assert router_state(_window(p95=950.0), TH, "", slow_usually) == ("ok", "p95 950 ms is usual 900 ms at this hour")
|
||
state, why = router_state(_window(p95=3000.0), TH, "", slow_usually)
|
||
assert state == "warn" and why == "p95 latency is 3000 ms, usual 900 ms at this hour"
|
||
|
||
|
||
def test_without_a_normal_the_fixed_cutoffs_still_apply():
|
||
assert router_state(_window(e4=5.0), TH, "")[0] == "warn"
|
||
assert router_state(_window(p95=450.0), TH, "")[0] == "warn"
|
||
|
||
|
||
def test_5xx_keeps_its_cutoff_whatever_the_history_or_traffic():
|
||
usual_5xx = {"error_rate_4xx_pct": _norm(50.0)}
|
||
assert router_state(_window(e5=40.0), TH, "", usual_5xx)[0] == "crit"
|
||
assert router_state(_window(e5=40.0, requests=4.0), TH, "")[0] == "crit"
|
||
|
||
|
||
def test_a_4xx_share_over_a_handful_of_requests_is_not_graded():
|
||
assert router_state(_window(e4=33.0, requests=3.0), TH, "") == ("ok", None)
|
||
assert router_state(_window(e4=33.0, requests=300.0), TH, "")[0] == "warn"
|
||
|
||
|
||
def test_grade_routers_hands_each_router_its_own_normals():
|
||
routers = [{"name": "errors@docker", "latest": _latest(e4=60.0)},
|
||
{"name": "app@docker", "latest": _latest(e4=60.0)}]
|
||
window = {"errors@docker": _window(e4=60.0), "app@docker": _window(e4=60.0)}
|
||
grade_routers(routers, window, TH, {"errors@docker": {"error_rate_4xx_pct": _norm(58.0)}})
|
||
assert [r["state"] for r in routers] == ["ok", "warn"]
|
||
|
||
|
||
def test_the_fewest_requests_to_judge_is_the_thresholds_setting():
|
||
fewer = to_thresholds_cfg({"traefik.threshold.min_requests": 5})
|
||
assert router_state(_window(e4=33.0, requests=10.0), TH, "")[0] == "ok" # default 30
|
||
assert router_state(_window(e4=33.0, requests=10.0), fewer, "")[0] == "warn"
|