feat(traefik): a 4xx share is worth a look only once it is a lot of 4xx (#6105)
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 1m29s
CI / integration (push) Successful in 1m42s
CI / publish (push) Successful in 1m9s

Three 404s on a quiet router read 7.5% and were raised as an anomaly; most of
the 4xx anomalies on review stood for 3 to 7 errors in 15 minutes. A new
Settings → Thresholds minimum, traefik.threshold.min_4xx (default 20, more
than one a minute held for the window), applies to both the router's grade
on its page and to anomaly detection, as min_5xx does for 5xx.

learn_normal takes `fewest`: for a share gated on a per-second rate, the
fewest events (rate × window × share) it must stand for to be judged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-19 16:02:17 -04:00
co-authored by Claude Opus 5.5
parent 4639ac6e52
commit b75b883326
11 changed files with 81 additions and 15 deletions
+13 -1
View File
@@ -9,7 +9,7 @@ from zoneinfo import ZoneInfo
from quart import render_template, session
from steward.core import mail
from steward.core.anomalies import DEPART, AnomalyView, departs
from steward.core.anomalies import DEPART, WINDOW, AnomalyView, _gate_open, departs
from steward.core.norms import Norm
from tests.test_style_tokens import hex_colours
@@ -146,3 +146,15 @@ def test_a_week_without_anomalies_leaves_the_section_out():
quiet = {**REPORT, "anomalies": {"seen": [], "lapsed": [], "to_review": 0}}
_, text, html = mail.report_email({}, quiet, to)
assert "anomal" not in (text + html).lower()
def test_a_share_gated_on_a_rate_is_judged_only_once_it_stands_for_enough_events():
"""#6105: three 404s on a quiet router read 7%. With `fewest`, the share
is judged only once rate × window × share reaches it."""
learned = SimpleNamespace(gate=("req", lambda: 30 / WINDOW.total_seconds()), fewest=lambda: 20)
key = ("traefik", "r", "e4")
per_s = 100 / WINDOW.total_seconds() # 100 requests in the window
assert not _gate_open(learned, key, {key: 7.5, ("traefik", "r", "req"): per_s}) # 7.5 errors
assert _gate_open(learned, key, {key: 25.0, ("traefik", "r", "req"): per_s}) # 25
assert not _gate_open(learned, key, {key: 90.0, ("traefik", "r", "req"): 10 / WINDOW.total_seconds()})
assert _gate_open(SimpleNamespace(gate=("req", 0.0), fewest=None), key, {key: 1.0})
+9 -1
View File
@@ -91,6 +91,14 @@ def test_grade_routers_hands_each_router_its_own_normals():
def test_the_fewest_requests_to_judge_is_the_thresholds_setting():
fewer = to_thresholds_cfg({"traefik.threshold.min_requests": 5})
fewer = to_thresholds_cfg({"traefik.threshold.min_requests": 5, "traefik.threshold.min_4xx": 1})
assert router_state(_window(e4=33.0, requests=10.0), TH, "")[0] == "ok" # default 30
assert router_state(_window(e4=33.0, requests=10.0), fewer, "")[0] == "warn"
def test_a_4xx_share_is_worth_a_look_only_once_enough_4xx_were_answered():
# librarian@swarm on 2026-10-19: 7.5% of about 100 requests, so 7 errors (#6105)
assert router_state(_window(e4=7.5, requests=100.0), TH, "") == ("ok", None)
assert router_state(_window(e4=7.5, requests=300.0), TH, "")[0] == "warn" # 22 errors
fewer = to_thresholds_cfg({"traefik.threshold.min_4xx": 5})
assert router_state(_window(e4=7.5, requests=100.0), fewer, "")[0] == "warn"