From b75b883326ede0e723821a9e97befe57a901e141 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 19 Oct 2026 16:02:17 -0400 Subject: [PATCH] feat(traefik): a 4xx share is worth a look only once it is a lot of 4xx (#6105) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three 404s on a quiet router read 7.5% and were raised as an anomaly; most of the 4xx anomalies on review stood for 3 to 7 errors in 15 minutes. A new Settings → Thresholds minimum, traefik.threshold.min_4xx (default 20, more than one a minute held for the window), applies to both the router's grade on its page and to anomaly detection, as min_5xx does for 5xx. learn_normal takes `fewest`: for a share gated on a per-second rate, the fewest events (rate × window × share) it must stand for to be judged. Co-Authored-By: Claude Opus 5.5 --- plugins/traefik/__init__.py | 8 +++++--- plugins/traefik/mcp_tools.py | 3 ++- plugins/traefik/routes.py | 17 ++++++++++++++--- steward/core/anomalies.py | 11 +++++++++-- steward/core/norms.py | 13 ++++++++++--- steward/core/settings.py | 9 +++++++++ steward/mcp/tools/anomalies.py | 4 +++- steward/settings/routes.py | 1 + steward/templates/settings/thresholds.html | 6 ++++++ tests/core/test_anomalies.py | 14 +++++++++++++- tests/plugins/traefik/test_normals.py | 10 +++++++++- 11 files changed, 81 insertions(+), 15 deletions(-) diff --git a/plugins/traefik/__init__.py b/plugins/traefik/__init__.py index 335734a..0949f85 100644 --- a/plugins/traefik/__init__.py +++ b/plugins/traefik/__init__.py @@ -22,15 +22,17 @@ def setup(app: "Quart") -> None: prune_by_age(TraefikAccessSummary.period_end) # Each router's 4xx share and p95 learn their normal (#5741), judged only # while the router answers the Thresholds minimum of requests a window - # (read at each pass, so a saved change applies at once). Latency is + # (read at each pass, so a saved change applies at once), and the 4xx + # share only once it stands for the minimum of 4xx (#6105). Latency is # learned from the scrape-by-scrape figures only: before #5755 it was a # percentile of every response since Traefik started. from datetime import datetime, timezone from steward.core.norms import floor_setting, learn_normal - from .routes import STATE_WINDOW, min_requests + from .routes import STATE_WINDOW, min_4xx, min_requests gate = ("request_rate", lambda: min_requests(app.config.get("THRESHOLDS", {})) / STATE_WINDOW.total_seconds()) - learn_normal("traefik", "error_rate_4xx_pct", floor_setting(app, "router_4xx_pct"), gate=gate) + learn_normal("traefik", "error_rate_4xx_pct", floor_setting(app, "router_4xx_pct"), gate=gate, + fewest=lambda: min_4xx(app.config.get("THRESHOLDS", {}))) learn_normal("traefik", "latency_p95_ms", floor_setting(app, "router_p95_ms"), gate=gate, since=datetime(2026, 10, 14, 18, 47, tzinfo=timezone.utc)) diff --git a/plugins/traefik/mcp_tools.py b/plugins/traefik/mcp_tools.py index ddb6e02..5bbd280 100644 --- a/plugins/traefik/mcp_tools.py +++ b/plugins/traefik/mcp_tools.py @@ -117,7 +117,8 @@ async def list_routers(state: str = "", limit: int = 100) -> dict: out = bounded(rows, clamp_limit(limit, 100, 500), key="routers", more="Pass a larger `limit`, or a `state`.") out["cutoffs"] = {"error_5xx_pct": error_5xx, "error_4xx_pct": error_4xx, - "min_requests_15m": page.min_requests(thresholds)} + "min_requests_15m": page.min_requests(thresholds), + "min_5xx_15m": page.min_5xx(thresholds), "min_4xx_15m": page.min_4xx(thresholds)} out["gone"] = [{"router": r["name"], "last_scraped_at": iso(r["latest"].scraped_at)} for r in gone[:20]] return out diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py index d0c6245..3721263 100644 --- a/plugins/traefik/routes.py +++ b/plugins/traefik/routes.py @@ -65,6 +65,8 @@ def error_cutoffs(thresholds: dict) -> tuple[float, float]: # keeps its fixed cutoff whatever the history; it shares the 4xx share's # minimum of traffic (the operator, #5859), and is failing only once # min_5xx() errors were answered: one 502 of 70 requests is 1.4% (#5865). +# A 4xx share is worth a look only once min_4xx() were answered: three 404s +# on a quiet router read 7%, and some 4xx are part of any day (#6105). NORMED = ("error_rate_4xx_pct", "latency_p95_ms") @@ -80,6 +82,13 @@ def min_5xx(thresholds: dict) -> float: return float(thresholds.get("router_5xx", {}).get("warn", 3)) +def min_4xx(thresholds: dict) -> float: + """Fewest 4xx answered over the STATE_WINDOW for a 4xx share to be worth a + look, on the page or as an anomaly (Settings → Thresholds, + traefik.threshold.min_4xx).""" + return float(thresholds.get("router_4xx", {}).get("warn", 20)) + + def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = None) -> tuple[str, str | None]: """(grade, why) for a router's reading (recent_window's, or one scrape's): crit while it serves 5xx past its cutoff, warn on a p95 past the latency @@ -87,8 +96,9 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = otherwise ok. `norms` ({metric: Norm}) are the router's normals for this hour: a p95 or 4xx share past its cutoff but within its normal is ok, and `why` then says it is usual. `over` names the span the reading covers. - Neither share is graded over fewer than min_requests() requests, and a - 5xx share is failing only once about min_5xx() errors were answered.""" + Neither share is graded over fewer than min_requests() requests, a 5xx + share is failing only once about min_5xx() errors were answered, and a + 4xx share is worth a look only once about min_4xx() were.""" norms = norms or {} error_5xx, error_4xx = error_cutoffs(thresholds) requests = getattr(reading, "requests", None) @@ -106,7 +116,8 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = return "warn", f"p95 latency is {p95:.0f} ms{over}" + (f", {said}" if said else "") usual.append(f"p95 {p95:.0f} ms is {said}") e4 = reading.error_rate_4xx_pct or 0 - if enough and e4 > error_4xx: + errors4 = None if requests is None else round(requests * e4 / 100.0) + if enough and e4 > error_4xx and (errors4 is None or errors4 >= min_4xx(thresholds)): out, said = anom.against(e4, norms.get("error_rate_4xx_pct"), floor_of(thresholds, "router_4xx_pct"), "%") if out: diff --git a/steward/core/anomalies.py b/steward/core/anomalies.py index c034d31..1ed45ef 100644 --- a/steward/core/anomalies.py +++ b/steward/core/anomalies.py @@ -38,11 +38,18 @@ MARKS = (NORMAL, REAL) def _gate_open(learned, key: tuple[str, str, str], reported: dict) -> bool: """Whether `key` may be judged: ungated, or its resource's gate metric - averaged at least the gate's least over the window.""" + averaged at least the gate's least over the window, and the share stands + for at least `fewest` events in it.""" if learned.gate is None: return True metric, least = learned.gate - return reported.get((key[0], key[1], metric), 0.0) >= (least() if callable(least) else least) + rate = reported.get((key[0], key[1], metric), 0.0) + if rate < (least() if callable(least) else least): + return False + if learned.fewest is None: + return True + fewest = learned.fewest() if callable(learned.fewest) else learned.fewest + return rate * WINDOW.total_seconds() * reported.get(key, 0.0) / 100.0 >= fewest def departs(level: float, norm: Norm, floor: float) -> bool: diff --git a/steward/core/norms.py b/steward/core/norms.py index a6b2cd2..712b12b 100644 --- a/steward/core/norms.py +++ b/steward/core/norms.py @@ -57,7 +57,11 @@ class Learned: `gate` is (metric, least): the resource is judged only while that metric of its own averages at least `least` over the window, as a router's error share means nothing over a handful of requests (#5741). `least` - may be a function, read at each pass, for a minimum that is a setting. `since`: + may be a function, read at each pass, for a minimum that is a setting. + `fewest`, for a share (%) gated on a per-second rate: the fewest events + the share stands for over the window (rate × window × share / 100), as + three 404s are 7% of a quiet router and no news (#6105); a number or a + function, like `least`. `since`: history before it is not learned from, for a metric whose meaning changed then. `parts=False` learns only whole resources, not their parts (":core3"): a host's CPU has a normal, each of its cores doesn't.""" @@ -67,6 +71,7 @@ class Learned: gate: tuple[str, float | Callable[[], float]] | None = None since: datetime | None = None parts: bool = True + fewest: float | Callable[[], float] | None = None @property def least_departure(self) -> float: @@ -83,11 +88,13 @@ LEARNED: dict[tuple[str, str], Learned] = {} def learn_normal(source_module: str, metric_name: str, floor: float | Callable[[], float], *, gate: tuple[str, float | Callable[[], float]] | None = None, - since: datetime | None = None, parts: bool = True) -> None: + since: datetime | None = None, parts: bool = True, + fewest: float | Callable[[], float] | None = None) -> None: """Have Steward learn this metric's normal for every resource reporting it. A plugin calls this from setup(), as it calls prune_by_age.""" LEARNED[(source_module, metric_name)] = Learned( - source_module, metric_name, floor if callable(floor) else float(floor), gate, since, parts) + source_module, metric_name, floor if callable(floor) else float(floor), gate, since, parts, + fewest) def floor_setting(app, name: str) -> Callable[[], float]: diff --git a/steward/core/settings.py b/steward/core/settings.py index 165d560..13338ba 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -103,6 +103,12 @@ DEFAULTS: dict[str, Any] = { # ...and a 5xx share past its cutoff is failing only once this many 5xx # were answered in those 15 minutes: one 502 of 70 is 1.4% (#5865). "traefik.threshold.min_5xx": 3, + # ...and a 4xx share past its cutoff, or departing its normal, is worth a + # look only once this many 4xx were answered: some 404s and failed + # sign-ins are expected all day; it is a lot of them that needs acting on. + # 20 is more than one a minute held for the window; the blips it drops + # were 3 to 7 errors (#6105). + "traefik.threshold.min_4xx": 20, # The smallest departure from a metric's normal worth calling an anomaly # (#5742), on top of being a quarter over its usual hold: a share's floor # is in percentage points, a response time's in ms. 5 points is the @@ -560,6 +566,9 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict: # Fewer 5xx than this in 15 minutes and a router isn't failing, whatever the share. "router_5xx": {"warn": g("traefik.threshold.min_5xx", 3), "crit": g("traefik.threshold.min_5xx", 3), "dir": "low"}, + # Fewer 4xx than this in 15 minutes and a router's 4xx share isn't news. + "router_4xx": {"warn": g("traefik.threshold.min_4xx", 20), + "crit": g("traefik.threshold.min_4xx", 20), "dir": "low"}, # Not a cutoff: each normal's floor (norms.floor.*), read by floor_of(). "floors": {name: float(g(f"norms.floor.{name}", DEFAULTS[f"norms.floor.{name}"])) for name in NORM_FLOORS}, diff --git a/steward/mcp/tools/anomalies.py b/steward/mcp/tools/anomalies.py index 2c3cd31..c5881e3 100644 --- a/steward/mcp/tools/anomalies.py +++ b/steward/mcp/tools/anomalies.py @@ -312,7 +312,9 @@ async def get_metric_normal(source_module: str, resource: str, metric: str) -> d "learned": bool(slots), "kind": kind, "floor": rnd(floor, 3), "floor_setting": learned.floor_name, "only_while": ({"metric": learned.gate[0], - "at_least": rnd(learned.gate[1]() if callable(learned.gate[1]) else learned.gate[1], 3)} + "at_least": rnd(learned.gate[1]() if callable(learned.gate[1]) else learned.gate[1], 3), + "fewest_events": (None if learned.fewest is None else + rnd(learned.fewest() if callable(learned.fewest) else learned.fewest, 3))} if learned.gate else None), "now": ({"slot": current.slot, "label": slot_label(current.kind, current.slot), "usual_hold": rnd(current.usual_hold, 3), "usual_peak": rnd(current.usual_peak, 3), diff --git a/steward/settings/routes.py b/steward/settings/routes.py index 9aae29f..4122965 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -147,6 +147,7 @@ _THRESHOLD_FIELDS = [ ("error_4xx_pct", "traefik.threshold.error_4xx_pct", True), ("min_requests", "traefik.threshold.min_requests", False), ("min_5xx", "traefik.threshold.min_5xx", False), + ("min_4xx", "traefik.threshold.min_4xx", False), # Each normal's floor (#5742), named by its own setting key. *[(f"norms.floor.{name}", f"norms.floor.{name}", True) for name in NORM_FLOORS], ] diff --git a/steward/templates/settings/thresholds.html b/steward/templates/settings/thresholds.html index 635ed54..c00d1fb 100644 --- a/steward/templates/settings/thresholds.html +++ b/steward/templates/settings/thresholds.html @@ -92,6 +92,12 @@ value="{{ settings['traefik.threshold.min_5xx'] }}" style="max-width:110px;margin-top:0.25rem;">
A router past the 5xx share above is failing only once it has answered this many 5xx: one 502 among 70 requests is 1.4%, but it is one error.
+
+ + +
A router's 4xx share is flagged, on its page or as an anomaly, only once it has answered this many 4xx: a few 404s and failed sign-ins are part of a normal day; a lot of them is what needs acting on.
+
{% endif %} diff --git a/tests/core/test_anomalies.py b/tests/core/test_anomalies.py index 246483d..004d786 100644 --- a/tests/core/test_anomalies.py +++ b/tests/core/test_anomalies.py @@ -9,7 +9,7 @@ from zoneinfo import ZoneInfo from quart import render_template, session from steward.core import mail -from steward.core.anomalies import DEPART, AnomalyView, departs +from steward.core.anomalies import DEPART, WINDOW, AnomalyView, _gate_open, departs from steward.core.norms import Norm from tests.test_style_tokens import hex_colours @@ -146,3 +146,15 @@ def test_a_week_without_anomalies_leaves_the_section_out(): quiet = {**REPORT, "anomalies": {"seen": [], "lapsed": [], "to_review": 0}} _, text, html = mail.report_email({}, quiet, to) assert "anomal" not in (text + html).lower() + + +def test_a_share_gated_on_a_rate_is_judged_only_once_it_stands_for_enough_events(): + """#6105: three 404s on a quiet router read 7%. With `fewest`, the share + is judged only once rate × window × share reaches it.""" + learned = SimpleNamespace(gate=("req", lambda: 30 / WINDOW.total_seconds()), fewest=lambda: 20) + key = ("traefik", "r", "e4") + per_s = 100 / WINDOW.total_seconds() # 100 requests in the window + assert not _gate_open(learned, key, {key: 7.5, ("traefik", "r", "req"): per_s}) # 7.5 errors + assert _gate_open(learned, key, {key: 25.0, ("traefik", "r", "req"): per_s}) # 25 + assert not _gate_open(learned, key, {key: 90.0, ("traefik", "r", "req"): 10 / WINDOW.total_seconds()}) + assert _gate_open(SimpleNamespace(gate=("req", 0.0), fewest=None), key, {key: 1.0}) diff --git a/tests/plugins/traefik/test_normals.py b/tests/plugins/traefik/test_normals.py index 95f0705..123042f 100644 --- a/tests/plugins/traefik/test_normals.py +++ b/tests/plugins/traefik/test_normals.py @@ -91,6 +91,14 @@ def test_grade_routers_hands_each_router_its_own_normals(): def test_the_fewest_requests_to_judge_is_the_thresholds_setting(): - fewer = to_thresholds_cfg({"traefik.threshold.min_requests": 5}) + fewer = to_thresholds_cfg({"traefik.threshold.min_requests": 5, "traefik.threshold.min_4xx": 1}) assert router_state(_window(e4=33.0, requests=10.0), TH, "")[0] == "ok" # default 30 assert router_state(_window(e4=33.0, requests=10.0), fewer, "")[0] == "warn" + + +def test_a_4xx_share_is_worth_a_look_only_once_enough_4xx_were_answered(): + # librarian@swarm on 2026-10-19: 7.5% of about 100 requests, so 7 errors (#6105) + assert router_state(_window(e4=7.5, requests=100.0), TH, "") == ("ok", None) + assert router_state(_window(e4=7.5, requests=300.0), TH, "")[0] == "warn" # 22 errors + fewer = to_thresholds_cfg({"traefik.threshold.min_4xx": 5}) + assert router_state(_window(e4=7.5, requests=100.0), fewer, "")[0] == "warn"