diff --git a/docs/plugins/traefik.md b/docs/plugins/traefik.md index b1e75c2..50c9c01 100644 --- a/docs/plugins/traefik.md +++ b/docs/plugins/traefik.md @@ -89,6 +89,8 @@ The full Traefik page at `/plugins/traefik/` reads top to bottom, for the time r A router is graded on its last 15 minutes against Settings → Thresholds. When a problem started is read from the router's history over the range. +Once a router has 5 days of history, its 4xx share and p95 latency are judged against its own normal for that hour (see [Normals and anomalies](../architecture.md)) instead of the fixed cutoffs. One over the cutoff but within its normal is healthy, and the healthy table says it is usual for that router (an error-pages service that answers 4xx all day, a login wall's steady 401s). One out of its normal is a warning that says what is usual, and a metric anomaly to review; the Traefik page lists the routers' anomalies of the last 30 days. A 4xx share over fewer requests in the 15 minutes than Settings → Thresholds allows (30 by default) isn't judged at all. The 5xx share is a failure and always keeps its fixed cutoff. Latency is learned only from #5755's per-scrape figures, so its normal arrives 5 days after that release. + --- ## File Locations diff --git a/plugins/traefik/__init__.py b/plugins/traefik/__init__.py index 6ab0550..6785af6 100644 --- a/plugins/traefik/__init__.py +++ b/plugins/traefik/__init__.py @@ -20,6 +20,19 @@ def setup(app: "Quart") -> None: prune_by_age(TraefikStatusCode.scraped_at) prune_by_age(TraefikGlobalStat.scraped_at) prune_by_age(TraefikAccessSummary.period_end) + # Each router's 4xx share and p95 learn their normal (#5741), judged only + # while the router answers the Thresholds minimum of requests a window + # (read at each pass, so a saved change applies at once). Latency is + # learned from the scrape-by-scrape figures only: before #5755 it was a + # percentile of every response since Traefik started. + from datetime import datetime, timezone + from steward.core.norms import learn_normal + from .routes import FLOOR_4XX, FLOOR_P95, STATE_WINDOW, min_requests + gate = ("request_rate", + lambda: min_requests(app.config.get("THRESHOLDS", {})) / STATE_WINDOW.total_seconds()) + learn_normal("traefik", "error_rate_4xx_pct", FLOOR_4XX, gate=gate) + learn_normal("traefik", "latency_p95_ms", FLOOR_P95, gate=gate, + since=datetime(2026, 10, 14, 18, 47, tzinfo=timezone.utc)) def get_scheduled_tasks() -> list: diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py index 88c2023..54a8eda 100644 --- a/plugins/traefik/routes.py +++ b/plugins/traefik/routes.py @@ -11,7 +11,9 @@ import time from steward.auth.middleware import require_role from steward.models.users import UserRole from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample, time_bucket +from steward.core import anomalies as anom from steward.core.formatting import cert_left, status_class +from steward.core.norms import norms_at from steward.models.metrics import PluginMetricLatest from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode from .scheduler import router_domains @@ -52,21 +54,67 @@ def error_cutoffs(thresholds: dict) -> tuple[float, float]: float(thresholds.get("error_4xx", {}).get("warn", 1.0))) -def router_state(reading, thresholds: dict, over: str = "") -> tuple[str, str | None]: +# Each router's 4xx share and p95 are judged against its own normal once it +# has one (#5741, System 182): an error-pages service answers 4xx all day and +# that is its normal, not a problem. Departing by less than the floor is never +# news. A share over fewer than min_requests() in the window is not graded: one +# 404 of three requests is 33%. The 5xx share is a failure and keeps its +# fixed cutoff whatever the history or traffic (the operator's ruling). +FLOOR_4XX = 2.0 # percentage points +FLOOR_P95 = 50.0 # ms +NORMED = ("error_rate_4xx_pct", "latency_p95_ms") + + +def min_requests(thresholds: dict) -> float: + """Fewest requests over the STATE_WINDOW for a 4xx share to be graded + (Settings → Thresholds, traefik.threshold.min_requests).""" + return float(thresholds.get("router_requests", {}).get("warn", 30)) + + +def _against(value: float, norm, floor: float, unit: str) -> tuple[bool, str]: + """(departs, what is usual) for a reading over its fixed cutoff.""" + if norm is None: + return True, "" + usual = f"usual {norm.usual_hold:.1f}{unit}" if unit == "%" else f"usual {norm.usual_hold:.0f}{unit}" + return anom.departs(value, norm, floor), f"{usual} at this hour" + + +def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = None) -> tuple[str, str | None]: """(grade, why) for a router's reading (recent_window's, or one scrape's): crit while it serves 5xx past its cutoff, warn on a p95 past the latency critical cutoff or a 4xx share past its cutoff (all Settings → Thresholds), - otherwise ok with nothing to say. `over` names the span the reading - covers, for the why.""" + otherwise ok. `norms` ({metric: Norm}) are the router's normals for this + hour: a p95 or 4xx share past its cutoff but within its normal is ok, and + `why` then says it is usual. `over` names the span the reading covers.""" + norms = norms or {} error_5xx, error_4xx = error_cutoffs(thresholds) if (reading.error_rate_5xx_pct or 0) > error_5xx: return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}" + usual = [] p95 = reading.latency_p95_ms if p95 is not None and status_class(p95, "latency", thresholds) == "crit": - return "warn", f"p95 latency is {p95:.0f} ms{over}" - if (reading.error_rate_4xx_pct or 0) > error_4xx: - return "warn", f"{reading.error_rate_4xx_pct:.1f}% of responses are 4xx{over}" - return "ok", None + out, said = _against(p95, norms.get("latency_p95_ms"), FLOOR_P95, " ms") + if out: + return "warn", f"p95 latency is {p95:.0f} ms{over}" + (f", {said}" if said else "") + usual.append(f"p95 {p95:.0f} ms is {said}") + requests = getattr(reading, "requests", None) + e4 = reading.error_rate_4xx_pct or 0 + if e4 > error_4xx and (requests is None or requests >= min_requests(thresholds)): + out, said = _against(e4, norms.get("error_rate_4xx_pct"), FLOOR_4XX, "%") + if out: + return "warn", f"{e4:.1f}% of responses are 4xx{over}" + (f", {said}" if said else "") + usual.append(f"4xx {e4:.1f}% is {said}") + return "ok", ("; ".join(usual) or None) + + +async def router_norms(db, names: list[str], now: datetime) -> dict[str, dict]: + """{router: {metric: Norm}}: each router's learned normals for this hour. + A router without enough history yet is absent, and graded on the cutoffs.""" + got = await norms_at(db, [("traefik", n, m) for n in names for m in NORMED], now) + out: dict[str, dict] = {} + for (_s, router, metric), norm in got.items(): + out.setdefault(router, {})[metric] = norm + return out # A router is graded on its last STATE_WINDOW of traffic, not on one scrape: @@ -81,7 +129,8 @@ STATE_WINDOW_WORDS = " over the last 15 min" async def recent_window(db, names: list[str], now: datetime) -> dict[str, SimpleNamespace]: """{router: one reading of its last STATE_WINDOW}: the 4xx and 5xx shares weighted by each scrape's traffic (an idle scrape's 100% counts for - little), p95 averaged. A router with no rows in the window is absent.""" + little), p95 averaged, and about how many requests it answered. A router + with no rows in the window is absent.""" if not names: return {} rate = TraefikMetric.request_rate @@ -94,6 +143,7 @@ async def recent_window(db, names: list[str], now: datetime) -> dict[str, Simple func.avg(TraefikMetric.error_rate_4xx_pct).label("e4"), func.avg(TraefikMetric.error_rate_5xx_pct).label("e5"), func.avg(TraefikMetric.latency_p95_ms).label("p95"), + func.avg(rate).label("rate"), ) .where(TraefikMetric.router_name.in_(names)) .where(TraefikMetric.scraped_at >= now - STATE_WINDOW) @@ -106,6 +156,7 @@ async def recent_window(db, names: list[str], now: datetime) -> dict[str, Simple error_rate_4xx_pct=r.e4w / req if req > 0 else r.e4, error_rate_5xx_pct=r.e5w / req if req > 0 else r.e5, latency_p95_ms=r.p95, + requests=(r.rate or 0.0) * STATE_WINDOW.total_seconds(), ) return out @@ -129,13 +180,15 @@ async def status_codes(db, names: list[str], since: datetime) -> dict[str, list[ return out -def grade_routers(routers: list[dict], window: dict, thresholds: dict) -> None: +def grade_routers(routers: list[dict], window: dict, thresholds: dict, norms: dict | None = None) -> None: """Set each router's state and why from its recent window, or from its - latest scrape when the window holds nothing.""" + latest scrape when the window holds nothing, against its normals + (router_norms') where it has them.""" + norms = norms or {} for r in routers: - reading = window.get(r["name"]) - r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS) if reading - else router_state(r["latest"], thresholds)) + reading, own = window.get(r["name"]), norms.get(r["name"]) + r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS, own) if reading + else router_state(r["latest"], thresholds, norms=own)) # A router missing from this many scrapes in a row is no longer reported: @@ -291,8 +344,12 @@ async def rows(): names = [r["name"] for r in current] history = await router_history(db, names, since, bucket_seconds(since)) phases.mark("history") - grade_routers(current, await recent_window(db, names, now), thresholds) + grade_routers(current, await recent_window(db, names, now), thresholds, + await router_norms(db, names, now)) phases.mark("window") + # The routers' metric anomalies, last 30 days (#5741). + anomalies = await anom.views(db, await anom.list_anomalies( + db, source="traefik", since=now - timedelta(days=30), limit=20), links=False) for r in current: points = history[r["name"]] r["since"], r["since_before"] = (problem_since(points, thresholds) if r["state"] != "ok" @@ -364,6 +421,9 @@ async def rows(): certs=certs, old_certs=old_certs, range_key=range_key, + anomalies=anomalies, + back="/plugins/traefik/", + base_resource=None, ) phases.mark("render") phases.report("rows", f"routers={len(current)} range={range_key}") @@ -388,6 +448,7 @@ async def widget(): phases.mark("latest") current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape) window = await recent_window(db, [r["name"] for r in current], now) + norms = await router_norms(db, [r["name"] for r in current], now) phases.mark("window") certs = (await db.execute(select(TraefikCert))).scalars().all() phases.mark("certs") @@ -396,7 +457,7 @@ async def widget(): served, _old = current_certs(certs, scrape) domains = router_domains() by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now) - grade_routers(current, window, thresholds) + grade_routers(current, window, thresholds, norms) for r in current: r["cert"] = by_router.get(r["name"]) if r["cert"]: diff --git a/plugins/traefik/templates/traefik/rows.html b/plugins/traefik/templates/traefik/rows.html index f0f1278..3bbc585 100644 --- a/plugins/traefik/templates/traefik/rows.html +++ b/plugins/traefik/templates/traefik/rows.html @@ -1,7 +1,8 @@ {# traefik/rows.html — HTMX fragment for the Traefik page: the verdict (swapped into #traefik-verdict above the request chart, #5710), routers that need - attention (routes.router_state), the proxy's own figures, the healthy - routers folded into one table, then certificates by soonest expiry. #} + attention (routes.router_state), their metric anomalies, the proxy's own + figures, the healthy routers folded into one table (one over a cutoff but + within its normal says so, #5741), then certificates by soonest expiry. #} {% from "_macros.html" import stat_tile, status_badge %} {# ── Verdict: what needs a look and since when, then the counts ─────────── #} @@ -77,6 +78,10 @@ {% endif %} +{# A router's 4xx share or p95 out of its own normal (#5741), and the last + 30 days' anomalies, to review here. #} +{% include "_anomaly_card.html" %} + {{ proxy_figures() }} {# ── Healthy routers, folded ─────────────────────────────────────────────── #} @@ -89,7 +94,7 @@ {% for r in healthy %} - {{ r.name }} + {{ r.name }}{% if r.why %}
{{ r.why }}
{% endif %} {{ "%.2f"|format(r.latest.request_rate) }} {{ r.latest.response_bytes_rate|rate }} {{ latency_cell(r.latest.latency_p95_ms) }} diff --git a/plugins/traefik/templates/traefik/widget.html b/plugins/traefik/templates/traefik/widget.html index 0909475..776854c 100644 --- a/plugins/traefik/templates/traefik/widget.html +++ b/plugins/traefik/templates/traefik/widget.html @@ -24,7 +24,7 @@ {{- status_dot(r.state) -}} {{ r.name.split('@')[0] }} - {%- if r.why %}{{ r.why }}{% endif %} + {%- if r.why and r.state != "ok" %}{{ r.why }}{% endif %} {% if m.request_rate is none %}—{% else %}{{ "%.2f"|format(m.request_rate) }}{% endif %} {% if m.response_bytes_rate is none %}—{% else %}{{ m.response_bytes_rate|rate }}{% endif %} diff --git a/steward/core/anomalies.py b/steward/core/anomalies.py index 69dd6b5..213096c 100644 --- a/steward/core/anomalies.py +++ b/steward/core/anomalies.py @@ -36,6 +36,15 @@ REVIEWS = (TO_REVIEW, NORMAL, REAL, LAPSED) MARKS = (NORMAL, REAL) +def _gate_open(learned, key: tuple[str, str, str], reported: dict) -> bool: + """Whether `key` may be judged: ungated, or its resource's gate metric + averaged at least the gate's least over the window.""" + if learned.gate is None: + return True + metric, least = learned.gate + return reported.get((key[0], key[1], metric), 0.0) >= (least() if callable(least) else least) + + def departs(level: float, norm: Norm, floor: float) -> bool: """Whether `level` is out of `norm`: above its usual hold by DEPART times and by at least `floor`, so a 0.1% → 0.3% share is never news.""" @@ -60,8 +69,10 @@ WHERE l.source_module || '/' || l.metric_name = ANY(:keys) async def levels_now(session, now: datetime) -> dict[tuple[str, str, str], float]: - """{(source, resource, metric): its average over the last WINDOW}.""" - keys = [f"{s}/{m}" for s, m in LEARNED] + """{(source, resource, metric): its average over the last WINDOW}, for + every learned metric and every metric one is gated on.""" + keys = sorted({f"{s}/{m}" for s, m in LEARNED} + | {f"{k.source_module}/{k.gate[0]}" for k in LEARNED.values() if k.gate}) if not keys: return {} rows = await session.execute(_LEVELS, {"keys": keys, "since": now - WINDOW, "now": now}) @@ -72,7 +83,9 @@ async def detect(session, now: datetime | None = None) -> dict: """One pass: open, extend and close occurrences. Runs inside the caller's transaction. Returns {"opened", "closed"} counts.""" now = now or datetime.now(timezone.utc) - levels = await levels_now(session, now) + reported = await levels_now(session, now) + levels = {key: level for key, level in reported.items() + if (key[0], key[2]) in LEARNED and _gate_open(LEARNED[(key[0], key[2])], key, reported)} norms = await norms_at(session, list(levels), now) open_now = { (a.source_module, a.resource_name, a.metric_name): (span, a) @@ -97,8 +110,8 @@ async def detect(session, now: datetime | None = None) -> dict: elif out: await _occur(session, key, level, norm, now) opened += 1 - # An open occurrence whose key stopped reporting, or lost its normal, - # can't be judged any more: it ends now. + # An open occurrence whose key stopped reporting, lost its normal or fell + # below its gate can't be judged any more: it ends now. for span, anomaly in open_now.values(): _close(anomaly, span, now) closed += 1 diff --git a/steward/core/norms.py b/steward/core/norms.py index 08103bd..f6c4872 100644 --- a/steward/core/norms.py +++ b/steward/core/norms.py @@ -31,10 +31,11 @@ absorbed on its own, and a lapsed anomaly is never learned. """ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from sqlalchemy import ARRAY, String, bindparam, delete, select, text, tuple_ +from sqlalchemy import ARRAY, DateTime, String, bindparam, delete, select, text, tuple_ from steward.models.norms import MetricNorm @@ -50,19 +51,31 @@ USUAL = 0.9 # "usual" is this percentile across past days: one odd d class Learned: """A metric that learns a normal. `floor` is the smallest departure, in the metric's own unit, worth calling an anomaly (#5740), so that - 0.1% → 0.3% of a share or 2 ms → 4 ms of latency is never news.""" + 0.1% → 0.3% of a share or 2 ms → 4 ms of latency is never news. + + `gate` is (metric, least): the resource is judged only while that metric + of its own averages at least `least` over the window, as a router's + error share means nothing over a handful of requests (#5741). `least` + may be a function, read at each pass, for a minimum that is a setting. `since`: + history before it is not learned from, for a metric whose meaning + changed then.""" source_module: str metric_name: str floor: float + gate: tuple[str, float | Callable[[], float]] | None = None + since: datetime | None = None LEARNED: dict[tuple[str, str], Learned] = {} -def learn_normal(source_module: str, metric_name: str, floor: float) -> None: +def learn_normal(source_module: str, metric_name: str, floor: float, *, + gate: tuple[str, float | Callable[[], float]] | None = None, + since: datetime | None = None) -> None: """Have Steward learn this metric's normal for every resource reporting it. A plugin calls this from setup(), as it calls prune_by_age.""" - LEARNED[(source_module, metric_name)] = Learned(source_module, metric_name, float(floor)) + LEARNED[(source_module, metric_name)] = Learned(source_module, metric_name, float(floor), + gate, since) @dataclass(frozen=True) @@ -89,17 +102,21 @@ def slot_of(at: datetime, kind: str) -> int: # each slot's usual hold and peak across past days, every hour counted in its # own slot and its neighbours'. _REBUILD = text(""" -WITH all_hours AS ( - SELECT source_module AS s, resource_name AS r, metric_name AS m, bucket AS b, - COALESCE(value_p95, value_avg) AS hold, value_max AS peak - FROM plugin_metrics_hourly - WHERE bucket >= :week_since AND source_module || '/' || metric_name = ANY(:keys) +WITH learned AS ( + SELECT key, since FROM unnest(CAST(:keys AS text[]), CAST(:starts AS timestamptz[])) AS k(key, since) +), +all_hours AS ( + SELECT h.source_module AS s, h.resource_name AS r, h.metric_name AS m, h.bucket AS b, + COALESCE(h.value_p95, h.value_avg) AS hold, h.value_max AS peak + FROM plugin_metrics_hourly h + JOIN learned k ON k.key = h.source_module || '/' || h.metric_name + WHERE h.bucket >= k.since UNION ALL - SELECT source_module, resource_name, metric_name, date_trunc('hour', recorded_at), - percentile_cont(0.95) WITHIN GROUP (ORDER BY value), max(value) - FROM plugin_metrics - WHERE recorded_at >= :week_since AND recorded_at < :this_hour - AND source_module || '/' || metric_name = ANY(:keys) + SELECT p.source_module, p.resource_name, p.metric_name, date_trunc('hour', p.recorded_at), + percentile_cont(0.95) WITHIN GROUP (ORDER BY p.value), max(p.value) + FROM plugin_metrics p + JOIN learned k ON k.key = p.source_module || '/' || p.metric_name + WHERE p.recorded_at >= k.since AND p.recorded_at < :this_hour GROUP BY 1, 2, 3, 4 ), hours AS ( @@ -139,7 +156,8 @@ SELECT s, r, m, (slot + n.off + slots) % slots, kind, days, :now FROM slotted CROSS JOIN generate_series(-CAST(:neighbours AS int), CAST(:neighbours AS int)) AS n(off) GROUP BY s, r, m, (slot + n.off + slots) % slots, kind, days -""").bindparams(bindparam("keys", type_=ARRAY(String))) +""").bindparams(bindparam("keys", type_=ARRAY(String)), + bindparam("starts", type_=ARRAY(DateTime(timezone=True)))) async def rebuild_norms(session, now: datetime | None = None) -> int: @@ -147,12 +165,13 @@ async def rebuild_norms(session, now: datetime | None = None) -> int: inside the caller's transaction. Returns how many slot rows it wrote.""" now = now or datetime.now(timezone.utc) await session.execute(delete(MetricNorm)) - keys = [f"{s}/{m}" for s, m in LEARNED] - if not keys: + if not LEARNED: return 0 + week_since = now - WEEK_WINDOW result = await session.execute(_REBUILD, { - "keys": keys, - "week_since": now - WEEK_WINDOW, + "keys": [f"{s}/{m}" for s, m in LEARNED], + # Each key learns from WEEK_WINDOW back, or from its own `since`. + "starts": [max(week_since, k.since) if k.since else week_since for k in LEARNED.values()], "day_since": now - DAY_WINDOW, "this_hour": now.replace(minute=0, second=0, microsecond=0), "week_ready": now - timedelta(days=WEEK_MIN_DAYS), diff --git a/steward/core/settings.py b/steward/core/settings.py index 51a8798..51a5d8e 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -97,6 +97,9 @@ DEFAULTS: dict[str, Any] = { # than this share of 4xx is worth a look (#5380). "traefik.threshold.error_5xx_pct": 0.1, "traefik.threshold.error_4xx_pct": 1.0, + # A router's 4xx share over fewer requests than this in 15 minutes isn't + # graded, on the page or for anomalies: one 404 of three is 33% (#5741). + "traefik.threshold.min_requests": 30, # Degraded/critical cutoffs for metric coloring (warn=amber, crit=red). # cpu/mem/disk/load are percentages (load is load-per-core %); temp in °C. # uptime is "higher is better" so its warn/crit are floors. Ping latency @@ -534,6 +537,9 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict: "crit": g("traefik.threshold.error_5xx_pct", 0.1), "dir": "high"}, "error_4xx": {"warn": g("traefik.threshold.error_4xx_pct", 1.0), "crit": g("traefik.threshold.error_4xx_pct", 1.0), "dir": "high"}, + # Fewer requests than this in 15 minutes and a router's 4xx share isn't graded. + "router_requests": {"warn": g("traefik.threshold.min_requests", 30), + "crit": g("traefik.threshold.min_requests", 30), "dir": "low"}, } diff --git a/steward/settings/routes.py b/steward/settings/routes.py index 7ac8603..02ce91e 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -144,6 +144,7 @@ _THRESHOLD_FIELDS = [ ("crash_loop_deaths", "thresholds.crash_loop_deaths", False), ("error_5xx_pct", "traefik.threshold.error_5xx_pct", True), ("error_4xx_pct", "traefik.threshold.error_4xx_pct", True), + ("min_requests", "traefik.threshold.min_requests", False), ] # Docker time-series retention windows (days). Read fresh by the cleanup task, diff --git a/steward/templates/settings/thresholds.html b/steward/templates/settings/thresholds.html index 38f14ed..caa6cf9 100644 --- a/steward/templates/settings/thresholds.html +++ b/steward/templates/settings/thresholds.html @@ -40,7 +40,7 @@ {{ pair("Disk usage", "disk_warn", "thresholds.disk_warn", "disk_crit", "thresholds.disk_crit", "%", "Filesystem usage.") }} {{ pair("Load", "load_warn", "thresholds.load_warn", "load_crit", "thresholds.load_crit", "% per core", "Load average normalized by core count (100% = one full core per core).") }} {{ pair("Temperature", "temp_warn", "thresholds.temp_warn", "temp_crit", "thresholds.temp_crit", "°C", "Hottest reported sensor per host.") }} - {{ pair("Response time", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "How long a monitor's check takes to answer. Also grades Traefik routers' p95 and the UniFi WAN round trip.") }} + {{ pair("Response time", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "How long a monitor's check takes to answer. Also grades Traefik routers' p95 (until a router has its own normal) and the UniFi WAN round trip.") }} {{ pair("Uptime / SLA", "uptime_warn", "thresholds.uptime_warn", "uptime_crit", "thresholds.uptime_crit", "%", "Lower is worse — amber below warn, red below crit.", step="0.1") }} {{ pair("TLS certificate expiry", "tls_warn", "thresholds.tls_warn_days", "tls_crit", "thresholds.tls_crit_days", "days left", "HTTPS monitors whose certificate expires within these many days are flagged. Lower is worse.") }} {{ pair("Clock drift", "clock_warn", "thresholds.clock_warn_s", "clock_crit", "thresholds.clock_crit_s", "seconds", "How far a host's clock may be from Steward's, either way, before its page and the Hosts list call it out.") }} @@ -69,13 +69,21 @@ -
Above this share, a router is failing (red).
+
Above this share, a router is failing (red), whatever its history.
-
Above this share, a router is worth a look (amber): missing routes, failed sign-ins.
+
Above this share, a router is worth a look (amber): missing routes, failed sign-ins. + Once a router has 5 days of history it is judged against its own normal for the hour instead, as is its p95, + so one that always answers some 4xx stops being listed.
+
+
+ + +
Below this, a router's 4xx share isn't judged, on its page or as an anomaly: one 404 of three requests is 33%. 5xx is always judged.
{% endif %} diff --git a/tests/integration/test_anomalies.py b/tests/integration/test_anomalies.py index 86d092c..90275bf 100644 --- a/tests/integration/test_anomalies.py +++ b/tests/integration/test_anomalies.py @@ -271,3 +271,65 @@ def test_cleanup_prunes_reviewed_anomalies_and_never_one_to_review(app, src): )).scalars()} assert asyncio.run(_go()) == {kept} + + +def test_a_gated_metric_is_judged_only_while_its_gate_is_open(app, src): + """A router's 4xx share means nothing over a handful of requests (#5741): + learned with gate=("req", 1.0), it is judged only while the resource's + own "req" averages at least 1 over the window.""" + from steward.core import norms + from steward.core.anomalies import WINDOW + from steward.models.metrics import PluginMetric, PluginMetricLatest + norms.learn_normal(src, METRIC, 5, gate=("req", 1.0)) + res = _res("gated") + quiet, busy = NOW, NOW + timedelta(hours=2) + + async def _req(s, until, value): + n = int(WINDOW.total_seconds() // 60) + s.add_all([PluginMetric(source_module=src, resource_name=res, metric_name="req", + value=value, recorded_at=until - timedelta(minutes=m)) for m in range(n)]) + await s.merge(PluginMetricLatest(source_module=src, resource_name=res, metric_name="req", + value=value, recorded_at=until)) + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + await _req(s, quiet, 0.2) + await _req(s, busy, 5.0) + return await _steps(app, src, res, [(quiet, 60.0), (busy, 60.0)]) + + rows, spans = asyncio.run(_go()) + (a,) = rows # only the busy window opened one + assert spans[a.id][0].started_at == busy - WINDOW + + +def test_a_metric_learns_only_from_its_since(app, src): + """A metric whose meaning changed (Traefik latency, #5755) learns only + from history after `since`: seven days held 90, but only the last three + are after it, under the five a normal needs.""" + from steward.core import norms + from steward.core.norms import norms_at, rebuild_norms + from steward.models.metrics import PluginMetricHourly + res = _res("since") + rows = [PluginMetricHourly(source_module=src, resource_name=res, metric_name=METRIC, + bucket=NOW.replace(hour=0) - timedelta(days=d) + timedelta(hours=h), + value_avg=90.0, value_max=90.0, value_p95=90.0, sample_count=60) + for d in range(1, 8) for h in range(24)] + + async def _go(since): + norms.learn_normal(src, METRIC, 5, since=since) + async with app.db_sessionmaker() as s: + async with s.begin(): + await rebuild_norms(s, NOW) + got = await norms_at(s, [(src, res, METRIC)], NOW.replace(hour=14)) + return got.get((src, res, METRIC)) + + async def _both(): + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add_all(rows) + return await _go(None), await _go(NOW.replace(hour=0) - timedelta(days=3)) + + learned, held_back = asyncio.run(_both()) + assert learned is not None and learned.usual_hold == 90.0 + assert held_back is None diff --git a/tests/plugins/traefik/test_normals.py b/tests/plugins/traefik/test_normals.py new file mode 100644 index 0000000..c8b98f4 --- /dev/null +++ b/tests/plugins/traefik/test_normals.py @@ -0,0 +1,85 @@ +"""A router's 4xx share and p95 are graded against its own normal once it has +one (#5741, System 182): a router over the fixed cutoff but inside its normal +is healthy and says so; one out of its normal is a warning that says what is +usual. The 5xx share keeps its fixed cutoff, and a 4xx share over too few +requests isn't graded.""" +from types import SimpleNamespace as NS + +from plugins.traefik.routes import STATE_WINDOW_WORDS, grade_routers, router_state +from steward.core.norms import Norm +from steward.core.settings import to_thresholds_cfg +from tests.plugins.traefik.test_traefik_page import _latest + +TH = to_thresholds_cfg({}) # 4xx warn over 1%, 5xx crit over 0.1%, latency crit 200 ms + + +def _norm(hold, peak=None): + return Norm(kind="day", slot=14, usual_hold=hold, usual_peak=peak or hold * 1.5, days_seen=7) + + +def _window(e4=0.0, e5=0.0, p95=20.0, requests=500.0): + return NS(error_rate_4xx_pct=e4, error_rate_5xx_pct=e5, latency_p95_ms=p95, requests=requests) + + +def test_an_error_pages_router_answering_4xx_all_day_reads_healthy(): + state, why = router_state(_window(e4=62.0), TH, STATE_WINDOW_WORDS, + {"error_rate_4xx_pct": _norm(60.0)}) + assert state == "ok" + assert why == "4xx 62.0% is usual 60.0% at this hour" + + +def test_a_router_that_flapped_on_the_cutoff_stays_healthy_inside_its_normal(): + """authentik-like: a login wall's 401s sit around 1–3%, either side of the + 1% cutoff, so it came and went from the problem list every few polls.""" + norm = {"error_rate_4xx_pct": _norm(2.8)} + for e4 in (0.6, 1.4, 2.9, 1.1, 3.2, 0.9): + assert router_state(_window(e4=e4), TH, STATE_WINDOW_WORDS, norm)[0] == "ok", e4 + + +def test_out_of_its_normal_is_a_warning_that_says_what_is_usual(): + state, why = router_state(_window(e4=12.0), TH, STATE_WINDOW_WORDS, + {"error_rate_4xx_pct": _norm(2.8)}) + assert state == "warn" + assert why == "12.0% of responses are 4xx over the last 15 min, usual 2.8% at this hour" + + +def test_a_departure_smaller_than_the_floor_is_no_news(): + # 1.25 × 1.2% is 1.5%, but 2.0% is only 0.8 points over: under the 2-point floor. + assert router_state(_window(e4=2.0), TH, "", {"error_rate_4xx_pct": _norm(1.2)})[0] == "ok" + + +def test_latency_is_graded_against_its_normal_too(): + slow_usually = {"latency_p95_ms": _norm(900.0)} + assert router_state(_window(p95=950.0), TH, "", slow_usually) == ("ok", "p95 950 ms is usual 900 ms at this hour") + state, why = router_state(_window(p95=3000.0), TH, "", slow_usually) + assert state == "warn" and why == "p95 latency is 3000 ms, usual 900 ms at this hour" + + +def test_without_a_normal_the_fixed_cutoffs_still_apply(): + assert router_state(_window(e4=5.0), TH, "")[0] == "warn" + assert router_state(_window(p95=450.0), TH, "")[0] == "warn" + + +def test_5xx_keeps_its_cutoff_whatever_the_history_or_traffic(): + usual_5xx = {"error_rate_4xx_pct": _norm(50.0)} + assert router_state(_window(e5=40.0), TH, "", usual_5xx)[0] == "crit" + assert router_state(_window(e5=40.0, requests=4.0), TH, "")[0] == "crit" + + +def test_a_4xx_share_over_a_handful_of_requests_is_not_graded(): + assert router_state(_window(e4=33.0, requests=3.0), TH, "") == ("ok", None) + assert router_state(_window(e4=33.0, requests=300.0), TH, "")[0] == "warn" + + +def test_grade_routers_hands_each_router_its_own_normals(): + routers = [{"name": "errors@docker", "latest": _latest(e4=60.0)}, + {"name": "app@docker", "latest": _latest(e4=60.0)}] + window = {"errors@docker": _window(e4=60.0), "app@docker": _window(e4=60.0)} + grade_routers(routers, window, TH, {"errors@docker": {"error_rate_4xx_pct": _norm(58.0)}}) + assert [r["state"] for r in routers] == ["ok", "warn"] + + +def test_the_fewest_requests_to_judge_is_the_thresholds_setting(): + fewer = to_thresholds_cfg({"traefik.threshold.min_requests": 5}) + assert router_state(_window(e4=33.0, requests=10.0), TH, "")[0] == "ok" # default 30 + assert router_state(_window(e4=33.0, requests=10.0), fewer, "")[0] == "warn"