feat(traefik): routers are graded against their own normal (#5741)

#5741 "Traefik routers are graded against their own normal", milestone 513.

- A router's 4xx share and p95 learn their normal (learn_normal in the
  plugin's setup). Once a router has 5 days of history, a reading over the
  fixed cutoff but inside its normal is healthy, and the healthy table says
  it is usual for that router. One out of its normal is a warning that says
  what is usual, and a metric anomaly.
- The 5xx share is a failure: fixed cutoff always, whatever the history or
  traffic (System 182's ruling).
- A 4xx share over fewer requests than Settings -> Thresholds' new "Fewest
  requests to judge 4xx" (30 per 15 minutes) isn't graded on the page, and
  isn't judged for anomalies.
- Core: learn_normal(gate=(metric, least)) judges a key only while its
  resource's gate metric averages at least `least`; `least` may be a
  function, so it follows a setting. learn_normal(since=...) learns only
  from history after a moment: Traefik latency learns from #5755's
  per-scrape figures only.
- The Traefik page lists the routers' metric anomalies of the last 30 days
  (_anomaly_card.html). The dashboard widget keeps its why to problems.
- Thresholds and docs say when the cutoffs give way to a normal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/anomalies.py

@@ -36,6 +36,15 @@ REVIEWS = (TO_REVIEW, NORMAL, REAL, LAPSED)
 MARKS = (NORMAL, REAL)
 
 
+def _gate_open(learned, key: tuple[str, str, str], reported: dict) -> bool:
+    """Whether `key` may be judged: ungated, or its resource's gate metric
+    averaged at least the gate's least over the window."""
+    if learned.gate is None:
+        return True
+    metric, least = learned.gate
+    return reported.get((key[0], key[1], metric), 0.0) >= (least() if callable(least) else least)
+
+
 def departs(level: float, norm: Norm, floor: float) -> bool:
     """Whether `level` is out of `norm`: above its usual hold by DEPART times
     and by at least `floor`, so a 0.1% → 0.3% share is never news."""
@@ -60,8 +69,10 @@ WHERE l.source_module || '/' || l.metric_name = ANY(:keys)
 
 
 async def levels_now(session, now: datetime) -> dict[tuple[str, str, str], float]:
-    """{(source, resource, metric): its average over the last WINDOW}."""
-    keys = [f"{s}/{m}" for s, m in LEARNED]
+    """{(source, resource, metric): its average over the last WINDOW}, for
+    every learned metric and every metric one is gated on."""
+    keys = sorted({f"{s}/{m}" for s, m in LEARNED}
+                  | {f"{k.source_module}/{k.gate[0]}" for k in LEARNED.values() if k.gate})
     if not keys:
         return {}
     rows = await session.execute(_LEVELS, {"keys": keys, "since": now - WINDOW, "now": now})
@@ -72,7 +83,9 @@ async def detect(session, now: datetime | None = None) -> dict:
     """One pass: open, extend and close occurrences. Runs inside the caller's
     transaction. Returns {"opened", "closed"} counts."""
     now = now or datetime.now(timezone.utc)
-    levels = await levels_now(session, now)
+    reported = await levels_now(session, now)
+    levels = {key: level for key, level in reported.items()
+              if (key[0], key[2]) in LEARNED and _gate_open(LEARNED[(key[0], key[2])], key, reported)}
     norms = await norms_at(session, list(levels), now)
     open_now = {
         (a.source_module, a.resource_name, a.metric_name): (span, a)
@@ -97,8 +110,8 @@ async def detect(session, now: datetime | None = None) -> dict:
         elif out:
             await _occur(session, key, level, norm, now)
             opened += 1
-    # An open occurrence whose key stopped reporting, or lost its normal,
-    # can't be judged any more: it ends now.
+    # An open occurrence whose key stopped reporting, lost its normal or fell
+    # below its gate can't be judged any more: it ends now.
     for span, anomaly in open_now.values():
         _close(anomaly, span, now)
         closed += 1

steward/core/norms.py

@@ -31,10 +31,11 @@ absorbed on its own, and a lapsed anomaly is never learned.
 """
 from __future__ import annotations
 
+from collections.abc import Callable
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 
-from sqlalchemy import ARRAY, String, bindparam, delete, select, text, tuple_
+from sqlalchemy import ARRAY, DateTime, String, bindparam, delete, select, text, tuple_
 
 from steward.models.norms import MetricNorm
 
@@ -50,19 +51,31 @@ USUAL = 0.9             # "usual" is this percentile across past days: one odd d
 class Learned:
     """A metric that learns a normal. `floor` is the smallest departure,
     in the metric's own unit, worth calling an anomaly (#5740), so that
-    0.1% → 0.3% of a share or 2 ms → 4 ms of latency is never news."""
+    0.1% → 0.3% of a share or 2 ms → 4 ms of latency is never news.
+
+    `gate` is (metric, least): the resource is judged only while that metric
+    of its own averages at least `least` over the window, as a router's
+    error share means nothing over a handful of requests (#5741). `least`
+    may be a function, read at each pass, for a minimum that is a setting. `since`:
+    history before it is not learned from, for a metric whose meaning
+    changed then."""
     source_module: str
     metric_name: str
     floor: float
+    gate: tuple[str, float | Callable[[], float]] | None = None
+    since: datetime | None = None
 
 
 LEARNED: dict[tuple[str, str], Learned] = {}
 
 
-def learn_normal(source_module: str, metric_name: str, floor: float) -> None:
+def learn_normal(source_module: str, metric_name: str, floor: float, *,
+                 gate: tuple[str, float | Callable[[], float]] | None = None,
+                 since: datetime | None = None) -> None:
     """Have Steward learn this metric's normal for every resource reporting
     it. A plugin calls this from setup(), as it calls prune_by_age."""
-    LEARNED[(source_module, metric_name)] = Learned(source_module, metric_name, float(floor))
+    LEARNED[(source_module, metric_name)] = Learned(source_module, metric_name, float(floor),
+                                                    gate, since)
 
 
 @dataclass(frozen=True)
@@ -89,17 +102,21 @@ def slot_of(at: datetime, kind: str) -> int:
 # each slot's usual hold and peak across past days, every hour counted in its
 # own slot and its neighbours'.
 _REBUILD = text("""
-WITH all_hours AS (
-    SELECT source_module AS s, resource_name AS r, metric_name AS m, bucket AS b,
-           COALESCE(value_p95, value_avg) AS hold, value_max AS peak
-    FROM plugin_metrics_hourly
-    WHERE bucket >= :week_since AND source_module || '/' || metric_name = ANY(:keys)
+WITH learned AS (
+    SELECT key, since FROM unnest(CAST(:keys AS text[]), CAST(:starts AS timestamptz[])) AS k(key, since)
+),
+all_hours AS (
+    SELECT h.source_module AS s, h.resource_name AS r, h.metric_name AS m, h.bucket AS b,
+           COALESCE(h.value_p95, h.value_avg) AS hold, h.value_max AS peak
+    FROM plugin_metrics_hourly h
+    JOIN learned k ON k.key = h.source_module || '/' || h.metric_name
+    WHERE h.bucket >= k.since
     UNION ALL
-    SELECT source_module, resource_name, metric_name, date_trunc('hour', recorded_at),
-           percentile_cont(0.95) WITHIN GROUP (ORDER BY value), max(value)
-    FROM plugin_metrics
-    WHERE recorded_at >= :week_since AND recorded_at < :this_hour
-      AND source_module || '/' || metric_name = ANY(:keys)
+    SELECT p.source_module, p.resource_name, p.metric_name, date_trunc('hour', p.recorded_at),
+           percentile_cont(0.95) WITHIN GROUP (ORDER BY p.value), max(p.value)
+    FROM plugin_metrics p
+    JOIN learned k ON k.key = p.source_module || '/' || p.metric_name
+    WHERE p.recorded_at >= k.since AND p.recorded_at < :this_hour
     GROUP BY 1, 2, 3, 4
 ),
 hours AS (
@@ -139,7 +156,8 @@ SELECT s, r, m, (slot + n.off + slots) % slots, kind,
        days, :now
 FROM slotted CROSS JOIN generate_series(-CAST(:neighbours AS int), CAST(:neighbours AS int)) AS n(off)
 GROUP BY s, r, m, (slot + n.off + slots) % slots, kind, days
-""").bindparams(bindparam("keys", type_=ARRAY(String)))
+""").bindparams(bindparam("keys", type_=ARRAY(String)),
+               bindparam("starts", type_=ARRAY(DateTime(timezone=True))))
 
 
 async def rebuild_norms(session, now: datetime | None = None) -> int:
@@ -147,12 +165,13 @@ async def rebuild_norms(session, now: datetime | None = None) -> int:
     inside the caller's transaction. Returns how many slot rows it wrote."""
     now = now or datetime.now(timezone.utc)
     await session.execute(delete(MetricNorm))
-    keys = [f"{s}/{m}" for s, m in LEARNED]
-    if not keys:
+    if not LEARNED:
         return 0
+    week_since = now - WEEK_WINDOW
     result = await session.execute(_REBUILD, {
-        "keys": keys,
-        "week_since": now - WEEK_WINDOW,
+        "keys": [f"{s}/{m}" for s, m in LEARNED],
+        # Each key learns from WEEK_WINDOW back, or from its own `since`.
+        "starts": [max(week_since, k.since) if k.since else week_since for k in LEARNED.values()],
         "day_since": now - DAY_WINDOW,
         "this_hour": now.replace(minute=0, second=0, microsecond=0),
         "week_ready": now - timedelta(days=WEEK_MIN_DAYS),

steward/core/settings.py

@@ -97,6 +97,9 @@ DEFAULTS: dict[str, Any] = {
     # than this share of 4xx is worth a look (#5380).
     "traefik.threshold.error_5xx_pct": 0.1,
     "traefik.threshold.error_4xx_pct": 1.0,
+    # A router's 4xx share over fewer requests than this in 15 minutes isn't
+    # graded, on the page or for anomalies: one 404 of three is 33% (#5741).
+    "traefik.threshold.min_requests": 30,
     # Degraded/critical cutoffs for metric coloring (warn=amber, crit=red).
     # cpu/mem/disk/load are percentages (load is load-per-core %); temp in °C.
     # uptime is "higher is better" so its warn/crit are floors. Ping latency
@@ -534,6 +537,9 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict:
                       "crit": g("traefik.threshold.error_5xx_pct", 0.1), "dir": "high"},
         "error_4xx": {"warn": g("traefik.threshold.error_4xx_pct", 1.0),
                       "crit": g("traefik.threshold.error_4xx_pct", 1.0), "dir": "high"},
+        # Fewer requests than this in 15 minutes and a router's 4xx share isn't graded.
+        "router_requests": {"warn": g("traefik.threshold.min_requests", 30),
+                            "crit": g("traefik.threshold.min_requests", 30), "dir": "low"},
     }
 
 

steward/settings/routes.py

@@ -144,6 +144,7 @@ _THRESHOLD_FIELDS = [
     ("crash_loop_deaths", "thresholds.crash_loop_deaths", False),
     ("error_5xx_pct", "traefik.threshold.error_5xx_pct", True),
     ("error_4xx_pct", "traefik.threshold.error_4xx_pct", True),
+    ("min_requests", "traefik.threshold.min_requests", False),
 ]
 
 # Docker time-series retention windows (days). Read fresh by the cleanup task,

steward/templates/settings/thresholds.html

@@ -40,7 +40,7 @@
   {{ pair("Disk usage",   "disk_warn", "thresholds.disk_warn", "disk_crit", "thresholds.disk_crit", "%", "Filesystem usage.") }}
   {{ pair("Load",         "load_warn", "thresholds.load_warn", "load_crit", "thresholds.load_crit", "% per core", "Load average normalized by core count (100% = one full core per core).") }}
   {{ pair("Temperature",  "temp_warn", "thresholds.temp_warn", "temp_crit", "thresholds.temp_crit", "°C", "Hottest reported sensor per host.") }}
-  {{ pair("Response time", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "How long a monitor's check takes to answer. Also grades Traefik routers' p95 and the UniFi WAN round trip.") }}
+  {{ pair("Response time", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "How long a monitor's check takes to answer. Also grades Traefik routers' p95 (until a router has its own normal) and the UniFi WAN round trip.") }}
   {{ pair("Uptime / SLA", "uptime_warn", "thresholds.uptime_warn", "uptime_crit", "thresholds.uptime_crit", "%", "Lower is worse — amber below warn, red below crit.", step="0.1") }}
   {{ pair("TLS certificate expiry", "tls_warn", "thresholds.tls_warn_days", "tls_crit", "thresholds.tls_crit_days", "days left", "HTTPS monitors whose certificate expires within these many days are flagged. Lower is worse.") }}
   {{ pair("Clock drift", "clock_warn", "thresholds.clock_warn_s", "clock_crit", "thresholds.clock_crit_s", "seconds", "How far a host's clock may be from Steward's, either way, before its page and the Hosts list call it out.") }}
@@ -69,13 +69,21 @@
     <label for="error_5xx_pct">Server errors (5xx) <span style="color:var(--text-muted);font-size:0.8rem;">(% of responses)</span></label>
     <input type="number" id="error_5xx_pct" name="error_5xx_pct" min="0" max="100" step="0.01"
            value="{{ settings['traefik.threshold.error_5xx_pct'] }}" style="max-width:110px;margin-top:0.25rem;">
-    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">Above this share, a router is failing (red).</div>
+    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">Above this share, a router is failing (red), whatever its history.</div>
   </div>
   <div class="form-group" style="margin-bottom:0.25rem;">
     <label for="error_4xx_pct">Client errors (4xx) <span style="color:var(--text-muted);font-size:0.8rem;">(% of responses)</span></label>
     <input type="number" id="error_4xx_pct" name="error_4xx_pct" min="0" max="100" step="0.1"
            value="{{ settings['traefik.threshold.error_4xx_pct'] }}" style="max-width:110px;margin-top:0.25rem;">
-    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">Above this share, a router is worth a look (amber): missing routes, failed sign-ins.</div>
+    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">Above this share, a router is worth a look (amber): missing routes, failed sign-ins.
+      Once a router has 5 days of history it is judged against its own normal for the hour instead, as is its p95,
+      so one that always answers some 4xx stops being listed.</div>
+  </div>
+  <div class="form-group" style="margin:1.1rem 0 0.25rem;">
+    <label for="min_requests">Fewest requests to judge 4xx <span style="color:var(--text-muted);font-size:0.8rem;">(in 15 minutes)</span></label>
+    <input type="number" id="min_requests" name="min_requests" min="0" step="1"
+           value="{{ settings['traefik.threshold.min_requests'] }}" style="max-width:110px;margin-top:0.25rem;">
+    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">Below this, a router's 4xx share isn't judged, on its page or as an anomaly: one 404 of three requests is 33%. 5xx is always judged.</div>
   </div>
 </div>
 {% endif %}