feat(norms): hosts, containers and monitor response times are graded against their own normal (#5742)

Host CPU, memory and load per core, container CPU and memory, and every
monitor's response time learn a normal per hour (milestone 513, System 182).
A reading past its fixed cutoff but within its normal reads healthy and says
what is usual at this hour; one well out of it keeps its grade and is a
metric anomaly. Disk, a monitor down and an agent offline keep their cutoffs.

- Each normal's floor, the smallest departure that counts, is a setting:
  Settings -> Thresholds -> Normals (norms.floor.*). Defaults: host CPU 10
  points, host memory 5, load per core 10, container CPU 10 points of one
  core, container memory 5, monitor response 100 ms; Traefik's 2-point 4xx
  and 50 ms p95 floors move there from constants. learn_normal takes a
  floor function (floor_setting) read at each pass.
- learn_normal(parts=False): a host learns, its cores don't.
- Ingest records load_pct (load_1m / cores x 100), the figure pages grade.
- Shared grading: anomalies.against / graded, worst_reading(normals=),
  hosts' host_grades, docker usage(thresholds=, norms=), monitors'
  latency_grade. Applied to host vitals, the Hosts widget, MCP overview,
  Docker usage, and the monitor list, widget and page.
- The monitor and container pages list their anomalies.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/__init__.py

@@ -26,12 +26,12 @@ def setup(app: "Quart") -> None:
     # learned from the scrape-by-scrape figures only: before #5755 it was a
     # percentile of every response since Traefik started.
     from datetime import datetime, timezone
-    from steward.core.norms import learn_normal
-    from .routes import FLOOR_4XX, FLOOR_P95, STATE_WINDOW, min_requests
+    from steward.core.norms import floor_setting, learn_normal
+    from .routes import STATE_WINDOW, min_requests
     gate = ("request_rate",
             lambda: min_requests(app.config.get("THRESHOLDS", {})) / STATE_WINDOW.total_seconds())
-    learn_normal("traefik", "error_rate_4xx_pct", FLOOR_4XX, gate=gate)
-    learn_normal("traefik", "latency_p95_ms", FLOOR_P95, gate=gate,
+    learn_normal("traefik", "error_rate_4xx_pct", floor_setting(app, "router_4xx_pct"), gate=gate)
+    learn_normal("traefik", "latency_p95_ms", floor_setting(app, "router_p95_ms"), gate=gate,
                  since=datetime(2026, 10, 14, 18, 47, tzinfo=timezone.utc))
 
 

plugins/traefik/routes.py

@@ -14,6 +14,7 @@ from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds,
 from steward.core import anomalies as anom
 from steward.core.formatting import cert_left, status_class
 from steward.core.norms import norms_at
+from steward.core.settings import floor_of
 from steward.models.metrics import PluginMetricLatest
 from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
 from .scheduler import router_domains
@@ -56,12 +57,11 @@ def error_cutoffs(thresholds: dict) -> tuple[float, float]:
 
 # Each router's 4xx share and p95 are judged against its own normal once it
 # has one (#5741, System 182): an error-pages service answers 4xx all day and
-# that is its normal, not a problem. Departing by less than the floor is never
-# news. A share over fewer than min_requests() in the window is not graded: one
-# 404 of three requests is 33%. The 5xx share is a failure and keeps its
-# fixed cutoff whatever the history or traffic (the operator's ruling).
-FLOOR_4XX = 2.0           # percentage points
-FLOOR_P95 = 50.0          # ms
+# that is its normal, not a problem. Departing by less than the floor
+# (Settings → Thresholds, norms.floor.router_*) is never news. A share over
+# fewer than min_requests() in the window is not graded: one 404 of three
+# requests is 33%. The 5xx share is a failure and keeps its fixed cutoff
+# whatever the history or traffic (the operator's ruling).
 NORMED = ("error_rate_4xx_pct", "latency_p95_ms")
 
 
@@ -71,14 +71,6 @@ def min_requests(thresholds: dict) -> float:
     return float(thresholds.get("router_requests", {}).get("warn", 30))
 
 
-def _against(value: float, norm, floor: float, unit: str) -> tuple[bool, str]:
-    """(departs, what is usual) for a reading over its fixed cutoff."""
-    if norm is None:
-        return True, ""
-    usual = f"usual {norm.usual_hold:.1f}{unit}" if unit == "%" else f"usual {norm.usual_hold:.0f}{unit}"
-    return anom.departs(value, norm, floor), f"{usual} at this hour"
-
-
 def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = None) -> tuple[str, str | None]:
     """(grade, why) for a router's reading (recent_window's, or one scrape's):
     crit while it serves 5xx past its cutoff, warn on a p95 past the latency
@@ -93,14 +85,16 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None =
     usual = []
     p95 = reading.latency_p95_ms
     if p95 is not None and status_class(p95, "latency", thresholds) == "crit":
-        out, said = _against(p95, norms.get("latency_p95_ms"), FLOOR_P95, " ms")
+        out, said = anom.against(p95, norms.get("latency_p95_ms"),
+                                  floor_of(thresholds, "router_p95_ms"), " ms")
         if out:
             return "warn", f"p95 latency is {p95:.0f} ms{over}" + (f", {said}" if said else "")
         usual.append(f"p95 {p95:.0f} ms is {said}")
     requests = getattr(reading, "requests", None)
     e4 = reading.error_rate_4xx_pct or 0
     if e4 > error_4xx and (requests is None or requests >= min_requests(thresholds)):
-        out, said = _against(e4, norms.get("error_rate_4xx_pct"), FLOOR_4XX, "%")
+        out, said = anom.against(e4, norms.get("error_rate_4xx_pct"),
+                                  floor_of(thresholds, "router_4xx_pct"), "%")
         if out:
             return "warn", f"{e4:.1f}% of responses are 4xx{over}" + (f", {said}" if said else "")
         usual.append(f"4xx {e4:.1f}% is {said}")