feat(norms): hosts, containers and monitor response times are graded against their own normal (#5742)

Host CPU, memory and load per core, container CPU and memory, and every
monitor's response time learn a normal per hour (milestone 513, System 182).
A reading past its fixed cutoff but within its normal reads healthy and says
what is usual at this hour; one well out of it keeps its grade and is a
metric anomaly. Disk, a monitor down and an agent offline keep their cutoffs.

- Each normal's floor, the smallest departure that counts, is a setting:
  Settings -> Thresholds -> Normals (norms.floor.*). Defaults: host CPU 10
  points, host memory 5, load per core 10, container CPU 10 points of one
  core, container memory 5, monitor response 100 ms; Traefik's 2-point 4xx
  and 50 ms p95 floors move there from constants. learn_normal takes a
  floor function (floor_setting) read at each pass.
- learn_normal(parts=False): a host learns, its cores don't.
- Ingest records load_pct (load_1m / cores x 100), the figure pages grade.
- Shared grading: anomalies.against / graded, worst_reading(normals=),
  hosts' host_grades, docker usage(thresholds=, norms=), monitors'
  latency_grade. Applied to host vitals, the Hosts widget, MCP overview,
  Docker usage, and the monitor list, widget and page.
- The monitor and container pages list their anomalies.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/monitors/routes.py

@@ -6,12 +6,12 @@ Host (host_id) or watch a free-standing custom destination (host_id = NULL).
 """
 from __future__ import annotations
 import uuid
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 
 from quart import Blueprint, current_app, render_template, request, redirect, url_for
 
 from steward.auth.middleware import require_role
-from steward.core import renames
+from steward.core import anomalies as anom, renames
 from steward.core.audit import audit_request
 from steward.models.users import UserRole
 from steward.models.monitors import Monitor, MonitorType, MONITOR_TYPE_VALUES
@@ -125,6 +125,10 @@ async def detail(monitor_id: str):
         st = (await monitor_svc.monitor_states(db, [m]))[m.id]
         failures = await monitor_svc.recent_failures(db, m.id)
         history = await monitor_svc.response_history(db, m.id, datetime.now(timezone.utc))
+        norm = (await monitor_svc.latency_norms(db, [m])).get(m.id)
+        anomalies = await anom.views(db, await anom.list_anomalies(
+            db, source=m.type, resource=m.name,
+            since=datetime.now(timezone.utc) - timedelta(days=30), limit=20), links=False)
         host = await db.get(Host, m.host_id) if m.host_id else None
     thresholds = current_app.config.get("THRESHOLDS", {})
     latest = st["latest"]
@@ -136,7 +140,8 @@ async def detail(monitor_id: str):
         tls_days = (latest.tls_expires_at - datetime.now(timezone.utc)).total_seconds() / 86400.0
     return await render_template(
         "monitors/detail.html", monitor=m, host=host, latest=latest, failures=failures,
-        state=monitor_svc.row_state(m, latest, st["uptime"].get("24h"), thresholds),
+        state=monitor_svc.row_state(m, latest, st["uptime"].get("24h"), thresholds, norm),
+        latency=monitor_svc.latency_grade(latest, thresholds, norm), anomalies=anomalies,
         since=since, since_at_least=since_at_least, uptime=st["uptime"], tls_days=tls_days,
         heartbeat=[{"state": "up" if r.is_up else "down", "title": _heartbeat_title(m.type, r)}
                    for r in st["recent"]],

steward/monitors/service.py

@@ -13,7 +13,10 @@ from datetime import datetime, timedelta, timezone
 
 from sqlalchemy import func, select
 
+from steward.core import anomalies as anom
 from steward.core.formatting import status_class
+from steward.core.norms import Norm, floor_setting, learn_normal, norms_at
+from steward.core.settings import floor_of
 from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor
 from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor, MonitorResult, MonitorType
 
@@ -145,12 +148,41 @@ async def check_name_free(db, mtype: str, name: str, exclude_id: str | None = No
 STATE_ORDER = {"down": 0, "degraded": 1, "pending": 2, "up": 3, "paused": 4}
 
 
-def row_state(monitor: Monitor, latest, uptime_pct, thresholds: dict) -> str:
+def learn_response_times(app) -> None:
+    """Each monitor's response time learns its normal (#5742), by type (the
+    metric's source), with its floor from Settings → Thresholds. A monitor
+    down keeps its fixed meaning whatever its history (System 182)."""
+    for mtype in MONITOR_TYPE_VALUES:
+        learn_normal(mtype, "response_ms", floor_setting(app, "monitor_ms"))
+
+
+async def latency_norms(db, monitors: list[Monitor], now: datetime | None = None) -> dict[str, Norm]:
+    """{monitor_id: its response time's normal for this hour}; a monitor with
+    too little history yet is absent, and graded on the fixed cutoffs."""
+    got = await norms_at(db, [(m.type, m.name, "response_ms") for m in monitors],
+                         now or datetime.now(timezone.utc))
+    return {m.id: got[(m.type, m.name, "response_ms")] for m in monitors
+            if (m.type, m.name, "response_ms") in got}
+
+
+def latency_grade(latest, thresholds: dict, norm: Norm | None = None) -> tuple[str, str | None]:
+    """(status class, why) for the latest response time: against the latency
+    cutoffs, except within its own normal for the hour, which is "ok" and says
+    what is usual."""
+    if latest is None or latest.response_ms is None:
+        return "unknown", None
+    return anom.graded(latest.response_ms, "latency", thresholds, norm,
+                       floor_of(thresholds, "monitor_ms"), " ms")
+
+
+def row_state(monitor: Monitor, latest, uptime_pct, thresholds: dict,
+              norm: Norm | None = None) -> str:
     """paused, pending (never checked), down, degraded or up.
 
     Degraded is up but worth a look: uptime over the window at or below its
     warning cutoff, or the latest response time past its critical one
-    (Settings → Thresholds). A slow-but-tolerable reply is coloured, not sorted.
+    (Settings → Thresholds) and out of its normal (`norm`, #5742). A
+    slow-but-tolerable reply is coloured, not sorted.
     """
     if not monitor.enabled:
         return "paused"
@@ -159,7 +191,7 @@ def row_state(monitor: Monitor, latest, uptime_pct, thresholds: dict) -> str:
     if not latest.is_up:
         return "down"
     if (status_class(uptime_pct, "uptime", thresholds) in ("warn", "crit")
-            or status_class(latest.response_ms, "latency", thresholds) == "crit"):
+            or latency_grade(latest, thresholds, norm)[0] == "crit"):
         return "degraded"
     return "up"
 
@@ -197,11 +229,13 @@ async def monitor_rows(db, thresholds: dict):
     uptime = await _uptime_by_monitor(db, ids)
 
     now = datetime.now(timezone.utc)
+    norms = await latency_norms(db, monitors, now)
     data = []
     for m in monitors:
         rows = recent.get(m.id, [])
         latest = rows[-1] if rows else None
         windows = uptime.get(m.id, {})
+        latency, usual = latency_grade(latest, thresholds, norms.get(m.id))
         tls_days = None
         if latest and latest.tls_expires_at:
             tls_days = (latest.tls_expires_at - now).total_seconds() / 86400.0
@@ -209,7 +243,11 @@ async def monitor_rows(db, thresholds: dict):
         data.append({
             "monitor": m,
             "latest": latest,
-            "state": row_state(m, latest, windows.get("24h"), thresholds),
+            "state": row_state(m, latest, windows.get("24h"), thresholds, norms.get(m.id)),
+            # The latest response time's grade, and what is usual when that
+            # quieted it (#5742).
+            "latency": latency,
+            "latency_usual": usual,
             "heartbeat": [{"state": "up" if r.is_up else "down",
                            "title": _heartbeat_title(m.type, r)} for r in rows],
             "spark": sparkline_svg(times, width=56, height=16, stroke="currentColor") if len(times) > 1 else "",