feat(norms): hosts, containers and monitor response times are graded against their own normal (#5742)

Host CPU, memory and load per core, container CPU and memory, and every
monitor's response time learn a normal per hour (milestone 513, System 182).
A reading past its fixed cutoff but within its normal reads healthy and says
what is usual at this hour; one well out of it keeps its grade and is a
metric anomaly. Disk, a monitor down and an agent offline keep their cutoffs.

- Each normal's floor, the smallest departure that counts, is a setting:
  Settings -> Thresholds -> Normals (norms.floor.*). Defaults: host CPU 10
  points, host memory 5, load per core 10, container CPU 10 points of one
  core, container memory 5, monitor response 100 ms; Traefik's 2-point 4xx
  and 50 ms p95 floors move there from constants. learn_normal takes a
  floor function (floor_setting) read at each pass.
- learn_normal(parts=False): a host learns, its cores don't.
- Ingest records load_pct (load_1m / cores x 100), the figure pages grade.
- Shared grading: anomalies.against / graded, worst_reading(normals=),
  hosts' host_grades, docker usage(thresholds=, norms=), monitors'
  latency_grade. Applied to host vitals, the Hosts widget, MCP overview,
  Docker usage, and the monitor list, widget and page.
- The monitor and container pages list their anomalies.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/app.py

@@ -129,6 +129,10 @@ def create_app(
     app.jinja_env.globals["threshold_style"] = (
         lambda value, kind: _ts_for(value, kind, app.config.get("THRESHOLDS", {}))
     )
+    # `grade_style(grade)` — the same styling for a grade worked out already
+    # (one graded against its own normal, #5742).
+    from .core.settings import grade_style as _grade_style
+    app.jinja_env.globals["grade_style"] = _grade_style
 
     # One way to write a size, a rate, a span and a moment, and to grade a
     # reading ('ok' / 'warn' / 'crit' / 'unknown') against the same cutoffs.
@@ -445,6 +449,11 @@ def _register_core_tasks(app: Quart) -> None:
         from .backup.job import tick
         await tick(app)
 
+    # Monitor response times learn their normal (#5742); plugins register
+    # their own metrics from setup().
+    from .monitors.service import learn_response_times
+    learn_response_times(app)
+
     async def run_metric_norms():
         from .core.norms import run_norms
         await run_norms(app)

steward/core/anomalies.py

@@ -51,6 +51,33 @@ def departs(level: float, norm: Norm, floor: float) -> bool:
     return level > norm.usual_hold * DEPART and level - norm.usual_hold >= floor
 
 
+def against(value: float, norm: Norm | None, floor: float, unit: str) -> tuple[bool, str]:
+    """(departs, what is usual) for a page's reading past its fixed cutoff
+    (#5741, #5742). Without a normal it departs, and the cutoff stands; within
+    one it is usual for that resource, and the page says so. `unit` is "%"
+    (one decimal) or " ms" (whole)."""
+    if norm is None:
+        return True, ""
+    usual = f"{norm.usual_hold:.1f}{unit}" if unit == "%" else f"{norm.usual_hold:.0f}{unit}"
+    return departs(value, norm, floor), f"usual {usual} at this hour"
+
+
+def graded(value: float | None, kind: str, thresholds: dict, norm: Norm | None,
+           floor: float, unit: str) -> tuple[str, str | None]:
+    """(status class, why) for a behaviour reading on a page (#5742): graded
+    against Settings → Thresholds, except that one past its cutoff but within
+    its own normal for the hour is "ok", and `why` says what is usual. Only
+    for behaviour (CPU, memory, load, latency); a failure keeps its cutoff
+    whatever its history (System 182)."""
+    from steward.core.formatting import status_class
+    cls = status_class(value, kind, thresholds)
+    if cls in ("warn", "crit") and norm is not None:
+        out, said = against(value, norm, floor, unit)
+        if not out:
+            return "ok", said
+    return cls, None
+
+
 # Each learned key reporting in the window, with the window's average. Keys
 # come from plugin_metric_latest (one row each), and each average is a range
 # read of the series index, so the pass never scans the raw table.
@@ -64,18 +91,22 @@ CROSS JOIN LATERAL (
       AND p.recorded_at <= :now
 ) v
 WHERE l.source_module || '/' || l.metric_name = ANY(:keys)
+  AND (strpos(l.resource_name, ':') = 0 OR NOT l.source_module || '/' || l.metric_name = ANY(:whole))
   AND l.recorded_at >= :since AND v.level IS NOT NULL
-""").bindparams(bindparam("keys", type_=ARRAY(String)))
+""").bindparams(bindparam("keys", type_=ARRAY(String)), bindparam("whole", type_=ARRAY(String)))
 
 
 async def levels_now(session, now: datetime) -> dict[tuple[str, str, str], float]:
     """{(source, resource, metric): its average over the last WINDOW}, for
-    every learned metric and every metric one is gated on."""
+    every learned metric and every metric one is gated on. A metric learned
+    without its parts reads its whole resources only."""
     keys = sorted({f"{s}/{m}" for s, m in LEARNED}
                   | {f"{k.source_module}/{k.gate[0]}" for k in LEARNED.values() if k.gate})
     if not keys:
         return {}
-    rows = await session.execute(_LEVELS, {"keys": keys, "since": now - WINDOW, "now": now})
+    whole = sorted(f"{k.source_module}/{k.metric_name}" for k in LEARNED.values() if not k.parts)
+    rows = await session.execute(_LEVELS, {"keys": keys, "whole": whole,
+                                           "since": now - WINDOW, "now": now})
     return {(s, r, m): float(level) for s, r, m, level in rows}
 
 
@@ -98,7 +129,7 @@ async def detect(session, now: datetime | None = None) -> dict:
     for key, level in levels.items():
         norm = norms.get(key)
         learned = LEARNED.get((key[0], key[2]))
-        out = norm is not None and learned is not None and departs(level, norm, learned.floor)
+        out = norm is not None and learned is not None and departs(level, norm, learned.least_departure)
         current = open_now.pop(key, None)
         if current is not None:
             span, anomaly = current

steward/core/formatting.py

@@ -233,16 +233,23 @@ def required(value: Any, what: str) -> Any:
 _SEVERITY = {"crit": 2, "warn": 1}
 
 
-def worst_reading(readings, thresholds: dict):
+def worst_reading(readings, thresholds: dict, normals: dict | None = None):
     """The reading furthest past its cutoff, as (label, value, grade), or None
     when every reading is within its cutoffs or missing.
 
     `readings` is (label, value, kind) triples, graded like status_class; the
     first of equally bad readings wins, so list them in the order a reader
-    should hear about them."""
+    should hear about them. `normals` ({kind: (Norm, floor)}) are the
+    resource's own normals for this hour: a reading past its cutoff but
+    within its normal is not past it (#5742)."""
+    from steward.core.anomalies import graded
+    normals = normals or {}
     worst, rank = None, 0
     for label, value, kind in readings:
-        grade = status_class(value, kind, thresholds)
+        if kind in normals:
+            grade = graded(value, kind, thresholds, *normals[kind], "%")[0]
+        else:
+            grade = status_class(value, kind, thresholds)
         if _SEVERITY.get(grade, 0) > rank:
             worst, rank = (label, value, grade), _SEVERITY[grade]
     return worst
@@ -327,6 +334,7 @@ METRICS: dict[str, tuple[str, str]] = {
     "load_1m": ("Load (1 min)", "runnable processes, averaged over a minute"),
     "load_5m": ("Load (5 min)", "runnable processes, averaged over 5 minutes"),
     "load_15m": ("Load (15 min)", "runnable processes, averaged over 15 minutes"),
+    "load_pct": ("Load per core", "1-minute load as a % of the host's cores; 100% keeps every core busy"),
     "uptime_secs": ("Uptime", "time since the host booted"),
     "seconds_since_seen": ("Time since the agent reported", "over 3 minutes means the agent is offline"),
     "clock_off_secs": ("Clock off by", "how far the host's clock is from Steward's, either way"),

steward/core/norms.py

@@ -35,7 +35,7 @@ from collections.abc import Callable
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 
-from sqlalchemy import ARRAY, DateTime, String, bindparam, delete, select, text, tuple_
+from sqlalchemy import ARRAY, Boolean, DateTime, String, bindparam, delete, select, text, tuple_
 
 from steward.models.norms import MetricNorm
 
@@ -51,31 +51,45 @@ USUAL = 0.9             # "usual" is this percentile across past days: one odd d
 class Learned:
     """A metric that learns a normal. `floor` is the smallest departure,
     in the metric's own unit, worth calling an anomaly (#5740), so that
-    0.1% → 0.3% of a share or 2 ms → 4 ms of latency is never news.
+    0.1% → 0.3% of a share or 2 ms → 4 ms of latency is never news. It may
+    be a function, read at each pass, for a floor that is a setting (#5742).
 
     `gate` is (metric, least): the resource is judged only while that metric
     of its own averages at least `least` over the window, as a router's
     error share means nothing over a handful of requests (#5741). `least`
     may be a function, read at each pass, for a minimum that is a setting. `since`:
     history before it is not learned from, for a metric whose meaning
-    changed then."""
+    changed then. `parts=False` learns only whole resources, not their parts
+    ("<host>:core3"): a host's CPU has a normal, each of its cores doesn't."""
     source_module: str
     metric_name: str
-    floor: float
+    floor: float | Callable[[], float]
     gate: tuple[str, float | Callable[[], float]] | None = None
     since: datetime | None = None
+    parts: bool = True
+
+    @property
+    def least_departure(self) -> float:
+        return float(self.floor() if callable(self.floor) else self.floor)
 
 
 LEARNED: dict[tuple[str, str], Learned] = {}
 
 
-def learn_normal(source_module: str, metric_name: str, floor: float, *,
+def learn_normal(source_module: str, metric_name: str, floor: float | Callable[[], float], *,
                  gate: tuple[str, float | Callable[[], float]] | None = None,
-                 since: datetime | None = None) -> None:
+                 since: datetime | None = None, parts: bool = True) -> None:
     """Have Steward learn this metric's normal for every resource reporting
     it. A plugin calls this from setup(), as it calls prune_by_age."""
-    LEARNED[(source_module, metric_name)] = Learned(source_module, metric_name, float(floor),
-                                                    gate, since)
+    LEARNED[(source_module, metric_name)] = Learned(
+        source_module, metric_name, floor if callable(floor) else float(floor), gate, since, parts)
+
+
+def floor_setting(app, name: str) -> Callable[[], float]:
+    """A floor read from Settings → Thresholds (norms.floor.<name>) at each
+    pass, so a saved change applies at once."""
+    from steward.core.settings import floor_of
+    return lambda: floor_of(app.config.get("THRESHOLDS", {}), name)
 
 
 @dataclass(frozen=True)
@@ -103,20 +117,23 @@ def slot_of(at: datetime, kind: str) -> int:
 # own slot and its neighbours'.
 _REBUILD = text("""
 WITH learned AS (
-    SELECT key, since FROM unnest(CAST(:keys AS text[]), CAST(:starts AS timestamptz[])) AS k(key, since)
+    SELECT key, since, parts
+    FROM unnest(CAST(:keys AS text[]), CAST(:starts AS timestamptz[]), CAST(:parts AS boolean[]))
+         AS k(key, since, parts)
 ),
 all_hours AS (
     SELECT h.source_module AS s, h.resource_name AS r, h.metric_name AS m, h.bucket AS b,
            COALESCE(h.value_p95, h.value_avg) AS hold, h.value_max AS peak
     FROM plugin_metrics_hourly h
     JOIN learned k ON k.key = h.source_module || '/' || h.metric_name
-    WHERE h.bucket >= k.since
+    WHERE h.bucket >= k.since AND (k.parts OR strpos(h.resource_name, ':') = 0)
     UNION ALL
     SELECT p.source_module, p.resource_name, p.metric_name, date_trunc('hour', p.recorded_at),
            percentile_cont(0.95) WITHIN GROUP (ORDER BY p.value), max(p.value)
     FROM plugin_metrics p
     JOIN learned k ON k.key = p.source_module || '/' || p.metric_name
     WHERE p.recorded_at >= k.since AND p.recorded_at < :this_hour
+      AND (k.parts OR strpos(p.resource_name, ':') = 0)
     GROUP BY 1, 2, 3, 4
 ),
 hours AS (
@@ -157,7 +174,8 @@ SELECT s, r, m, (slot + n.off + slots) % slots, kind,
 FROM slotted CROSS JOIN generate_series(-CAST(:neighbours AS int), CAST(:neighbours AS int)) AS n(off)
 GROUP BY s, r, m, (slot + n.off + slots) % slots, kind, days
 """).bindparams(bindparam("keys", type_=ARRAY(String)),
-               bindparam("starts", type_=ARRAY(DateTime(timezone=True))))
+               bindparam("starts", type_=ARRAY(DateTime(timezone=True))),
+               bindparam("parts", type_=ARRAY(Boolean)))
 
 
 async def rebuild_norms(session, now: datetime | None = None) -> int:
@@ -172,6 +190,7 @@ async def rebuild_norms(session, now: datetime | None = None) -> int:
         "keys": [f"{s}/{m}" for s, m in LEARNED],
         # Each key learns from WEEK_WINDOW back, or from its own `since`.
         "starts": [max(week_since, k.since) if k.since else week_since for k in LEARNED.values()],
+        "parts": [k.parts for k in LEARNED.values()],
         "day_since": now - DAY_WINDOW,
         "this_hour": now.replace(minute=0, second=0, microsecond=0),
         "week_ready": now - timedelta(days=WEEK_MIN_DAYS),

steward/core/settings.py

@@ -100,6 +100,19 @@ DEFAULTS: dict[str, Any] = {
     # A router's 4xx share over fewer requests than this in 15 minutes isn't
     # graded, on the page or for anomalies: one 404 of three is 33% (#5741).
     "traefik.threshold.min_requests": 30,
+    # The smallest departure from a metric's normal worth calling an anomaly
+    # (#5742), on top of being a quarter over its usual hold: a share's floor
+    # is in percentage points, a response time's in ms. 5 points is the
+    # baseline; bursty figures (CPU, load) get 10 so an ordinary build or
+    # backup minute isn't news.
+    "norms.floor.host_cpu_pct": 10,
+    "norms.floor.host_mem_pct": 5,
+    "norms.floor.host_load_pct": 10,
+    "norms.floor.container_cpu_pct": 10,
+    "norms.floor.container_mem_pct": 5,
+    "norms.floor.monitor_ms": 100,
+    "norms.floor.router_4xx_pct": 2,
+    "norms.floor.router_p95_ms": 50,
     # Degraded/critical cutoffs for metric coloring (warn=amber, crit=red).
     # cpu/mem/disk/load are percentages (load is load-per-core %); temp in °C.
     # uptime is "higher is better" so its warn/crit are floors. Ping latency
@@ -540,9 +553,23 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict:
         # Fewer requests than this in 15 minutes and a router's 4xx share isn't graded.
         "router_requests": {"warn": g("traefik.threshold.min_requests", 30),
                             "crit": g("traefik.threshold.min_requests", 30), "dir": "low"},
+        # Not a cutoff: each normal's floor (norms.floor.*), read by floor_of().
+        "floors": {name: float(g(f"norms.floor.{name}", DEFAULTS[f"norms.floor.{name}"]))
+                   for name in NORM_FLOORS},
     }
 
 
+NORM_FLOORS = ("host_cpu_pct", "host_mem_pct", "host_load_pct", "container_cpu_pct",
+               "container_mem_pct", "monitor_ms", "router_4xx_pct", "router_p95_ms")
+
+
+def floor_of(thresholds: dict, name: str) -> float:
+    """A normal's floor (one of NORM_FLOORS) from to_thresholds_cfg's output:
+    the smallest departure from it worth calling an anomaly (#5742)."""
+    got = (thresholds.get("floors") or {}).get(name)
+    return float(got if got is not None else DEFAULTS[f"norms.floor.{name}"])
+
+
 _STATUS_STYLE = {"warn": "color:var(--yellow);font-weight:500;",
                  "crit": "color:var(--red);font-weight:500;"}
 
@@ -557,6 +584,12 @@ def threshold_style_for(value: Any, kind: str, thresholds: dict) -> str:
     return _STATUS_STYLE.get(status_class(value, kind, thresholds), "")
 
 
+def grade_style(grade: str | None) -> str:
+    """threshold_style_for's style for a grade already worked out ("warn" /
+    "crit"); empty for anything else."""
+    return _STATUS_STYLE.get(grade or "", "")
+
+
 def to_plugins_cfg(settings: dict[str, Any]) -> dict:
     """Assemble {plugin_name: {...config}} from all plugin.* keys."""
     result = {}

steward/hosts/service.py

@@ -9,6 +9,8 @@ from datetime import datetime, timedelta, timezone
 
 from sqlalchemy import and_, case, func, select
 
+from steward.core.norms import norms_at
+from steward.core.settings import floor_of
 from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor
 from steward.models.hosts import Host
 from steward.models.monitors import Monitor, MonitorResult
@@ -249,8 +251,9 @@ async def agent_overview_by_host(
     """Per-host agent glance (cpu/mem/disk/load + freshness) from plugin_metric_latest.
 
     {host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root, load_1m,
-                 cores, load_per_core, last_at, fresh, clock_off_secs}} for each host with any
-    agent samples. Cores are counted from the per-core CPU sub-resources
+                 cores, load_per_core, last_at, fresh, clock_off_secs, norms}} for each host
+    with any agent samples. `norms` is {kind: Norm}, the host's CPU, memory and
+    load normals for this hour (host_normals grades by them). Cores are counted from the per-core CPU sub-resources
     ("<host>:core<N>"), as on the host page; load_per_core is load_1m as a %
     of them, the figure Settings → Thresholds grades "load" by.
 
@@ -294,10 +297,50 @@ async def agent_overview_by_host(
         d["cores"] = cores.get(name)
         load = d.get("load_1m")
         d["load_per_core"] = round(load / d["cores"] * 100) if (load is not None and d["cores"]) else None
+        d["norms"] = {}
         out[name] = d
+    learned = await norms_at(db, [("host_agent", h, metric) for h in out
+                                  for metric, _f in HOST_NORMED.values()], now)
+    for (_s, host, metric), norm in learned.items():
+        out[host]["norms"][_KIND_OF[metric]] = norm
     return out
 
 
+# The host figures graded against their own normal (#5742): the threshold
+# kind, the metric that learns it and its floor's setting (norms.floor.*).
+# Disk keeps its cutoff: a nearly full disk is a failure (System 182).
+HOST_NORMED = {"cpu": ("cpu_pct", "host_cpu_pct"), "mem": ("mem_used_pct", "host_mem_pct"),
+               "load": ("load_pct", "host_load_pct")}
+_KIND_OF = {metric: kind for kind, (metric, _f) in HOST_NORMED.items()}
+
+
+def host_normals(norms: dict, thresholds: dict) -> dict:
+    """{kind: (Norm, floor)} for worst_reading and anomalies.graded, from an
+    agent glance's `norms`."""
+    return {kind: (norm, floor_of(thresholds, HOST_NORMED[kind][1]))
+            for kind, norm in (norms or {}).items()}
+
+
+async def host_norms(db, host_name: str, now: datetime | None = None) -> dict:
+    """{kind: Norm}: one host's CPU, memory and load normals for this hour."""
+    learned = await norms_at(db, [("host_agent", host_name, metric)
+                                  for metric, _f in HOST_NORMED.values()],
+                             now or datetime.now(timezone.utc))
+    return {_KIND_OF[metric]: norm for (_s, _h, metric), norm in learned.items()}
+
+
+def host_grades(values: dict, norms: dict, thresholds: dict) -> dict:
+    """{kind: (status class, why)} for a host's readings ({kind: value}):
+    against Settings → Thresholds, except CPU, memory and load within the
+    host's own normal, which are "ok" and say what is usual (#5742)."""
+    from steward.core.anomalies import graded
+    from steward.core.formatting import status_class
+    normals = host_normals(norms, thresholds)
+    return {kind: (graded(value, kind, thresholds, *normals[kind], "%") if kind in normals
+                   else (status_class(value, kind, thresholds), None))
+            for kind, value in values.items()}
+
+
 def clock_trouble(agent: dict[str, dict], limit: float) -> dict | None:
     """Reporting hosts whose clock is more than `limit` seconds from Steward's
     (Settings → Thresholds' clock drift warning, milestone 490),
@@ -326,19 +369,24 @@ def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict
     Each is {host, state, worst, agent}. State is "down" when a linked monitor
     is down, "stale" when the agent has stopped reporting, else the grade of
     the reading furthest past its cutoff ("crit" / "warn": CPU, memory, disk /
-    or load per core, against Settings → Thresholds), else "up"; "unknown" for
+    or load per core, against Settings → Thresholds; CPU, memory and load
+    within the host's own normal are not past it), else "up"; "unknown" for
     a host with neither monitors nor agent. `worst` is (label, value, grade)
-    or None, from formatting.worst_reading."""
+    or None, from formatting.worst_reading; `grades` is host_grades' {kind:
+    (grade, why)} for a fresh agent, else {}."""
     from steward.core.formatting import worst_reading
     rows = []
     for h in hosts:
         st, a = status.get(h.id), agent.get(h.name)
-        worst = None
+        worst, grades = None, {}
         if a and a.get("fresh"):
+            grades = host_grades({"cpu": a.get("cpu_pct"), "mem": a.get("mem_used_pct"),
+                                  "disk": a.get("disk_root"), "load": a.get("load_per_core")},
+                                 a.get("norms"), thresholds)
             worst = worst_reading((
                 ("CPU", a.get("cpu_pct"), "cpu"), ("Memory", a.get("mem_used_pct"), "mem"),
                 ("Disk /", a.get("disk_root"), "disk"), ("Load", a.get("load_per_core"), "load")),
-                thresholds)
+                thresholds, host_normals(a.get("norms"), thresholds))
         if st and st["state"] == "down":
             state = "down"
         elif a and not a.get("fresh"):
@@ -349,5 +397,5 @@ def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict
             state = "up"
         else:
             state = "unknown"
-        rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}})
+        rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}, "grades": grades})
     return sorted(rows, key=lambda r: (GLANCE_ORDER[r["state"]], r["host"].name.lower()))

steward/mcp/tools/overview.py

@@ -111,6 +111,9 @@ def agent_findings(hosts, agent: dict, thresholds: dict) -> list[dict]:
                 "host agent has stopped reporting; its metrics are out of date",
                 since=a.get("last_at"), ids=ids, next=f'get_host(host="{h.id}")'))
             continue  # stale values would only add noise
+        # CPU and memory within the host's own normal for the hour aren't
+        # findings (#5742); the fullest disk keeps its cutoff.
+        normals = host_svc.host_normals(a.get("norms"), thresholds)
         for metric, kind, label in _AGENT_CHECKS:
             value = a.get(metric)
             th = thresholds.get(kind) or {}
@@ -122,6 +125,8 @@ def agent_findings(hosts, agent: dict, thresholds: dict) -> list[dict]:
                 sev = "warning"
             else:
                 continue
+            if kind in normals and not anom.against(value, normals[kind][0], normals[kind][1], "%")[0]:
+                continue
             out.append(finding(
                 sev, f"host_{kind}_high", h.name,
                 f"{label} at {rnd(value, 1)}% (warn {th['warn']}%, crit {th.get('crit')}%)",

steward/monitors/routes.py

@@ -6,12 +6,12 @@ Host (host_id) or watch a free-standing custom destination (host_id = NULL).
 """
 from __future__ import annotations
 import uuid
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 
 from quart import Blueprint, current_app, render_template, request, redirect, url_for
 
 from steward.auth.middleware import require_role
-from steward.core import renames
+from steward.core import anomalies as anom, renames
 from steward.core.audit import audit_request
 from steward.models.users import UserRole
 from steward.models.monitors import Monitor, MonitorType, MONITOR_TYPE_VALUES
@@ -125,6 +125,10 @@ async def detail(monitor_id: str):
         st = (await monitor_svc.monitor_states(db, [m]))[m.id]
         failures = await monitor_svc.recent_failures(db, m.id)
         history = await monitor_svc.response_history(db, m.id, datetime.now(timezone.utc))
+        norm = (await monitor_svc.latency_norms(db, [m])).get(m.id)
+        anomalies = await anom.views(db, await anom.list_anomalies(
+            db, source=m.type, resource=m.name,
+            since=datetime.now(timezone.utc) - timedelta(days=30), limit=20), links=False)
         host = await db.get(Host, m.host_id) if m.host_id else None
     thresholds = current_app.config.get("THRESHOLDS", {})
     latest = st["latest"]
@@ -136,7 +140,8 @@ async def detail(monitor_id: str):
         tls_days = (latest.tls_expires_at - datetime.now(timezone.utc)).total_seconds() / 86400.0
     return await render_template(
         "monitors/detail.html", monitor=m, host=host, latest=latest, failures=failures,
-        state=monitor_svc.row_state(m, latest, st["uptime"].get("24h"), thresholds),
+        state=monitor_svc.row_state(m, latest, st["uptime"].get("24h"), thresholds, norm),
+        latency=monitor_svc.latency_grade(latest, thresholds, norm), anomalies=anomalies,
         since=since, since_at_least=since_at_least, uptime=st["uptime"], tls_days=tls_days,
         heartbeat=[{"state": "up" if r.is_up else "down", "title": _heartbeat_title(m.type, r)}
                    for r in st["recent"]],

steward/monitors/service.py

@@ -13,7 +13,10 @@ from datetime import datetime, timedelta, timezone
 
 from sqlalchemy import func, select
 
+from steward.core import anomalies as anom
 from steward.core.formatting import status_class
+from steward.core.norms import Norm, floor_setting, learn_normal, norms_at
+from steward.core.settings import floor_of
 from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor
 from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor, MonitorResult, MonitorType
 
@@ -145,12 +148,41 @@ async def check_name_free(db, mtype: str, name: str, exclude_id: str | None = No
 STATE_ORDER = {"down": 0, "degraded": 1, "pending": 2, "up": 3, "paused": 4}
 
 
-def row_state(monitor: Monitor, latest, uptime_pct, thresholds: dict) -> str:
+def learn_response_times(app) -> None:
+    """Each monitor's response time learns its normal (#5742), by type (the
+    metric's source), with its floor from Settings → Thresholds. A monitor
+    down keeps its fixed meaning whatever its history (System 182)."""
+    for mtype in MONITOR_TYPE_VALUES:
+        learn_normal(mtype, "response_ms", floor_setting(app, "monitor_ms"))
+
+
+async def latency_norms(db, monitors: list[Monitor], now: datetime | None = None) -> dict[str, Norm]:
+    """{monitor_id: its response time's normal for this hour}; a monitor with
+    too little history yet is absent, and graded on the fixed cutoffs."""
+    got = await norms_at(db, [(m.type, m.name, "response_ms") for m in monitors],
+                         now or datetime.now(timezone.utc))
+    return {m.id: got[(m.type, m.name, "response_ms")] for m in monitors
+            if (m.type, m.name, "response_ms") in got}
+
+
+def latency_grade(latest, thresholds: dict, norm: Norm | None = None) -> tuple[str, str | None]:
+    """(status class, why) for the latest response time: against the latency
+    cutoffs, except within its own normal for the hour, which is "ok" and says
+    what is usual."""
+    if latest is None or latest.response_ms is None:
+        return "unknown", None
+    return anom.graded(latest.response_ms, "latency", thresholds, norm,
+                       floor_of(thresholds, "monitor_ms"), " ms")
+
+
+def row_state(monitor: Monitor, latest, uptime_pct, thresholds: dict,
+              norm: Norm | None = None) -> str:
     """paused, pending (never checked), down, degraded or up.
 
     Degraded is up but worth a look: uptime over the window at or below its
     warning cutoff, or the latest response time past its critical one
-    (Settings → Thresholds). A slow-but-tolerable reply is coloured, not sorted.
+    (Settings → Thresholds) and out of its normal (`norm`, #5742). A
+    slow-but-tolerable reply is coloured, not sorted.
     """
     if not monitor.enabled:
         return "paused"
@@ -159,7 +191,7 @@ def row_state(monitor: Monitor, latest, uptime_pct, thresholds: dict) -> str:
     if not latest.is_up:
         return "down"
     if (status_class(uptime_pct, "uptime", thresholds) in ("warn", "crit")
-            or status_class(latest.response_ms, "latency", thresholds) == "crit"):
+            or latency_grade(latest, thresholds, norm)[0] == "crit"):
         return "degraded"
     return "up"
 
@@ -197,11 +229,13 @@ async def monitor_rows(db, thresholds: dict):
     uptime = await _uptime_by_monitor(db, ids)
 
     now = datetime.now(timezone.utc)
+    norms = await latency_norms(db, monitors, now)
     data = []
     for m in monitors:
         rows = recent.get(m.id, [])
         latest = rows[-1] if rows else None
         windows = uptime.get(m.id, {})
+        latency, usual = latency_grade(latest, thresholds, norms.get(m.id))
         tls_days = None
         if latest and latest.tls_expires_at:
             tls_days = (latest.tls_expires_at - now).total_seconds() / 86400.0
@@ -209,7 +243,11 @@ async def monitor_rows(db, thresholds: dict):
         data.append({
             "monitor": m,
             "latest": latest,
-            "state": row_state(m, latest, windows.get("24h"), thresholds),
+            "state": row_state(m, latest, windows.get("24h"), thresholds, norms.get(m.id)),
+            # The latest response time's grade, and what is usual when that
+            # quieted it (#5742).
+            "latency": latency,
+            "latency_usual": usual,
             "heartbeat": [{"state": "up" if r.is_up else "down",
                            "title": _heartbeat_title(m.type, r)} for r in rows],
             "spark": sparkline_svg(times, width=56, height=16, stroke="currentColor") if len(times) > 1 else "",

steward/settings/routes.py

@@ -15,7 +15,7 @@ from steward.core.settings import (
     get_stored_plugin_names, find_orphaned_plugins,
     to_smtp_cfg, to_webhook_cfg, to_ansible_cfg, to_plugins_cfg,
     to_oidc_cfg, to_ldap_cfg, to_thresholds_cfg, to_access_cfg, DEFAULTS, seal_source_tokens,
-    is_plugin_secret,
+    is_plugin_secret, NORM_FLOORS,
 )
 
 settings_bp = Blueprint("settings", __name__, url_prefix="/settings")
@@ -145,6 +145,8 @@ _THRESHOLD_FIELDS = [
     ("error_5xx_pct", "traefik.threshold.error_5xx_pct", True),
     ("error_4xx_pct", "traefik.threshold.error_4xx_pct", True),
     ("min_requests", "traefik.threshold.min_requests", False),
+    # Each normal's floor (#5742), named by its own setting key.
+    *[(f"norms.floor.{name}", f"norms.floor.{name}", True) for name in NORM_FLOORS],
 ]
 
 # Docker time-series retention windows (days). Read fresh by the cleanup task,
@@ -165,7 +167,8 @@ async def thresholds():
         settings = await get_all_settings(db)
     from steward.core.plugin_manager import _LOADED_PLUGINS
     return await render_template("settings/thresholds.html", settings=settings,
-                                 traefik_on="traefik" in _LOADED_PLUGINS)
+                                 traefik_on="traefik" in _LOADED_PLUGINS,
+                                 plugins_on=set(_LOADED_PLUGINS))
 
 
 @settings_bp.post("/thresholds/")

steward/templates/hosts/overview_widget.html

@@ -1,6 +1,8 @@
 {# hosts/overview_widget.html — the dashboard Hosts widget (#5251): a verdict,
    then one row per host, worst first, with its resources under one header.
-   Load is shown against the host's cores and graded per core. Reachability
+   Load is shown against the host's cores and graded per core. CPU, memory
+   and load within the host's own normal for the hour read healthy (#5742),
+   with what is usual on hover. Reachability
    and uptime belong to the Monitors widget, so they aren't repeated here. #}
 {% from "_macros.html" import status_dot %}
 {% if not rows %}
@@ -30,10 +32,12 @@
       {%- elif not a %}<span class="w-sub">no agent</span>{% endif %}
     </td>
     {% for v, kind in ((a.get('cpu_pct'), 'cpu'), (a.get('mem_used_pct'), 'mem'), (a.get('disk_root'), 'disk')) %}
-    <td style="{{ threshold_style(v, kind) if fresh and v is not none }}">{% if v is none %}—{% else %}{{ "%.0f"|format(v) }}%{% endif %}</td>
+    {% set g = (r.grades or {}).get(kind) or (none, none) %}
+    <td style="{{ grade_style(g[0]) }}"{% if g[1] %} title="{{ g[1]|capitalize }}"{% endif %}>{% if v is none %}—{% else %}{{ "%.0f"|format(v) }}%{% endif %}</td>
     {% endfor %}
-    <td style="{{ threshold_style(a.get('load_per_core'), 'load') if fresh and a.get('load_per_core') is not none }}"
-        title="{% if a.get('cores') %}Load over 1 minute on {{ a.cores }} cores: {{ a.load_per_core }}% of them busy or waiting{% endif %}">
+    {% set g = (r.grades or {}).get('load') or (none, none) %}
+    <td style="{{ grade_style(g[0]) }}"
+        title="{% if a.get('cores') %}Load over 1 minute on {{ a.cores }} cores: {{ a.load_per_core }}% of them busy or waiting{% endif %}{% if g[1] %}; {{ g[1] }}{% endif %}">
       {%- if a.get('load_1m') is none %}—{% elif a.get('cores') %}{{ "%.1f"|format(a.load_1m) }} / {{ a.cores }} cores{% else %}{{ "%.1f"|format(a.load_1m) }}{% endif -%}
     </td>
   </tr>

steward/templates/monitors/detail.html

@@ -1,7 +1,8 @@
 {% extends "base.html" %}
 {% from "_macros.html" import crumbs, page_header, stat_tile %}
 {# One monitor: its state and for how long, latency and uptime against
-   Settings → Thresholds, its response time over the last day, its last
+   Settings → Thresholds (latency against its own normal once it has one,
+   with its anomalies), its response time over the last day, its last
    checks, the failures behind them and how it is set up. #}
 {% block title %}{{ monitor.name }} — Monitors — Steward{% endblock %}
 {% block breadcrumb %}{{ crumbs([("Monitors", "/monitors/"), (monitor.name, "")]) }}{% endblock %}
@@ -23,7 +24,7 @@
   {% set since_hint = ((("for at least " if since_at_least else "for ") ~ (since|ago|striptags|replace(" ago", ""))) if since and state in ("up", "down", "degraded") else none) %}
   {{ stat_tile("State", status_info(state)[1], state=state, hint=since_hint) }}
   {% if latest and latest.response_ms is not none %}
-  {{ stat_tile("Latency", "%.0f ms"|format(latest.response_ms), state=status_class(latest.response_ms, "latency")) }}
+  {{ stat_tile("Latency", "%.0f ms"|format(latest.response_ms), state=latency[0], hint=latency[1]) }}
   {% endif %}
   {% for window in ("24h", "7d", "30d") %}
   {% set pct = uptime.get(window) %}
@@ -34,6 +35,8 @@
   {% endif %}
 </div>
 
+{% with back="/monitors/" ~ monitor.id, base_resource=monitor.name %}{% include "_anomaly_card.html" %}{% endwith %}
+
 {% set timed = history | selectattr("ms", "ne", none) | list %}
 {% if timed|length > 1 %}
 <section class="card" aria-labelledby="m-chart">

steward/templates/monitors/rows.html

@@ -51,7 +51,7 @@
     {% if d.state in ('down', 'paused', 'pending') %}{{ status_badge(d.state) }}
     {% elif latest.response_ms is not none %}
       {% if d.spark %}<span class="mon-spark" title="Response time over the last 30 checks">{{ d.spark|safe }}</span>{% endif %}
-      <span class="tone-{{ status_class(latest.response_ms, 'latency') }}">{{ "%.0f"|format(latest.response_ms) }} ms</span>
+      <span class="tone-{{ d.latency }}"{% if d.latency_usual %} title="{{ d.latency_usual|capitalize }}"{% endif %}>{{ "%.0f"|format(latest.response_ms) }} ms</span>
     {% elif m.type == 'dns' and latest.resolved_ip %}<span class="tone-ok">{{ latest.resolved_ip }}</span>
     {% else %}{{ status_badge('up') }}{% endif %}
   </div>

steward/templates/monitors/widget.html

@@ -26,10 +26,10 @@
       {{- status_dot(d.state) -}}
       {% if session.user_id %}<a href="/monitors/{{ m.id }}">{{ m.name }}</a>{% else %}{{ m.name }}{% endif %}
       {%- if d.state == 'down' %}<span class="w-sub tone-crit">went down {{ d.down_since|ago if d.down_since else '' }}{% if d.down_reason %} · {{ d.down_reason }}{% endif %}</span>
-      {%- elif d.state == 'degraded' %}<span class="w-sub tone-warn">{% if latest and status_class(latest.response_ms, 'latency') == 'crit' %}slow replies{% else %}checks failing now and then{% endif %}</span>
+      {%- elif d.state == 'degraded' %}<span class="w-sub tone-warn">{% if d.latency == 'crit' %}slow replies{% else %}checks failing now and then{% endif %}</span>
       {%- elif d.state == 'pending' %}<span class="w-sub">not checked yet</span>{% endif %}
     </td>
-    <td style="{{ threshold_style(latest.response_ms, 'latency') if latest and latest.is_up and latest.response_ms is not none }}">
+    <td class="{{ 'tone-' ~ d.latency if d.latency in ('warn', 'crit') }}"{% if d.latency_usual %} title="{{ d.latency_usual|capitalize }}"{% endif %}>
       {%- if latest and latest.is_up and latest.response_ms is not none %}{{ "%.0f"|format(latest.response_ms) }} ms{% else %}—{% endif -%}
     </td>
     {% for k in ("24h", "7d", "30d") %}{% set u = d.uptime.get(k) %}

steward/templates/settings/thresholds.html

@@ -35,12 +35,12 @@
     Higher is worse for everything except uptime, where the cutoffs are floors.
   </p>
 
-  {{ pair("CPU usage",    "cpu_warn",  "thresholds.cpu_warn",  "cpu_crit",  "thresholds.cpu_crit",  "%", "Average CPU utilization per host.") }}
-  {{ pair("Memory usage", "mem_warn",  "thresholds.mem_warn",  "mem_crit",  "thresholds.mem_crit",  "%", "Memory in use per host.") }}
+  {{ pair("CPU usage",    "cpu_warn",  "thresholds.cpu_warn",  "cpu_crit",  "thresholds.cpu_crit",  "%", "Average CPU utilization per host. A host or container with its own normal is judged against that instead (see Normals below).") }}
+  {{ pair("Memory usage", "mem_warn",  "thresholds.mem_warn",  "mem_crit",  "thresholds.mem_crit",  "%", "Memory in use per host, or a container's share of its limit. Judged against a normal once there is one.") }}
   {{ pair("Disk usage",   "disk_warn", "thresholds.disk_warn", "disk_crit", "thresholds.disk_crit", "%", "Filesystem usage.") }}
-  {{ pair("Load",         "load_warn", "thresholds.load_warn", "load_crit", "thresholds.load_crit", "% per core", "Load average normalized by core count (100% = one full core per core).") }}
+  {{ pair("Load",         "load_warn", "thresholds.load_warn", "load_crit", "thresholds.load_crit", "% per core", "Load average normalized by core count (100% = one full core per core). Judged against a host's normal once it has one.") }}
   {{ pair("Temperature",  "temp_warn", "thresholds.temp_warn", "temp_crit", "thresholds.temp_crit", "°C", "Hottest reported sensor per host.") }}
-  {{ pair("Response time", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "How long a monitor's check takes to answer. Also grades Traefik routers' p95 (until a router has its own normal) and the UniFi WAN round trip.") }}
+  {{ pair("Response time", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "How long a monitor's check takes to answer. Also grades Traefik routers' p95 and the UniFi WAN round trip. A monitor or router with its own normal is judged against that instead.") }}
   {{ pair("Uptime / SLA", "uptime_warn", "thresholds.uptime_warn", "uptime_crit", "thresholds.uptime_crit", "%", "Lower is worse — amber below warn, red below crit.", step="0.1") }}
   {{ pair("TLS certificate expiry", "tls_warn", "thresholds.tls_warn_days", "tls_crit", "thresholds.tls_crit_days", "days left", "HTTPS monitors whose certificate expires within these many days are flagged. Lower is worse.") }}
   {{ pair("Clock drift", "clock_warn", "thresholds.clock_warn_s", "clock_crit", "thresholds.clock_crit_s", "seconds", "How far a host's clock may be from Steward's, either way, before its page and the Hosts list call it out.") }}
@@ -99,6 +99,42 @@
 </div>
 {% endmacro %}
 
+{# Each normal's floor (#5742): the smallest departure from it worth calling an anomaly. #}
+{% macro floor(label, key, unit, hint, step="1") %}
+<div class="form-group" style="margin-bottom:1.1rem;">
+  <label for="{{ key }}">{{ label }} <span style="color:var(--text-muted);font-size:0.8rem;">({{ unit }})</span></label>
+  <input type="number" id="{{ key }}" name="{{ key }}" min="0" step="{{ step }}"
+         value="{{ settings[key] }}" style="max-width:110px;margin-top:0.25rem;display:block;">
+  {% if hint %}<div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">{{ hint }}</div>{% endif %}
+</div>
+{% endmacro %}
+
+<div class="card" id="normals" style="max-width:640px;margin-top:1.5rem;">
+  <h2 class="section-title" style="margin-bottom:0.5rem;">Normals</h2>
+  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
+    Once one has 5 days of history, these learn what is normal for each host, container, monitor or router
+    at each hour, and are judged against that instead of the cutoffs above: a reading over a cutoff but usual
+    for that hour shows healthy, and one well above its usual level is a warning and an anomaly to review.
+    Disk space, a monitor that is down, an agent that stopped reporting and a router's 5xx share are failures
+    and always keep their cutoffs. A reading counts as out of its normal only when it is a quarter above its
+    usual level <em>and</em> at least this much above it:
+  </p>
+  {% if "host_agent" in (plugins_on or ()) %}
+  {{ floor("Host CPU", "norms.floor.host_cpu_pct", "% points", "CPU is bursty: a build or a backup moves it this much without anything being wrong.") }}
+  {{ floor("Host memory", "norms.floor.host_mem_pct", "% points", "Memory moves slowly, so a smaller step already means something changed.") }}
+  {{ floor("Host load per core", "norms.floor.host_load_pct", "% points", "Bursty, like CPU.") }}
+  {% endif %}
+  {% if "docker" in (plugins_on or ()) %}
+  {{ floor("Container CPU", "norms.floor.container_cpu_pct", "% of one core", "10 is a tenth of a core; with many containers, smaller steps are everyday noise.") }}
+  {{ floor("Container memory", "norms.floor.container_mem_pct", "% points", "Of its limit, or of the host's memory when it has none, so a container without a limit only counts large growth.") }}
+  {% endif %}
+  {{ floor("Monitor response time", "norms.floor.monitor_ms", "ms", "One check per interval is noisy; a slower reply than this over its usual is worth a look.") }}
+  {% if traefik_on %}
+  {{ floor("Router 4xx share", "norms.floor.router_4xx_pct", "% points", "", step="0.1") }}
+  {{ floor("Router p95", "norms.floor.router_p95_ms", "ms", "") }}
+  {% endif %}
+</div>
+
 <div class="card" style="max-width:640px;margin-top:1.5rem;">
   <h2 class="section-title" style="margin-bottom:0.5rem;">Metric anomalies</h2>
   <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">