feat(norms): hosts, containers and monitor response times are graded against their own normal (#5742)

Host CPU, memory and load per core, container CPU and memory, and every
monitor's response time learn a normal per hour (milestone 513, System 182).
A reading past its fixed cutoff but within its normal reads healthy and says
what is usual at this hour; one well out of it keeps its grade and is a
metric anomaly. Disk, a monitor down and an agent offline keep their cutoffs.

- Each normal's floor, the smallest departure that counts, is a setting:
  Settings -> Thresholds -> Normals (norms.floor.*). Defaults: host CPU 10
  points, host memory 5, load per core 10, container CPU 10 points of one
  core, container memory 5, monitor response 100 ms; Traefik's 2-point 4xx
  and 50 ms p95 floors move there from constants. learn_normal takes a
  floor function (floor_setting) read at each pass.
- learn_normal(parts=False): a host learns, its cores don't.
- Ingest records load_pct (load_1m / cores x 100), the figure pages grade.
- Shared grading: anomalies.against / graded, worst_reading(normals=),
  hosts' host_grades, docker usage(thresholds=, norms=), monitors'
  latency_grade. Applied to host vitals, the Hosts widget, MCP overview,
  Docker usage, and the monitor list, widget and page.
- The monitor and container pages list their anomalies.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/hosts/service.py

@@ -9,6 +9,8 @@ from datetime import datetime, timedelta, timezone
 
 from sqlalchemy import and_, case, func, select
 
+from steward.core.norms import norms_at
+from steward.core.settings import floor_of
 from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor
 from steward.models.hosts import Host
 from steward.models.monitors import Monitor, MonitorResult
@@ -249,8 +251,9 @@ async def agent_overview_by_host(
     """Per-host agent glance (cpu/mem/disk/load + freshness) from plugin_metric_latest.
 
     {host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root, load_1m,
-                 cores, load_per_core, last_at, fresh, clock_off_secs}} for each host with any
-    agent samples. Cores are counted from the per-core CPU sub-resources
+                 cores, load_per_core, last_at, fresh, clock_off_secs, norms}} for each host
+    with any agent samples. `norms` is {kind: Norm}, the host's CPU, memory and
+    load normals for this hour (host_normals grades by them). Cores are counted from the per-core CPU sub-resources
     ("<host>:core<N>"), as on the host page; load_per_core is load_1m as a %
     of them, the figure Settings → Thresholds grades "load" by.
 
@@ -294,10 +297,50 @@ async def agent_overview_by_host(
         d["cores"] = cores.get(name)
         load = d.get("load_1m")
         d["load_per_core"] = round(load / d["cores"] * 100) if (load is not None and d["cores"]) else None
+        d["norms"] = {}
         out[name] = d
+    learned = await norms_at(db, [("host_agent", h, metric) for h in out
+                                  for metric, _f in HOST_NORMED.values()], now)
+    for (_s, host, metric), norm in learned.items():
+        out[host]["norms"][_KIND_OF[metric]] = norm
     return out
 
 
+# The host figures graded against their own normal (#5742): the threshold
+# kind, the metric that learns it and its floor's setting (norms.floor.*).
+# Disk keeps its cutoff: a nearly full disk is a failure (System 182).
+HOST_NORMED = {"cpu": ("cpu_pct", "host_cpu_pct"), "mem": ("mem_used_pct", "host_mem_pct"),
+               "load": ("load_pct", "host_load_pct")}
+_KIND_OF = {metric: kind for kind, (metric, _f) in HOST_NORMED.items()}
+
+
+def host_normals(norms: dict, thresholds: dict) -> dict:
+    """{kind: (Norm, floor)} for worst_reading and anomalies.graded, from an
+    agent glance's `norms`."""
+    return {kind: (norm, floor_of(thresholds, HOST_NORMED[kind][1]))
+            for kind, norm in (norms or {}).items()}
+
+
+async def host_norms(db, host_name: str, now: datetime | None = None) -> dict:
+    """{kind: Norm}: one host's CPU, memory and load normals for this hour."""
+    learned = await norms_at(db, [("host_agent", host_name, metric)
+                                  for metric, _f in HOST_NORMED.values()],
+                             now or datetime.now(timezone.utc))
+    return {_KIND_OF[metric]: norm for (_s, _h, metric), norm in learned.items()}
+
+
+def host_grades(values: dict, norms: dict, thresholds: dict) -> dict:
+    """{kind: (status class, why)} for a host's readings ({kind: value}):
+    against Settings → Thresholds, except CPU, memory and load within the
+    host's own normal, which are "ok" and say what is usual (#5742)."""
+    from steward.core.anomalies import graded
+    from steward.core.formatting import status_class
+    normals = host_normals(norms, thresholds)
+    return {kind: (graded(value, kind, thresholds, *normals[kind], "%") if kind in normals
+                   else (status_class(value, kind, thresholds), None))
+            for kind, value in values.items()}
+
+
 def clock_trouble(agent: dict[str, dict], limit: float) -> dict | None:
     """Reporting hosts whose clock is more than `limit` seconds from Steward's
     (Settings → Thresholds' clock drift warning, milestone 490),
@@ -326,19 +369,24 @@ def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict
     Each is {host, state, worst, agent}. State is "down" when a linked monitor
     is down, "stale" when the agent has stopped reporting, else the grade of
     the reading furthest past its cutoff ("crit" / "warn": CPU, memory, disk /
-    or load per core, against Settings → Thresholds), else "up"; "unknown" for
+    or load per core, against Settings → Thresholds; CPU, memory and load
+    within the host's own normal are not past it), else "up"; "unknown" for
     a host with neither monitors nor agent. `worst` is (label, value, grade)
-    or None, from formatting.worst_reading."""
+    or None, from formatting.worst_reading; `grades` is host_grades' {kind:
+    (grade, why)} for a fresh agent, else {}."""
     from steward.core.formatting import worst_reading
     rows = []
     for h in hosts:
         st, a = status.get(h.id), agent.get(h.name)
-        worst = None
+        worst, grades = None, {}
         if a and a.get("fresh"):
+            grades = host_grades({"cpu": a.get("cpu_pct"), "mem": a.get("mem_used_pct"),
+                                  "disk": a.get("disk_root"), "load": a.get("load_per_core")},
+                                 a.get("norms"), thresholds)
             worst = worst_reading((
                 ("CPU", a.get("cpu_pct"), "cpu"), ("Memory", a.get("mem_used_pct"), "mem"),
                 ("Disk /", a.get("disk_root"), "disk"), ("Load", a.get("load_per_core"), "load")),
-                thresholds)
+                thresholds, host_normals(a.get("norms"), thresholds))
         if st and st["state"] == "down":
             state = "down"
         elif a and not a.get("fresh"):
@@ -349,5 +397,5 @@ def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict
             state = "up"
         else:
             state = "unknown"
-        rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}})
+        rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}, "grades": grades})
     return sorted(rows, key=lambda r: (GLANCE_ORDER[r["state"]], r["host"].name.lower()))