feat(norms): hosts, containers and monitor response times are graded against their own normal (#5742)

Host CPU, memory and load per core, container CPU and memory, and every
monitor's response time learn a normal per hour (milestone 513, System 182).
A reading past its fixed cutoff but within its normal reads healthy and says
what is usual at this hour; one well out of it keeps its grade and is a
metric anomaly. Disk, a monitor down and an agent offline keep their cutoffs.

- Each normal's floor, the smallest departure that counts, is a setting:
  Settings -> Thresholds -> Normals (norms.floor.*). Defaults: host CPU 10
  points, host memory 5, load per core 10, container CPU 10 points of one
  core, container memory 5, monitor response 100 ms; Traefik's 2-point 4xx
  and 50 ms p95 floors move there from constants. learn_normal takes a
  floor function (floor_setting) read at each pass.
- learn_normal(parts=False): a host learns, its cores don't.
- Ingest records load_pct (load_1m / cores x 100), the figure pages grade.
- Shared grading: anomalies.against / graded, worst_reading(normals=),
  hosts' host_grades, docker usage(thresholds=, norms=), monitors'
  latency_grade. Applied to host vitals, the Hosts widget, MCP overview,
  Docker usage, and the monitor list, widget and page.
- The monitor and container pages list their anomalies.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/__init__.py

@@ -11,6 +11,14 @@ _app: "Quart | None" = None
 def setup(app: "Quart") -> None:
     global _app
     _app = app
+    # A host's CPU, memory and load per core learn their normal (#5742), each
+    # with its floor from Settings → Thresholds. Parts don't: a host's CPU has
+    # a normal, each of its cores doesn't. Disks stay on their fixed cutoffs:
+    # a nearly full disk is a failure whatever its history (System 182).
+    from steward.core.norms import floor_setting, learn_normal
+    learn_normal("host_agent", "cpu_pct", floor_setting(app, "host_cpu_pct"), parts=False)
+    learn_normal("host_agent", "mem_used_pct", floor_setting(app, "host_mem_pct"), parts=False)
+    learn_normal("host_agent", "load_pct", floor_setting(app, "host_load_pct"), parts=False)
 
 
 def get_scheduled_tasks() -> list:
@@ -38,9 +46,10 @@ def get_alert_metrics() -> dict[str, list[str]]:
     disk_used_pct per mount, resource "<host>:<mount>"). seconds_since_seen
     comes from the scheduler, not the agent: "agent offline" is > 180.
     clock_off_secs is how far the host's clock is from Steward's, either way
-    (agent ≥ 1.12.0): "clock drifting" is > 30."""
+    (agent ≥ 1.12.0): "clock drifting" is > 30. load_pct is load_1m as a %
+    of the host's cores."""
     return {"host_agent": [
         "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
-        "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m",
+        "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m", "load_pct",
         "uptime_secs", "seconds_since_seen", "clock_off_secs",
     ]}

plugins/host_agent/routes.py

@@ -21,6 +21,7 @@ from steward.core.health import ingest as ingest_tally
 from steward.core.metric_series import points as metric_points
 from steward.core.settings import get_setting, public_base_url
 from steward.core.time_range import parse_range, RANGE_OPTIONS
+from steward.hosts import service as host_svc
 from steward.models.hosts import Host
 from steward.models.metrics import PluginMetric
 from .models import HostAgentRegistration
@@ -118,6 +119,10 @@ def _expand_sample_to_metrics(
     for key in ("1m", "5m", "15m"):
         if key in load:
             row(f"load_{key}", host_name, load[key])
+    # Load per core as a %, the figure the pages grade, so it can learn a
+    # normal (#5742): a load of 6 is idle on 32 cores and busy on 4.
+    if "1m" in load and sample.get("cpu_cores"):
+        row("load_pct", host_name, 100.0 * load["1m"] / len(sample["cpu_cores"]))
 
     if sample.get("uptime_secs") is not None:
         row("uptime_secs", host_name, sample["uptime_secs"])
@@ -725,6 +730,7 @@ async def host_vitals(host_id: str):
         reg = (await db.execute(select(HostAgentRegistration).where(
             HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
         latest = await _latest_metrics_for_host(db, host.name) if reg else {}
+        norms = await host_svc.host_norms(db, host.name) if reg else {}
 
         # The last 6 h for the at-a-glance sparklines (cpu/mem/load host-level,
         # disk from the root mount), averaged into ~120 points in SQL, so the
@@ -759,13 +765,23 @@ async def host_vitals(host_id: str):
     reporting = bool(reg) and ls is not None
     stale = reporting and (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()
     cpu, mem = hostlvl.get("cpu_pct"), hostlvl.get("mem_used_pct")
+    thresholds = current_app.config.get("THRESHOLDS", {})
     worst = worst_reading((("CPU", cpu, "cpu"), ("Memory", mem, "mem"), ("Disk /", disk_root, "disk"),
                            ("Load per core", load_per_core, "load")),
-                          current_app.config.get("THRESHOLDS", {}))
+                          thresholds, host_svc.host_normals(norms, thresholds))
+    # Each tile's grade; CPU, memory and load within the host's own normal
+    # for the hour read healthy, and say what is usual (#5742).
+    grades = host_svc.host_grades({"cpu": cpu, "mem": mem, "disk": disk_root, "load": load_per_core},
+                                  norms, thresholds)
+    usual = [f"{label} {value:.0f}% is {grades[kind][1]}"
+             for label, value, kind in (("CPU", cpu, "cpu"), ("Memory", mem, "mem"),
+                                        ("load per core", load_per_core, "load"))
+             if grades[kind][1]]
     return await render_template(
         "_host_vitals.html",
         reg=reg, cpu=cpu, mem=mem, disk_root=disk_root, load_per_core=load_per_core,
         sparks=sparks, psi=psi, reporting=reporting, stale=stale, worst=worst,
+        grades=grades, usual=usual,
         clock=clock_words(reg.clock_offset_s if reg else None, current_app.config.get("THRESHOLDS", {})),
     )
 

plugins/host_agent/templates/_host_vitals.html

@@ -1,6 +1,7 @@
 {# Host health header — live polled into the host hub: live or stale, the worst
    reading against Settings → Thresholds, last seen, then CPU / memory / disk /
-   load tiles graded by the same cutoffs. Renders nothing until the agent
+   load tiles graded by the same cutoffs (`grades`; CPU, memory and load within
+   the host's own normal for the hour read healthy, and `usual` says so). Renders nothing until the agent
    reports (the Agent panel below then shows provisioning). #}
 {% from "_macros.html" import status_badge, stat_tile %}
 {% macro pct(v) %}{{ '%.0f%%'|format(v) if v is not none else '—' }}{% endmacro %}
@@ -11,6 +12,7 @@
     <span class="host-health-summary">
       {%- if stale %}No report since {{ reg.last_seen_at|ago }}; the readings below are from then.
       {%- elif worst %}<span class="tone-{{ worst[2] }}">{{ worst[0] }} {{ pct(worst[1]) }}</span> is past its {{ "critical" if worst[2] == "crit" else "warning" }} threshold.
+      {%- elif usual %}Every reading is within its thresholds or its usual level for this hour.
       {%- else %}Every reading is within its thresholds.{% endif -%}
     </span>
     <span class="problem-detail">
@@ -20,11 +22,14 @@
     </span>
   </div>
   <div class="stat-strip host-health-tiles" style="margin-bottom:0;">
-    {{ stat_tile("CPU", pct(cpu), state=status_class(cpu, 'cpu'), hint=sparks.cpu|safe) }}
-    {{ stat_tile("Memory", pct(mem), state=status_class(mem, 'mem'), hint=sparks.mem|safe) }}
-    {{ stat_tile("Disk /", pct(disk_root), state=status_class(disk_root, 'disk'), hint=sparks.disk|safe) }}
-    {{ stat_tile("Load per core", pct(load_per_core), state=status_class(load_per_core, 'load'), hint=sparks.load|safe) }}
+    {{ stat_tile("CPU", pct(cpu), state=grades.cpu[0], hint=sparks.cpu|safe) }}
+    {{ stat_tile("Memory", pct(mem), state=grades.mem[0], hint=sparks.mem|safe) }}
+    {{ stat_tile("Disk /", pct(disk_root), state=grades.disk[0], hint=sparks.disk|safe) }}
+    {{ stat_tile("Load per core", pct(load_per_core), state=grades.load[0], hint=sparks.load|safe) }}
   </div>
+  {% if usual and not stale %}
+  <p class="field-hint" style="margin-top:0.6rem;">{{ usual | join("; ") | capitalize }}.</p>
+  {% endif %}
   {% if psi.cpu is not none or psi.mem is not none or psi.io is not none %}
   <p class="field-hint" style="margin-top:0.6rem;" title="Pressure stall: the share of the last 10 s that tasks waited for the resource">
     Pressure (10 s): cpu {{ pct(psi.cpu) }} · mem {{ pct(psi.mem) }} · io {{ pct(psi.io) }}