feat(norms): hosts, containers and monitor response times are graded against their own normal (#5742)

Host CPU, memory and load per core, container CPU and memory, and every
monitor's response time learn a normal per hour (milestone 513, System 182).
A reading past its fixed cutoff but within its normal reads healthy and says
what is usual at this hour; one well out of it keeps its grade and is a
metric anomaly. Disk, a monitor down and an agent offline keep their cutoffs.

- Each normal's floor, the smallest departure that counts, is a setting:
  Settings -> Thresholds -> Normals (norms.floor.*). Defaults: host CPU 10
  points, host memory 5, load per core 10, container CPU 10 points of one
  core, container memory 5, monitor response 100 ms; Traefik's 2-point 4xx
  and 50 ms p95 floors move there from constants. learn_normal takes a
  floor function (floor_setting) read at each pass.
- learn_normal(parts=False): a host learns, its cores don't.
- Ingest records load_pct (load_1m / cores x 100), the figure pages grade.
- Shared grading: anomalies.against / graded, worst_reading(normals=),
  hosts' host_grades, docker usage(thresholds=, norms=), monitors'
  latency_grade. Applied to host vitals, the Hosts widget, MCP overview,
  Docker usage, and the monitor list, widget and page.
- The monitor and container pages list their anomalies.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_norm_grading.py

@@ -0,0 +1,135 @@
+"""Hosts, containers and monitor response times are graded against their own
+normal (#5742, System 182): a reading past its fixed cutoff but within its
+normal for the hour reads healthy and says what is usual; one well out of it
+keeps its grade. How far out counts is each normal's floor, a setting on
+Settings → Thresholds. Disk keeps its cutoff whatever the history."""
+from types import SimpleNamespace as NS
+
+from quart import render_template, session
+
+from plugins.docker.service import usage
+from plugins.host_agent.routes import _expand_sample_to_metrics
+from steward.core import norms
+from steward.core.anomalies import graded
+from steward.core.formatting import worst_reading
+from steward.core.norms import Norm, floor_setting
+from steward.core.settings import DEFAULTS, NORM_FLOORS, floor_of, to_thresholds_cfg
+from steward.hosts.service import host_glance, host_grades
+from steward.monitors.service import latency_grade, row_state
+
+TH = to_thresholds_cfg({})      # cpu 80/90, mem 80/90, disk 80/90, load 80/100, latency 50/200 ms
+
+
+def _norm(hold):
+    return Norm(kind="day", slot=14, usual_hold=hold, usual_peak=hold * 1.2, days_seen=7)
+
+
+def test_each_floor_is_a_setting_with_the_chosen_default():
+    assert {n: floor_of(TH, n) for n in NORM_FLOORS} == {
+        "host_cpu_pct": 10.0, "host_mem_pct": 5.0, "host_load_pct": 10.0,
+        "container_cpu_pct": 10.0, "container_mem_pct": 5.0, "monitor_ms": 100.0,
+        "router_4xx_pct": 2.0, "router_p95_ms": 50.0}
+    assert floor_of(to_thresholds_cfg({"norms.floor.host_cpu_pct": 25}), "host_cpu_pct") == 25.0
+    assert floor_of({}, "monitor_ms") == DEFAULTS["norms.floor.monitor_ms"]
+
+
+def test_a_floor_from_settings_is_read_at_each_pass():
+    app = NS(config={"THRESHOLDS": to_thresholds_cfg({})})
+    norms.learn_normal("t5742", "x_pct", floor_setting(app, "host_mem_pct"))
+    try:
+        learned = norms.LEARNED[("t5742", "x_pct")]
+        assert learned.least_departure == 5.0
+        app.config["THRESHOLDS"] = to_thresholds_cfg({"norms.floor.host_mem_pct": 12})
+        assert learned.least_departure == 12.0
+    finally:
+        norms.LEARNED.pop(("t5742", "x_pct"), None)
+
+
+def test_past_the_cutoff_but_within_its_normal_reads_healthy():
+    assert graded(93.0, "cpu", TH, _norm(88.0), 10.0, "%") == ("ok", "usual 88.0% at this hour")
+    assert graded(93.0, "cpu", TH, None, 10.0, "%") == ("crit", None)
+    # Out of its normal: a quarter over the usual hold and past the floor.
+    assert graded(93.0, "cpu", TH, _norm(40.0), 10.0, "%") == ("crit", None)
+    # Under the cutoff there is nothing to quiet.
+    assert graded(30.0, "cpu", TH, _norm(5.0), 10.0, "%") == ("ok", None)
+
+
+def test_a_departure_smaller_than_the_floor_is_still_usual():
+    # 1.25 × 70 is 87.5 and 91 is over it, but only 21 over: under a 25-point floor.
+    assert graded(91.0, "cpu", TH, _norm(70.0), 25.0, "%")[0] == "ok"
+    assert graded(91.0, "cpu", TH, _norm(70.0), 10.0, "%")[0] == "crit"
+
+
+def test_a_hosts_worst_reading_skips_what_is_usual_for_it():
+    readings = (("CPU", 95.0, "cpu"), ("Memory", 85.0, "mem"), ("Disk /", 50.0, "disk"))
+    assert worst_reading(readings, TH) == ("CPU", 95.0, "crit")
+    usual = {"cpu": (_norm(92.0), 10.0), "mem": (_norm(84.0), 5.0)}
+    assert worst_reading(readings, TH, usual) is None
+
+
+def test_disk_keeps_its_cutoff_whatever_the_history():
+    grades = host_grades({"cpu": 95.0, "disk": 95.0}, {"cpu": _norm(93.0)}, TH)
+    assert grades["cpu"] == ("ok", "usual 93.0% at this hour")
+    assert grades["disk"] == ("crit", None)
+
+
+def test_the_hosts_glance_grades_each_host_by_its_own_normal():
+    hosts = [NS(id="b", name="builder"), NS(id="w", name="web")]
+    agent = {"builder": {"cpu_pct": 95.0, "load_per_core": 120, "fresh": True,
+                         "norms": {"cpu": _norm(90.0), "load": _norm(110.0)}},
+             "web": {"cpu_pct": 95.0, "fresh": True, "norms": {"cpu": _norm(20.0)}}}
+    rows = {r["host"].name: r for r in host_glance(hosts, {}, agent, TH)}
+    assert rows["builder"]["state"] == "up" and rows["builder"]["worst"] is None
+    assert rows["builder"]["grades"]["load"] == ("ok", "usual 110.0% at this hour")
+    assert rows["web"]["state"] == "crit" and rows["web"]["worst"][0] == "CPU"
+
+
+def test_a_monitor_slow_as_usual_is_up_not_degraded():
+    monitor = NS(enabled=True)
+    slow = NS(is_up=True, response_ms=450.0)
+    assert row_state(monitor, slow, 100.0, TH) == "degraded"
+    assert row_state(monitor, slow, 100.0, TH, _norm(400.0)) == "up"
+    assert latency_grade(slow, TH, _norm(400.0)) == ("ok", "usual 400 ms at this hour")
+    # 1.25 × 300 = 375, and 150 over is past the 100 ms floor.
+    assert row_state(monitor, slow, 100.0, TH, _norm(300.0)) == "degraded"
+
+
+def test_a_monitor_down_stays_down_whatever_its_normal():
+    down = NS(is_up=False, response_ms=None)
+    assert row_state(NS(enabled=True), down, 100.0, TH, _norm(400.0)) == "down"
+
+
+def test_a_containers_cpu_is_learned_per_core_and_said_in_cores():
+    # 3.8 cores of a 4-core host is 95% of it: crit, unless 3.6 cores is usual.
+    c = NS(cpu_pct=380.0, mem_pct=50.0, mem_usage_bytes=2**30, mem_limit_bytes=2 * 2**30)
+    u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"cpu_pct": _norm(360.0)})
+    assert u["cpu_grade"] == "ok" and u["cpu_usual"] == "usual 3.60 cores at this hour"
+    assert usage(c, 4, 64 * 2**30, thresholds=TH)["cpu_grade"] == "crit"
+    assert usage(c, 4, 64 * 2**30)["cpu_grade"] is None          # no thresholds: the page grades
+
+
+def test_a_containers_memory_within_its_normal_reads_healthy():
+    c = NS(cpu_pct=1.0, mem_pct=92.0, mem_usage_bytes=2**30, mem_limit_bytes=2 * 2**30)
+    u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"mem_pct": _norm(90.0)})
+    assert (u["mem_grade"], u["mem_usual"]) == ("ok", "usual 90.0% at this hour")
+    assert usage(c, 4, 64 * 2**30, thresholds=TH)["mem_grade"] == "crit"
+
+
+def test_ingest_records_load_per_core():
+    sample = {"load": {"1m": 6.0, "5m": 5.0, "15m": 4.0}, "cpu_cores": [10.0] * 8}
+    rows = _expand_sample_to_metrics(sample, "h", None)
+    (load_pct,) = [r for r in rows if r.metric_name == "load_pct"]
+    assert load_pct.resource_name == "h" and load_pct.value == 75.0
+    assert not [r for r in _expand_sample_to_metrics({"load": {"1m": 1.0}}, "h", None)
+                if r.metric_name == "load_pct"]                # no cores reported: no figure
+
+
+async def test_the_thresholds_page_offers_each_running_kinds_floor(app):
+    async with app.test_request_context("/settings/thresholds/"):
+        session["user_role"] = "admin"
+        page = await render_template("settings/thresholds.html", settings=dict(DEFAULTS),
+                                     traefik_on=False, plugins_on={"host_agent"})
+    assert 'id="normals"' in page
+    assert 'name="norms.floor.host_cpu_pct"' in page and 'name="norms.floor.monitor_ms"' in page
+    assert 'name="norms.floor.container_cpu_pct"' not in page         # docker isn't running
+    assert 'name="norms.floor.router_4xx_pct"' not in page