feat(norms): hosts, containers and monitor response times are graded against their own normal (#5742)

Host CPU, memory and load per core, container CPU and memory, and every
monitor's response time learn a normal per hour (milestone 513, System 182).
A reading past its fixed cutoff but within its normal reads healthy and says
what is usual at this hour; one well out of it keeps its grade and is a
metric anomaly. Disk, a monitor down and an agent offline keep their cutoffs.

- Each normal's floor, the smallest departure that counts, is a setting:
  Settings -> Thresholds -> Normals (norms.floor.*). Defaults: host CPU 10
  points, host memory 5, load per core 10, container CPU 10 points of one
  core, container memory 5, monitor response 100 ms; Traefik's 2-point 4xx
  and 50 ms p95 floors move there from constants. learn_normal takes a
  floor function (floor_setting) read at each pass.
- learn_normal(parts=False): a host learns, its cores don't.
- Ingest records load_pct (load_1m / cores x 100), the figure pages grade.
- Shared grading: anomalies.against / graded, worst_reading(normals=),
  hosts' host_grades, docker usage(thresholds=, norms=), monitors'
  latency_grade. Applied to host vitals, the Hosts widget, MCP overview,
  Docker usage, and the monitor list, widget and page.
- The monitor and container pages list their anomalies.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/__init__.py

@@ -43,6 +43,11 @@ def setup(app: "Quart") -> None:
         description="Keep each container's logs within the age window and byte cap.",
         required_role=UserRole.viewer,
     )
+    # Each container's CPU (100% is one core) and memory share learn their
+    # normal (#5742), each with its floor from Settings → Thresholds.
+    from steward.core.norms import floor_setting, learn_normal
+    learn_normal("docker", "cpu_pct", floor_setting(app, "container_cpu_pct"))
+    learn_normal("docker", "mem_pct", floor_setting(app, "container_mem_pct"))
 
 
 def get_scheduled_tasks() -> list:

plugins/docker/routes.py

@@ -12,6 +12,7 @@ from sqlalchemy import func, select
 from steward.auth.middleware import require_role
 from steward.models.users import UserRole
 from steward.models.hosts import Host
+from steward.core import anomalies as anom
 from steward.core.formatting import human_bytes, human_duration, seconds_since
 from steward.core.time_range import parse_range, DEFAULT_RANGE
 from . import service as docker_svc
@@ -159,16 +160,20 @@ async def rows():
         host_name = {h.id: h.name for h in hosts.values()}
         swarm_view = ctx["swarm_view"]
         capacity = await docker_svc.host_capacity(db, {c.host_id for c in containers})
+        norms = await docker_svc.container_norms(
+            db, {(c.host_id, c.name) for c in containers if not c.service_name}, now)
         failures = docker_svc.failures_24h(
             await docker_svc.failure_events(db, now - timedelta(hours=24)),
             swarm_view["services"], host_name)
 
+    thresholds = current_app.config.get("THRESHOLDS", {})
     service_rows = [service_row(s, now) for s in swarm_view["services"]]
     standalone = [c for c in containers if not c.service_name]
     container_rows = [
         container_row(
             c, label=docker_svc.display_name(c), host=host_name.get(c.host_id, c.host_id),
-            usage=docker_svc.usage(c, *capacity.get(c.host_id, (None, None))),
+            usage=docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), thresholds=thresholds,
+                                   norms=norms.get((c.host_id, c.name))),
             ports=json.loads(c.ports_json) if c.ports_json else [],
             stale=_is_stale(c, now, stale_after, docker_down),
             exit_failed=docker_svc.exit_is_failure(c.exit_code, c.oom_killed), now=now)
@@ -314,6 +319,8 @@ async def widget_resources():
             db, {(c.host_id, c.name) for c in busiest}, slots[0], width)
         hosts = await docker_svc.host_map(db, {c.host_id for c in busiest})
         capacity = await docker_svc.host_capacity(db, {c.host_id for c in busiest})
+        norms = await docker_svc.container_norms(db, {(c.host_id, c.name) for c in busiest})
+    thresholds = current_app.config.get("THRESHOLDS", {})
 
     def bars(c, i, scale, floor):
         seen = history.get((c.host_id, c.name), {})
@@ -325,7 +332,8 @@ async def widget_resources():
         "c": c,
         "label": docker_svc.display_name(c),
         "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id,
-        "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c)),
+        "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c),
+                                  thresholds=thresholds, norms=norms.get((c.host_id, c.name))),
         "cpu_bars": bars(c, 0, 0.01, cpu_floor),            # cpu_pct → cores
         "mem_bars": bars(c, 1, 1.0, mem_floor * 2**30),     # bytes; floor in GiB
     } for c in busiest]
@@ -365,7 +373,9 @@ async def host_panel(host_id: str):
         nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
         failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
         capacity = await docker_svc.host_capacity(db, {host_id})
+        norms = await docker_svc.container_norms(db, {(c.host_id, c.name) for c in containers}, now)
 
+    thresholds = current_app.config.get("THRESHOLDS", {})
     replaced = replaced_tasks(containers, services)
     node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
     hour_ago = now - timedelta(hours=1)
@@ -380,7 +390,8 @@ async def host_panel(host_id: str):
         rows.append({
             "c": c, "label": docker_svc.task_label(c, node_hostname), "problems": problems,
             "since": docker_svc.problem_since(problems, events, hour_ago),
-            "usage": docker_svc.usage(c, *capacity.get(host_id, (None, None))),
+            "usage": docker_svc.usage(c, *capacity.get(host_id, (None, None)), thresholds=thresholds,
+                                      norms=norms.get((c.host_id, c.name))),
             "uptime": _uptime(c.started_at) if c.status == "running" else None,
         })
     shown, folded = host_panel_rows(rows)
@@ -445,6 +456,13 @@ async def container_detail(host_id: str, name: str):
             )).scalar()
         cpus, mem_total = (await docker_svc.host_capacity(db, [host_id])).get(host_id, (None, None)) \
             if c is not None else (None, None)
+        # Its CPU and memory against their normal, and its anomalies (#5742).
+        norms = (await docker_svc.container_norms(db, {(host_id, name)})).get((host_id, name))
+        resource = docker_svc.container_resource(host.name, name) if host is not None else None
+        anomalies = await anom.views(db, await anom.list_anomalies(
+            db, source="docker", resource=resource,
+            since=datetime.now(timezone.utc) - timedelta(days=30), limit=20),
+            links=False) if resource else []
 
     if c is None:
         return await render_template(
@@ -460,7 +478,10 @@ async def container_detail(host_id: str, name: str):
         name=name,
         label=docker_svc.task_label(c, {c.node_id: node_name} if node_name else None),
         node_name=node_name,
-        usage=docker_svc.usage(c, cpus, mem_total),
+        usage=docker_svc.usage(c, cpus, mem_total, thresholds=current_app.config.get("THRESHOLDS", {}),
+                               norms=norms),
+        anomalies=anomalies, anomaly_back=f"/plugins/docker/container/{host_id}/{name}",
+        anomaly_resource=resource,
         exit_failed=docker_svc.exit_is_failure(c.exit_code, c.oom_killed),
         ports=json.loads(c.ports_json) if c.ports_json else [],
         uptime=_uptime(c.started_at) if c.status == "running" else None,
@@ -888,6 +909,8 @@ async def service_detail(name: str):
         s = next((s for s in ctx["swarm_view"]["services"] if s["name"] == name), None)
         capacity = await docker_svc.host_capacity(
             db, {r["host_id"] for r in (s["replicas"] if s else []) if not r["ghost"]})
+        norms = await docker_svc.container_norms(
+            db, {(r["host_id"], r["name"]) for r in (s["replicas"] if s else []) if not r["ghost"]}, now)
     if s is None and not task_containers:
         return await render_template("docker/service.html", name=name, service=None), 404
 
@@ -899,7 +922,9 @@ async def service_detail(name: str):
     for r in (s["replicas"] if s else []):
         c = None if r["ghost"] else by_key.get((r["host_id"], r["name"]))
         replicas.append(dict(r, task_id=getattr(c, "task_id", None) if c is not None else None,
-                             usage=docker_svc.usage(c, *capacity.get(r["host_id"], (None, None)))
+                             usage=docker_svc.usage(c, *capacity.get(r["host_id"], (None, None)),
+                                                    thresholds=current_app.config.get("THRESHOLDS", {}),
+                                                    norms=norms.get((c.host_id, c.name)))
                              if c is not None else None))
     node_host = {c.node_id: c.host_id for c in ctx["containers"] if c.node_id}
     table = _service_tasks(name, tasks, replicas, node_host, ctx["host_name"])

plugins/docker/service.py

@@ -155,7 +155,8 @@ def display_name(c) -> str:
     return c.name
 
 
-def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = None) -> dict:
+def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = None, *,
+          thresholds: dict | None = None, norms: dict | None = None) -> dict:
     """A container's CPU and memory in terms a reader can weigh.
 
     Docker's CPU% is per core (100% = one core busy), so it is shown as cores
@@ -164,11 +165,16 @@ def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = No
     is shown in bytes, against a limit only when one is actually set; mem_pct
     is then a share of that limit, otherwise of the host's memory. `cpu_pct`
     replaces the container's last sample (an average, say) when given.
+
+    With `thresholds`, cpu_grade and mem_grade are the grades to show: past
+    the cutoff but within the container's own normal for the hour (`norms`,
+    {metric: Norm}, container_norms) is "ok", and cpu_usual / mem_usual say
+    what is usual (#5742). Without, they are None and the page grades.
     """
     cpu = getattr(c, "cpu_pct", None) if cpu_pct is None else cpu_pct
     limit = getattr(c, "mem_limit_bytes", None)
     limited = bool(limit and mem_total and limit < mem_total * 0.95)
-    return {
+    out = {
         "cores": None if cpu is None else cpu / 100.0,
         "cpu_share": None if cpu is None or not cpus else cpu / cpus,
         "cpus": cpus,
@@ -176,7 +182,51 @@ def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = No
         "mem_limit": limit if limited else None,
         "mem_pct": getattr(c, "mem_pct", None),
         "mem_of": "its limit" if limited else ("the host's memory" if mem_total else None),
+        "cpu_grade": None, "cpu_usual": None, "mem_grade": None, "mem_usual": None,
     }
+    if thresholds is not None:
+        _grade(out, cpu, thresholds, norms or {})
+    return out
+
+
+# Container figures graded against their own normal (#5742): the metric that
+# learns it and its floor's setting (norms.floor.*).
+NORMED = {"cpu": ("cpu_pct", "container_cpu_pct"), "mem": ("mem_pct", "container_mem_pct")}
+
+
+def _grade(u: dict, cpu: float | None, thresholds: dict, norms: dict) -> None:
+    """Fill usage()'s grades. CPU is graded as a share of the host but learned
+    per core (Docker's own figure), so its normal is read in cores."""
+    from steward.core.anomalies import departs, graded
+    from steward.core.formatting import status_class
+    from steward.core.settings import floor_of
+    grade = status_class(u["cpu_share"], "cpu", thresholds)
+    norm = norms.get("cpu_pct")
+    if grade in ("warn", "crit") and norm is not None and cpu is not None \
+            and not departs(cpu, norm, floor_of(thresholds, "container_cpu_pct")):
+        grade = "ok"
+        u["cpu_usual"] = f"usual {norm.usual_hold / 100:.2f} cores at this hour"
+    u["cpu_grade"] = grade
+    if u["mem_of"]:
+        u["mem_grade"], u["mem_usual"] = graded(
+            u["mem_pct"], "mem", thresholds, norms.get("mem_pct"),
+            floor_of(thresholds, "container_mem_pct"), "%")
+    else:
+        u["mem_grade"] = "dim"
+
+
+async def container_norms(db, keys, now: datetime | None = None) -> dict:
+    """{(host_id, container_name): {metric: Norm}}: each container's CPU and
+    memory normals for this hour. One without enough history is absent."""
+    from steward.core.norms import norms_at
+    resources = await container_resources(db, keys)
+    got = await norms_at(db, [("docker", r, metric) for r in resources
+                              for metric, _f in NORMED.values()],
+                         now or datetime.now(timezone.utc))
+    out: dict = {}
+    for (_s, resource, metric), norm in got.items():
+        out.setdefault(resources[resource], {})[metric] = norm
+    return out
 
 
 async def host_capacity(db, host_ids) -> dict[str, tuple[int | None, int | None]]:

plugins/docker/templates/docker/_usage.html

@@ -3,7 +3,9 @@
     Docker counts CPU per core (100% = one core busy), so CPU reads as cores,
     toned by its share of the host's CPUs when the host's core count is known.
     Memory reads in bytes, against a limit only when one is actually set;
-    mem_pct is then a share of that limit, otherwise of the host's memory. -#}
+    mem_pct is then a share of that limit, otherwise of the host's memory.
+    Each is toned by usage()'s grade when it has one: within the container's
+    own normal for the hour reads healthy, and says what is usual (#5742). -#}
 
 {% macro cores(n) -%}
 {{ "%.2f"|format(n) if n < 1 else "%.1f"|format(n) }} {{ "core" if n == 1 else "cores" }}
@@ -12,15 +14,15 @@
 {% macro cpu(u) -%}
 {%- if u.cores is none -%}<span class="tone-dim">—</span>
 {%- else -%}
-<span class="tone-{{ status_class(u.cpu_share, 'cpu') }}"
-      title="Docker counts CPU per core: {{ '%.0f'|format(u.cores * 100) }}% of one core{% if u.cpus %}, {{ '%.1f'|format(u.cpu_share) }}% of this host's {{ u.cpus }} core{{ 's' if u.cpus != 1 }}{% endif %}">{{ cores(u.cores) }}</span>
+<span class="tone-{{ u.cpu_grade or status_class(u.cpu_share, 'cpu') }}"
+      title="Docker counts CPU per core: {{ '%.0f'|format(u.cores * 100) }}% of one core{% if u.cpus %}, {{ '%.1f'|format(u.cpu_share) }}% of this host's {{ u.cpus }} core{{ 's' if u.cpus != 1 }}{% endif %}{% if u.cpu_usual %}; {{ u.cpu_usual }}{% endif %}">{{ cores(u.cores) }}</span>
 {%- endif -%}
 {%- endmacro %}
 
 {% macro mem(u) -%}
 {%- if u.mem_bytes is none -%}<span class="tone-dim">—</span>
 {%- else -%}
-<span class="tone-{{ status_class(u.mem_pct, 'mem') if u.mem_of else 'dim' }}"
-      title="{% if u.mem_pct is not none and u.mem_of %}{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% else %}No memory limit set{% endif %}">{{ u.mem_bytes|bytes }}{% if u.mem_limit %} / {{ u.mem_limit|bytes }}{% endif %}</span>
+<span class="tone-{{ u.mem_grade or (status_class(u.mem_pct, 'mem') if u.mem_of else 'dim') }}"
+      title="{% if u.mem_pct is not none and u.mem_of %}{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% if u.mem_usual %}, {{ u.mem_usual }}{% endif %}{% else %}No memory limit set{% endif %}">{{ u.mem_bytes|bytes }}{% if u.mem_limit %} / {{ u.mem_limit|bytes }}{% endif %}</span>
 {%- endif -%}
 {%- endmacro %}

plugins/docker/templates/docker/container_detail.html

@@ -1,7 +1,8 @@
 {# docker/container_detail.html — one container (#5310), shaped like a
    service's page: a header that says whether it is well and what it is; why
    it stopped, only while it has; what it moved since it started; its CPU and
-   memory over time (the one home for what it uses now); its lifecycle.
+   memory over time (the one home for what it uses now), with its metric
+   anomalies above when it has any; its lifecycle.
    A swarm task is titled by its service and slot, the raw name under it. #}
 {% extends "base.html" %}
 {% from "_macros.html" import crumbs, status_badge %}
@@ -63,6 +64,9 @@
 </section>
 {% endif %}
 
+{# ── Its CPU or memory out of its normal for the hour, and past anomalies (#5742) #}
+{% with back=anomaly_back, base_resource=anomaly_resource %}{% include "_anomaly_card.html" %}{% endwith %}
+
 {# ── What it is set up with, and what it moved since it started ─────────── #}
 <div class="card">
   <dl class="kv">

plugins/docker/templates/docker/widget_resources.html

@@ -33,13 +33,13 @@
       {%- if session.user_id %}<a href="/plugins/docker/container/{{ r.c.host_id }}/{{ r.c.name }}">{{ r.label }}</a>{% else %}{{ r.label }}{% endif -%}
     </td>
     <td class="w-wide-only w-text" title="{{ r.host_name }}">{{ r.host_name }}</td>
-    <td class="tone-{{ status_class(u.cpu_share, 'cpu') if u.cpus and u.cores is not none else 'dim' }}"
-        {% if u.cpus and u.cores is not none %}title="{{ '%.1f'|format(u.cpu_share) }}% of {{ r.host_name }}'s {{ u.cpus }} cores"{% endif %}>
+    <td class="tone-{{ (u.cpu_grade or status_class(u.cpu_share, 'cpu')) if u.cpus and u.cores is not none else 'dim' }}"
+        {% if u.cpus and u.cores is not none %}title="{{ '%.1f'|format(u.cpu_share) }}% of {{ r.host_name }}'s {{ u.cpus }} cores{% if u.cpu_usual %}; {{ u.cpu_usual }}{% endif %}"{% endif %}>
       {{- bars(r.cpu_bars, (cores(r.cpu_bars.peak) ~ " cores") if r.cpu_bars.peak is not none else "", cores(r.cpu_bars.roof) ~ " cores") -}}
       <span class="w-fig">{% if u.cores is none %}—{% else %}{{ cores(u.cores) }} cores{% endif %}</span>
     </td>
-    <td class="tone-{{ status_class(u.mem_pct, 'mem') if u.mem_of and u.mem_bytes is not none else 'dim' }}"
-        {% if u.mem_pct is not none and u.mem_of and u.mem_bytes is not none %}title="{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}"{% endif %}>
+    <td class="tone-{{ (u.mem_grade or status_class(u.mem_pct, 'mem')) if u.mem_of and u.mem_bytes is not none else 'dim' }}"
+        {% if u.mem_pct is not none and u.mem_of and u.mem_bytes is not none %}title="{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% if u.mem_usual %}, {{ u.mem_usual }}{% endif %}"{% endif %}>
       {{- bars(r.mem_bars, r.mem_bars.peak|bytes if r.mem_bars.peak is not none else "", r.mem_bars.roof|bytes) -}}
       <span class="w-fig">{% if u.mem_bytes is none %}—{% else %}{{ u.mem_bytes|bytes }}{% endif %}</span>
     </td>

plugins/host_agent/__init__.py

@@ -11,6 +11,14 @@ _app: "Quart | None" = None
 def setup(app: "Quart") -> None:
     global _app
     _app = app
+    # A host's CPU, memory and load per core learn their normal (#5742), each
+    # with its floor from Settings → Thresholds. Parts don't: a host's CPU has
+    # a normal, each of its cores doesn't. Disks stay on their fixed cutoffs:
+    # a nearly full disk is a failure whatever its history (System 182).
+    from steward.core.norms import floor_setting, learn_normal
+    learn_normal("host_agent", "cpu_pct", floor_setting(app, "host_cpu_pct"), parts=False)
+    learn_normal("host_agent", "mem_used_pct", floor_setting(app, "host_mem_pct"), parts=False)
+    learn_normal("host_agent", "load_pct", floor_setting(app, "host_load_pct"), parts=False)
 
 
 def get_scheduled_tasks() -> list:
@@ -38,9 +46,10 @@ def get_alert_metrics() -> dict[str, list[str]]:
     disk_used_pct per mount, resource "<host>:<mount>"). seconds_since_seen
     comes from the scheduler, not the agent: "agent offline" is > 180.
     clock_off_secs is how far the host's clock is from Steward's, either way
-    (agent ≥ 1.12.0): "clock drifting" is > 30."""
+    (agent ≥ 1.12.0): "clock drifting" is > 30. load_pct is load_1m as a %
+    of the host's cores."""
     return {"host_agent": [
         "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
-        "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m",
+        "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m", "load_pct",
         "uptime_secs", "seconds_since_seen", "clock_off_secs",
     ]}

plugins/host_agent/routes.py

@@ -21,6 +21,7 @@ from steward.core.health import ingest as ingest_tally
 from steward.core.metric_series import points as metric_points
 from steward.core.settings import get_setting, public_base_url
 from steward.core.time_range import parse_range, RANGE_OPTIONS
+from steward.hosts import service as host_svc
 from steward.models.hosts import Host
 from steward.models.metrics import PluginMetric
 from .models import HostAgentRegistration
@@ -118,6 +119,10 @@ def _expand_sample_to_metrics(
     for key in ("1m", "5m", "15m"):
         if key in load:
             row(f"load_{key}", host_name, load[key])
+    # Load per core as a %, the figure the pages grade, so it can learn a
+    # normal (#5742): a load of 6 is idle on 32 cores and busy on 4.
+    if "1m" in load and sample.get("cpu_cores"):
+        row("load_pct", host_name, 100.0 * load["1m"] / len(sample["cpu_cores"]))
 
     if sample.get("uptime_secs") is not None:
         row("uptime_secs", host_name, sample["uptime_secs"])
@@ -725,6 +730,7 @@ async def host_vitals(host_id: str):
         reg = (await db.execute(select(HostAgentRegistration).where(
             HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
         latest = await _latest_metrics_for_host(db, host.name) if reg else {}
+        norms = await host_svc.host_norms(db, host.name) if reg else {}
 
         # The last 6 h for the at-a-glance sparklines (cpu/mem/load host-level,
         # disk from the root mount), averaged into ~120 points in SQL, so the
@@ -759,13 +765,23 @@ async def host_vitals(host_id: str):
     reporting = bool(reg) and ls is not None
     stale = reporting and (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()
     cpu, mem = hostlvl.get("cpu_pct"), hostlvl.get("mem_used_pct")
+    thresholds = current_app.config.get("THRESHOLDS", {})
     worst = worst_reading((("CPU", cpu, "cpu"), ("Memory", mem, "mem"), ("Disk /", disk_root, "disk"),
                            ("Load per core", load_per_core, "load")),
-                          current_app.config.get("THRESHOLDS", {}))
+                          thresholds, host_svc.host_normals(norms, thresholds))
+    # Each tile's grade; CPU, memory and load within the host's own normal
+    # for the hour read healthy, and say what is usual (#5742).
+    grades = host_svc.host_grades({"cpu": cpu, "mem": mem, "disk": disk_root, "load": load_per_core},
+                                  norms, thresholds)
+    usual = [f"{label} {value:.0f}% is {grades[kind][1]}"
+             for label, value, kind in (("CPU", cpu, "cpu"), ("Memory", mem, "mem"),
+                                        ("load per core", load_per_core, "load"))
+             if grades[kind][1]]
     return await render_template(
         "_host_vitals.html",
         reg=reg, cpu=cpu, mem=mem, disk_root=disk_root, load_per_core=load_per_core,
         sparks=sparks, psi=psi, reporting=reporting, stale=stale, worst=worst,
+        grades=grades, usual=usual,
         clock=clock_words(reg.clock_offset_s if reg else None, current_app.config.get("THRESHOLDS", {})),
     )
 

plugins/host_agent/templates/_host_vitals.html

@@ -1,6 +1,7 @@
 {# Host health header — live polled into the host hub: live or stale, the worst
    reading against Settings → Thresholds, last seen, then CPU / memory / disk /
-   load tiles graded by the same cutoffs. Renders nothing until the agent
+   load tiles graded by the same cutoffs (`grades`; CPU, memory and load within
+   the host's own normal for the hour read healthy, and `usual` says so). Renders nothing until the agent
    reports (the Agent panel below then shows provisioning). #}
 {% from "_macros.html" import status_badge, stat_tile %}
 {% macro pct(v) %}{{ '%.0f%%'|format(v) if v is not none else '—' }}{% endmacro %}
@@ -11,6 +12,7 @@
     <span class="host-health-summary">
       {%- if stale %}No report since {{ reg.last_seen_at|ago }}; the readings below are from then.
       {%- elif worst %}<span class="tone-{{ worst[2] }}">{{ worst[0] }} {{ pct(worst[1]) }}</span> is past its {{ "critical" if worst[2] == "crit" else "warning" }} threshold.
+      {%- elif usual %}Every reading is within its thresholds or its usual level for this hour.
       {%- else %}Every reading is within its thresholds.{% endif -%}
     </span>
     <span class="problem-detail">
@@ -20,11 +22,14 @@
     </span>
   </div>
   <div class="stat-strip host-health-tiles" style="margin-bottom:0;">
-    {{ stat_tile("CPU", pct(cpu), state=status_class(cpu, 'cpu'), hint=sparks.cpu|safe) }}
-    {{ stat_tile("Memory", pct(mem), state=status_class(mem, 'mem'), hint=sparks.mem|safe) }}
-    {{ stat_tile("Disk /", pct(disk_root), state=status_class(disk_root, 'disk'), hint=sparks.disk|safe) }}
-    {{ stat_tile("Load per core", pct(load_per_core), state=status_class(load_per_core, 'load'), hint=sparks.load|safe) }}
+    {{ stat_tile("CPU", pct(cpu), state=grades.cpu[0], hint=sparks.cpu|safe) }}
+    {{ stat_tile("Memory", pct(mem), state=grades.mem[0], hint=sparks.mem|safe) }}
+    {{ stat_tile("Disk /", pct(disk_root), state=grades.disk[0], hint=sparks.disk|safe) }}
+    {{ stat_tile("Load per core", pct(load_per_core), state=grades.load[0], hint=sparks.load|safe) }}
   </div>
+  {% if usual and not stale %}
+  <p class="field-hint" style="margin-top:0.6rem;">{{ usual | join("; ") | capitalize }}.</p>
+  {% endif %}
   {% if psi.cpu is not none or psi.mem is not none or psi.io is not none %}
   <p class="field-hint" style="margin-top:0.6rem;" title="Pressure stall: the share of the last 10 s that tasks waited for the resource">
     Pressure (10 s): cpu {{ pct(psi.cpu) }} · mem {{ pct(psi.mem) }} · io {{ pct(psi.io) }}

plugins/traefik/__init__.py

@@ -26,12 +26,12 @@ def setup(app: "Quart") -> None:
     # learned from the scrape-by-scrape figures only: before #5755 it was a
     # percentile of every response since Traefik started.
     from datetime import datetime, timezone
-    from steward.core.norms import learn_normal
-    from .routes import FLOOR_4XX, FLOOR_P95, STATE_WINDOW, min_requests
+    from steward.core.norms import floor_setting, learn_normal
+    from .routes import STATE_WINDOW, min_requests
     gate = ("request_rate",
             lambda: min_requests(app.config.get("THRESHOLDS", {})) / STATE_WINDOW.total_seconds())
-    learn_normal("traefik", "error_rate_4xx_pct", FLOOR_4XX, gate=gate)
-    learn_normal("traefik", "latency_p95_ms", FLOOR_P95, gate=gate,
+    learn_normal("traefik", "error_rate_4xx_pct", floor_setting(app, "router_4xx_pct"), gate=gate)
+    learn_normal("traefik", "latency_p95_ms", floor_setting(app, "router_p95_ms"), gate=gate,
                  since=datetime(2026, 10, 14, 18, 47, tzinfo=timezone.utc))
 
 

plugins/traefik/routes.py

@@ -14,6 +14,7 @@ from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds,
 from steward.core import anomalies as anom
 from steward.core.formatting import cert_left, status_class
 from steward.core.norms import norms_at
+from steward.core.settings import floor_of
 from steward.models.metrics import PluginMetricLatest
 from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
 from .scheduler import router_domains
@@ -56,12 +57,11 @@ def error_cutoffs(thresholds: dict) -> tuple[float, float]:
 
 # Each router's 4xx share and p95 are judged against its own normal once it
 # has one (#5741, System 182): an error-pages service answers 4xx all day and
-# that is its normal, not a problem. Departing by less than the floor is never
-# news. A share over fewer than min_requests() in the window is not graded: one
-# 404 of three requests is 33%. The 5xx share is a failure and keeps its
-# fixed cutoff whatever the history or traffic (the operator's ruling).
-FLOOR_4XX = 2.0           # percentage points
-FLOOR_P95 = 50.0          # ms
+# that is its normal, not a problem. Departing by less than the floor
+# (Settings → Thresholds, norms.floor.router_*) is never news. A share over
+# fewer than min_requests() in the window is not graded: one 404 of three
+# requests is 33%. The 5xx share is a failure and keeps its fixed cutoff
+# whatever the history or traffic (the operator's ruling).
 NORMED = ("error_rate_4xx_pct", "latency_p95_ms")
 
 
@@ -71,14 +71,6 @@ def min_requests(thresholds: dict) -> float:
     return float(thresholds.get("router_requests", {}).get("warn", 30))
 
 
-def _against(value: float, norm, floor: float, unit: str) -> tuple[bool, str]:
-    """(departs, what is usual) for a reading over its fixed cutoff."""
-    if norm is None:
-        return True, ""
-    usual = f"usual {norm.usual_hold:.1f}{unit}" if unit == "%" else f"usual {norm.usual_hold:.0f}{unit}"
-    return anom.departs(value, norm, floor), f"{usual} at this hour"
-
-
 def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = None) -> tuple[str, str | None]:
     """(grade, why) for a router's reading (recent_window's, or one scrape's):
     crit while it serves 5xx past its cutoff, warn on a p95 past the latency
@@ -93,14 +85,16 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None =
     usual = []
     p95 = reading.latency_p95_ms
     if p95 is not None and status_class(p95, "latency", thresholds) == "crit":
-        out, said = _against(p95, norms.get("latency_p95_ms"), FLOOR_P95, " ms")
+        out, said = anom.against(p95, norms.get("latency_p95_ms"),
+                                  floor_of(thresholds, "router_p95_ms"), " ms")
         if out:
             return "warn", f"p95 latency is {p95:.0f} ms{over}" + (f", {said}" if said else "")
         usual.append(f"p95 {p95:.0f} ms is {said}")
     requests = getattr(reading, "requests", None)
     e4 = reading.error_rate_4xx_pct or 0
     if e4 > error_4xx and (requests is None or requests >= min_requests(thresholds)):
-        out, said = _against(e4, norms.get("error_rate_4xx_pct"), FLOOR_4XX, "%")
+        out, said = anom.against(e4, norms.get("error_rate_4xx_pct"),
+                                  floor_of(thresholds, "router_4xx_pct"), "%")
         if out:
             return "warn", f"{e4:.1f}% of responses are 4xx{over}" + (f", {said}" if said else "")
         usual.append(f"4xx {e4:.1f}% is {said}")