feat(norms): hosts, containers and monitor response times are graded against their own normal (#5742)

Host CPU, memory and load per core, container CPU and memory, and every
monitor's response time learn a normal per hour (milestone 513, System 182).
A reading past its fixed cutoff but within its normal reads healthy and says
what is usual at this hour; one well out of it keeps its grade and is a
metric anomaly. Disk, a monitor down and an agent offline keep their cutoffs.

- Each normal's floor, the smallest departure that counts, is a setting:
  Settings -> Thresholds -> Normals (norms.floor.*). Defaults: host CPU 10
  points, host memory 5, load per core 10, container CPU 10 points of one
  core, container memory 5, monitor response 100 ms; Traefik's 2-point 4xx
  and 50 ms p95 floors move there from constants. learn_normal takes a
  floor function (floor_setting) read at each pass.
- learn_normal(parts=False): a host learns, its cores don't.
- Ingest records load_pct (load_1m / cores x 100), the figure pages grade.
- Shared grading: anomalies.against / graded, worst_reading(normals=),
  hosts' host_grades, docker usage(thresholds=, norms=), monitors'
  latency_grade. Applied to host vitals, the Hosts widget, MCP overview,
  Docker usage, and the monitor list, widget and page.
- The monitor and container pages list their anomalies.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/routes.py

@@ -12,6 +12,7 @@ from sqlalchemy import func, select
 from steward.auth.middleware import require_role
 from steward.models.users import UserRole
 from steward.models.hosts import Host
+from steward.core import anomalies as anom
 from steward.core.formatting import human_bytes, human_duration, seconds_since
 from steward.core.time_range import parse_range, DEFAULT_RANGE
 from . import service as docker_svc
@@ -159,16 +160,20 @@ async def rows():
         host_name = {h.id: h.name for h in hosts.values()}
         swarm_view = ctx["swarm_view"]
         capacity = await docker_svc.host_capacity(db, {c.host_id for c in containers})
+        norms = await docker_svc.container_norms(
+            db, {(c.host_id, c.name) for c in containers if not c.service_name}, now)
         failures = docker_svc.failures_24h(
             await docker_svc.failure_events(db, now - timedelta(hours=24)),
             swarm_view["services"], host_name)
 
+    thresholds = current_app.config.get("THRESHOLDS", {})
     service_rows = [service_row(s, now) for s in swarm_view["services"]]
     standalone = [c for c in containers if not c.service_name]
     container_rows = [
         container_row(
             c, label=docker_svc.display_name(c), host=host_name.get(c.host_id, c.host_id),
-            usage=docker_svc.usage(c, *capacity.get(c.host_id, (None, None))),
+            usage=docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), thresholds=thresholds,
+                                   norms=norms.get((c.host_id, c.name))),
             ports=json.loads(c.ports_json) if c.ports_json else [],
             stale=_is_stale(c, now, stale_after, docker_down),
             exit_failed=docker_svc.exit_is_failure(c.exit_code, c.oom_killed), now=now)
@@ -314,6 +319,8 @@ async def widget_resources():
             db, {(c.host_id, c.name) for c in busiest}, slots[0], width)
         hosts = await docker_svc.host_map(db, {c.host_id for c in busiest})
         capacity = await docker_svc.host_capacity(db, {c.host_id for c in busiest})
+        norms = await docker_svc.container_norms(db, {(c.host_id, c.name) for c in busiest})
+    thresholds = current_app.config.get("THRESHOLDS", {})
 
     def bars(c, i, scale, floor):
         seen = history.get((c.host_id, c.name), {})
@@ -325,7 +332,8 @@ async def widget_resources():
         "c": c,
         "label": docker_svc.display_name(c),
         "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id,
-        "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c)),
+        "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c),
+                                  thresholds=thresholds, norms=norms.get((c.host_id, c.name))),
         "cpu_bars": bars(c, 0, 0.01, cpu_floor),            # cpu_pct → cores
         "mem_bars": bars(c, 1, 1.0, mem_floor * 2**30),     # bytes; floor in GiB
     } for c in busiest]
@@ -365,7 +373,9 @@ async def host_panel(host_id: str):
         nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
         failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
         capacity = await docker_svc.host_capacity(db, {host_id})
+        norms = await docker_svc.container_norms(db, {(c.host_id, c.name) for c in containers}, now)
 
+    thresholds = current_app.config.get("THRESHOLDS", {})
     replaced = replaced_tasks(containers, services)
     node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
     hour_ago = now - timedelta(hours=1)
@@ -380,7 +390,8 @@ async def host_panel(host_id: str):
         rows.append({
             "c": c, "label": docker_svc.task_label(c, node_hostname), "problems": problems,
             "since": docker_svc.problem_since(problems, events, hour_ago),
-            "usage": docker_svc.usage(c, *capacity.get(host_id, (None, None))),
+            "usage": docker_svc.usage(c, *capacity.get(host_id, (None, None)), thresholds=thresholds,
+                                      norms=norms.get((c.host_id, c.name))),
             "uptime": _uptime(c.started_at) if c.status == "running" else None,
         })
     shown, folded = host_panel_rows(rows)
@@ -445,6 +456,13 @@ async def container_detail(host_id: str, name: str):
             )).scalar()
         cpus, mem_total = (await docker_svc.host_capacity(db, [host_id])).get(host_id, (None, None)) \
             if c is not None else (None, None)
+        # Its CPU and memory against their normal, and its anomalies (#5742).
+        norms = (await docker_svc.container_norms(db, {(host_id, name)})).get((host_id, name))
+        resource = docker_svc.container_resource(host.name, name) if host is not None else None
+        anomalies = await anom.views(db, await anom.list_anomalies(
+            db, source="docker", resource=resource,
+            since=datetime.now(timezone.utc) - timedelta(days=30), limit=20),
+            links=False) if resource else []
 
     if c is None:
         return await render_template(
@@ -460,7 +478,10 @@ async def container_detail(host_id: str, name: str):
         name=name,
         label=docker_svc.task_label(c, {c.node_id: node_name} if node_name else None),
         node_name=node_name,
-        usage=docker_svc.usage(c, cpus, mem_total),
+        usage=docker_svc.usage(c, cpus, mem_total, thresholds=current_app.config.get("THRESHOLDS", {}),
+                               norms=norms),
+        anomalies=anomalies, anomaly_back=f"/plugins/docker/container/{host_id}/{name}",
+        anomaly_resource=resource,
         exit_failed=docker_svc.exit_is_failure(c.exit_code, c.oom_killed),
         ports=json.loads(c.ports_json) if c.ports_json else [],
         uptime=_uptime(c.started_at) if c.status == "running" else None,
@@ -888,6 +909,8 @@ async def service_detail(name: str):
         s = next((s for s in ctx["swarm_view"]["services"] if s["name"] == name), None)
         capacity = await docker_svc.host_capacity(
             db, {r["host_id"] for r in (s["replicas"] if s else []) if not r["ghost"]})
+        norms = await docker_svc.container_norms(
+            db, {(r["host_id"], r["name"]) for r in (s["replicas"] if s else []) if not r["ghost"]}, now)
     if s is None and not task_containers:
         return await render_template("docker/service.html", name=name, service=None), 404
 
@@ -899,7 +922,9 @@ async def service_detail(name: str):
     for r in (s["replicas"] if s else []):
         c = None if r["ghost"] else by_key.get((r["host_id"], r["name"]))
         replicas.append(dict(r, task_id=getattr(c, "task_id", None) if c is not None else None,
-                             usage=docker_svc.usage(c, *capacity.get(r["host_id"], (None, None)))
+                             usage=docker_svc.usage(c, *capacity.get(r["host_id"], (None, None)),
+                                                    thresholds=current_app.config.get("THRESHOLDS", {}),
+                                                    norms=norms.get((c.host_id, c.name)))
                              if c is not None else None))
     node_host = {c.node_id: c.host_id for c in ctx["containers"] if c.node_id}
     table = _service_tasks(name, tasks, replicas, node_host, ctx["host_name"])