feat(docker): readable names, CPU in cores and memory in bytes, failures listed with since-times (#5209)

#5209 "Docker views: readable names, CPU/memory in cores and bytes, failures
listed with since-time" (milestone 477 "UI problems first").

- A swarm task reads as its service and slot ("codex_db · 1", or the node
  for a global service), with the raw name in the title: Docker page, host
  panel, busiest widget, container detail.
- CPU reads in cores (Docker's 100% = one core), toned by its share of the
  host's CPUs from the agent's per-core series. Memory reads in bytes, against
  a limit only when one is really set (Docker reports host RAM otherwise).
- Exit 137/143 without an OOM kill is a stop, not a failure: no
  exited_nonzero, no crash-loop death, no "failed (24h)", dim exit code.
- "Failed (24h)" is a list (Docker page #failures, linked from the widget
  and the tile): plain containers by their deaths, swarm services by Swarm's
  failed tasks; a second manager's copy of a task death counts once.
- Failing swarm services lead Needs attention; services sort problems first
  with since-times on the page, the widget and the Swarm page; nodes that
  are down or drained sort first.
- Busiest widget orders by a 5-minute CPU average so the list holds still.
- Host panel: a table, problems first with what is wrong and since when;
  replaced swarm tasks hidden and counted.
- Container detail: node hostname, I/O labelled "since start", Restarts
  hidden for swarm tasks with a note on where task failures are.
- Widget names: "Docker — Services & failures", "Docker — Busiest containers".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/mcp_tools.py

@@ -194,7 +194,8 @@ async def list_containers(host: str = "", status: str = "", problem: str = "",
       "restarting", "crash_looping" (died repeatedly within the last hour;
       the count is set under Settings → Thresholds, default 3), "unhealthy"
       (failing its healthcheck), "oom_killed", "exited_nonzero" (died with an
-      error in the last 24h); empty for no filter.
+      error in the last 24h; exit 137/143 without an OOM kill is a stop, not
+      an error); empty for no filter.
     compose_project: only containers from this Compose project.
     limit: default 100, max 500.
 

plugins/docker/routes.py

@@ -16,9 +16,12 @@ from steward.core.formatting import human_bytes, human_duration, seconds_since
 from steward.core.time_range import parse_range, DEFAULT_RANGE
 from . import service as docker_svc
 from .dedup import dedup_by_container_id
-from .swarm_view import build_swarm_services, parse_tasks, task_summary
+from .swarm_view import (
+    build_swarm_services, failing_services, parse_tasks, replaced_tasks, services_problems_first,
+    task_summary, trouble_since,
+)
 from .models import (
-    DockerContainer, DockerDiskUsage, DockerEvent, DockerHostStatus, DockerImage,
+    DockerContainer, DockerDiskUsage, DockerHostStatus, DockerImage, DockerMetric,
     DockerSwarmNode, DockerSwarmService,
 )
 
@@ -109,6 +112,10 @@ async def rows():
         swarm_view = build_swarm_services(
             containers, services, node_hostname, host_name,
             crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))
+        capacity = await docker_svc.host_capacity(db, {c.host_id for c in containers})
+        failures = docker_svc.failures_24h(
+            await docker_svc.failure_events(db, now - timedelta(hours=24)),
+            swarm_view["services"], host_name)
 
         standalone = [c for c in containers if not c.service_name]
         groups: dict[str, dict] = {}
@@ -127,6 +134,8 @@ async def rows():
                 "ports": json.loads(c.ports_json) if c.ports_json else [],
                 "uptime": _uptime(c.started_at) if c.status == "running" else None,
                 "stale": _is_stale(c, now, stale_after, docker_down),
+                "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None))),
+                "exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed),
             })
             if c.status == "running":
                 g["running"] += 1
@@ -151,15 +160,20 @@ async def rows():
         g["grouped"] = any(s["label"] for s in g["subgroups"])
 
     host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
-    swarm_services = swarm_view["services"]
+    swarm_services = services_problems_first(swarm_view["services"])
     standalone_running = sum(g["running"] for g in host_groups)
-    ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"])
-    attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down))
+    attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down),
+                      label=docker_svc.task_label(r["container"], node_hostname))
                  for r in docker_svc.needs_attention(ranked)]
+    failing = failing_services(swarm_services)
     return await render_template(
         "docker/rows.html",
         host_groups=host_groups,
         swarm_services=swarm_services,
+        failing_services=failing,
+        failures=failures,
+        services_healthy=sum(s["state"] == "healthy" for s in swarm_services),
+        docker_hosts=len({c.host_id for c in containers}),
         docker_down=sorted(
             ({"host_id": h, "host_name": hosts[h].name if h in hosts else h, "status": d.status,
               "detail": d.detail, "since": d.since} for h, d in docker_down.items()),
@@ -168,9 +182,8 @@ async def rows():
         unhealthy=sum("unhealthy" in r["problems"] for r in attention),
         restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention),
         stale=sum(1 for g in host_groups for i in g["containers"] if i["stale"]),
-        running=standalone_running + sum(s["running"] for s in swarm_services),
+        running=standalone_running,
         stopped=len(standalone) - standalone_running,
-        total=len(containers) + ghost_total,
     )
 
 
@@ -185,55 +198,55 @@ def _is_stale(c, now: datetime, stale_after: int, docker_down=()) -> bool:
 @docker_bp.get("/widget")
 @require_role(UserRole.viewer)
 async def widget():
-    """HTMX dashboard widget: container status overview, grouped by host."""
+    """HTMX dashboard widget: swarm services and containers, problems first,
+    with how many things failed in the last 24h (linked to the list)."""
     show_stopped = request.args.get("show_stopped", "no") == "yes"
     widget_id = request.args.get("wid", "0")
-    since_24h = datetime.now(timezone.utc) - timedelta(hours=24)
+    now = datetime.now(timezone.utc)
 
     async with current_app.db_sessionmaker() as db:
-        all_containers = dedup_by_container_id(list((await db.execute(
-            select(DockerContainer).order_by(
-                (DockerContainer.status == "running").desc(),
-                DockerContainer.name,
-            )
-        )).scalars()))
+        all_containers = dedup_by_container_id(await docker_svc.list_containers(db))
         services = list((await db.execute(select(DockerSwarmService))).scalars())
         nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
-        # Recent failures are more actionable than a static "stopped" tally. Read
-        # DISTINCT container names so the two managers' duplicate die/oom events
-        # don't double it; swarm tasks are counted on their service below.
-        dead_names = list((await db.execute(
-            select(func.distinct(DockerEvent.container_name))
-            .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since_24h)
-        )).scalars())
+        events = await docker_svc.failure_events(db, now - timedelta(hours=24))
         hosts = await docker_svc.host_map(
             db,
             {c.host_id for c in all_containers}
             | {n.host_id for n in nodes} | {s.host_id for s in services},
         )
+        capacity = await docker_svc.host_capacity(db, {c.host_id for c in all_containers})
 
     # Swarm tasks collapse into per-service rows (each replica tagged with its
     # host); non-swarm containers stay grouped by host.
     node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
     host_name = {h.id: h.name for h in hosts.values()}
-    swarm_services = build_swarm_services(
+    swarm_services = services_problems_first(build_swarm_services(
         all_containers, services, node_hostname, host_name,
-        crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"]
-    failed_24h = docker_svc.failed_24h(dead_names, swarm_services)
+        crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"])
+    failures = docker_svc.failures_24h(events, swarm_services, host_name)
 
     standalone = [c for c in all_containers if not c.service_name]
     running = [c for c in standalone if c.status == "running"]
     display = standalone if show_stopped else running
 
+    def trouble(c) -> int:
+        if (c.status or "") == "restarting" or c.health == "unhealthy":
+            return 0
+        if c.status != "running" and docker_svc.exit_is_failure(c.exit_code, c.oom_killed):
+            return 1
+        return 2 if c.status == "running" else 3
+
     # Group for display so multi-host fleets read clearly; single-host stays flat.
     groups: dict[str, dict] = {}
-    for c in display:
+    for c in sorted(display, key=lambda c: (trouble(c), c.name)):
         g = groups.setdefault(c.host_id, {
             "host_id": c.host_id,
             "host_name": host_name.get(c.host_id, c.host_id),
             "containers": [],
         })
-        g["containers"].append(c)
+        g["containers"].append({"c": c, "usage": docker_svc.usage(
+            c, *capacity.get(c.host_id, (None, None))),
+            "exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed)})
     host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
 
     return await render_template(
@@ -241,39 +254,59 @@ async def widget():
         host_groups=host_groups,
         multi_host=len(host_groups) > 1,
         swarm_services=swarm_services,
-        running_count=len(running) + sum(s["running"] for s in swarm_services),
-        failed_24h=failed_24h,
+        services_healthy=sum(s["state"] == "healthy" for s in swarm_services),
+        running_count=len(running),
+        failures=failures,
         total_count=len(all_containers) + len(swarm_services),
         show_stopped=show_stopped,
         widget_id=widget_id,
     )
 
 
+CPU_AVERAGE_MINUTES = 5
+
+
 @docker_bp.get("/widget/resources")
 @require_role(UserRole.viewer)
 async def widget_resources():
-    """HTMX dashboard widget: CPU + memory usage for the busiest containers."""
+    """HTMX dashboard widget: the containers using the most CPU, averaged over
+    the last few minutes so the order holds still between polls, with their
+    CPU in cores and memory in bytes."""
     limit = max(1, min(20, int(request.args.get("limit", 10) or 10)))
     widget_id = request.args.get("wid", "0")
+    since = datetime.now(timezone.utc) - timedelta(minutes=CPU_AVERAGE_MINUTES)
 
     async with current_app.db_sessionmaker() as db:
         # Dedup before the limit, else swarm tasks reported by both managers can
         # fill the list with duplicates of the same container.
-        containers = dedup_by_container_id(list((await db.execute(
-            select(DockerContainer)
-            .where(DockerContainer.status == "running")
-            .order_by(DockerContainer.cpu_pct.desc().nullslast())
-        )).scalars()))[:limit]
-        hosts = await docker_svc.host_map(db, {c.host_id for c in containers})
+        running = dedup_by_container_id([
+            c for c in await docker_svc.list_containers(db) if c.status == "running"])
+        average = {(h, n): v for h, n, v in (await db.execute(
+            select(DockerMetric.host_id, DockerMetric.container_name, func.avg(DockerMetric.cpu_pct))
+            .where(DockerMetric.scraped_at >= since)
+            .group_by(DockerMetric.host_id, DockerMetric.container_name)
+        )).all()}
+        nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
+        hosts = await docker_svc.host_map(db, {c.host_id for c in running})
+        capacity = await docker_svc.host_capacity(db, {c.host_id for c in running})
 
-    rows_data = [
-        {"c": c, "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id}
-        for c in containers
-    ]
+    def cpu_of(c):
+        v = average.get((c.host_id, c.name))
+        return c.cpu_pct if v is None else float(v)
+
+    busiest = sorted(running, key=lambda c: (-(cpu_of(c) or 0), c.name))[:limit]
+    node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
+    rows_data = [{
+        "c": c,
+        "label": docker_svc.task_label(c, node_hostname),
+        "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id,
+        "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c)),
+    } for c in busiest]
     return await render_template(
         "docker/widget_resources.html",
         rows=rows_data,
-        multi_host=len({c.host_id for c in containers}) > 1,
+        minutes=CPU_AVERAGE_MINUTES,
+        multi_host=len({c.host_id for c in busiest}) > 1,
         widget_id=widget_id,
     )
 
@@ -281,22 +314,50 @@ async def widget_resources():
 @docker_bp.get("/host/<host_id>")
 @require_role(UserRole.viewer)
 async def host_panel(host_id: str):
-    """Per-host Docker fragment embedded on the core Hosts hub via HTMX.
+    """Per-host Docker fragment embedded on the core Hosts hub via HTMX:
+    problems first (with exit code and since when), then running, then
+    stopped. Swarm tasks Swarm has already replaced are history and left out.
 
     Renders nothing when the host reports no containers, so hosts without Docker
     don't carry an empty card.
     """
+    now = datetime.now(timezone.utc)
     async with current_app.db_sessionmaker() as db:
         containers = await docker_svc.list_containers(db, host_id)
+        if not containers:
+            return ""
+        services = list((await db.execute(select(DockerSwarmService))).scalars())
+        nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
+        failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
+        capacity = await docker_svc.host_capacity(db, {host_id})
 
-    if not containers:
-        return ""
-    running = sum(1 for c in containers if c.status == "running")
+    replaced = replaced_tasks(containers, services)
+    node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
+    hour_ago = now - timedelta(hours=1)
+    crash = docker_svc.crash_loop_deaths(current_app.config)
+    rows = []
+    for c in containers:
+        if (c.host_id, c.name) in replaced:
+            continue
+        events = failures.get((c.host_id, c.name), [])
+        recent = sum(1 for e in events if e.at >= hour_ago)
+        problems = docker_svc.container_problems(c, recent, bool(events), crash)
+        rows.append({
+            "c": c, "label": docker_svc.task_label(c, node_hostname), "problems": problems,
+            "since": docker_svc.problem_since(problems, events, hour_ago),
+            "usage": docker_svc.usage(c, *capacity.get(host_id, (None, None))),
+            "uptime": _uptime(c.started_at) if c.status == "running" else None,
+        })
+    rows.sort(key=lambda r: (
+        docker_svc.PROBLEMS.index(r["problems"][0]) if r["problems"] else len(docker_svc.PROBLEMS),
+        r["c"].status != "running", r["label"].lower()))
+    running = sum(1 for r in rows if r["c"].status == "running")
     return await render_template(
         "docker/host_panel.html",
-        containers=containers,
-        running=running,
-        stopped=len(containers) - running,
+        rows=rows, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
+        running=running, stopped=len(rows) - running,
+        problems=sum(1 for r in rows if r["problems"]),
+        replaced=len(replaced),
     )
 
 
@@ -319,6 +380,15 @@ async def container_detail(host_id: str, name: str):
         c = await db.get(DockerContainer, (host_id, name))
         host = await db.get(Host, host_id)
         events = await docker_svc.container_events(db, host_id, name) if c is not None else []
+        node_name = None
+        if c is not None and c.node_id:
+            node_name = (await db.execute(
+                select(DockerSwarmNode.hostname)
+                .where(DockerSwarmNode.node_id == c.node_id, DockerSwarmNode.hostname != "")
+                .limit(1)
+            )).scalar()
+        cpus, mem_total = (await docker_svc.host_capacity(db, [host_id])).get(host_id, (None, None)) \
+            if c is not None else (None, None)
 
     if c is None:
         return await render_template(
@@ -332,6 +402,10 @@ async def container_detail(host_id: str, name: str):
         host=host,
         host_id=host_id,
         name=name,
+        label=docker_svc.task_label(c, {c.node_id: node_name} if node_name else None),
+        node_name=node_name,
+        usage=docker_svc.usage(c, cpus, mem_total),
+        exit_failed=docker_svc.exit_is_failure(c.exit_code, c.oom_killed),
         ports=json.loads(c.ports_json) if c.ports_json else [],
         uptime=_uptime(c.started_at) if c.status == "running" else None,
         net_rx=human_bytes(c.net_rx_bytes), net_tx=human_bytes(c.net_tx_bytes),
@@ -415,7 +489,11 @@ async def swarm():
         now = datetime.now(timezone.utc)
         deaths = docker_svc.crash_loop_deaths(current_app.config)
         svc_dicts = []
-        for s in sorted(svc_by_name.values(), key=lambda s: s.service_name):
+        for s in svc_by_name.values():
+            tasks = parse_tasks(s)
+            summary = task_summary(tasks, now, deaths, node_names)
+            state = ("healthy" if s.running >= s.desired and s.desired > 0
+                     else "degraded" if s.running > 0 else "down")
             placement = []
             for p in (json.loads(s.placement_json) if s.placement_json else []):
                 nid = p.get("node_id", "")
@@ -429,18 +507,21 @@ async def swarm():
                 "healthy": s.running >= s.desired and s.desired > 0,
                 "degraded": 0 < s.running < s.desired,
                 "down": s.running == 0 and s.desired > 0,
-                "state": ("healthy" if s.running >= s.desired and s.desired > 0
-                          else "degraded" if s.running > 0 else "down"),
+                "state": state,
                 "placement": placement,
-                **task_summary(parse_tasks(s), now, deaths, node_names),
+                **summary,
+                "trouble_since": trouble_since(tasks, state, summary["crash_looping"], now),
             })
+        # Problems first: a node that is down or not taking work, then by role.
         node_rows = sorted(
-            node_by_id.values(), key=lambda n: (n.role, (n.hostname or n.node_id).lower())
+            node_by_id.values(), key=lambda n: (
+                n.status == "ready" and n.availability == "active",
+                n.role, (n.hostname or n.node_id).lower())
         )
         manager_names = sorted(hosts[m].name if m in hosts else m for m in members)
         swarms.append({
             "managers": manager_names,
-            "services": svc_dicts,
+            "services": services_problems_first(svc_dicts),
             "nodes": node_rows,
         })
 

plugins/docker/service.py

@@ -6,7 +6,7 @@ Quart's pipeline) call the same code.
 """
 from __future__ import annotations
 
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 from sqlalchemy import func, select
 
@@ -30,25 +30,159 @@ def crash_loop_deaths(config) -> int:
     return int(th.get("warn") or CRASH_LOOP_DEATHS)
 
 
-def failed_24h(dead_names, swarm_services) -> int:
-    """Failures in the last 24h for the dashboard: distinct plain containers
-    that died or were OOM-killed, plus each swarm service's failed tasks.
+# Exit codes a deliberate stop leaves: SIGTERM (143), and SIGKILL (137) when
+# the process outlives the stop timeout. Without an OOM kill they are someone
+# stopping the container, not the container failing.
+STOP_EXITS = (137, 143)
 
-    A swarm task's container dies whenever Swarm replaces it (a rolling update
-    kills it with 137), so its die event is not a failure. Services with
-    Swarm's task records count their failed tasks instead; a service without
-    them (an agent older than 1.8.0) falls back to its tasks' die events.
+
+def exit_is_failure(exit_code, oom_killed: bool = False) -> bool:
+    """Whether a container's exit is a failure: an OOM kill, or a non-zero
+    exit other than a stop's."""
+    return bool(oom_killed) or exit_code not in (None, 0, *STOP_EXITS)
+
+
+def _event_exit(detail) -> int | None:
+    """The exit code in a die event's detail ("exit 1"), or None."""
+    text = (detail or "").strip()
+    if text.startswith("exit "):
+        try:
+            return int(text[5:])
+        except ValueError:
+            return None
+    return None
+
+
+def event_is_failure(e) -> bool:
+    """Whether a die/oom event is a failure. An oom always is; a die is unless
+    its exit is a stop's (or clean)."""
+    if e.event == "oom":
+        return True
+    return e.event == "die" and exit_is_failure(_event_exit(e.detail))
+
+
+def failures_24h(events, swarm_services, host_name: dict) -> list[dict]:
+    """What failed in the last 24h, newest first, for the dashboard count and
+    the Docker page's list of it: one row per plain container that died or was
+    OOM-killed, and one per swarm service with failed tasks.
+
+    `events` are die/oom events of the last 24h. A swarm task's container dies
+    whenever Swarm replaces it, so a service with Swarm's task records counts
+    its failed tasks instead of its containers' deaths; a service without them
+    (an agent older than 1.8.0) falls back to its tasks' deaths.
+
+    Each row: {kind: "container"|"service", name, host_id, host, count, at,
+    detail}.
     """
     by_prefix = {s["name"] + ".": s for s in swarm_services}
-    plain, fallback = 0, set()
-    for name in dead_names:
-        svc = next((s for p, s in by_prefix.items() if name.startswith(p)), None)
-        if svc is None:
-            plain += 1
-        elif not svc["tasks_known"]:
-            fallback.add(name)
-    tasks = sum(s["failures_24h"] or 0 for s in swarm_services if s["tasks_known"])
-    return plain + len(fallback) + tasks
+    rows: dict[tuple, dict] = {}
+    # Each manager reports every swarm task, so one task's death can arrive
+    # once per manager: a service counts each dead task container once.
+    task_deaths: dict[tuple, set] = {}
+    for e in sorted(events, key=lambda e: e.at, reverse=True):
+        if not event_is_failure(e):
+            continue
+        svc = next((s for p, s in by_prefix.items() if e.container_name.startswith(p)), None)
+        if svc is not None and svc["tasks_known"]:
+            continue
+        kind, name = ("service", svc["name"]) if svc else ("container", e.container_name)
+        key = (kind, name) if svc else (kind, e.host_id, name)
+        if svc is not None:
+            seen = task_deaths.setdefault(key, set())
+            if e.container_name in seen:
+                continue
+            seen.add(e.container_name)
+        row = rows.get(key)
+        if row is None:
+            rows[key] = {"kind": kind, "name": name, "host_id": e.host_id,
+                         "host": host_name.get(e.host_id, e.host_id), "count": 1, "at": e.at,
+                         "detail": "OOM-killed" if e.event == "oom" else e.detail}
+        else:
+            row["count"] += 1
+    from .swarm_view import task_time
+    for s in swarm_services:
+        if s["tasks_known"] and s["failures_24h"]:
+            err = s["last_error"] or {}
+            rows[("service", s["name"])] = {
+                "kind": "service", "name": s["name"], "host_id": None, "host": err.get("node"),
+                "count": s["failures_24h"], "at": task_time(err) if err else None,
+                "detail": err.get("error") or err.get("message") or err.get("state")}
+    epoch = datetime.min.replace(tzinfo=timezone.utc)
+    return sorted(rows.values(), key=lambda r: r["at"] or epoch, reverse=True)
+
+
+def task_label(c, node_hostname: dict | None = None) -> str:
+    """How a container reads to a person. A swarm task is its service and
+    slot: "codex_db · 1" for a replica, "agent · node-3" for a global
+    service's task on a node. Anything else is its own name."""
+    from .swarm_view import _slot_key
+    svc = getattr(c, "service_name", None)
+    if not svc:
+        return c.name
+    slot = _slot_key(c)
+    if slot.isdigit():
+        return f"{svc} · {slot}"
+    host = (node_hostname or {}).get(slot) or (node_hostname or {}).get(getattr(c, "node_id", None))
+    return f"{svc} · {host}" if host else svc
+
+
+def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = None) -> dict:
+    """A container's CPU and memory in terms a reader can weigh.
+
+    Docker's CPU% is per core (100% = one core busy), so it is shown as cores
+    and graded as a share of the host's CPUs when the host's core count is
+    known. Docker's memory limit is the host's RAM when none is set, so memory
+    is shown in bytes, against a limit only when one is actually set; mem_pct
+    is then a share of that limit, otherwise of the host's memory. `cpu_pct`
+    replaces the container's last sample (an average, say) when given.
+    """
+    cpu = getattr(c, "cpu_pct", None) if cpu_pct is None else cpu_pct
+    limit = getattr(c, "mem_limit_bytes", None)
+    limited = bool(limit and mem_total and limit < mem_total * 0.95)
+    return {
+        "cores": None if cpu is None else cpu / 100.0,
+        "cpu_share": None if cpu is None or not cpus else cpu / cpus,
+        "cpus": cpus,
+        "mem_bytes": getattr(c, "mem_usage_bytes", None),
+        "mem_limit": limit if limited else None,
+        "mem_pct": getattr(c, "mem_pct", None),
+        "mem_of": "its limit" if limited else ("the host's memory" if mem_total else None),
+    }
+
+
+async def host_capacity(db, host_ids) -> dict[str, tuple[int | None, int | None]]:
+    """{host_id: (CPU cores, memory bytes)} from the host agent's latest
+    readings: cores counted from its per-core CPU series, memory from
+    available ÷ (1 − used%). (None, None) for a host without an agent."""
+    from sqlalchemy import or_
+    from steward.models.metrics import PluginMetricLatest
+    hosts = await host_map(db, set(host_ids))
+    if not hosts:
+        return {}
+    names = {h.name: hid for hid, h in hosts.items()}
+    rows = (await db.execute(
+        select(PluginMetricLatest.resource_name, PluginMetricLatest.metric_name,
+               PluginMetricLatest.value)
+        .where(PluginMetricLatest.source_module == "host_agent",
+               or_(PluginMetricLatest.resource_name.in_(list(names)),
+                   *(PluginMetricLatest.resource_name.like(escape_like(n) + ":core%", escape="\\")
+                     for n in names)))
+    )).all()
+    cores: dict[str, int] = {}
+    mem: dict[str, dict] = {}
+    for res, metric, value in rows:
+        if res in names:
+            mem.setdefault(res, {})[metric] = value
+        elif metric == "cpu_pct":
+            host = res.rsplit(":core", 1)[0]
+            cores[host] = cores.get(host, 0) + 1
+    out = {}
+    for name, hid in names.items():
+        m = mem.get(name, {})
+        avail, used = m.get("mem_available_bytes"), m.get("mem_used_pct")
+        total = int(avail / (1 - used / 100)) if avail and used is not None and used < 99.5 else None
+        out[hid] = (cores.get(name), total)
+    return out
 
 
 def bucket(values: list[float], target: int = 40) -> list[float]:
@@ -210,7 +344,8 @@ async def log_context(db, host_id: str, name: str, ts, n: int) -> tuple[list, li
 
 
 async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]:
-    """{(host_id, container_name): [die/oom events since `since`, newest first]}."""
+    """{(host_id, container_name): [die/oom events since `since`, newest first]},
+    leaving out a die that was a stop (event_is_failure)."""
     rows = (await db.execute(
         select(DockerEvent)
         .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
@@ -218,10 +353,21 @@ async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]
     )).scalars()
     out: dict[tuple[str, str], list] = {}
     for e in rows:
-        out.setdefault((e.host_id, e.container_name), []).append(e)
+        if event_is_failure(e):
+            out.setdefault((e.host_id, e.container_name), []).append(e)
     return out
 
 
+async def failure_events(db, since) -> list:
+    """Die and OOM events since `since`, newest first; failures_24h() decides
+    which of them are failures."""
+    return list((await db.execute(
+        select(DockerEvent)
+        .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
+        .order_by(DockerEvent.at.desc())
+    )).scalars())
+
+
 async def retained_names(db, want: str, *, exact: bool, host_id: str | None = None,
                          limit: int = 20) -> list[tuple[str, str, datetime]]:
     """(host_id, container_name, last seen) of containers Steward still holds
@@ -303,7 +449,7 @@ def container_problems(c, failures_1h: int, failed_24h: bool,
         out.append("unhealthy")
     if c.oom_killed:
         out.append("oom_killed")
-    if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h:
+    if status in ("exited", "dead") and exit_is_failure(c.exit_code) and failed_24h:
         out.append("exited_nonzero")
     return out
 

plugins/docker/swarm_view.py

@@ -350,3 +350,31 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
     standalone = sorted(groups.values(), key=lambda g: g["host"].lower())
 
     return {"services": out_services, "standalone": standalone, "superseded": superseded}
+
+
+def service_trouble(s: dict) -> int | None:
+    """How bad a built service is, for sorting and for Needs attention: 0 a
+    crash loop, 1 no replica running, 2 short of replicas, 3 healthy but with
+    failed tasks in the last 24h; None when nothing is wrong."""
+    if s.get("crash_looping"):
+        return 0
+    if s.get("state") == "down":
+        return 1
+    if s.get("state") == "degraded":
+        return 2
+    if s.get("failures_24h"):
+        return 3
+    return None
+
+
+def services_problems_first(services: list[dict]) -> list[dict]:
+    """Services in trouble first, worst first, then the rest by name."""
+    return sorted(services, key=lambda s: (
+        4 if service_trouble(s) is None else service_trouble(s), s["name"].lower()))
+
+
+def failing_services(services: list[dict]) -> list[dict]:
+    """The services that belong in Needs attention: a crash loop, or short of
+    their desired replicas. A healthy service whose failed tasks Swarm has
+    already replaced is history, shown on the service, not a problem now."""
+    return [s for s in services_problems_first(services) if (service_trouble(s) or 9) <= 2]

plugins/docker/templates/docker/_usage.html

@@ -0,0 +1,26 @@
+{#- A container's CPU and memory as service.usage() describes them (#5209).
+
+    Docker counts CPU per core (100% = one core busy), so CPU reads as cores,
+    toned by its share of the host's CPUs when the host's core count is known.
+    Memory reads in bytes, against a limit only when one is actually set;
+    mem_pct is then a share of that limit, otherwise of the host's memory. -#}
+
+{% macro cores(n) -%}
+{{ "%.2f"|format(n) if n < 1 else "%.1f"|format(n) }} {{ "core" if n == 1 else "cores" }}
+{%- endmacro %}
+
+{% macro cpu(u) -%}
+{%- if u.cores is none -%}<span class="tone-dim">—</span>
+{%- else -%}
+<span class="tone-{{ status_class(u.cpu_share, 'cpu') }}"
+      title="Docker counts CPU per core: {{ '%.0f'|format(u.cores * 100) }}% of one core{% if u.cpus %}, {{ '%.1f'|format(u.cpu_share) }}% of this host's {{ u.cpus }} core{{ 's' if u.cpus != 1 }}{% endif %}">{{ cores(u.cores) }}</span>
+{%- endif -%}
+{%- endmacro %}
+
+{% macro mem(u) -%}
+{%- if u.mem_bytes is none -%}<span class="tone-dim">—</span>
+{%- else -%}
+<span class="tone-{{ status_class(u.mem_pct, 'mem') if u.mem_of else 'dim' }}"
+      title="{% if u.mem_pct is not none and u.mem_of %}{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% else %}No memory limit set{% endif %}">{{ u.mem_bytes|bytes }}{% if u.mem_limit %} / {{ u.mem_limit|bytes }}{% endif %}</span>
+{%- endif -%}
+{%- endmacro %}

plugins/docker/templates/docker/container_detail.html

@@ -1,8 +1,11 @@
-{# docker/container_detail.html — full detail page for one container #}
+{# docker/container_detail.html — full detail page for one container. A swarm
+   task is titled by its service and slot, with the raw container name under
+   it; CPU reads in cores and memory in bytes (see _usage.html). #}
 {% extends "base.html" %}
 {% from "_macros.html" import crumbs %}
-{% block title %}{{ name }} — Docker — Steward{% endblock %}
-{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %}
+{% from "docker/_usage.html" import cpu, mem %}
+{% block title %}{{ label or name }} — Docker — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (label or name, "")]) }}{% endblock %}
 {% block content %}
 
 {% if container is none %}
@@ -18,11 +21,14 @@
 {# ── Header ──────────────────────────────────────────────────────────────── #}
 <div style="display:flex;align-items:center;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
   <span class="dot {% if container.status == 'running' %}dot-up{% elif container.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
-  <h1 class="page-title" style="margin-bottom:0;">{{ container.name }}</h1>
+  <h1 class="page-title" style="margin-bottom:0;">{{ label }}</h1>
   {% if container.health == 'healthy' %}<span style="font-size:0.72rem;color:var(--green);border:1px solid var(--green);border-radius:3px;padding:0.05rem 0.4rem;">healthy</span>
   {% elif container.health == 'unhealthy' %}<span style="font-size:0.72rem;color:var(--red);border:1px solid var(--red);border-radius:3px;padding:0.05rem 0.4rem;">unhealthy</span>
   {% elif container.health == 'starting' %}<span style="font-size:0.72rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0.05rem 0.4rem;">starting</span>{% endif %}
 </div>
+{% if label != container.name %}
+<div style="font-size:0.75rem;color:var(--text-dim);font-family:ui-monospace,monospace;margin-bottom:0.25rem;overflow-wrap:anywhere;">{{ container.name }}</div>
+{% endif %}
 <div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
   {{ container.status }}{% if uptime %} · up {{ uptime }}{% endif %}
   {% if host %} · on <a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %}
@@ -38,16 +44,27 @@
 {% endmacro %}
 
 <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
-  {{ fact("CPU", "%.1f%%" | format(container.cpu_pct) if container.cpu_pct is not none else "—") }}
-  {{ fact("Memory", "%.1f%%" | format(container.mem_pct) if container.mem_pct is not none else "—") }}
+  {{ fact("CPU", cpu(usage) if container.status == 'running' else "—") }}
+  {{ fact("Memory", mem(usage) if container.status == 'running' else "—") }}
+  {#- Swarm never restarts a task's container: it replaces the task with a new
+      one, so restart_count is always 0 there and the service's task failures
+      (Swarm page) are the count that means something. #}
+  {% if not container.service_name %}
   {{ fact("Restarts", container.restart_count, "var(--orange)" if container.restart_count else "") }}
+  {% endif %}
   {{ fact("Last exit", (container.exit_code ~ (" (OOM)" if container.oom_killed else "")) if container.exit_code is not none else "—",
-          "var(--red)" if (container.exit_code is not none and container.exit_code != 0) else "") }}
-  {{ fact("Net in", net_rx) }}
-  {{ fact("Net out", net_tx) }}
-  {{ fact("Block read", blk_read) }}
-  {{ fact("Block write", blk_write) }}
+          "var(--red)" if exit_failed else "") }}
+  {{ fact("Net in since start", net_rx) }}
+  {{ fact("Net out since start", net_tx) }}
+  {{ fact("Disk read since start", blk_read) }}
+  {{ fact("Disk written since start", blk_write) }}
 </div>
+{% if container.service_name %}
+<div style="font-size:0.78rem;color:var(--text-muted);margin:-0.75rem 0 1.5rem;">
+  Swarm replaces a failed task with a new container rather than restarting it; the
+  service's failed tasks are on the <a href="/plugins/docker/swarm">Swarm page</a>.
+</div>
+{% endif %}
 
 {# ── Image / placement ───────────────────────────────────────────────────── #}
 <div class="card" style="margin-bottom:1.5rem;">
@@ -61,7 +78,9 @@
     <span style="color:var(--text-muted);">Swarm service</span><span>{{ container.service_name }}</span>
     {% endif %}
     {% if container.node_id %}
-    <span style="color:var(--text-muted);">Node</span><span style="font-family:ui-monospace,monospace;">{{ container.node_id }}</span>
+    <span style="color:var(--text-muted);">Node</span>
+    {% if node_name %}<span title="node id {{ container.node_id }}">{{ node_name }}</span>
+    {% else %}<span style="font-family:ui-monospace,monospace;">{{ container.node_id }}</span>{% endif %}
     {% endif %}
     {% if ports %}
     <span style="color:var(--text-muted);">Ports</span>

plugins/docker/templates/docker/host_panel.html

@@ -1,22 +1,58 @@
-{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub #}
+{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub:
+   problems first with what is wrong and since when, then running, then
+   stopped. Tasks Swarm has replaced are left out (counted in the header). #}
+{% from "_macros.html" import status_badge, status_dot %}
+{% from "docker/_usage.html" import cpu, mem %}
 <div class="card">
-  <div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;">
+  <div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;flex-wrap:wrap;">
     <h3 class="section-title" style="margin-bottom:0;">Docker</h3>
     <span style="font-size:0.78rem;color:var(--text-muted);">
+      {% if problems %}<span class="tone-crit">{{ problems }} with problems</span> · {% endif %}
       {{ running }} running{% if stopped %} · {{ stopped }} stopped{% endif %}
+      {% if replaced %}<span title="Old swarm tasks Swarm has replaced with new ones: history, not containers that need anything"> · {{ replaced }} replaced task{{ "s" if replaced != 1 }} hidden</span>{% endif %}
       <a href="/plugins/docker/" style="margin-left:0.6rem;color:var(--text-muted);">All →</a>
     </span>
   </div>
-  <div style="display:grid;gap:2px;">
-    {% for c in containers %}
-    <div style="display:flex;align-items:center;gap:0.5rem;font-size:0.85rem;padding:0.2rem 0;">
-      <span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
-      <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;" title="{{ c.image }}">{{ c.name }}</a>
-      <span style="font-size:0.74rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
-        {% if c.cpu_pct is not none %}{{ "%.1f" | format(c.cpu_pct) }}%{% endif %}
-        {% if c.mem_pct is not none %} · {{ "%.1f" | format(c.mem_pct) }}%{% endif %}
-      </span>
-    </div>
-    {% endfor %}
-  </div>
+  <table class="table">
+    <thead>
+      <tr>
+        <th>Container</th>
+        <th title="Docker counts CPU per core: 1 core = one core fully busy">CPU</th>
+        <th title="Used, and the limit when one is set">Memory</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for r in rows %}
+      {% set c = r.c %}
+      <tr>
+        <td>
+          <div style="display:flex;align-items:center;gap:0.5rem;min-width:0;">
+            {{ status_dot('crit' if r.problems else ('up' if c.status == 'running' else 'inactive'), c.status|capitalize) }}
+            <div style="min-width:0;">
+              <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" title="{{ c.name }} · {{ c.image }}" style="font-weight:500;color:inherit;text-decoration:none;overflow-wrap:anywhere;">{{ r.label }}</a>
+              {% if r.problems %}
+              {% set worst = r.problems[0] %}
+              <div style="font-size:0.75rem;margin-top:0.1rem;">
+                {{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }}
+                <span style="color:var(--text-muted);">
+                  {%- for p in r.problems %} {{ says[p] }}{{ " and" if not loop.last }}{% endfor %}
+                  {%- if c.exit_code %} · exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}{% endif %}
+                  {%- if r.since %} · since {{ r.since|ago }}{% endif -%}
+                </span>
+              </div>
+              {% else %}
+              <div style="font-size:0.72rem;color:var(--text-muted);">
+                {{ c.status }}{% if r.uptime %} · up {{ r.uptime }}{% endif %}
+                {%- if c.status != 'running' and c.exit_code %} · exit {{ c.exit_code }}{% endif %}
+              </div>
+              {% endif %}
+            </div>
+          </div>
+        </td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">{% if c.status == 'running' %}{{ cpu(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">{% if c.status == 'running' %}{{ mem(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
 </div>

plugins/docker/templates/docker/index.html

@@ -13,4 +13,17 @@
      hx-swap="innerHTML">
   <div class="card"><p class="field-hint">Loading…</p></div>
 </div>
+{# A link to a section (the dashboard's "failed (24h)" → #failures) lands
+   before the rows exist; scroll to it once, when they first arrive. #}
+<script>
+(function () {
+  var rows = document.getElementById('docker-rows');
+  if (!location.hash || !rows) return;
+  rows.addEventListener('htmx:afterSwap', function once() {
+    rows.removeEventListener('htmx:afterSwap', once);
+    var target = document.getElementById(location.hash.slice(1));
+    if (target) target.scrollIntoView({ block: 'start' });
+  });
+})();
+</script>
 {% endblock %}

plugins/docker/templates/docker/rows.html

@@ -2,18 +2,36 @@
    first (service.needs_attention, the ranking the MCP tools use), the counts,
    then swarm services and each host's containers. #}
 {% from "_macros.html" import stat_tile, status_badge, status_dot %}
+{% from "docker/_usage.html" import cpu, mem, cores %}
 
 {# ── Needs attention ──────────────────────────────────────────────────────── #}
-{% if attention %}
+{% if attention or failing_services %}
 <section class="card problem-card" aria-labelledby="docker-problems">
   <h2 id="docker-problems" class="section-title" style="margin-bottom:0.6rem;">Needs attention</h2>
   <ul class="problem-list">
+    {# A swarm service in trouble: Swarm replaces its failed tasks, so the
+       service, not any one container, is what is failing. #}
+    {% for s in failing_services %}
+    {% set e = s.last_error %}
+    <li>
+      {% if s.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}
+      {% elif s.state == 'down' %}{{ status_badge('crit', 'Down') }}
+      {% else %}{{ status_badge('warn', 'Degraded') }}{% endif %}
+      <a href="/plugins/docker/swarm">{{ s.name }}</a>
+      <span class="problem-detail">
+        {{- s.running }} of {{ s.desired }} running
+        {%- if s.crash_looping %} · {{ s.failures_1h }} tasks failed in the last hour{% endif %}
+        {%- if e %} · last task {{ e.state }}: {{ e.error or e.message or e.state }}{% if e.exit_code %} (exit {{ e.exit_code }}){% endif %}{% if e.node %} on {{ e.node }}{% endif %}{% endif %}
+        {%- if s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif -%}
+      </span>
+    </li>
+    {% endfor %}
     {% for r in attention %}
     {% set c = r.container %}
     {% set worst = r.problems[0] %}
     <li>
       {{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }}
-      <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}">{{ c.name }}</a>
+      <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" title="{{ c.name }}">{{ r.label }}</a>
       <span class="problem-detail">
         {%- for p in r.problems %}{{ says[p] }}{{ " and " if not loop.last }}{% endfor %}
         {%- if c.exit_code and worst in ('exited_nonzero', 'crash_looping', 'restarting') %} · exit {{ c.exit_code }}{% endif %}
@@ -42,18 +60,45 @@
 </section>
 {% endif %}
 
-{# ── Counts ───────────────────────────────────────────────────────────────── #}
+{# ── Counts ──────────────────────────────────────────────────────────────
+   Swarm services are counted as services (a task's container comes and goes);
+   the container counts are the containers outside any swarm service. ────── #}
 <div class="stat-strip">
-  {{ stat_tile("Needs attention", attention|length, state="crit" if attention else "inactive") }}
+  {{ stat_tile("Needs attention", attention|length + failing_services|length, state="crit" if attention or failing_services else "inactive") }}
   {{ stat_tile("Unhealthy", unhealthy, state="crit" if unhealthy else "inactive") }}
   {{ stat_tile("Restarting", restarting, state="crit" if restarting else "inactive", hint="restarting or crash-looping") }}
-  {{ stat_tile("Running", running, state="up" if running else "inactive") }}
-  {{ stat_tile("Stopped", stopped, state="inactive") }}
+  {{ stat_tile("Failed (24h)", failures|length, state="warn" if failures else "inactive", hint="containers and services; listed below", href="#failures" if failures else none) }}
+  {% if swarm_services %}{{ stat_tile("Swarm services", services_healthy ~ "/" ~ swarm_services|length, state="up" if services_healthy == swarm_services|length else "warn", hint="healthy", href="/plugins/docker/swarm") }}{% endif %}
+  {{ stat_tile("Containers running", running, state="up" if running else "inactive", hint="outside swarm services" if swarm_services else none) }}
+  {{ stat_tile("Containers stopped", stopped, state="inactive") }}
   {% if stale %}{{ stat_tile("Stale", stale, state="stale", hint="no recent report") }}{% endif %}
-  {% if swarm_services %}{{ stat_tile("Services", swarm_services|length, href="/plugins/docker/swarm") }}{% endif %}
-  {{ stat_tile("Hosts", host_groups|length) }}
+  {{ stat_tile("Docker hosts", docker_hosts) }}
 </div>
 
+{# ── What failed in the last 24h: what the dashboard's "failed (24h)" counts.
+   A plain container's death, or a swarm service's failed tasks; a container
+   someone stopped (exit 137/143) is not a failure. ───────────────────────── #}
+{% if failures %}
+<section id="failures" class="card" aria-labelledby="docker-failures" style="margin-bottom:1.5rem;">
+  <h2 id="docker-failures" class="section-title" style="margin-bottom:0.6rem;">Failed in the last 24h</h2>
+  <ul class="problem-list">
+    {% for f in failures %}
+    <li>
+      {{ status_badge('warn', 'Service' if f.kind == 'service' else 'Container') }}
+      {% if f.kind == 'service' %}<a href="/plugins/docker/swarm">{{ f.name }}</a>
+      {% else %}<a href="/plugins/docker/container/{{ f.host_id }}/{{ f.name }}">{{ f.name }}</a>{% endif %}
+      <span class="problem-detail">
+        {{- f.count }} {{ "failure" if f.count == 1 else "failures" }}
+        {%- if f.detail %} · last: {{ f.detail }}{% endif %}
+        {%- if f.host %} · {{ f.host }}{% endif %}
+        {%- if f.at %} · {{ f.at|ago }}{% endif -%}
+      </span>
+    </li>
+    {% endfor %}
+  </ul>
+</section>
+{% endif %}
+
 {# ── Swarm services: collapsed per-service, each current replica labelled with
    its host (tasks Swarm replaced are left out). Ghost replicas (tasks on nodes
    with no Steward agent) come from the managers' placement data so the picture
@@ -72,7 +117,7 @@
         <span style="font-weight:500;font-size:0.9rem;">{{ s.name }}</span>
         <span style="font-size:0.72rem;color:var(--text-dim);">{{ s.mode }}</span>
         <span class="tone-{{ 'ok' if s.state == 'healthy' else ('warn' if s.state == 'degraded' else 'crit') }}" style="font-size:0.78rem;font-variant-numeric:tabular-nums;">
-          {{ s.running }}/{{ s.desired }} running
+          {{ s.running }}/{{ s.desired }} running{% if s.state != 'healthy' and s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif %}
         </span>
       </div>
       {% with style="padding-left:1.1rem;margin-bottom:0.35rem;" %}{% include "docker/_swarm_failures.html" %}{% endwith %}
@@ -88,7 +133,7 @@
         <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px solid var(--border);border-radius:5px;padding:0.3rem 0.55rem;">
           <span class="dot {% if r.status == 'running' %}dot-up{% elif r.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
           <a href="/plugins/docker/container/{{ r.host_id }}/{{ r.name }}" style="font-weight:500;color:inherit;text-decoration:none;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;" title="{{ r.name }}">{{ r.host }}</a>
-          {% if r.cpu_pct is not none %}<span style="color:var(--text-muted);font-family:ui-monospace,monospace;margin-left:auto;">{{ "%.0f" | format(r.cpu_pct) }}%</span>{% endif %}
+          {% if r.cpu_pct is not none %}<span style="color:var(--text-muted);font-family:ui-monospace,monospace;margin-left:auto;" title="Docker counts CPU per core">{{ cores(r.cpu_pct / 100) }}</span>{% endif %}
         </div>
         {% endif %}
         {% endfor %}
@@ -119,8 +164,8 @@
         <th>Container</th>
         <th>Image</th>
         <th>Ports</th>
-        <th>CPU %</th>
-        <th>Mem %</th>
+        <th title="Docker counts CPU per core: 1 core = one core fully busy">CPU</th>
+        <th title="Used, and the limit when one is set">Memory</th>
       </tr>
     </thead>
     <tbody>
@@ -144,7 +189,7 @@
               <div style="font-size:0.73rem;color:var(--text-muted);">
                 {{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %}
                 {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
-                  · <span class="tone-crit">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
+                  · <span class="tone-{{ 'crit' if item.exit_failed else 'dim' }}" title="{{ 'last exit code' if item.exit_failed else 'stopped by a signal; not a failure' }}">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
                 {% endif %}
                 {% if c.restart_count %} · <span class="tone-warn" title="Restarted {{ c.restart_count }} times">⟳{{ c.restart_count }}</span>{% endif %}
               </div>
@@ -164,20 +209,8 @@
           <div style="color:var(--text-muted);">{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}</div>
           {% endfor %}
         </td>
-        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
-          {% if c.cpu_pct is not none %}
-          <span class="tone-{{ status_class(c.cpu_pct, 'cpu') }}">{{ "%.1f" | format(c.cpu_pct) }}%</span>
-          {% else %}
-          <span style="color:var(--text-dim);">—</span>
-          {% endif %}
-        </td>
-        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
-          {% if c.mem_pct is not none %}
-          <span class="tone-{{ status_class(c.mem_pct, 'mem') }}">{{ "%.1f" | format(c.mem_pct) }}%</span>
-          {% else %}
-          <span style="color:var(--text-dim);">—</span>
-          {% endif %}
-        </td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;white-space:nowrap;">{{ cpu(item.usage) }}</td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;white-space:nowrap;">{{ mem(item.usage) }}</td>
       </tr>
       {% endfor %}
       {% endfor %}

plugins/docker/templates/docker/swarm.html

@@ -1,4 +1,6 @@
-{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement #}
+{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement.
+   Services in trouble come first (crash loop, down, short), with since-times;
+   nodes that are down or not taking work come first. #}
 {% extends "base.html" %}
 {% from "_macros.html" import crumbs %}
 {% block title %}Swarm — Docker — Steward{% endblock %}
@@ -34,6 +36,7 @@
             <span style="color:{% if s.healthy %}var(--green){% elif s.down %}var(--red){% elif s.degraded %}var(--orange){% else %}var(--text-muted){% endif %};">
               {{ s.running }}/{{ s.desired }}
             </span>
+            {% if s.state != 'healthy' and s.trouble_since %}<div style="font-family:inherit;font-size:0.72rem;color:var(--text-dim);" title="{{ s.trouble_since }}">since {{ s.trouble_since|ago }}</div>{% endif %}
           </td>
           <td style="font-size:0.8rem;color:var(--text-muted);max-width:220px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.image or "—" }}</td>
           <td style="font-size:0.78rem;color:var(--text-muted);">

plugins/docker/templates/docker/widget.html

@@ -1,21 +1,32 @@
-{# docker/widget.html — dashboard widget: container status overview, by host #}
+{# docker/widget.html — dashboard widget: what Docker is running and what has
+   failed. Swarm services first, troubled ones at the top with since-times; then
+   the containers outside any service, by host, problems first. The failed count
+   links to the Docker page's list of exactly what it counts. #}
+{% from "docker/_usage.html" import cpu %}
 {% if total_count == 0 %}
 <p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No containers reported yet.</p>
 {% else %}
 
-<div style="display:flex;gap:1.25rem;margin-bottom:0.65rem;">
-  <div>
+<div style="display:flex;flex-wrap:wrap;gap:0.4rem 1.25rem;margin-bottom:0.65rem;">
+  {% if swarm_services %}
+  <div title="Swarm services with every replica running">
+    <span class="tone-{{ 'ok' if services_healthy == swarm_services|length else 'warn' }}" style="font-size:1.4rem;font-weight:500;line-height:1;">{{ services_healthy }}/{{ swarm_services|length }}</span>
+    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">services healthy</span>
+  </div>
+  {% endif %}
+  <div title="Containers running outside any swarm service">
     <span style="font-size:1.4rem;font-weight:500;line-height:1;color:var(--green);">{{ running_count }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">running</span>
-  </div>
-  <div title="Containers that died or were OOM-killed in the last 24h, plus swarm tasks that failed (a task Swarm replaced is not a failure)">
-    <span style="font-size:1.4rem;font-weight:500;line-height:1;color:{% if failed_24h %}var(--red){% else %}var(--text-muted){% endif %};">{{ failed_24h }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">failed (24h)</span>
+    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">containers running</span>
   </div>
+  <a href="/plugins/docker/#failures" style="color:inherit;text-decoration:none;"
+     title="Containers that died or were OOM-killed in the last 24h, and swarm services with failed tasks. A container someone stopped is not counted. Click for the list.">
+    <span class="tone-{{ 'crit' if failures else 'dim' }}" style="font-size:1.4rem;font-weight:500;line-height:1;">{{ failures|length }}</span>
+    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;border-bottom:1px dotted var(--border-mid);">failed in 24h</span>
+  </a>
 </div>
 
-{# Swarm services — collapsed, each with its replicas' host chips (dashed = a
-   node with no Steward agent, count-only from placement). #}
+{# Swarm services — troubled first, each with its replicas' host chips
+   (dashed = a node with no Steward agent, count-only from placement). #}
 {% if swarm_services %}
 <div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">Swarm services</div>
 <div style="display:grid;gap:0.4rem;margin-bottom:0.4rem;">
@@ -24,8 +35,9 @@
     <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.82rem;">
       <span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
       <span style="font-weight:500;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.name }}</span>
-      {% if s.crash_looping %}<span style="color:var(--red);font-size:0.7rem;font-weight:500;flex-shrink:0;" title="{{ s.failures_1h }} tasks failed in the last hour{% if s.last_error %}: {{ s.last_error.error or s.last_error.message or s.last_error.state }}{% endif %}">crash-looping</span>{% endif %}
-      <span style="color:var(--text-muted);font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;">{{ s.running }}/{{ s.desired }}</span>
+      {% if s.crash_looping %}<span class="tone-crit" style="font-size:0.7rem;font-weight:500;flex-shrink:0;" title="{{ s.failures_1h }} tasks failed in the last hour{% if s.last_error %}: {{ s.last_error.error or s.last_error.message or s.last_error.state }}{% endif %}">crash-looping</span>{% endif %}
+      {% if s.state != 'healthy' and s.trouble_since %}<span class="tone-dim" style="font-size:0.7rem;flex-shrink:0;">since {{ s.trouble_since|ago }}</span>{% endif %}
+      <span class="tone-{{ 'ok' if s.state == 'healthy' else ('warn' if s.state == 'degraded' else 'crit') }}" style="font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
     </div>
     <div style="display:flex;flex-wrap:wrap;gap:0.25rem;margin:0.2rem 0 0 1rem;">
       {% for r in s.replicas %}
@@ -41,25 +53,25 @@
 </div>
 {% endif %}
 
+{% if host_groups and swarm_services %}
+<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.6rem 0 0.2rem;">Other containers</div>
+{% endif %}
 {% for g in host_groups %}
 {% if multi_host %}
-<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">{{ g.host_name }}</div>
+<div style="font-size:0.7rem;color:var(--text-dim);margin:0.4rem 0 0.15rem;">{{ g.host_name }}</div>
 {% endif %}
 <div style="display:grid;gap:2px;">
-  {% for c in g.containers %}
+  {% for item in g.containers %}
+  {% set c = item.c %}
   <div style="display:flex;align-items:center;gap:0.5rem;font-size:0.82rem;padding:0.15rem 0;">
-    <span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
+    <span class="dot {% if c.status == 'running' and c.health != 'unhealthy' %}dot-up{% elif c.status in ('paused', 'restarting') %}dot-warn{% else %}dot-down{% endif %}"></span>
     <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;">
-      {{ c.name }}{% if c.health == 'unhealthy' %}<span title="unhealthy" style="color:var(--red);"> ●</span>{% elif c.health == 'healthy' %}<span title="healthy" style="color:var(--green);"> ●</span>{% endif %}{% if c.restart_count %}<span title="restarts" style="color:var(--orange);font-size:0.7rem;"> ⟳{{ c.restart_count }}</span>{% endif %}
+      {{ c.name }}{% if c.health == 'unhealthy' %} <span class="tone-crit" style="font-size:0.7rem;">unhealthy</span>{% endif %}{% if c.status == 'restarting' %} <span class="tone-warn" style="font-size:0.7rem;">restarting</span>{% endif %}
     </a>
     {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
-    <span title="last exit code" style="font-size:0.7rem;color:var(--red);flex-shrink:0;">exit {{ c.exit_code }}</span>
-    {% endif %}
-    {% if c.cpu_pct is not none %}
-    <span style="font-size:0.72rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
-      {{ "%.1f" | format(c.cpu_pct) }}%
-    </span>
+    <span title="{{ 'last exit code' if item.exit_failed else 'stopped by a signal; not a failure' }}" class="tone-{{ 'crit' if item.exit_failed else 'dim' }}" style="font-size:0.7rem;flex-shrink:0;">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
     {% endif %}
+    {% if c.status == 'running' %}<span style="font-size:0.72rem;font-family:ui-monospace,monospace;flex-shrink:0;">{{ cpu(item.usage) }}</span>{% endif %}
   </div>
   {% endfor %}
 </div>

plugins/docker/templates/docker/widget_resources.html

@@ -1,28 +1,41 @@
-{# docker/widget_resources.html — dashboard widget: the busiest running containers
-   by CPU, each with labeled CPU + memory utilisation bars. #}
+{# docker/widget_resources.html — dashboard widget: the containers using the
+   most CPU (averaged over the last few minutes, so the order holds still),
+   each with its CPU in cores and memory in bytes. The CPU bar is a share of
+   the host's CPUs; the memory bar a share of the container's limit, or of
+   the host's memory when no limit is set. #}
+{% from "docker/_usage.html" import cpu, mem %}
 {% if not rows %}
 <div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No running containers.</div>
 {% else %}
-{% macro bar(label, pct, warn, crit) %}
-<div style="display:flex;align-items:center;gap:0.45rem;margin-bottom:2px;">
+{% macro bar(label, pct, tone, text, title) %}
+<div style="display:flex;align-items:center;gap:0.45rem;margin-bottom:2px;" title="{{ title }}">
   <span style="font-size:0.62rem;color:var(--text-dim);width:1.9rem;flex-shrink:0;letter-spacing:0.03em;">{{ label }}</span>
   <div style="flex:1;height:6px;background:var(--bg-elevated);border-radius:3px;overflow:hidden;">
-    <div style="height:100%;border-radius:3px;width:{{ [pct, 100] | min }}%;
-                background:{% if pct > crit %}var(--red){% elif pct > warn %}var(--orange){% else %}var(--accent){% endif %};
+    {% if pct is not none %}
+    <div style="height:100%;border-radius:3px;width:{{ [[pct, 0]|max, 100]|min }}%;
+                background:{% if tone == 'crit' %}var(--red){% elif tone == 'warn' %}var(--orange){% else %}var(--accent){% endif %};
                 transition:width 0.4s;"></div>
+    {% endif %}
   </div>
-  <span style="font-size:0.7rem;color:var(--text-muted);font-family:ui-monospace,monospace;width:3rem;text-align:right;flex-shrink:0;">{{ "%.1f" | format(pct) }}%</span>
+  <span style="font-size:0.7rem;color:var(--text-muted);font-family:ui-monospace,monospace;min-width:4.5rem;text-align:right;flex-shrink:0;white-space:nowrap;">{{ text }}</span>
 </div>
 {% endmacro %}
+<div style="font-size:0.7rem;color:var(--text-dim);margin-bottom:0.45rem;">Busiest by CPU, {{ minutes }}-minute average</div>
 <div style="display:grid;gap:0.6rem;">
   {% for r in rows %}
-  {% set c = r.c %}
+  {% set u = r.usage %}
   <div>
-    <div style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-bottom:0.25rem;">
-      {{ c.name }}{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
+    <div style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-bottom:0.25rem;" title="{{ r.c.name }}">
+      <a href="/plugins/docker/container/{{ r.c.host_id }}/{{ r.c.name }}" style="color:inherit;text-decoration:none;">{{ r.label }}</a>{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
     </div>
-    {% if c.cpu_pct is not none %}{{ bar("CPU", c.cpu_pct, 50, 80) }}{% endif %}
-    {% if c.mem_pct is not none %}{{ bar("MEM", c.mem_pct, 70, 90) }}{% endif %}
+    {% if u.cores is not none %}
+    {{ bar("CPU", u.cpu_share, status_class(u.cpu_share, 'cpu'), cpu(u),
+           ("%.1f%% of the host's %d core%s" % (u.cpu_share, u.cpus, "" if u.cpus == 1 else "s")) if u.cpus else "Host core count unknown") }}
+    {% endif %}
+    {% if u.mem_bytes is not none %}
+    {{ bar("MEM", u.mem_pct if u.mem_of else none, status_class(u.mem_pct, 'mem') if u.mem_of else 'unknown', mem(u),
+           ("%.1f%% of %s" % (u.mem_pct, u.mem_of)) if u.mem_of and u.mem_pct is not none else "") }}
+    {% endif %}
   </div>
   {% endfor %}
 </div>