feat(docker): readable names, CPU in cores and memory in bytes, failures listed with since-times (#5209)

#5209 "Docker views: readable names, CPU/memory in cores and bytes, failures
listed with since-time" (milestone 477 "UI problems first").

- A swarm task reads as its service and slot ("codex_db · 1", or the node
  for a global service), with the raw name in the title: Docker page, host
  panel, busiest widget, container detail.
- CPU reads in cores (Docker's 100% = one core), toned by its share of the
  host's CPUs from the agent's per-core series. Memory reads in bytes, against
  a limit only when one is really set (Docker reports host RAM otherwise).
- Exit 137/143 without an OOM kill is a stop, not a failure: no
  exited_nonzero, no crash-loop death, no "failed (24h)", dim exit code.
- "Failed (24h)" is a list (Docker page #failures, linked from the widget
  and the tile): plain containers by their deaths, swarm services by Swarm's
  failed tasks; a second manager's copy of a task death counts once.
- Failing swarm services lead Needs attention; services sort problems first
  with since-times on the page, the widget and the Swarm page; nodes that
  are down or drained sort first.
- Busiest widget orders by a 5-minute CPU average so the list holds still.
- Host panel: a table, problems first with what is wrong and since when;
  replaced swarm tasks hidden and counted.
- Container detail: node hostname, I/O labelled "since start", Restarts
  hidden for swarm tasks with a note on where task failures are.
- Widget names: "Docker — Services & failures", "Docker — Busiest containers".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/mcp_tools.py

@@ -194,7 +194,8 @@ async def list_containers(host: str = "", status: str = "", problem: str = "",
       "restarting", "crash_looping" (died repeatedly within the last hour;
       the count is set under Settings → Thresholds, default 3), "unhealthy"
       (failing its healthcheck), "oom_killed", "exited_nonzero" (died with an
-      error in the last 24h); empty for no filter.
+      error in the last 24h; exit 137/143 without an OOM kill is a stop, not
+      an error); empty for no filter.
     compose_project: only containers from this Compose project.
     limit: default 100, max 500.
 

plugins/docker/routes.py

@@ -16,9 +16,12 @@ from steward.core.formatting import human_bytes, human_duration, seconds_since
 from steward.core.time_range import parse_range, DEFAULT_RANGE
 from . import service as docker_svc
 from .dedup import dedup_by_container_id
-from .swarm_view import build_swarm_services, parse_tasks, task_summary
+from .swarm_view import (
+    build_swarm_services, failing_services, parse_tasks, replaced_tasks, services_problems_first,
+    task_summary, trouble_since,
+)
 from .models import (
-    DockerContainer, DockerDiskUsage, DockerEvent, DockerHostStatus, DockerImage,
+    DockerContainer, DockerDiskUsage, DockerHostStatus, DockerImage, DockerMetric,
     DockerSwarmNode, DockerSwarmService,
 )
 
@@ -109,6 +112,10 @@ async def rows():
         swarm_view = build_swarm_services(
             containers, services, node_hostname, host_name,
             crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))
+        capacity = await docker_svc.host_capacity(db, {c.host_id for c in containers})
+        failures = docker_svc.failures_24h(
+            await docker_svc.failure_events(db, now - timedelta(hours=24)),
+            swarm_view["services"], host_name)
 
         standalone = [c for c in containers if not c.service_name]
         groups: dict[str, dict] = {}
@@ -127,6 +134,8 @@ async def rows():
                 "ports": json.loads(c.ports_json) if c.ports_json else [],
                 "uptime": _uptime(c.started_at) if c.status == "running" else None,
                 "stale": _is_stale(c, now, stale_after, docker_down),
+                "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None))),
+                "exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed),
             })
             if c.status == "running":
                 g["running"] += 1
@@ -151,15 +160,20 @@ async def rows():
         g["grouped"] = any(s["label"] for s in g["subgroups"])
 
     host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
-    swarm_services = swarm_view["services"]
+    swarm_services = services_problems_first(swarm_view["services"])
     standalone_running = sum(g["running"] for g in host_groups)
-    ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"])
-    attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down))
+    attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down),
+                      label=docker_svc.task_label(r["container"], node_hostname))
                  for r in docker_svc.needs_attention(ranked)]
+    failing = failing_services(swarm_services)
     return await render_template(
         "docker/rows.html",
         host_groups=host_groups,
         swarm_services=swarm_services,
+        failing_services=failing,
+        failures=failures,
+        services_healthy=sum(s["state"] == "healthy" for s in swarm_services),
+        docker_hosts=len({c.host_id for c in containers}),
         docker_down=sorted(
             ({"host_id": h, "host_name": hosts[h].name if h in hosts else h, "status": d.status,
               "detail": d.detail, "since": d.since} for h, d in docker_down.items()),
@@ -168,9 +182,8 @@ async def rows():
         unhealthy=sum("unhealthy" in r["problems"] for r in attention),
         restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention),
         stale=sum(1 for g in host_groups for i in g["containers"] if i["stale"]),
-        running=standalone_running + sum(s["running"] for s in swarm_services),
+        running=standalone_running,
         stopped=len(standalone) - standalone_running,
-        total=len(containers) + ghost_total,
     )
 
 
@@ -185,55 +198,55 @@ def _is_stale(c, now: datetime, stale_after: int, docker_down=()) -> bool:
 @docker_bp.get("/widget")
 @require_role(UserRole.viewer)
 async def widget():
-    """HTMX dashboard widget: container status overview, grouped by host."""
+    """HTMX dashboard widget: swarm services and containers, problems first,
+    with how many things failed in the last 24h (linked to the list)."""
     show_stopped = request.args.get("show_stopped", "no") == "yes"
     widget_id = request.args.get("wid", "0")
-    since_24h = datetime.now(timezone.utc) - timedelta(hours=24)
+    now = datetime.now(timezone.utc)
 
     async with current_app.db_sessionmaker() as db:
-        all_containers = dedup_by_container_id(list((await db.execute(
-            select(DockerContainer).order_by(
-                (DockerContainer.status == "running").desc(),
-                DockerContainer.name,
-            )
-        )).scalars()))
+        all_containers = dedup_by_container_id(await docker_svc.list_containers(db))
         services = list((await db.execute(select(DockerSwarmService))).scalars())
         nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
-        # Recent failures are more actionable than a static "stopped" tally. Read
-        # DISTINCT container names so the two managers' duplicate die/oom events
-        # don't double it; swarm tasks are counted on their service below.
-        dead_names = list((await db.execute(
-            select(func.distinct(DockerEvent.container_name))
-            .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since_24h)
-        )).scalars())
+        events = await docker_svc.failure_events(db, now - timedelta(hours=24))
         hosts = await docker_svc.host_map(
             db,
             {c.host_id for c in all_containers}
             | {n.host_id for n in nodes} | {s.host_id for s in services},
         )
+        capacity = await docker_svc.host_capacity(db, {c.host_id for c in all_containers})
 
     # Swarm tasks collapse into per-service rows (each replica tagged with its
     # host); non-swarm containers stay grouped by host.
     node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
     host_name = {h.id: h.name for h in hosts.values()}
-    swarm_services = build_swarm_services(
+    swarm_services = services_problems_first(build_swarm_services(
         all_containers, services, node_hostname, host_name,
-        crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"]
-    failed_24h = docker_svc.failed_24h(dead_names, swarm_services)
+        crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"])
+    failures = docker_svc.failures_24h(events, swarm_services, host_name)
 
     standalone = [c for c in all_containers if not c.service_name]
     running = [c for c in standalone if c.status == "running"]
     display = standalone if show_stopped else running
 
+    def trouble(c) -> int:
+        if (c.status or "") == "restarting" or c.health == "unhealthy":
+            return 0
+        if c.status != "running" and docker_svc.exit_is_failure(c.exit_code, c.oom_killed):
+            return 1
+        return 2 if c.status == "running" else 3
+
     # Group for display so multi-host fleets read clearly; single-host stays flat.
     groups: dict[str, dict] = {}
-    for c in display:
+    for c in sorted(display, key=lambda c: (trouble(c), c.name)):
         g = groups.setdefault(c.host_id, {
             "host_id": c.host_id,
             "host_name": host_name.get(c.host_id, c.host_id),
             "containers": [],
         })
-        g["containers"].append(c)
+        g["containers"].append({"c": c, "usage": docker_svc.usage(
+            c, *capacity.get(c.host_id, (None, None))),
+            "exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed)})
     host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
 
     return await render_template(
@@ -241,39 +254,59 @@ async def widget():
         host_groups=host_groups,
         multi_host=len(host_groups) > 1,
         swarm_services=swarm_services,
-        running_count=len(running) + sum(s["running"] for s in swarm_services),
-        failed_24h=failed_24h,
+        services_healthy=sum(s["state"] == "healthy" for s in swarm_services),
+        running_count=len(running),
+        failures=failures,
         total_count=len(all_containers) + len(swarm_services),
         show_stopped=show_stopped,
         widget_id=widget_id,
     )
 
 
+CPU_AVERAGE_MINUTES = 5
+
+
 @docker_bp.get("/widget/resources")
 @require_role(UserRole.viewer)
 async def widget_resources():
-    """HTMX dashboard widget: CPU + memory usage for the busiest containers."""
+    """HTMX dashboard widget: the containers using the most CPU, averaged over
+    the last few minutes so the order holds still between polls, with their
+    CPU in cores and memory in bytes."""
     limit = max(1, min(20, int(request.args.get("limit", 10) or 10)))
     widget_id = request.args.get("wid", "0")
+    since = datetime.now(timezone.utc) - timedelta(minutes=CPU_AVERAGE_MINUTES)
 
     async with current_app.db_sessionmaker() as db:
         # Dedup before the limit, else swarm tasks reported by both managers can
         # fill the list with duplicates of the same container.
-        containers = dedup_by_container_id(list((await db.execute(
-            select(DockerContainer)
-            .where(DockerContainer.status == "running")
-            .order_by(DockerContainer.cpu_pct.desc().nullslast())
-        )).scalars()))[:limit]
-        hosts = await docker_svc.host_map(db, {c.host_id for c in containers})
+        running = dedup_by_container_id([
+            c for c in await docker_svc.list_containers(db) if c.status == "running"])
+        average = {(h, n): v for h, n, v in (await db.execute(
+            select(DockerMetric.host_id, DockerMetric.container_name, func.avg(DockerMetric.cpu_pct))
+            .where(DockerMetric.scraped_at >= since)
+            .group_by(DockerMetric.host_id, DockerMetric.container_name)
+        )).all()}
+        nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
+        hosts = await docker_svc.host_map(db, {c.host_id for c in running})
+        capacity = await docker_svc.host_capacity(db, {c.host_id for c in running})
 
-    rows_data = [
-        {"c": c, "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id}
-        for c in containers
-    ]
+    def cpu_of(c):
+        v = average.get((c.host_id, c.name))
+        return c.cpu_pct if v is None else float(v)
+
+    busiest = sorted(running, key=lambda c: (-(cpu_of(c) or 0), c.name))[:limit]
+    node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
+    rows_data = [{
+        "c": c,
+        "label": docker_svc.task_label(c, node_hostname),
+        "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id,
+        "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c)),
+    } for c in busiest]
     return await render_template(
         "docker/widget_resources.html",
         rows=rows_data,
-        multi_host=len({c.host_id for c in containers}) > 1,
+        minutes=CPU_AVERAGE_MINUTES,
+        multi_host=len({c.host_id for c in busiest}) > 1,
         widget_id=widget_id,
     )
 
@@ -281,22 +314,50 @@ async def widget_resources():
 @docker_bp.get("/host/<host_id>")
 @require_role(UserRole.viewer)
 async def host_panel(host_id: str):
-    """Per-host Docker fragment embedded on the core Hosts hub via HTMX.
+    """Per-host Docker fragment embedded on the core Hosts hub via HTMX:
+    problems first (with exit code and since when), then running, then
+    stopped. Swarm tasks Swarm has already replaced are history and left out.
 
     Renders nothing when the host reports no containers, so hosts without Docker
     don't carry an empty card.
     """
+    now = datetime.now(timezone.utc)
     async with current_app.db_sessionmaker() as db:
         containers = await docker_svc.list_containers(db, host_id)
+        if not containers:
+            return ""
+        services = list((await db.execute(select(DockerSwarmService))).scalars())
+        nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
+        failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
+        capacity = await docker_svc.host_capacity(db, {host_id})
 
-    if not containers:
-        return ""
-    running = sum(1 for c in containers if c.status == "running")
+    replaced = replaced_tasks(containers, services)
+    node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
+    hour_ago = now - timedelta(hours=1)
+    crash = docker_svc.crash_loop_deaths(current_app.config)
+    rows = []
+    for c in containers:
+        if (c.host_id, c.name) in replaced:
+            continue
+        events = failures.get((c.host_id, c.name), [])
+        recent = sum(1 for e in events if e.at >= hour_ago)
+        problems = docker_svc.container_problems(c, recent, bool(events), crash)
+        rows.append({
+            "c": c, "label": docker_svc.task_label(c, node_hostname), "problems": problems,
+            "since": docker_svc.problem_since(problems, events, hour_ago),
+            "usage": docker_svc.usage(c, *capacity.get(host_id, (None, None))),
+            "uptime": _uptime(c.started_at) if c.status == "running" else None,
+        })
+    rows.sort(key=lambda r: (
+        docker_svc.PROBLEMS.index(r["problems"][0]) if r["problems"] else len(docker_svc.PROBLEMS),
+        r["c"].status != "running", r["label"].lower()))
+    running = sum(1 for r in rows if r["c"].status == "running")
     return await render_template(
         "docker/host_panel.html",
-        containers=containers,
-        running=running,
-        stopped=len(containers) - running,
+        rows=rows, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
+        running=running, stopped=len(rows) - running,
+        problems=sum(1 for r in rows if r["problems"]),
+        replaced=len(replaced),
     )
 
 
@@ -319,6 +380,15 @@ async def container_detail(host_id: str, name: str):
         c = await db.get(DockerContainer, (host_id, name))
         host = await db.get(Host, host_id)
         events = await docker_svc.container_events(db, host_id, name) if c is not None else []
+        node_name = None
+        if c is not None and c.node_id:
+            node_name = (await db.execute(
+                select(DockerSwarmNode.hostname)
+                .where(DockerSwarmNode.node_id == c.node_id, DockerSwarmNode.hostname != "")
+                .limit(1)
+            )).scalar()
+        cpus, mem_total = (await docker_svc.host_capacity(db, [host_id])).get(host_id, (None, None)) \
+            if c is not None else (None, None)
 
     if c is None:
         return await render_template(
@@ -332,6 +402,10 @@ async def container_detail(host_id: str, name: str):
         host=host,
         host_id=host_id,
         name=name,
+        label=docker_svc.task_label(c, {c.node_id: node_name} if node_name else None),
+        node_name=node_name,
+        usage=docker_svc.usage(c, cpus, mem_total),
+        exit_failed=docker_svc.exit_is_failure(c.exit_code, c.oom_killed),
         ports=json.loads(c.ports_json) if c.ports_json else [],
         uptime=_uptime(c.started_at) if c.status == "running" else None,
         net_rx=human_bytes(c.net_rx_bytes), net_tx=human_bytes(c.net_tx_bytes),
@@ -415,7 +489,11 @@ async def swarm():
         now = datetime.now(timezone.utc)
         deaths = docker_svc.crash_loop_deaths(current_app.config)
         svc_dicts = []
-        for s in sorted(svc_by_name.values(), key=lambda s: s.service_name):
+        for s in svc_by_name.values():
+            tasks = parse_tasks(s)
+            summary = task_summary(tasks, now, deaths, node_names)
+            state = ("healthy" if s.running >= s.desired and s.desired > 0
+                     else "degraded" if s.running > 0 else "down")
             placement = []
             for p in (json.loads(s.placement_json) if s.placement_json else []):
                 nid = p.get("node_id", "")
@@ -429,18 +507,21 @@ async def swarm():
                 "healthy": s.running >= s.desired and s.desired > 0,
                 "degraded": 0 < s.running < s.desired,
                 "down": s.running == 0 and s.desired > 0,
-                "state": ("healthy" if s.running >= s.desired and s.desired > 0
-                          else "degraded" if s.running > 0 else "down"),
+                "state": state,
                 "placement": placement,
-                **task_summary(parse_tasks(s), now, deaths, node_names),
+                **summary,
+                "trouble_since": trouble_since(tasks, state, summary["crash_looping"], now),
             })
+        # Problems first: a node that is down or not taking work, then by role.
         node_rows = sorted(
-            node_by_id.values(), key=lambda n: (n.role, (n.hostname or n.node_id).lower())
+            node_by_id.values(), key=lambda n: (
+                n.status == "ready" and n.availability == "active",
+                n.role, (n.hostname or n.node_id).lower())
         )
         manager_names = sorted(hosts[m].name if m in hosts else m for m in members)
         swarms.append({
             "managers": manager_names,
-            "services": svc_dicts,
+            "services": services_problems_first(svc_dicts),
             "nodes": node_rows,
         })
 

plugins/docker/service.py

@@ -6,7 +6,7 @@ Quart's pipeline) call the same code.
 """
 from __future__ import annotations
 
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 from sqlalchemy import func, select
 
@@ -30,25 +30,159 @@ def crash_loop_deaths(config) -> int:
     return int(th.get("warn") or CRASH_LOOP_DEATHS)
 
 
-def failed_24h(dead_names, swarm_services) -> int:
-    """Failures in the last 24h for the dashboard: distinct plain containers
-    that died or were OOM-killed, plus each swarm service's failed tasks.
+# Exit codes a deliberate stop leaves: SIGTERM (143), and SIGKILL (137) when
+# the process outlives the stop timeout. Without an OOM kill they are someone
+# stopping the container, not the container failing.
+STOP_EXITS = (137, 143)
 
-    A swarm task's container dies whenever Swarm replaces it (a rolling update
-    kills it with 137), so its die event is not a failure. Services with
-    Swarm's task records count their failed tasks instead; a service without
-    them (an agent older than 1.8.0) falls back to its tasks' die events.
+
+def exit_is_failure(exit_code, oom_killed: bool = False) -> bool:
+    """Whether a container's exit is a failure: an OOM kill, or a non-zero
+    exit other than a stop's."""
+    return bool(oom_killed) or exit_code not in (None, 0, *STOP_EXITS)
+
+
+def _event_exit(detail) -> int | None:
+    """The exit code in a die event's detail ("exit 1"), or None."""
+    text = (detail or "").strip()
+    if text.startswith("exit "):
+        try:
+            return int(text[5:])
+        except ValueError:
+            return None
+    return None
+
+
+def event_is_failure(e) -> bool:
+    """Whether a die/oom event is a failure. An oom always is; a die is unless
+    its exit is a stop's (or clean)."""
+    if e.event == "oom":
+        return True
+    return e.event == "die" and exit_is_failure(_event_exit(e.detail))
+
+
+def failures_24h(events, swarm_services, host_name: dict) -> list[dict]:
+    """What failed in the last 24h, newest first, for the dashboard count and
+    the Docker page's list of it: one row per plain container that died or was
+    OOM-killed, and one per swarm service with failed tasks.
+
+    `events` are die/oom events of the last 24h. A swarm task's container dies
+    whenever Swarm replaces it, so a service with Swarm's task records counts
+    its failed tasks instead of its containers' deaths; a service without them
+    (an agent older than 1.8.0) falls back to its tasks' deaths.
+
+    Each row: {kind: "container"|"service", name, host_id, host, count, at,
+    detail}.
     """
     by_prefix = {s["name"] + ".": s for s in swarm_services}
-    plain, fallback = 0, set()
-    for name in dead_names:
-        svc = next((s for p, s in by_prefix.items() if name.startswith(p)), None)
-        if svc is None:
-            plain += 1
-        elif not svc["tasks_known"]:
-            fallback.add(name)
-    tasks = sum(s["failures_24h"] or 0 for s in swarm_services if s["tasks_known"])
-    return plain + len(fallback) + tasks
+    rows: dict[tuple, dict] = {}
+    # Each manager reports every swarm task, so one task's death can arrive
+    # once per manager: a service counts each dead task container once.
+    task_deaths: dict[tuple, set] = {}
+    for e in sorted(events, key=lambda e: e.at, reverse=True):
+        if not event_is_failure(e):
+            continue
+        svc = next((s for p, s in by_prefix.items() if e.container_name.startswith(p)), None)
+        if svc is not None and svc["tasks_known"]:
+            continue
+        kind, name = ("service", svc["name"]) if svc else ("container", e.container_name)
+        key = (kind, name) if svc else (kind, e.host_id, name)
+        if svc is not None:
+            seen = task_deaths.setdefault(key, set())
+            if e.container_name in seen:
+                continue
+            seen.add(e.container_name)
+        row = rows.get(key)
+        if row is None:
+            rows[key] = {"kind": kind, "name": name, "host_id": e.host_id,
+                         "host": host_name.get(e.host_id, e.host_id), "count": 1, "at": e.at,
+                         "detail": "OOM-killed" if e.event == "oom" else e.detail}
+        else:
+            row["count"] += 1
+    from .swarm_view import task_time
+    for s in swarm_services:
+        if s["tasks_known"] and s["failures_24h"]:
+            err = s["last_error"] or {}
+            rows[("service", s["name"])] = {
+                "kind": "service", "name": s["name"], "host_id": None, "host": err.get("node"),
+                "count": s["failures_24h"], "at": task_time(err) if err else None,
+                "detail": err.get("error") or err.get("message") or err.get("state")}
+    epoch = datetime.min.replace(tzinfo=timezone.utc)
+    return sorted(rows.values(), key=lambda r: r["at"] or epoch, reverse=True)
+
+
+def task_label(c, node_hostname: dict | None = None) -> str:
+    """How a container reads to a person. A swarm task is its service and
+    slot: "codex_db · 1" for a replica, "agent · node-3" for a global
+    service's task on a node. Anything else is its own name."""
+    from .swarm_view import _slot_key
+    svc = getattr(c, "service_name", None)
+    if not svc:
+        return c.name
+    slot = _slot_key(c)
+    if slot.isdigit():
+        return f"{svc} · {slot}"
+    host = (node_hostname or {}).get(slot) or (node_hostname or {}).get(getattr(c, "node_id", None))
+    return f"{svc} · {host}" if host else svc
+
+
+def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = None) -> dict:
+    """A container's CPU and memory in terms a reader can weigh.
+
+    Docker's CPU% is per core (100% = one core busy), so it is shown as cores
+    and graded as a share of the host's CPUs when the host's core count is
+    known. Docker's memory limit is the host's RAM when none is set, so memory
+    is shown in bytes, against a limit only when one is actually set; mem_pct
+    is then a share of that limit, otherwise of the host's memory. `cpu_pct`
+    replaces the container's last sample (an average, say) when given.
+    """
+    cpu = getattr(c, "cpu_pct", None) if cpu_pct is None else cpu_pct
+    limit = getattr(c, "mem_limit_bytes", None)
+    limited = bool(limit and mem_total and limit < mem_total * 0.95)
+    return {
+        "cores": None if cpu is None else cpu / 100.0,
+        "cpu_share": None if cpu is None or not cpus else cpu / cpus,
+        "cpus": cpus,
+        "mem_bytes": getattr(c, "mem_usage_bytes", None),
+        "mem_limit": limit if limited else None,
+        "mem_pct": getattr(c, "mem_pct", None),
+        "mem_of": "its limit" if limited else ("the host's memory" if mem_total else None),
+    }
+
+
+async def host_capacity(db, host_ids) -> dict[str, tuple[int | None, int | None]]:
+    """{host_id: (CPU cores, memory bytes)} from the host agent's latest
+    readings: cores counted from its per-core CPU series, memory from
+    available ÷ (1 − used%). (None, None) for a host without an agent."""
+    from sqlalchemy import or_
+    from steward.models.metrics import PluginMetricLatest
+    hosts = await host_map(db, set(host_ids))
+    if not hosts:
+        return {}
+    names = {h.name: hid for hid, h in hosts.items()}
+    rows = (await db.execute(
+        select(PluginMetricLatest.resource_name, PluginMetricLatest.metric_name,
+               PluginMetricLatest.value)
+        .where(PluginMetricLatest.source_module == "host_agent",
+               or_(PluginMetricLatest.resource_name.in_(list(names)),
+                   *(PluginMetricLatest.resource_name.like(escape_like(n) + ":core%", escape="\\")
+                     for n in names)))
+    )).all()
+    cores: dict[str, int] = {}
+    mem: dict[str, dict] = {}
+    for res, metric, value in rows:
+        if res in names:
+            mem.setdefault(res, {})[metric] = value
+        elif metric == "cpu_pct":
+            host = res.rsplit(":core", 1)[0]
+            cores[host] = cores.get(host, 0) + 1
+    out = {}
+    for name, hid in names.items():
+        m = mem.get(name, {})
+        avail, used = m.get("mem_available_bytes"), m.get("mem_used_pct")
+        total = int(avail / (1 - used / 100)) if avail and used is not None and used < 99.5 else None
+        out[hid] = (cores.get(name), total)
+    return out
 
 
 def bucket(values: list[float], target: int = 40) -> list[float]:
@@ -210,7 +344,8 @@ async def log_context(db, host_id: str, name: str, ts, n: int) -> tuple[list, li
 
 
 async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]:
-    """{(host_id, container_name): [die/oom events since `since`, newest first]}."""
+    """{(host_id, container_name): [die/oom events since `since`, newest first]},
+    leaving out a die that was a stop (event_is_failure)."""
     rows = (await db.execute(
         select(DockerEvent)
         .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
@@ -218,10 +353,21 @@ async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]
     )).scalars()
     out: dict[tuple[str, str], list] = {}
     for e in rows:
-        out.setdefault((e.host_id, e.container_name), []).append(e)
+        if event_is_failure(e):
+            out.setdefault((e.host_id, e.container_name), []).append(e)
     return out
 
 
+async def failure_events(db, since) -> list:
+    """Die and OOM events since `since`, newest first; failures_24h() decides
+    which of them are failures."""
+    return list((await db.execute(
+        select(DockerEvent)
+        .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
+        .order_by(DockerEvent.at.desc())
+    )).scalars())
+
+
 async def retained_names(db, want: str, *, exact: bool, host_id: str | None = None,
                          limit: int = 20) -> list[tuple[str, str, datetime]]:
     """(host_id, container_name, last seen) of containers Steward still holds
@@ -303,7 +449,7 @@ def container_problems(c, failures_1h: int, failed_24h: bool,
         out.append("unhealthy")
     if c.oom_killed:
         out.append("oom_killed")
-    if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h:
+    if status in ("exited", "dead") and exit_is_failure(c.exit_code) and failed_24h:
         out.append("exited_nonzero")
     return out
 

plugins/docker/swarm_view.py

@@ -350,3 +350,31 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
     standalone = sorted(groups.values(), key=lambda g: g["host"].lower())
 
     return {"services": out_services, "standalone": standalone, "superseded": superseded}
+
+
+def service_trouble(s: dict) -> int | None:
+    """How bad a built service is, for sorting and for Needs attention: 0 a
+    crash loop, 1 no replica running, 2 short of replicas, 3 healthy but with
+    failed tasks in the last 24h; None when nothing is wrong."""
+    if s.get("crash_looping"):
+        return 0
+    if s.get("state") == "down":
+        return 1
+    if s.get("state") == "degraded":
+        return 2
+    if s.get("failures_24h"):
+        return 3
+    return None
+
+
+def services_problems_first(services: list[dict]) -> list[dict]:
+    """Services in trouble first, worst first, then the rest by name."""
+    return sorted(services, key=lambda s: (
+        4 if service_trouble(s) is None else service_trouble(s), s["name"].lower()))
+
+
+def failing_services(services: list[dict]) -> list[dict]:
+    """The services that belong in Needs attention: a crash loop, or short of
+    their desired replicas. A healthy service whose failed tasks Swarm has
+    already replaced is history, shown on the service, not a problem now."""
+    return [s for s in services_problems_first(services) if (service_trouble(s) or 9) <= 2]

plugins/docker/templates/docker/_usage.html

@@ -0,0 +1,26 @@
+{#- A container's CPU and memory as service.usage() describes them (#5209).
+
+    Docker counts CPU per core (100% = one core busy), so CPU reads as cores,
+    toned by its share of the host's CPUs when the host's core count is known.
+    Memory reads in bytes, against a limit only when one is actually set;
+    mem_pct is then a share of that limit, otherwise of the host's memory. -#}
+
+{% macro cores(n) -%}
+{{ "%.2f"|format(n) if n < 1 else "%.1f"|format(n) }} {{ "core" if n == 1 else "cores" }}
+{%- endmacro %}
+
+{% macro cpu(u) -%}
+{%- if u.cores is none -%}<span class="tone-dim">—</span>
+{%- else -%}
+<span class="tone-{{ status_class(u.cpu_share, 'cpu') }}"
+      title="Docker counts CPU per core: {{ '%.0f'|format(u.cores * 100) }}% of one core{% if u.cpus %}, {{ '%.1f'|format(u.cpu_share) }}% of this host's {{ u.cpus }} core{{ 's' if u.cpus != 1 }}{% endif %}">{{ cores(u.cores) }}</span>
+{%- endif -%}
+{%- endmacro %}
+
+{% macro mem(u) -%}
+{%- if u.mem_bytes is none -%}<span class="tone-dim">—</span>
+{%- else -%}
+<span class="tone-{{ status_class(u.mem_pct, 'mem') if u.mem_of else 'dim' }}"
+      title="{% if u.mem_pct is not none and u.mem_of %}{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% else %}No memory limit set{% endif %}">{{ u.mem_bytes|bytes }}{% if u.mem_limit %} / {{ u.mem_limit|bytes }}{% endif %}</span>
+{%- endif -%}
+{%- endmacro %}

plugins/docker/templates/docker/container_detail.html

@@ -1,8 +1,11 @@
-{# docker/container_detail.html — full detail page for one container #}
+{# docker/container_detail.html — full detail page for one container. A swarm
+   task is titled by its service and slot, with the raw container name under
+   it; CPU reads in cores and memory in bytes (see _usage.html). #}
 {% extends "base.html" %}
 {% from "_macros.html" import crumbs %}
-{% block title %}{{ name }} — Docker — Steward{% endblock %}
-{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %}
+{% from "docker/_usage.html" import cpu, mem %}
+{% block title %}{{ label or name }} — Docker — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (label or name, "")]) }}{% endblock %}
 {% block content %}
 
 {% if container is none %}
@@ -18,11 +21,14 @@
 {# ── Header ──────────────────────────────────────────────────────────────── #}
 <div style="display:flex;align-items:center;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
   <span class="dot {% if container.status == 'running' %}dot-up{% elif container.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
-  <h1 class="page-title" style="margin-bottom:0;">{{ container.name }}</h1>
+  <h1 class="page-title" style="margin-bottom:0;">{{ label }}</h1>
   {% if container.health == 'healthy' %}<span style="font-size:0.72rem;color:var(--green);border:1px solid var(--green);border-radius:3px;padding:0.05rem 0.4rem;">healthy</span>
   {% elif container.health == 'unhealthy' %}<span style="font-size:0.72rem;color:var(--red);border:1px solid var(--red);border-radius:3px;padding:0.05rem 0.4rem;">unhealthy</span>
   {% elif container.health == 'starting' %}<span style="font-size:0.72rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0.05rem 0.4rem;">starting</span>{% endif %}
 </div>
+{% if label != container.name %}
+<div style="font-size:0.75rem;color:var(--text-dim);font-family:ui-monospace,monospace;margin-bottom:0.25rem;overflow-wrap:anywhere;">{{ container.name }}</div>
+{% endif %}
 <div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
   {{ container.status }}{% if uptime %} · up {{ uptime }}{% endif %}
   {% if host %} · on <a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %}
@@ -38,16 +44,27 @@
 {% endmacro %}
 
 <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
-  {{ fact("CPU", "%.1f%%" | format(container.cpu_pct) if container.cpu_pct is not none else "—") }}
-  {{ fact("Memory", "%.1f%%" | format(container.mem_pct) if container.mem_pct is not none else "—") }}
+  {{ fact("CPU", cpu(usage) if container.status == 'running' else "—") }}
+  {{ fact("Memory", mem(usage) if container.status == 'running' else "—") }}
+  {#- Swarm never restarts a task's container: it replaces the task with a new
+      one, so restart_count is always 0 there and the service's task failures
+      (Swarm page) are the count that means something. #}
+  {% if not container.service_name %}
   {{ fact("Restarts", container.restart_count, "var(--orange)" if container.restart_count else "") }}
+  {% endif %}
   {{ fact("Last exit", (container.exit_code ~ (" (OOM)" if container.oom_killed else "")) if container.exit_code is not none else "—",
-          "var(--red)" if (container.exit_code is not none and container.exit_code != 0) else "") }}
-  {{ fact("Net in", net_rx) }}
-  {{ fact("Net out", net_tx) }}
-  {{ fact("Block read", blk_read) }}
-  {{ fact("Block write", blk_write) }}
+          "var(--red)" if exit_failed else "") }}
+  {{ fact("Net in since start", net_rx) }}
+  {{ fact("Net out since start", net_tx) }}
+  {{ fact("Disk read since start", blk_read) }}
+  {{ fact("Disk written since start", blk_write) }}
 </div>
+{% if container.service_name %}
+<div style="font-size:0.78rem;color:var(--text-muted);margin:-0.75rem 0 1.5rem;">
+  Swarm replaces a failed task with a new container rather than restarting it; the
+  service's failed tasks are on the <a href="/plugins/docker/swarm">Swarm page</a>.
+</div>
+{% endif %}
 
 {# ── Image / placement ───────────────────────────────────────────────────── #}
 <div class="card" style="margin-bottom:1.5rem;">
@@ -61,7 +78,9 @@
     <span style="color:var(--text-muted);">Swarm service</span><span>{{ container.service_name }}</span>
     {% endif %}
     {% if container.node_id %}
-    <span style="color:var(--text-muted);">Node</span><span style="font-family:ui-monospace,monospace;">{{ container.node_id }}</span>
+    <span style="color:var(--text-muted);">Node</span>
+    {% if node_name %}<span title="node id {{ container.node_id }}">{{ node_name }}</span>
+    {% else %}<span style="font-family:ui-monospace,monospace;">{{ container.node_id }}</span>{% endif %}
     {% endif %}
     {% if ports %}
     <span style="color:var(--text-muted);">Ports</span>

plugins/docker/templates/docker/host_panel.html

@@ -1,22 +1,58 @@
-{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub #}
+{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub:
+   problems first with what is wrong and since when, then running, then
+   stopped. Tasks Swarm has replaced are left out (counted in the header). #}
+{% from "_macros.html" import status_badge, status_dot %}
+{% from "docker/_usage.html" import cpu, mem %}
 <div class="card">
-  <div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;">
+  <div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;flex-wrap:wrap;">
     <h3 class="section-title" style="margin-bottom:0;">Docker</h3>
     <span style="font-size:0.78rem;color:var(--text-muted);">
+      {% if problems %}<span class="tone-crit">{{ problems }} with problems</span> · {% endif %}
       {{ running }} running{% if stopped %} · {{ stopped }} stopped{% endif %}
+      {% if replaced %}<span title="Old swarm tasks Swarm has replaced with new ones: history, not containers that need anything"> · {{ replaced }} replaced task{{ "s" if replaced != 1 }} hidden</span>{% endif %}
       <a href="/plugins/docker/" style="margin-left:0.6rem;color:var(--text-muted);">All →</a>
     </span>
   </div>
-  <div style="display:grid;gap:2px;">
-    {% for c in containers %}
-    <div style="display:flex;align-items:center;gap:0.5rem;font-size:0.85rem;padding:0.2rem 0;">
-      <span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
-      <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;" title="{{ c.image }}">{{ c.name }}</a>
-      <span style="font-size:0.74rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
-        {% if c.cpu_pct is not none %}{{ "%.1f" | format(c.cpu_pct) }}%{% endif %}
-        {% if c.mem_pct is not none %} · {{ "%.1f" | format(c.mem_pct) }}%{% endif %}
-      </span>
-    </div>
-    {% endfor %}
-  </div>
+  <table class="table">
+    <thead>
+      <tr>
+        <th>Container</th>
+        <th title="Docker counts CPU per core: 1 core = one core fully busy">CPU</th>
+        <th title="Used, and the limit when one is set">Memory</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for r in rows %}
+      {% set c = r.c %}
+      <tr>
+        <td>
+          <div style="display:flex;align-items:center;gap:0.5rem;min-width:0;">
+            {{ status_dot('crit' if r.problems else ('up' if c.status == 'running' else 'inactive'), c.status|capitalize) }}
+            <div style="min-width:0;">
+              <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" title="{{ c.name }} · {{ c.image }}" style="font-weight:500;color:inherit;text-decoration:none;overflow-wrap:anywhere;">{{ r.label }}</a>
+              {% if r.problems %}
+              {% set worst = r.problems[0] %}
+              <div style="font-size:0.75rem;margin-top:0.1rem;">
+                {{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }}
+                <span style="color:var(--text-muted);">
+                  {%- for p in r.problems %} {{ says[p] }}{{ " and" if not loop.last }}{% endfor %}
+                  {%- if c.exit_code %} · exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}{% endif %}
+                  {%- if r.since %} · since {{ r.since|ago }}{% endif -%}
+                </span>
+              </div>
+              {% else %}
+              <div style="font-size:0.72rem;color:var(--text-muted);">
+                {{ c.status }}{% if r.uptime %} · up {{ r.uptime }}{% endif %}
+                {%- if c.status != 'running' and c.exit_code %} · exit {{ c.exit_code }}{% endif %}
+              </div>
+              {% endif %}
+            </div>
+          </div>
+        </td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">{% if c.status == 'running' %}{{ cpu(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">{% if c.status == 'running' %}{{ mem(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
 </div>

plugins/docker/templates/docker/index.html

@@ -13,4 +13,17 @@
      hx-swap="innerHTML">
   <div class="card"><p class="field-hint">Loading…</p></div>
 </div>
+{# A link to a section (the dashboard's "failed (24h)" → #failures) lands
+   before the rows exist; scroll to it once, when they first arrive. #}
+<script>
+(function () {
+  var rows = document.getElementById('docker-rows');
+  if (!location.hash || !rows) return;
+  rows.addEventListener('htmx:afterSwap', function once() {
+    rows.removeEventListener('htmx:afterSwap', once);
+    var target = document.getElementById(location.hash.slice(1));
+    if (target) target.scrollIntoView({ block: 'start' });
+  });
+})();
+</script>
 {% endblock %}

plugins/docker/templates/docker/rows.html

@@ -2,18 +2,36 @@
    first (service.needs_attention, the ranking the MCP tools use), the counts,
    then swarm services and each host's containers. #}
 {% from "_macros.html" import stat_tile, status_badge, status_dot %}
+{% from "docker/_usage.html" import cpu, mem, cores %}
 
 {# ── Needs attention ──────────────────────────────────────────────────────── #}
-{% if attention %}
+{% if attention or failing_services %}
 <section class="card problem-card" aria-labelledby="docker-problems">
   <h2 id="docker-problems" class="section-title" style="margin-bottom:0.6rem;">Needs attention</h2>
   <ul class="problem-list">
+    {# A swarm service in trouble: Swarm replaces its failed tasks, so the
+       service, not any one container, is what is failing. #}
+    {% for s in failing_services %}
+    {% set e = s.last_error %}
+    <li>
+      {% if s.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}
+      {% elif s.state == 'down' %}{{ status_badge('crit', 'Down') }}
+      {% else %}{{ status_badge('warn', 'Degraded') }}{% endif %}
+      <a href="/plugins/docker/swarm">{{ s.name }}</a>
+      <span class="problem-detail">
+        {{- s.running }} of {{ s.desired }} running
+        {%- if s.crash_looping %} · {{ s.failures_1h }} tasks failed in the last hour{% endif %}
+        {%- if e %} · last task {{ e.state }}: {{ e.error or e.message or e.state }}{% if e.exit_code %} (exit {{ e.exit_code }}){% endif %}{% if e.node %} on {{ e.node }}{% endif %}{% endif %}
+        {%- if s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif -%}
+      </span>
+    </li>
+    {% endfor %}
     {% for r in attention %}
     {% set c = r.container %}
     {% set worst = r.problems[0] %}
     <li>
       {{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }}
-      <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}">{{ c.name }}</a>
+      <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" title="{{ c.name }}">{{ r.label }}</a>
       <span class="problem-detail">
         {%- for p in r.problems %}{{ says[p] }}{{ " and " if not loop.last }}{% endfor %}
         {%- if c.exit_code and worst in ('exited_nonzero', 'crash_looping', 'restarting') %} · exit {{ c.exit_code }}{% endif %}
@@ -42,18 +60,45 @@
 </section>
 {% endif %}
 
-{# ── Counts ───────────────────────────────────────────────────────────────── #}
+{# ── Counts ──────────────────────────────────────────────────────────────
+   Swarm services are counted as services (a task's container comes and goes);
+   the container counts are the containers outside any swarm service. ────── #}
 <div class="stat-strip">
-  {{ stat_tile("Needs attention", attention|length, state="crit" if attention else "inactive") }}
+  {{ stat_tile("Needs attention", attention|length + failing_services|length, state="crit" if attention or failing_services else "inactive") }}
   {{ stat_tile("Unhealthy", unhealthy, state="crit" if unhealthy else "inactive") }}
   {{ stat_tile("Restarting", restarting, state="crit" if restarting else "inactive", hint="restarting or crash-looping") }}
-  {{ stat_tile("Running", running, state="up" if running else "inactive") }}
-  {{ stat_tile("Stopped", stopped, state="inactive") }}
+  {{ stat_tile("Failed (24h)", failures|length, state="warn" if failures else "inactive", hint="containers and services; listed below", href="#failures" if failures else none) }}
+  {% if swarm_services %}{{ stat_tile("Swarm services", services_healthy ~ "/" ~ swarm_services|length, state="up" if services_healthy == swarm_services|length else "warn", hint="healthy", href="/plugins/docker/swarm") }}{% endif %}
+  {{ stat_tile("Containers running", running, state="up" if running else "inactive", hint="outside swarm services" if swarm_services else none) }}
+  {{ stat_tile("Containers stopped", stopped, state="inactive") }}
   {% if stale %}{{ stat_tile("Stale", stale, state="stale", hint="no recent report") }}{% endif %}
-  {% if swarm_services %}{{ stat_tile("Services", swarm_services|length, href="/plugins/docker/swarm") }}{% endif %}
-  {{ stat_tile("Hosts", host_groups|length) }}
+  {{ stat_tile("Docker hosts", docker_hosts) }}
 </div>
 
+{# ── What failed in the last 24h: what the dashboard's "failed (24h)" counts.
+   A plain container's death, or a swarm service's failed tasks; a container
+   someone stopped (exit 137/143) is not a failure. ───────────────────────── #}
+{% if failures %}
+<section id="failures" class="card" aria-labelledby="docker-failures" style="margin-bottom:1.5rem;">
+  <h2 id="docker-failures" class="section-title" style="margin-bottom:0.6rem;">Failed in the last 24h</h2>
+  <ul class="problem-list">
+    {% for f in failures %}
+    <li>
+      {{ status_badge('warn', 'Service' if f.kind == 'service' else 'Container') }}
+      {% if f.kind == 'service' %}<a href="/plugins/docker/swarm">{{ f.name }}</a>
+      {% else %}<a href="/plugins/docker/container/{{ f.host_id }}/{{ f.name }}">{{ f.name }}</a>{% endif %}
+      <span class="problem-detail">
+        {{- f.count }} {{ "failure" if f.count == 1 else "failures" }}
+        {%- if f.detail %} · last: {{ f.detail }}{% endif %}
+        {%- if f.host %} · {{ f.host }}{% endif %}
+        {%- if f.at %} · {{ f.at|ago }}{% endif -%}
+      </span>
+    </li>
+    {% endfor %}
+  </ul>
+</section>
+{% endif %}
+
 {# ── Swarm services: collapsed per-service, each current replica labelled with
    its host (tasks Swarm replaced are left out). Ghost replicas (tasks on nodes
    with no Steward agent) come from the managers' placement data so the picture
@@ -72,7 +117,7 @@
         <span style="font-weight:500;font-size:0.9rem;">{{ s.name }}</span>
         <span style="font-size:0.72rem;color:var(--text-dim);">{{ s.mode }}</span>
         <span class="tone-{{ 'ok' if s.state == 'healthy' else ('warn' if s.state == 'degraded' else 'crit') }}" style="font-size:0.78rem;font-variant-numeric:tabular-nums;">
-          {{ s.running }}/{{ s.desired }} running
+          {{ s.running }}/{{ s.desired }} running{% if s.state != 'healthy' and s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif %}
         </span>
       </div>
       {% with style="padding-left:1.1rem;margin-bottom:0.35rem;" %}{% include "docker/_swarm_failures.html" %}{% endwith %}
@@ -88,7 +133,7 @@
         <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px solid var(--border);border-radius:5px;padding:0.3rem 0.55rem;">
           <span class="dot {% if r.status == 'running' %}dot-up{% elif r.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
           <a href="/plugins/docker/container/{{ r.host_id }}/{{ r.name }}" style="font-weight:500;color:inherit;text-decoration:none;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;" title="{{ r.name }}">{{ r.host }}</a>
-          {% if r.cpu_pct is not none %}<span style="color:var(--text-muted);font-family:ui-monospace,monospace;margin-left:auto;">{{ "%.0f" | format(r.cpu_pct) }}%</span>{% endif %}
+          {% if r.cpu_pct is not none %}<span style="color:var(--text-muted);font-family:ui-monospace,monospace;margin-left:auto;" title="Docker counts CPU per core">{{ cores(r.cpu_pct / 100) }}</span>{% endif %}
         </div>
         {% endif %}
         {% endfor %}
@@ -119,8 +164,8 @@
         <th>Container</th>
         <th>Image</th>
         <th>Ports</th>
-        <th>CPU %</th>
-        <th>Mem %</th>
+        <th title="Docker counts CPU per core: 1 core = one core fully busy">CPU</th>
+        <th title="Used, and the limit when one is set">Memory</th>
       </tr>
     </thead>
     <tbody>
@@ -144,7 +189,7 @@
               <div style="font-size:0.73rem;color:var(--text-muted);">
                 {{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %}
                 {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
-                  · <span class="tone-crit">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
+                  · <span class="tone-{{ 'crit' if item.exit_failed else 'dim' }}" title="{{ 'last exit code' if item.exit_failed else 'stopped by a signal; not a failure' }}">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
                 {% endif %}
                 {% if c.restart_count %} · <span class="tone-warn" title="Restarted {{ c.restart_count }} times">⟳{{ c.restart_count }}</span>{% endif %}
               </div>
@@ -164,20 +209,8 @@
           <div style="color:var(--text-muted);">{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}</div>
           {% endfor %}
         </td>
-        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
-          {% if c.cpu_pct is not none %}
-          <span class="tone-{{ status_class(c.cpu_pct, 'cpu') }}">{{ "%.1f" | format(c.cpu_pct) }}%</span>
-          {% else %}
-          <span style="color:var(--text-dim);">—</span>
-          {% endif %}
-        </td>
-        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
-          {% if c.mem_pct is not none %}
-          <span class="tone-{{ status_class(c.mem_pct, 'mem') }}">{{ "%.1f" | format(c.mem_pct) }}%</span>
-          {% else %}
-          <span style="color:var(--text-dim);">—</span>
-          {% endif %}
-        </td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;white-space:nowrap;">{{ cpu(item.usage) }}</td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;white-space:nowrap;">{{ mem(item.usage) }}</td>
       </tr>
       {% endfor %}
       {% endfor %}

plugins/docker/templates/docker/swarm.html

@@ -1,4 +1,6 @@
-{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement #}
+{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement.
+   Services in trouble come first (crash loop, down, short), with since-times;
+   nodes that are down or not taking work come first. #}
 {% extends "base.html" %}
 {% from "_macros.html" import crumbs %}
 {% block title %}Swarm — Docker — Steward{% endblock %}
@@ -34,6 +36,7 @@
             <span style="color:{% if s.healthy %}var(--green){% elif s.down %}var(--red){% elif s.degraded %}var(--orange){% else %}var(--text-muted){% endif %};">
               {{ s.running }}/{{ s.desired }}
             </span>
+            {% if s.state != 'healthy' and s.trouble_since %}<div style="font-family:inherit;font-size:0.72rem;color:var(--text-dim);" title="{{ s.trouble_since }}">since {{ s.trouble_since|ago }}</div>{% endif %}
           </td>
           <td style="font-size:0.8rem;color:var(--text-muted);max-width:220px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.image or "—" }}</td>
           <td style="font-size:0.78rem;color:var(--text-muted);">

plugins/docker/templates/docker/widget.html

@@ -1,21 +1,32 @@
-{# docker/widget.html — dashboard widget: container status overview, by host #}
+{# docker/widget.html — dashboard widget: what Docker is running and what has
+   failed. Swarm services first, troubled ones at the top with since-times; then
+   the containers outside any service, by host, problems first. The failed count
+   links to the Docker page's list of exactly what it counts. #}
+{% from "docker/_usage.html" import cpu %}
 {% if total_count == 0 %}
 <p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No containers reported yet.</p>
 {% else %}
 
-<div style="display:flex;gap:1.25rem;margin-bottom:0.65rem;">
-  <div>
+<div style="display:flex;flex-wrap:wrap;gap:0.4rem 1.25rem;margin-bottom:0.65rem;">
+  {% if swarm_services %}
+  <div title="Swarm services with every replica running">
+    <span class="tone-{{ 'ok' if services_healthy == swarm_services|length else 'warn' }}" style="font-size:1.4rem;font-weight:500;line-height:1;">{{ services_healthy }}/{{ swarm_services|length }}</span>
+    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">services healthy</span>
+  </div>
+  {% endif %}
+  <div title="Containers running outside any swarm service">
     <span style="font-size:1.4rem;font-weight:500;line-height:1;color:var(--green);">{{ running_count }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">running</span>
-  </div>
-  <div title="Containers that died or were OOM-killed in the last 24h, plus swarm tasks that failed (a task Swarm replaced is not a failure)">
-    <span style="font-size:1.4rem;font-weight:500;line-height:1;color:{% if failed_24h %}var(--red){% else %}var(--text-muted){% endif %};">{{ failed_24h }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">failed (24h)</span>
+    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">containers running</span>
   </div>
+  <a href="/plugins/docker/#failures" style="color:inherit;text-decoration:none;"
+     title="Containers that died or were OOM-killed in the last 24h, and swarm services with failed tasks. A container someone stopped is not counted. Click for the list.">
+    <span class="tone-{{ 'crit' if failures else 'dim' }}" style="font-size:1.4rem;font-weight:500;line-height:1;">{{ failures|length }}</span>
+    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;border-bottom:1px dotted var(--border-mid);">failed in 24h</span>
+  </a>
 </div>
 
-{# Swarm services — collapsed, each with its replicas' host chips (dashed = a
-   node with no Steward agent, count-only from placement). #}
+{# Swarm services — troubled first, each with its replicas' host chips
+   (dashed = a node with no Steward agent, count-only from placement). #}
 {% if swarm_services %}
 <div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">Swarm services</div>
 <div style="display:grid;gap:0.4rem;margin-bottom:0.4rem;">
@@ -24,8 +35,9 @@
     <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.82rem;">
       <span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
       <span style="font-weight:500;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.name }}</span>
-      {% if s.crash_looping %}<span style="color:var(--red);font-size:0.7rem;font-weight:500;flex-shrink:0;" title="{{ s.failures_1h }} tasks failed in the last hour{% if s.last_error %}: {{ s.last_error.error or s.last_error.message or s.last_error.state }}{% endif %}">crash-looping</span>{% endif %}
-      <span style="color:var(--text-muted);font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;">{{ s.running }}/{{ s.desired }}</span>
+      {% if s.crash_looping %}<span class="tone-crit" style="font-size:0.7rem;font-weight:500;flex-shrink:0;" title="{{ s.failures_1h }} tasks failed in the last hour{% if s.last_error %}: {{ s.last_error.error or s.last_error.message or s.last_error.state }}{% endif %}">crash-looping</span>{% endif %}
+      {% if s.state != 'healthy' and s.trouble_since %}<span class="tone-dim" style="font-size:0.7rem;flex-shrink:0;">since {{ s.trouble_since|ago }}</span>{% endif %}
+      <span class="tone-{{ 'ok' if s.state == 'healthy' else ('warn' if s.state == 'degraded' else 'crit') }}" style="font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
     </div>
     <div style="display:flex;flex-wrap:wrap;gap:0.25rem;margin:0.2rem 0 0 1rem;">
       {% for r in s.replicas %}
@@ -41,25 +53,25 @@
 </div>
 {% endif %}
 
+{% if host_groups and swarm_services %}
+<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.6rem 0 0.2rem;">Other containers</div>
+{% endif %}
 {% for g in host_groups %}
 {% if multi_host %}
-<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">{{ g.host_name }}</div>
+<div style="font-size:0.7rem;color:var(--text-dim);margin:0.4rem 0 0.15rem;">{{ g.host_name }}</div>
 {% endif %}
 <div style="display:grid;gap:2px;">
-  {% for c in g.containers %}
+  {% for item in g.containers %}
+  {% set c = item.c %}
   <div style="display:flex;align-items:center;gap:0.5rem;font-size:0.82rem;padding:0.15rem 0;">
-    <span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
+    <span class="dot {% if c.status == 'running' and c.health != 'unhealthy' %}dot-up{% elif c.status in ('paused', 'restarting') %}dot-warn{% else %}dot-down{% endif %}"></span>
     <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;">
-      {{ c.name }}{% if c.health == 'unhealthy' %}<span title="unhealthy" style="color:var(--red);"> ●</span>{% elif c.health == 'healthy' %}<span title="healthy" style="color:var(--green);"> ●</span>{% endif %}{% if c.restart_count %}<span title="restarts" style="color:var(--orange);font-size:0.7rem;"> ⟳{{ c.restart_count }}</span>{% endif %}
+      {{ c.name }}{% if c.health == 'unhealthy' %} <span class="tone-crit" style="font-size:0.7rem;">unhealthy</span>{% endif %}{% if c.status == 'restarting' %} <span class="tone-warn" style="font-size:0.7rem;">restarting</span>{% endif %}
     </a>
     {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
-    <span title="last exit code" style="font-size:0.7rem;color:var(--red);flex-shrink:0;">exit {{ c.exit_code }}</span>
-    {% endif %}
-    {% if c.cpu_pct is not none %}
-    <span style="font-size:0.72rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
-      {{ "%.1f" | format(c.cpu_pct) }}%
-    </span>
+    <span title="{{ 'last exit code' if item.exit_failed else 'stopped by a signal; not a failure' }}" class="tone-{{ 'crit' if item.exit_failed else 'dim' }}" style="font-size:0.7rem;flex-shrink:0;">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
     {% endif %}
+    {% if c.status == 'running' %}<span style="font-size:0.72rem;font-family:ui-monospace,monospace;flex-shrink:0;">{{ cpu(item.usage) }}</span>{% endif %}
   </div>
   {% endfor %}
 </div>

plugins/docker/templates/docker/widget_resources.html

@@ -1,28 +1,41 @@
-{# docker/widget_resources.html — dashboard widget: the busiest running containers
-   by CPU, each with labeled CPU + memory utilisation bars. #}
+{# docker/widget_resources.html — dashboard widget: the containers using the
+   most CPU (averaged over the last few minutes, so the order holds still),
+   each with its CPU in cores and memory in bytes. The CPU bar is a share of
+   the host's CPUs; the memory bar a share of the container's limit, or of
+   the host's memory when no limit is set. #}
+{% from "docker/_usage.html" import cpu, mem %}
 {% if not rows %}
 <div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No running containers.</div>
 {% else %}
-{% macro bar(label, pct, warn, crit) %}
-<div style="display:flex;align-items:center;gap:0.45rem;margin-bottom:2px;">
+{% macro bar(label, pct, tone, text, title) %}
+<div style="display:flex;align-items:center;gap:0.45rem;margin-bottom:2px;" title="{{ title }}">
   <span style="font-size:0.62rem;color:var(--text-dim);width:1.9rem;flex-shrink:0;letter-spacing:0.03em;">{{ label }}</span>
   <div style="flex:1;height:6px;background:var(--bg-elevated);border-radius:3px;overflow:hidden;">
-    <div style="height:100%;border-radius:3px;width:{{ [pct, 100] | min }}%;
-                background:{% if pct > crit %}var(--red){% elif pct > warn %}var(--orange){% else %}var(--accent){% endif %};
+    {% if pct is not none %}
+    <div style="height:100%;border-radius:3px;width:{{ [[pct, 0]|max, 100]|min }}%;
+                background:{% if tone == 'crit' %}var(--red){% elif tone == 'warn' %}var(--orange){% else %}var(--accent){% endif %};
                 transition:width 0.4s;"></div>
+    {% endif %}
   </div>
-  <span style="font-size:0.7rem;color:var(--text-muted);font-family:ui-monospace,monospace;width:3rem;text-align:right;flex-shrink:0;">{{ "%.1f" | format(pct) }}%</span>
+  <span style="font-size:0.7rem;color:var(--text-muted);font-family:ui-monospace,monospace;min-width:4.5rem;text-align:right;flex-shrink:0;white-space:nowrap;">{{ text }}</span>
 </div>
 {% endmacro %}
+<div style="font-size:0.7rem;color:var(--text-dim);margin-bottom:0.45rem;">Busiest by CPU, {{ minutes }}-minute average</div>
 <div style="display:grid;gap:0.6rem;">
   {% for r in rows %}
-  {% set c = r.c %}
+  {% set u = r.usage %}
   <div>
-    <div style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-bottom:0.25rem;">
-      {{ c.name }}{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
+    <div style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-bottom:0.25rem;" title="{{ r.c.name }}">
+      <a href="/plugins/docker/container/{{ r.c.host_id }}/{{ r.c.name }}" style="color:inherit;text-decoration:none;">{{ r.label }}</a>{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
     </div>
-    {% if c.cpu_pct is not none %}{{ bar("CPU", c.cpu_pct, 50, 80) }}{% endif %}
-    {% if c.mem_pct is not none %}{{ bar("MEM", c.mem_pct, 70, 90) }}{% endif %}
+    {% if u.cores is not none %}
+    {{ bar("CPU", u.cpu_share, status_class(u.cpu_share, 'cpu'), cpu(u),
+           ("%.1f%% of the host's %d core%s" % (u.cpu_share, u.cpus, "" if u.cpus == 1 else "s")) if u.cpus else "Host core count unknown") }}
+    {% endif %}
+    {% if u.mem_bytes is not none %}
+    {{ bar("MEM", u.mem_pct if u.mem_of else none, status_class(u.mem_pct, 'mem') if u.mem_of else 'unknown', mem(u),
+           ("%.1f%% of %s" % (u.mem_pct, u.mem_of)) if u.mem_of and u.mem_pct is not none else "") }}
+    {% endif %}
   </div>
   {% endfor %}
 </div>

steward/core/widgets.py

@@ -209,8 +209,8 @@ WIDGET_REGISTRY: dict[str, dict] = {
     },
     "docker_containers": {
         "key": "docker_containers",
-        "label": "Docker — Containers",
-        "description": "Container status overview — running/stopped counts and per-container CPU",
+        "label": "Docker — Services & failures",
+        "description": "What Docker is running and what failed: swarm services and containers, problems first, with the last 24h's failures",
         "hx_url": "/plugins/docker/widget",
         "detail_url": "/plugins/docker/",
         "plugin": "docker",
@@ -230,8 +230,8 @@ WIDGET_REGISTRY: dict[str, dict] = {
     },
     "docker_resources": {
         "key": "docker_resources",
-        "label": "Docker — Resources",
-        "description": "CPU and memory usage bars for running containers",
+        "label": "Docker — Busiest containers",
+        "description": "The containers using the most CPU (cores, 5-minute average) and their memory",
         "hx_url": "/plugins/docker/widget/resources",
         "detail_url": "/plugins/docker/",
         "plugin": "docker",

tests/plugins/docker/test_docker_page.py

@@ -43,22 +43,24 @@ def test_stale_is_older_than_the_agents_window():
 async def test_a_crash_loop_comes_first_and_a_stale_row_says_so(app):
     loop = _c("api", status="restarting", exit_code=1, restart_count=7)
     old = _c("cache", scraped_at=NOW - timedelta(minutes=30), cpu_pct=95.0)
-    attention = [{"container": loop, "problems": ["restarting", "crash_looping"], "replaced": False,
+    attention = [{"container": loop, "label": "api", "problems": ["restarting", "crash_looping"], "replaced": False,
                   "since": datetime.now(timezone.utc) - timedelta(minutes=8), "stale": False}]
     groups = [{"host": None, "host_id": "h1", "host_name": "nas", "running": 1, "stopped": 1, "grouped": False,
                "subgroups": [{"label": None, "containers": [
-                   {"container": loop, "ports": [], "uptime": None, "stale": False},
-                   {"container": old, "ports": [], "uptime": "2h", "stale": True}]}]}]
+                   {"container": loop, "ports": [], "uptime": None, "stale": False,
+                    "usage": svc.usage(loop, 1, None), "exit_failed": True},
+                   {"container": old, "ports": [], "uptime": "2h", "stale": True,
+                    "usage": svc.usage(old, 1, None), "exit_failed": False}]}]}]
     async with app.test_request_context("/plugins/docker/rows"):
         page = await render_plugin_template(
             app, "docker", "docker/rows.html", attention=attention, says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
             unhealthy=0, restarting=1, stale=1, host_groups=groups, swarm_services=[],
-            running=1, stopped=1, total=2)
+            running=1, stopped=1, docker_hosts=1, failures=[], failing_services=[])
     page = re.sub(r">\s+<", "><", " ".join(page.split()))
 
     card = page[page.index("Needs attention"):page.index("</section>")]
     assert '<span class="badge badge-crit">Restarting</span>' in card
-    assert 'href="/plugins/docker/container/h1/api">api</a>' in card
+    assert 'href="/plugins/docker/container/h1/api" title="api">api</a>' in card
     assert "is restarting and keeps dying · exit 1 · since <time" in card and ">8m ago</time>" in card
     assert "cache" not in card
     assert page.index("Needs attention") < page.index("<table")
@@ -66,7 +68,9 @@ async def test_a_crash_loop_comes_first_and_a_stale_row_says_so(app):
     assert '<span class="stat-tile-label">Restarting</span><span class="stat-tile-value">1</span>' in page
     assert '<span class="stat-tile-label">Stale</span><span class="stat-tile-value">1</span>' in page
     assert 'class="badge badge-warn" title="Last report' in page          # the stale row
-    assert '<span class="tone-crit">95.0%</span>' in page                  # CPU through Settings → Thresholds
+    # CPU in cores, toned by its share of the host's CPUs through Settings → Thresholds.
+    assert ('<span class="tone-crit" title="Docker counts CPU per core: 95% of one core, '
+            '95.0% of this host\'s 1 core">0.95 cores</span>') in page
     assert "history" not in page.lower() and "<svg" not in page            # no sparklines on the overview
     assert not hex_colours(page), hex_colours(page)
 
@@ -75,7 +79,8 @@ async def test_a_healthy_board_has_no_attention_card(app):
     async with app.test_request_context("/plugins/docker/rows"):
         page = await render_plugin_template(
             app, "docker", "docker/rows.html", attention=[], says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
-            unhealthy=0, restarting=0, stale=0, host_groups=[], swarm_services=[], running=0, stopped=0, total=0)
+            unhealthy=0, restarting=0, stale=0, host_groups=[], swarm_services=[], running=0, stopped=0,
+            docker_hosts=0, failures=[], failing_services=[])
     assert "Needs attention</h2>" not in page and "No containers reported yet" in page
 
 
@@ -89,7 +94,7 @@ async def test_a_host_whose_docker_stopped_answering_says_so(app):
         page = await render_plugin_template(
             app, "docker", "docker/rows.html", docker_down=down, attention=[], says=svc.PROBLEM_SAYS,
             severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
-            swarm_services=[], running=0, stopped=0, total=0)
+            swarm_services=[], running=0, stopped=0, docker_hosts=0, failures=[], failing_services=[])
     page = re.sub(r">\s+<", "><", " ".join(page.split()))
     card = page[page.index("Docker not answering"):page.index("</section>")]
     assert '<span class="badge badge-warn">No Docker</span><a href="/hosts/h1">nas</a>' in card
@@ -102,5 +107,35 @@ async def test_no_card_while_every_docker_answers(app):
         page = await render_plugin_template(
             app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS,
             severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
-            swarm_services=[], running=0, stopped=0, total=0)
+            swarm_services=[], running=0, stopped=0, docker_hosts=0, failures=[], failing_services=[])
     assert "Docker not answering" not in page
+
+
+async def test_a_failing_service_leads_and_the_failures_are_listed(app):
+    now = datetime.now(timezone.utc)
+    sick = {"name": "codex_db", "mode": "replicated", "running": 0, "desired": 1, "state": "down",
+            "crash_looping": True, "failures_1h": 7, "failures_24h": 9, "tasks_known": True,
+            "trouble_since": now - timedelta(minutes=20), "replicas": [], "previous_tasks": 7,
+            "last_error": {"state": "failed", "error": "task: non-zero exit (1)", "exit_code": 1,
+                           "node": "nas", "at": None}}
+    failures = [{"kind": "service", "name": "codex_db", "host_id": None, "host": "nas", "count": 9,
+                 "at": now - timedelta(minutes=2), "detail": "task: non-zero exit (1)"},
+                {"kind": "container", "name": "backup", "host_id": "h1", "host": "nas", "count": 1,
+                 "at": now - timedelta(hours=3), "detail": "exit 2"}]
+    async with app.test_request_context("/plugins/docker/rows"):
+        page = await render_plugin_template(
+            app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS,
+            severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
+            swarm_services=[sick], services_healthy=0, failing_services=[sick], failures=failures,
+            running=0, stopped=0, docker_hosts=1)
+    page = re.sub(r">\s+<", "><", " ".join(page.split()))
+    card = page[page.index("Needs attention"):page.index("</section>")]
+    assert '<span class="badge badge-crit">Crash-looping</span><a href="/plugins/docker/swarm">codex_db</a>' in card
+    assert "0 of 1 running · 7 tasks failed in the last hour · last task failed: task: non-zero exit (1)" in card
+    assert "since <time" in card
+    assert '<span class="stat-tile-label">Swarm services</span><span class="stat-tile-value">0/1</span>' in page
+    listed = page[page.index('id="failures"'):]
+    listed = listed[:listed.index("</section>")]
+    assert listed.index("codex_db") < listed.index("backup")              # newest first
+    assert "9 failures · last: task: non-zero exit (1) · nas" in listed
+    assert 'href="/plugins/docker/container/h1/backup">backup</a>' in listed

tests/plugins/docker/test_mcp_tools.py

@@ -34,12 +34,21 @@ def test_container_problems_most_severe_first():
 
 
 def test_exited_nonzero_needs_a_recent_death():
-    crashed = _c(status="exited", exit_code=137)
+    crashed = _c(status="exited", exit_code=1)
     assert t.container_problems(crashed, 1, True) == ["exited_nonzero"]
     # A container that died weeks ago and was left stopped is not news.
     assert t.container_problems(crashed, 0, False) == []
 
 
+def test_a_stop_is_not_an_exit_failure():
+    # 137 (SIGKILL) and 143 (SIGTERM) are how `docker stop` ends a container;
+    # without an OOM kill that is someone stopping it, not it failing (#5209).
+    for code in (137, 143):
+        assert t.container_problems(_c(status="exited", exit_code=code), 1, True) == []
+    oom = _c(status="exited", exit_code=137, oom_killed=True)
+    assert t.container_problems(oom, 1, True) == ["oom_killed"]
+
+
 def test_cut_middle_keeps_both_ends_and_sizes_the_gap():
     assert t.cut_middle("short") == "short"
     line = "A" * 10 + "B" * 100 + "C" * 10

tests/plugins/docker/test_service.py

@@ -1,5 +1,11 @@
 """Unit tests for the docker plugin's shared query helpers (no DB)."""
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace as NS
+
 from plugins.docker import service as svc
+from plugins.docker import swarm_view as sv
+
+NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc)
 
 
 def test_escape_like_makes_wildcards_literal():
@@ -27,3 +33,78 @@ def test_prune_extra_vars_mapping():
     }
     # System prune stays conservative — no prune_all_images key.
     assert svc.prune_extra_vars("system") == {"prune_target": "system"}
+
+
+def test_a_stop_is_not_a_failure():
+    # docker stop ends a container with 143 (SIGTERM) or 137 (SIGKILL).
+    assert not svc.exit_is_failure(None) and not svc.exit_is_failure(0)
+    assert not svc.exit_is_failure(137) and not svc.exit_is_failure(143)
+    assert svc.exit_is_failure(1) and svc.exit_is_failure(2)
+    assert svc.exit_is_failure(137, oom_killed=True)
+
+
+def _ev(name, event, detail="", mins=5, host="h1"):
+    return NS(container_name=name, host_id=host, event=event, detail=detail, at=NOW - timedelta(minutes=mins))
+
+
+def test_event_is_failure_reads_the_exit_from_the_die_detail():
+    assert svc.event_is_failure(_ev("a", "die", "exit 1"))
+    assert not svc.event_is_failure(_ev("a", "die", "exit 143"))
+    assert not svc.event_is_failure(_ev("a", "die", "exit 0"))
+    assert svc.event_is_failure(_ev("a", "oom"))
+    assert not svc.event_is_failure(_ev("a", "start"))
+
+
+def _service(name, *, known=True, failures=0, error=None):
+    return {"name": name, "tasks_known": known, "failures_24h": failures if known else None,
+            "last_error": error}
+
+
+def test_failures_24h_counts_services_once_and_skips_stops():
+    events = [_ev("backup", "die", "exit 2", mins=30), _ev("backup", "die", "exit 2", mins=90),
+              _ev("web", "die", "exit 143", mins=10),                         # someone stopped it
+              _ev("app.1.abc", "die", "exit 1", mins=3),                      # Swarm counts its tasks
+              _ev("old.1.x", "die", "exit 1", mins=4), _ev("old.2.y", "die", "exit 1", mins=40),
+              _ev("old.1.x", "die", "exit 1", mins=4, host="h2")]               # a second manager's copy
+    services = [_service("app", failures=3, error={"state": "failed", "error": "boom", "node": "nas",
+                                                   "at": (NOW - timedelta(minutes=1)).isoformat()}),
+                _service("old", known=False)]                                 # agent before 1.8.0
+    rows = svc.failures_24h(events, services, {"h1": "nas"})
+    got = {(r["kind"], r["name"]): r for r in rows}
+    assert set(got) == {("container", "backup"), ("service", "app"), ("service", "old")}
+    assert got[("container", "backup")]["count"] == 2 and got[("container", "backup")]["host"] == "nas"
+    assert got[("service", "app")]["count"] == 3 and got[("service", "app")]["detail"] == "boom"
+    assert got[("service", "old")]["count"] == 2                              # its tasks' deaths
+    assert [r["name"] for r in rows] == ["app", "old", "backup"]              # newest first
+
+
+def test_task_label_names_the_service_and_slot():
+    assert svc.task_label(NS(name="nginx", service_name=None)) == "nginx"
+    assert svc.task_label(NS(name="codex_db.1.37ma", service_name="codex_db")) == "codex_db · 1"
+    agent = NS(name="agent.n0d3id.t4sk", service_name="agent", node_id="n0d3id")
+    assert svc.task_label(agent, {"n0d3id": "node-3"}) == "agent · node-3"
+    assert svc.task_label(agent) == "agent"
+
+
+def test_usage_reads_cpu_in_cores_and_memory_against_a_real_limit():
+    gib = 1024 ** 3
+    c = NS(cpu_pct=150.0, mem_usage_bytes=gib, mem_limit_bytes=16 * gib, mem_pct=6.25)
+    u = svc.usage(c, 4, 16 * gib)
+    assert u["cores"] == 1.5 and u["cpu_share"] == 37.5
+    # A limit of the host's whole memory is Docker saying "no limit".
+    assert u["mem_limit"] is None and u["mem_of"] == "the host's memory"
+    limited = svc.usage(NS(cpu_pct=None, mem_usage_bytes=gib, mem_limit_bytes=2 * gib, mem_pct=50.0), 4, 16 * gib)
+    assert limited["cores"] is None and limited["mem_limit"] == 2 * gib and limited["mem_of"] == "its limit"
+    # Unknown host size: cores still read, no share to grade against.
+    assert svc.usage(c, None, None)["cpu_share"] is None
+    assert svc.usage(c, 4, 16 * gib, cpu_pct=50.0)["cores"] == 0.5             # an average in place
+
+
+def test_services_in_trouble_come_first_worst_first():
+    def s(name, state, loop=False, f24=0):
+        return {"name": name, "state": state, "crash_looping": loop, "failures_24h": f24}
+    services = [s("alpha", "healthy"), s("beta", "healthy", f24=2), s("gamma", "degraded"),
+                s("delta", "down"), s("eps", "down", loop=True)]
+    assert [x["name"] for x in sv.services_problems_first(services)] == ["eps", "delta", "gamma", "beta", "alpha"]
+    # A recovered service's failed tasks are history: not in Needs attention.
+    assert [x["name"] for x in sv.failing_services(services)] == ["eps", "delta", "gamma"]