feat(docker): readable names, CPU in cores and memory in bytes, failures listed with since-times (#5209)

#5209 "Docker views: readable names, CPU/memory in cores and bytes, failures
listed with since-time" (milestone 477 "UI problems first").

- A swarm task reads as its service and slot ("codex_db · 1", or the node
  for a global service), with the raw name in the title: Docker page, host
  panel, busiest widget, container detail.
- CPU reads in cores (Docker's 100% = one core), toned by its share of the
  host's CPUs from the agent's per-core series. Memory reads in bytes, against
  a limit only when one is really set (Docker reports host RAM otherwise).
- Exit 137/143 without an OOM kill is a stop, not a failure: no
  exited_nonzero, no crash-loop death, no "failed (24h)", dim exit code.
- "Failed (24h)" is a list (Docker page #failures, linked from the widget
  and the tile): plain containers by their deaths, swarm services by Swarm's
  failed tasks; a second manager's copy of a task death counts once.
- Failing swarm services lead Needs attention; services sort problems first
  with since-times on the page, the widget and the Swarm page; nodes that
  are down or drained sort first.
- Busiest widget orders by a 5-minute CPU average so the list holds still.
- Host panel: a table, problems first with what is wrong and since when;
  replaced swarm tasks hidden and counted.
- Container detail: node hostname, I/O labelled "since start", Restarts
  hidden for swarm tasks with a note on where task failures are.
- Widget names: "Docker — Services & failures", "Docker — Busiest containers".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/service.py

@@ -6,7 +6,7 @@ Quart's pipeline) call the same code.
 """
 from __future__ import annotations
 
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 from sqlalchemy import func, select
 
@@ -30,25 +30,159 @@ def crash_loop_deaths(config) -> int:
     return int(th.get("warn") or CRASH_LOOP_DEATHS)
 
 
-def failed_24h(dead_names, swarm_services) -> int:
-    """Failures in the last 24h for the dashboard: distinct plain containers
-    that died or were OOM-killed, plus each swarm service's failed tasks.
+# Exit codes a deliberate stop leaves: SIGTERM (143), and SIGKILL (137) when
+# the process outlives the stop timeout. Without an OOM kill they are someone
+# stopping the container, not the container failing.
+STOP_EXITS = (137, 143)
 
-    A swarm task's container dies whenever Swarm replaces it (a rolling update
-    kills it with 137), so its die event is not a failure. Services with
-    Swarm's task records count their failed tasks instead; a service without
-    them (an agent older than 1.8.0) falls back to its tasks' die events.
+
+def exit_is_failure(exit_code, oom_killed: bool = False) -> bool:
+    """Whether a container's exit is a failure: an OOM kill, or a non-zero
+    exit other than a stop's."""
+    return bool(oom_killed) or exit_code not in (None, 0, *STOP_EXITS)
+
+
+def _event_exit(detail) -> int | None:
+    """The exit code in a die event's detail ("exit 1"), or None."""
+    text = (detail or "").strip()
+    if text.startswith("exit "):
+        try:
+            return int(text[5:])
+        except ValueError:
+            return None
+    return None
+
+
+def event_is_failure(e) -> bool:
+    """Whether a die/oom event is a failure. An oom always is; a die is unless
+    its exit is a stop's (or clean)."""
+    if e.event == "oom":
+        return True
+    return e.event == "die" and exit_is_failure(_event_exit(e.detail))
+
+
+def failures_24h(events, swarm_services, host_name: dict) -> list[dict]:
+    """What failed in the last 24h, newest first, for the dashboard count and
+    the Docker page's list of it: one row per plain container that died or was
+    OOM-killed, and one per swarm service with failed tasks.
+
+    `events` are die/oom events of the last 24h. A swarm task's container dies
+    whenever Swarm replaces it, so a service with Swarm's task records counts
+    its failed tasks instead of its containers' deaths; a service without them
+    (an agent older than 1.8.0) falls back to its tasks' deaths.
+
+    Each row: {kind: "container"|"service", name, host_id, host, count, at,
+    detail}.
     """
     by_prefix = {s["name"] + ".": s for s in swarm_services}
-    plain, fallback = 0, set()
-    for name in dead_names:
-        svc = next((s for p, s in by_prefix.items() if name.startswith(p)), None)
-        if svc is None:
-            plain += 1
-        elif not svc["tasks_known"]:
-            fallback.add(name)
-    tasks = sum(s["failures_24h"] or 0 for s in swarm_services if s["tasks_known"])
-    return plain + len(fallback) + tasks
+    rows: dict[tuple, dict] = {}
+    # Each manager reports every swarm task, so one task's death can arrive
+    # once per manager: a service counts each dead task container once.
+    task_deaths: dict[tuple, set] = {}
+    for e in sorted(events, key=lambda e: e.at, reverse=True):
+        if not event_is_failure(e):
+            continue
+        svc = next((s for p, s in by_prefix.items() if e.container_name.startswith(p)), None)
+        if svc is not None and svc["tasks_known"]:
+            continue
+        kind, name = ("service", svc["name"]) if svc else ("container", e.container_name)
+        key = (kind, name) if svc else (kind, e.host_id, name)
+        if svc is not None:
+            seen = task_deaths.setdefault(key, set())
+            if e.container_name in seen:
+                continue
+            seen.add(e.container_name)
+        row = rows.get(key)
+        if row is None:
+            rows[key] = {"kind": kind, "name": name, "host_id": e.host_id,
+                         "host": host_name.get(e.host_id, e.host_id), "count": 1, "at": e.at,
+                         "detail": "OOM-killed" if e.event == "oom" else e.detail}
+        else:
+            row["count"] += 1
+    from .swarm_view import task_time
+    for s in swarm_services:
+        if s["tasks_known"] and s["failures_24h"]:
+            err = s["last_error"] or {}
+            rows[("service", s["name"])] = {
+                "kind": "service", "name": s["name"], "host_id": None, "host": err.get("node"),
+                "count": s["failures_24h"], "at": task_time(err) if err else None,
+                "detail": err.get("error") or err.get("message") or err.get("state")}
+    epoch = datetime.min.replace(tzinfo=timezone.utc)
+    return sorted(rows.values(), key=lambda r: r["at"] or epoch, reverse=True)
+
+
+def task_label(c, node_hostname: dict | None = None) -> str:
+    """How a container reads to a person. A swarm task is its service and
+    slot: "codex_db · 1" for a replica, "agent · node-3" for a global
+    service's task on a node. Anything else is its own name."""
+    from .swarm_view import _slot_key
+    svc = getattr(c, "service_name", None)
+    if not svc:
+        return c.name
+    slot = _slot_key(c)
+    if slot.isdigit():
+        return f"{svc} · {slot}"
+    host = (node_hostname or {}).get(slot) or (node_hostname or {}).get(getattr(c, "node_id", None))
+    return f"{svc} · {host}" if host else svc
+
+
+def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = None) -> dict:
+    """A container's CPU and memory in terms a reader can weigh.
+
+    Docker's CPU% is per core (100% = one core busy), so it is shown as cores
+    and graded as a share of the host's CPUs when the host's core count is
+    known. Docker's memory limit is the host's RAM when none is set, so memory
+    is shown in bytes, against a limit only when one is actually set; mem_pct
+    is then a share of that limit, otherwise of the host's memory. `cpu_pct`
+    replaces the container's last sample (an average, say) when given.
+    """
+    cpu = getattr(c, "cpu_pct", None) if cpu_pct is None else cpu_pct
+    limit = getattr(c, "mem_limit_bytes", None)
+    limited = bool(limit and mem_total and limit < mem_total * 0.95)
+    return {
+        "cores": None if cpu is None else cpu / 100.0,
+        "cpu_share": None if cpu is None or not cpus else cpu / cpus,
+        "cpus": cpus,
+        "mem_bytes": getattr(c, "mem_usage_bytes", None),
+        "mem_limit": limit if limited else None,
+        "mem_pct": getattr(c, "mem_pct", None),
+        "mem_of": "its limit" if limited else ("the host's memory" if mem_total else None),
+    }
+
+
+async def host_capacity(db, host_ids) -> dict[str, tuple[int | None, int | None]]:
+    """{host_id: (CPU cores, memory bytes)} from the host agent's latest
+    readings: cores counted from its per-core CPU series, memory from
+    available ÷ (1 − used%). (None, None) for a host without an agent."""
+    from sqlalchemy import or_
+    from steward.models.metrics import PluginMetricLatest
+    hosts = await host_map(db, set(host_ids))
+    if not hosts:
+        return {}
+    names = {h.name: hid for hid, h in hosts.items()}
+    rows = (await db.execute(
+        select(PluginMetricLatest.resource_name, PluginMetricLatest.metric_name,
+               PluginMetricLatest.value)
+        .where(PluginMetricLatest.source_module == "host_agent",
+               or_(PluginMetricLatest.resource_name.in_(list(names)),
+                   *(PluginMetricLatest.resource_name.like(escape_like(n) + ":core%", escape="\\")
+                     for n in names)))
+    )).all()
+    cores: dict[str, int] = {}
+    mem: dict[str, dict] = {}
+    for res, metric, value in rows:
+        if res in names:
+            mem.setdefault(res, {})[metric] = value
+        elif metric == "cpu_pct":
+            host = res.rsplit(":core", 1)[0]
+            cores[host] = cores.get(host, 0) + 1
+    out = {}
+    for name, hid in names.items():
+        m = mem.get(name, {})
+        avail, used = m.get("mem_available_bytes"), m.get("mem_used_pct")
+        total = int(avail / (1 - used / 100)) if avail and used is not None and used < 99.5 else None
+        out[hid] = (cores.get(name), total)
+    return out
 
 
 def bucket(values: list[float], target: int = 40) -> list[float]:
@@ -210,7 +344,8 @@ async def log_context(db, host_id: str, name: str, ts, n: int) -> tuple[list, li
 
 
 async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]:
-    """{(host_id, container_name): [die/oom events since `since`, newest first]}."""
+    """{(host_id, container_name): [die/oom events since `since`, newest first]},
+    leaving out a die that was a stop (event_is_failure)."""
     rows = (await db.execute(
         select(DockerEvent)
         .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
@@ -218,10 +353,21 @@ async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]
     )).scalars()
     out: dict[tuple[str, str], list] = {}
     for e in rows:
-        out.setdefault((e.host_id, e.container_name), []).append(e)
+        if event_is_failure(e):
+            out.setdefault((e.host_id, e.container_name), []).append(e)
     return out
 
 
+async def failure_events(db, since) -> list:
+    """Die and OOM events since `since`, newest first; failures_24h() decides
+    which of them are failures."""
+    return list((await db.execute(
+        select(DockerEvent)
+        .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
+        .order_by(DockerEvent.at.desc())
+    )).scalars())
+
+
 async def retained_names(db, want: str, *, exact: bool, host_id: str | None = None,
                          limit: int = 20) -> list[tuple[str, str, datetime]]:
     """(host_id, container_name, last seen) of containers Steward still holds
@@ -303,7 +449,7 @@ def container_problems(c, failures_1h: int, failed_24h: bool,
         out.append("unhealthy")
     if c.oom_killed:
         out.append("oom_killed")
-    if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h:
+    if status in ("exited", "dead") and exit_is_failure(c.exit_code) and failed_24h:
         out.append("exited_nonzero")
     return out