feat(docker): readable names, CPU in cores and memory in bytes, failures listed with since-times (#5209)

#5209 "Docker views: readable names, CPU/memory in cores and bytes, failures
listed with since-time" (milestone 477 "UI problems first").

- A swarm task reads as its service and slot ("codex_db · 1", or the node
  for a global service), with the raw name in the title: Docker page, host
  panel, busiest widget, container detail.
- CPU reads in cores (Docker's 100% = one core), toned by its share of the
  host's CPUs from the agent's per-core series. Memory reads in bytes, against
  a limit only when one is really set (Docker reports host RAM otherwise).
- Exit 137/143 without an OOM kill is a stop, not a failure: no
  exited_nonzero, no crash-loop death, no "failed (24h)", dim exit code.
- "Failed (24h)" is a list (Docker page #failures, linked from the widget
  and the tile): plain containers by their deaths, swarm services by Swarm's
  failed tasks; a second manager's copy of a task death counts once.
- Failing swarm services lead Needs attention; services sort problems first
  with since-times on the page, the widget and the Swarm page; nodes that
  are down or drained sort first.
- Busiest widget orders by a 5-minute CPU average so the list holds still.
- Host panel: a table, problems first with what is wrong and since when;
  replaced swarm tasks hidden and counted.
- Container detail: node hostname, I/O labelled "since start", Restarts
  hidden for swarm tasks with a note on where task failures are.
- Widget names: "Docker — Services & failures", "Docker — Busiest containers".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/plugins/docker/test_docker_page.py

@@ -43,22 +43,24 @@ def test_stale_is_older_than_the_agents_window():
 async def test_a_crash_loop_comes_first_and_a_stale_row_says_so(app):
     loop = _c("api", status="restarting", exit_code=1, restart_count=7)
     old = _c("cache", scraped_at=NOW - timedelta(minutes=30), cpu_pct=95.0)
-    attention = [{"container": loop, "problems": ["restarting", "crash_looping"], "replaced": False,
+    attention = [{"container": loop, "label": "api", "problems": ["restarting", "crash_looping"], "replaced": False,
                   "since": datetime.now(timezone.utc) - timedelta(minutes=8), "stale": False}]
     groups = [{"host": None, "host_id": "h1", "host_name": "nas", "running": 1, "stopped": 1, "grouped": False,
                "subgroups": [{"label": None, "containers": [
-                   {"container": loop, "ports": [], "uptime": None, "stale": False},
-                   {"container": old, "ports": [], "uptime": "2h", "stale": True}]}]}]
+                   {"container": loop, "ports": [], "uptime": None, "stale": False,
+                    "usage": svc.usage(loop, 1, None), "exit_failed": True},
+                   {"container": old, "ports": [], "uptime": "2h", "stale": True,
+                    "usage": svc.usage(old, 1, None), "exit_failed": False}]}]}]
     async with app.test_request_context("/plugins/docker/rows"):
         page = await render_plugin_template(
             app, "docker", "docker/rows.html", attention=attention, says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
             unhealthy=0, restarting=1, stale=1, host_groups=groups, swarm_services=[],
-            running=1, stopped=1, total=2)
+            running=1, stopped=1, docker_hosts=1, failures=[], failing_services=[])
     page = re.sub(r">\s+<", "><", " ".join(page.split()))
 
     card = page[page.index("Needs attention"):page.index("</section>")]
     assert '<span class="badge badge-crit">Restarting</span>' in card
-    assert 'href="/plugins/docker/container/h1/api">api</a>' in card
+    assert 'href="/plugins/docker/container/h1/api" title="api">api</a>' in card
     assert "is restarting and keeps dying · exit 1 · since <time" in card and ">8m ago</time>" in card
     assert "cache" not in card
     assert page.index("Needs attention") < page.index("<table")
@@ -66,7 +68,9 @@ async def test_a_crash_loop_comes_first_and_a_stale_row_says_so(app):
     assert '<span class="stat-tile-label">Restarting</span><span class="stat-tile-value">1</span>' in page
     assert '<span class="stat-tile-label">Stale</span><span class="stat-tile-value">1</span>' in page
     assert 'class="badge badge-warn" title="Last report' in page          # the stale row
-    assert '<span class="tone-crit">95.0%</span>' in page                  # CPU through Settings → Thresholds
+    # CPU in cores, toned by its share of the host's CPUs through Settings → Thresholds.
+    assert ('<span class="tone-crit" title="Docker counts CPU per core: 95% of one core, '
+            '95.0% of this host\'s 1 core">0.95 cores</span>') in page
     assert "history" not in page.lower() and "<svg" not in page            # no sparklines on the overview
     assert not hex_colours(page), hex_colours(page)
 
@@ -75,7 +79,8 @@ async def test_a_healthy_board_has_no_attention_card(app):
     async with app.test_request_context("/plugins/docker/rows"):
         page = await render_plugin_template(
             app, "docker", "docker/rows.html", attention=[], says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
-            unhealthy=0, restarting=0, stale=0, host_groups=[], swarm_services=[], running=0, stopped=0, total=0)
+            unhealthy=0, restarting=0, stale=0, host_groups=[], swarm_services=[], running=0, stopped=0,
+            docker_hosts=0, failures=[], failing_services=[])
     assert "Needs attention</h2>" not in page and "No containers reported yet" in page
 
 
@@ -89,7 +94,7 @@ async def test_a_host_whose_docker_stopped_answering_says_so(app):
         page = await render_plugin_template(
             app, "docker", "docker/rows.html", docker_down=down, attention=[], says=svc.PROBLEM_SAYS,
             severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
-            swarm_services=[], running=0, stopped=0, total=0)
+            swarm_services=[], running=0, stopped=0, docker_hosts=0, failures=[], failing_services=[])
     page = re.sub(r">\s+<", "><", " ".join(page.split()))
     card = page[page.index("Docker not answering"):page.index("</section>")]
     assert '<span class="badge badge-warn">No Docker</span><a href="/hosts/h1">nas</a>' in card
@@ -102,5 +107,35 @@ async def test_no_card_while_every_docker_answers(app):
         page = await render_plugin_template(
             app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS,
             severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
-            swarm_services=[], running=0, stopped=0, total=0)
+            swarm_services=[], running=0, stopped=0, docker_hosts=0, failures=[], failing_services=[])
     assert "Docker not answering" not in page
+
+
+async def test_a_failing_service_leads_and_the_failures_are_listed(app):
+    now = datetime.now(timezone.utc)
+    sick = {"name": "codex_db", "mode": "replicated", "running": 0, "desired": 1, "state": "down",
+            "crash_looping": True, "failures_1h": 7, "failures_24h": 9, "tasks_known": True,
+            "trouble_since": now - timedelta(minutes=20), "replicas": [], "previous_tasks": 7,
+            "last_error": {"state": "failed", "error": "task: non-zero exit (1)", "exit_code": 1,
+                           "node": "nas", "at": None}}
+    failures = [{"kind": "service", "name": "codex_db", "host_id": None, "host": "nas", "count": 9,
+                 "at": now - timedelta(minutes=2), "detail": "task: non-zero exit (1)"},
+                {"kind": "container", "name": "backup", "host_id": "h1", "host": "nas", "count": 1,
+                 "at": now - timedelta(hours=3), "detail": "exit 2"}]
+    async with app.test_request_context("/plugins/docker/rows"):
+        page = await render_plugin_template(
+            app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS,
+            severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
+            swarm_services=[sick], services_healthy=0, failing_services=[sick], failures=failures,
+            running=0, stopped=0, docker_hosts=1)
+    page = re.sub(r">\s+<", "><", " ".join(page.split()))
+    card = page[page.index("Needs attention"):page.index("</section>")]
+    assert '<span class="badge badge-crit">Crash-looping</span><a href="/plugins/docker/swarm">codex_db</a>' in card
+    assert "0 of 1 running · 7 tasks failed in the last hour · last task failed: task: non-zero exit (1)" in card
+    assert "since <time" in card
+    assert '<span class="stat-tile-label">Swarm services</span><span class="stat-tile-value">0/1</span>' in page
+    listed = page[page.index('id="failures"'):]
+    listed = listed[:listed.index("</section>")]
+    assert listed.index("codex_db") < listed.index("backup")              # newest first
+    assert "9 failures · last: task: non-zero exit (1) · nas" in listed
+    assert 'href="/plugins/docker/container/h1/backup">backup</a>' in listed

tests/plugins/docker/test_mcp_tools.py

@@ -34,12 +34,21 @@ def test_container_problems_most_severe_first():
 
 
 def test_exited_nonzero_needs_a_recent_death():
-    crashed = _c(status="exited", exit_code=137)
+    crashed = _c(status="exited", exit_code=1)
     assert t.container_problems(crashed, 1, True) == ["exited_nonzero"]
     # A container that died weeks ago and was left stopped is not news.
     assert t.container_problems(crashed, 0, False) == []
 
 
+def test_a_stop_is_not_an_exit_failure():
+    # 137 (SIGKILL) and 143 (SIGTERM) are how `docker stop` ends a container;
+    # without an OOM kill that is someone stopping it, not it failing (#5209).
+    for code in (137, 143):
+        assert t.container_problems(_c(status="exited", exit_code=code), 1, True) == []
+    oom = _c(status="exited", exit_code=137, oom_killed=True)
+    assert t.container_problems(oom, 1, True) == ["oom_killed"]
+
+
 def test_cut_middle_keeps_both_ends_and_sizes_the_gap():
     assert t.cut_middle("short") == "short"
     line = "A" * 10 + "B" * 100 + "C" * 10

tests/plugins/docker/test_service.py

@@ -1,5 +1,11 @@
 """Unit tests for the docker plugin's shared query helpers (no DB)."""
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace as NS
+
 from plugins.docker import service as svc
+from plugins.docker import swarm_view as sv
+
+NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc)
 
 
 def test_escape_like_makes_wildcards_literal():
@@ -27,3 +33,78 @@ def test_prune_extra_vars_mapping():
     }
     # System prune stays conservative — no prune_all_images key.
     assert svc.prune_extra_vars("system") == {"prune_target": "system"}
+
+
+def test_a_stop_is_not_a_failure():
+    # docker stop ends a container with 143 (SIGTERM) or 137 (SIGKILL).
+    assert not svc.exit_is_failure(None) and not svc.exit_is_failure(0)
+    assert not svc.exit_is_failure(137) and not svc.exit_is_failure(143)
+    assert svc.exit_is_failure(1) and svc.exit_is_failure(2)
+    assert svc.exit_is_failure(137, oom_killed=True)
+
+
+def _ev(name, event, detail="", mins=5, host="h1"):
+    return NS(container_name=name, host_id=host, event=event, detail=detail, at=NOW - timedelta(minutes=mins))
+
+
+def test_event_is_failure_reads_the_exit_from_the_die_detail():
+    assert svc.event_is_failure(_ev("a", "die", "exit 1"))
+    assert not svc.event_is_failure(_ev("a", "die", "exit 143"))
+    assert not svc.event_is_failure(_ev("a", "die", "exit 0"))
+    assert svc.event_is_failure(_ev("a", "oom"))
+    assert not svc.event_is_failure(_ev("a", "start"))
+
+
+def _service(name, *, known=True, failures=0, error=None):
+    return {"name": name, "tasks_known": known, "failures_24h": failures if known else None,
+            "last_error": error}
+
+
+def test_failures_24h_counts_services_once_and_skips_stops():
+    events = [_ev("backup", "die", "exit 2", mins=30), _ev("backup", "die", "exit 2", mins=90),
+              _ev("web", "die", "exit 143", mins=10),                         # someone stopped it
+              _ev("app.1.abc", "die", "exit 1", mins=3),                      # Swarm counts its tasks
+              _ev("old.1.x", "die", "exit 1", mins=4), _ev("old.2.y", "die", "exit 1", mins=40),
+              _ev("old.1.x", "die", "exit 1", mins=4, host="h2")]               # a second manager's copy
+    services = [_service("app", failures=3, error={"state": "failed", "error": "boom", "node": "nas",
+                                                   "at": (NOW - timedelta(minutes=1)).isoformat()}),
+                _service("old", known=False)]                                 # agent before 1.8.0
+    rows = svc.failures_24h(events, services, {"h1": "nas"})
+    got = {(r["kind"], r["name"]): r for r in rows}
+    assert set(got) == {("container", "backup"), ("service", "app"), ("service", "old")}
+    assert got[("container", "backup")]["count"] == 2 and got[("container", "backup")]["host"] == "nas"
+    assert got[("service", "app")]["count"] == 3 and got[("service", "app")]["detail"] == "boom"
+    assert got[("service", "old")]["count"] == 2                              # its tasks' deaths
+    assert [r["name"] for r in rows] == ["app", "old", "backup"]              # newest first
+
+
+def test_task_label_names_the_service_and_slot():
+    assert svc.task_label(NS(name="nginx", service_name=None)) == "nginx"
+    assert svc.task_label(NS(name="codex_db.1.37ma", service_name="codex_db")) == "codex_db · 1"
+    agent = NS(name="agent.n0d3id.t4sk", service_name="agent", node_id="n0d3id")
+    assert svc.task_label(agent, {"n0d3id": "node-3"}) == "agent · node-3"
+    assert svc.task_label(agent) == "agent"
+
+
+def test_usage_reads_cpu_in_cores_and_memory_against_a_real_limit():
+    gib = 1024 ** 3
+    c = NS(cpu_pct=150.0, mem_usage_bytes=gib, mem_limit_bytes=16 * gib, mem_pct=6.25)
+    u = svc.usage(c, 4, 16 * gib)
+    assert u["cores"] == 1.5 and u["cpu_share"] == 37.5
+    # A limit of the host's whole memory is Docker saying "no limit".
+    assert u["mem_limit"] is None and u["mem_of"] == "the host's memory"
+    limited = svc.usage(NS(cpu_pct=None, mem_usage_bytes=gib, mem_limit_bytes=2 * gib, mem_pct=50.0), 4, 16 * gib)
+    assert limited["cores"] is None and limited["mem_limit"] == 2 * gib and limited["mem_of"] == "its limit"
+    # Unknown host size: cores still read, no share to grade against.
+    assert svc.usage(c, None, None)["cpu_share"] is None
+    assert svc.usage(c, 4, 16 * gib, cpu_pct=50.0)["cores"] == 0.5             # an average in place
+
+
+def test_services_in_trouble_come_first_worst_first():
+    def s(name, state, loop=False, f24=0):
+        return {"name": name, "state": state, "crash_looping": loop, "failures_24h": f24}
+    services = [s("alpha", "healthy"), s("beta", "healthy", f24=2), s("gamma", "degraded"),
+                s("delta", "down"), s("eps", "down", loop=True)]
+    assert [x["name"] for x in sv.services_problems_first(services)] == ["eps", "delta", "gamma", "beta", "alpha"]
+    # A recovered service's failed tasks are history: not in Needs attention.
+    assert [x["name"] for x in sv.failing_services(services)] == ["eps", "delta", "gamma"]