feat(docker): Services & Failures widget lists what is wrong, then packs the healthy (#5252)

A verdict line (services healthy · other containers running · failed in
24h), then "Needs a look": troubled services with why (crash-looping, N of
M running), since when and the last error, and troubled containers
(unhealthy, restarting, exited with an error, out of memory). Then "Failed
in the last 24h" by name, with how often, when and where. Then the healthy
rest one line each (name · hosts · replicas) in as many columns as the
widget is wide, collapsing after 40 to "+N more healthy".

Per-container CPU is gone from this widget: Busiest Containers owns it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/routes.py

@@ -214,7 +214,6 @@ async def widget():
             {c.host_id for c in all_containers}
             | {n.host_id for n in nodes} | {s.host_id for s in services},
         )
-        capacity = await docker_svc.host_capacity(db, {c.host_id for c in all_containers})
 
     # Swarm tasks collapse into per-service rows (each replica tagged with its
     # host); non-swarm containers stay grouped by host.
@@ -227,42 +226,41 @@ async def widget():
 
     standalone = [c for c in all_containers if not c.service_name]
     running = [c for c in standalone if c.status == "running"]
-    display = standalone if show_stopped else running
 
-    def trouble(c) -> int:
-        if (c.status or "") == "restarting" or c.health == "unhealthy":
-            return 0
-        if c.status != "running" and docker_svc.exit_is_failure(c.exit_code, c.oom_killed):
-            return 1
-        return 2 if c.status == "running" else 3
+    def troubled(c) -> bool:
+        return ((c.status or "") == "restarting" or c.health == "unhealthy"
+                or (c.status != "running" and docker_svc.exit_is_failure(c.exit_code, c.oom_killed)))
 
-    # Group for display so multi-host fleets read clearly; single-host stays flat.
-    groups: dict[str, dict] = {}
-    for c in sorted(display, key=lambda c: (trouble(c), c.name)):
-        g = groups.setdefault(c.host_id, {
-            "host_id": c.host_id,
-            "host_name": host_name.get(c.host_id, c.host_id),
-            "containers": [],
-        })
-        g["containers"].append({"c": c, "usage": docker_svc.usage(
-            c, *capacity.get(c.host_id, (None, None))),
-            "exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed)})
-    host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
+    # The widget-shape order (UI & Design ruling): what is wrong first, then
+    # the healthy rest one line each, collapsing past HEALTHY_SHOWN.
+    bad_containers = sorted((c for c in standalone if troubled(c)), key=lambda c: c.name)
+    healthy = ([{"kind": "service", "s": s} for s in swarm_services if s["state"] == "healthy"]
+               + [{"kind": "container", "c": c} for c in sorted(running, key=lambda c: c.name)
+                  if not troubled(c)]
+               + ([{"kind": "stopped", "c": c} for c in sorted(standalone, key=lambda c: c.name)
+                   if c.status != "running" and not troubled(c)] if show_stopped else []))
 
     return await render_template(
         "docker/widget.html",
-        host_groups=host_groups,
-        multi_host=len(host_groups) > 1,
-        swarm_services=swarm_services,
+        bad_services=[s for s in swarm_services if s["state"] != "healthy"],
+        bad_containers=bad_containers,
+        healthy=healthy[:HEALTHY_SHOWN], healthy_more=max(0, len(healthy) - HEALTHY_SHOWN),
+        services_total=len(swarm_services),
         services_healthy=sum(s["state"] == "healthy" for s in swarm_services),
         running_count=len(running),
         failures=failures,
+        host_name=host_name,
         total_count=len(all_containers) + len(swarm_services),
-        show_stopped=show_stopped,
         widget_id=widget_id,
     )
 
 
+# Healthy services and containers listed on the dashboard widget before the
+# rest collapse to "+N healthy"; the Docker page lists them all.
+HEALTHY_SHOWN = 40
+FAILURES_SHOWN = 5
+
+
 CPU_AVERAGE_MINUTES = 5
 
 

plugins/docker/templates/docker/widget.html

@@ -1,80 +1,80 @@
-{# docker/widget.html — dashboard widget: what Docker is running and what has
-   failed. Swarm services first, troubled ones at the top with since-times; then
-   the containers outside any service, by host, problems first. The failed count
-   links to the Docker page's list of exactly what it counts. #}
-{% from "docker/_usage.html" import cpu %}
+{# docker/widget.html — the dashboard Docker widget (#5252), in the widget-shape
+   order: a verdict line; the services and containers in trouble; what failed
+   in the last 24h, by name; then everything healthy, one line each, packed
+   into as many columns as the widget is wide. Resource use is the Busiest
+   Containers widget's, so it isn't repeated here. #}
+{% from "_macros.html" import status_dot %}
 {% if total_count == 0 %}
-<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No containers reported yet.</p>
+<div class="w-verdict" style="color:var(--text-muted);">No containers reported yet.</div>
 {% else %}
-
-<div style="display:flex;flex-wrap:wrap;gap:0.4rem 1.25rem;margin-bottom:0.65rem;">
-  {% if swarm_services %}
-  <div title="Swarm services with every replica running">
-    <span class="tone-{{ 'ok' if services_healthy == swarm_services|length else 'warn' }}" style="font-size:1.4rem;font-weight:500;line-height:1;">{{ services_healthy }}/{{ swarm_services|length }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">services healthy</span>
-  </div>
-  {% endif %}
-  <div title="Containers running outside any swarm service">
-    <span style="font-size:1.4rem;font-weight:500;line-height:1;color:var(--green);">{{ running_count }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">containers running</span>
-  </div>
-  <a href="/plugins/docker/#failures" style="color:inherit;text-decoration:none;"
-     title="Containers that died or were OOM-killed in the last 24h, and swarm services with failed tasks. A container someone stopped is not counted. Click for the list.">
-    <span class="tone-{{ 'crit' if failures else 'dim' }}" style="font-size:1.4rem;font-weight:500;line-height:1;">{{ failures|length }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;border-bottom:1px dotted var(--border-mid);">failed in 24h</span>
-  </a>
+{% set wrong = bad_services|length + bad_containers|length %}
+<div class="w-verdict {{ 'tone-crit' if wrong else ('tone-warn' if failures else 'tone-ok') }}">
+  {%- if services_total %}{{ services_healthy }}/{{ services_total }} services healthy · {% endif -%}
+  {{ running_count }} other container{{ 's' if running_count != 1 }} running ·
+  {% if failures %}<a href="/plugins/docker/#failures" style="color:inherit;">{{ failures|length }} failed in 24h</a>{% else %}nothing failed in 24h{% endif -%}
 </div>
 
-{# Swarm services — troubled first, each with its replicas' host chips
-   (dashed = a node with no Steward agent, count-only from placement). #}
-{% if swarm_services %}
-<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">Swarm services</div>
-<div style="display:grid;gap:0.4rem;margin-bottom:0.4rem;">
-  {% for s in swarm_services %}
-  <div>
-    <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.82rem;">
-      <span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
-      <span style="font-weight:500;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.name }}</span>
-      {% if s.crash_looping %}<span class="tone-crit" style="font-size:0.7rem;font-weight:500;flex-shrink:0;" title="{{ s.failures_1h }} tasks failed in the last hour{% if s.last_error %}: {{ s.last_error.error or s.last_error.message or s.last_error.state }}{% endif %}">crash-looping</span>{% endif %}
-      {% if s.state != 'healthy' and s.trouble_since %}<span class="tone-dim" style="font-size:0.7rem;flex-shrink:0;">since {{ s.trouble_since|ago }}</span>{% endif %}
-      <span class="tone-{{ 'ok' if s.state == 'healthy' else ('warn' if s.state == 'degraded' else 'crit') }}" style="font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
-    </div>
-    <div style="display:flex;flex-wrap:wrap;gap:0.25rem;margin:0.2rem 0 0 1rem;">
-      {% for r in s.replicas %}
-      {% if r.ghost %}
-      <span title="no Steward agent on this node — count from swarm placement" style="font-size:0.68rem;color:var(--text-dim);background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }} ×{{ r.count }}</span>
-      {% else %}
-      <span style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }}</span>
-      {% endif %}
-      {% endfor %}
-    </div>
-  </div>
-  {% endfor %}
+{% if wrong %}
+<div class="w-section">Needs a look</div>
+{% for s in bad_services %}
+<div class="w-item">
+  {{ status_dot('degraded' if s.state == 'degraded' else 'down') }}
+  <span class="w-label" title="{{ s.name }}">{{ s.name }}
+    <span class="w-sub {{ 'tone-crit' if s.state != 'degraded' else 'tone-warn' }}">
+      {%- if s.crash_looping %}crash-looping: {{ s.failures_1h }} failed in the last hour{% else %}{{ s.running }} of {{ s.desired }} running{% endif %}
+      {%- if s.trouble_since %}, since {{ s.trouble_since|ago }}{% endif %}
+      {%- if s.last_error and (s.last_error.error or s.last_error.message) %} · {{ s.last_error.error or s.last_error.message }}{% endif -%}
+    </span>
+  </span>
+  <span class="w-meta" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
 </div>
-{% endif %}
-
-{% if host_groups and swarm_services %}
-<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.6rem 0 0.2rem;">Other containers</div>
-{% endif %}
-{% for g in host_groups %}
-{% if multi_host %}
-<div style="font-size:0.7rem;color:var(--text-dim);margin:0.4rem 0 0.15rem;">{{ g.host_name }}</div>
-{% endif %}
-<div style="display:grid;gap:2px;">
-  {% for item in g.containers %}
-  {% set c = item.c %}
-  <div style="display:flex;align-items:center;gap:0.5rem;font-size:0.82rem;padding:0.15rem 0;">
-    <span class="dot {% if c.status == 'running' and c.health != 'unhealthy' %}dot-up{% elif c.status in ('paused', 'restarting') %}dot-warn{% else %}dot-down{% endif %}"></span>
-    <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;">
-      {{ c.name }}{% if c.health == 'unhealthy' %} <span class="tone-crit" style="font-size:0.7rem;">unhealthy</span>{% endif %}{% if c.status == 'restarting' %} <span class="tone-warn" style="font-size:0.7rem;">restarting</span>{% endif %}
-    </a>
-    {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
-    <span title="{{ 'last exit code' if item.exit_failed else 'stopped by a signal; not a failure' }}" class="tone-{{ 'crit' if item.exit_failed else 'dim' }}" style="font-size:0.7rem;flex-shrink:0;">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
-    {% endif %}
-    {% if c.status == 'running' %}<span style="font-size:0.72rem;font-family:ui-monospace,monospace;flex-shrink:0;">{{ cpu(item.usage) }}</span>{% endif %}
-  </div>
-  {% endfor %}
+{% endfor %}
+{% for c in bad_containers %}
+<div class="w-item">
+  {{ status_dot('warn' if c.status == 'restarting' else 'down') }}
+  <span class="w-label" title="{{ c.name }}">
+    {%- if session.user_id %}<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}">{{ c.name }}</a>{% else %}{{ c.name }}{% endif %}
+    <span class="w-sub tone-crit">
+      {%- if c.health == 'unhealthy' %}unhealthy{% elif c.status == 'restarting' %}restarting{% else %}exited {{ c.exit_code }}{% if c.oom_killed %}, out of memory{% endif %}{% endif %} · {{ host_name.get(c.host_id, c.host_id) }}
+    </span>
+  </span>
 </div>
 {% endfor %}
+{% endif %}
 
+{% if failures %}
+<div class="w-section">Failed in the last 24h</div>
+{% for f in failures[:5] %}
+<div class="w-item">
+  {{ status_dot('failed') }}
+  <span class="w-label" title="{{ f.name }}">{{ f.name }}
+    <span class="w-sub">{% if f.count > 1 %}{{ f.count }} times, last {% endif %}{{ f.at|ago }}{% if f.detail %} · {{ f.detail }}{% endif %} · {{ f.host }}</span>
+  </span>
+</div>
+{% endfor %}
+{% if failures|length > 5 %}<div class="w-more">+{{ failures|length - 5 }} more · <a href="/plugins/docker/#failures">the full list</a></div>{% endif %}
+{% endif %}
+
+{% if healthy %}
+<div class="w-section">Healthy</div>
+<div class="w-grid">
+  {% for h in healthy %}
+  <div class="w-item">
+    {% if h.kind == 'service' %}{% set s = h.s %}
+    {{ status_dot('up') }}
+    <span class="w-label" title="{{ s.name }} — on {{ s.replicas|map(attribute='host')|unique|join(', ') }}">{{ s.name }}
+      <span style="color:var(--text-dim);"> · {{ s.replicas|map(attribute='host')|unique|join(', ') }}</span></span>
+    <span class="w-meta" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
+    {% else %}{% set c = h.c %}
+    {{ status_dot('up' if h.kind == 'container' else 'unknown') }}
+    <span class="w-label" title="{{ c.name }}">
+      {%- if session.user_id %}<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}">{{ c.name }}</a>{% else %}{{ c.name }}{% endif %}
+      <span style="color:var(--text-dim);"> · {{ host_name.get(c.host_id, c.host_id) }}</span></span>
+    {% if h.kind == 'stopped' %}<span class="w-meta">stopped</span>{% endif %}
+    {% endif %}
+  </div>
+  {% endfor %}
+</div>
+{% if healthy_more %}<div class="w-more">+{{ healthy_more }} more healthy · <a href="/plugins/docker/">all of Docker</a></div>{% endif %}
+{% endif %}
 {% endif %}

steward/static/steward.css

@@ -396,6 +396,14 @@ body.dash-editing .widget-drawer { transform:translateY(0); }
 .w-table .dot { margin-right:0.4rem; vertical-align:middle; }
 .w-sub { display:block; font-size:0.72rem; color:var(--text-muted); white-space:nowrap; overflow:hidden; text-overflow:ellipsis; }
 .w-more { font-size:0.75rem; color:var(--text-dim); padding-top:0.3rem; }
+.w-section { font-size:var(--fs-size-tiny); color:var(--text-dim); text-transform:uppercase; letter-spacing:var(--fs-tracking-tiny); margin:0.5rem 0 0.2rem; }
+/* The healthy rest: one line per item, as many columns as the widget's width
+   allows (it is a size container, so this follows the widget, not the window). */
+.w-grid { display:grid; grid-template-columns:repeat(auto-fill, minmax(15rem, 1fr)); gap:0 1.25rem; }
+.w-item { display:flex; align-items:center; gap:0.4rem; min-width:0; font-size:0.82rem; padding:0.12rem 0; }
+.w-item .w-label { flex:1; min-width:0; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; }
+.w-item .w-label a { color:inherit; text-decoration:none; }
+.w-item .w-meta { flex-shrink:0; font-size:0.72rem; color:var(--text-muted); font-variant-numeric:tabular-nums; }
 
 
 /* Time range selector */

tests/core/test_monitors_page.py

@@ -153,7 +153,7 @@ async def test_the_widget_gives_a_verdict_then_problems_with_uptime_columns(app)
     and since when, then each monitor's latency and 24h/7d/30d under one header."""
     down = {**_row(_mon("api", mid="m2"), "down", _check(False)),
             "uptime": {"24h": 90.0, "7d": 97.5, "30d": 99.12},
-            "down_since": NOW - timedelta(minutes=12), "down_reason": "HTTP 503"}
+            "down_since": datetime.now(timezone.utc) - timedelta(minutes=12), "down_reason": "HTTP 503"}
     up = {**_row(_mon("web", mid="m1"), "up", _check(ms=31)), "uptime": {"24h": 100.0, "7d": 99.96}}
     page = await _widget(app, [down], [up], more=3)
     assert "1 of 5 monitors down." in page

tests/plugins/docker/test_docker_page.py

@@ -162,3 +162,49 @@ async def test_the_disk_page_explains_its_terms_and_names_untagged_images(app):
     assert "held by images no container uses" in page and "safe to clear" in page
     assert "Shared with other images" in page and "2 containers" in page and ">nothing<" in page
     assert "dangling" not in page
+
+
+def _widget_svc(name, state="healthy", running=1, desired=1, hosts=("VDNT-Docker02",), **kw):
+    base = dict(name=name, state=state, running=running, desired=desired, crash_looping=False,
+                failures_1h=0, last_error=None, trouble_since=None,
+                replicas=[{"host": h, "ghost": False} for h in hosts])
+    base.update(kw)
+    return base
+
+
+async def _services_widget(app, **ctx):
+    base = dict(bad_services=[], bad_containers=[], healthy=[], healthy_more=0, services_total=0,
+                services_healthy=0, running_count=0, failures=[], host_name={"h1": "VDNT-Docker02"},
+                total_count=1, widget_id="1")
+    base.update(ctx)
+    async with app.test_request_context("/plugins/docker/widget"):
+        return await render_plugin_template(app, "docker", "docker/widget.html", **base)
+
+
+async def test_the_services_widget_lists_what_is_wrong_then_packs_the_healthy(app):
+    """#5252: a verdict, the troubled service with why and since when, the
+    failures by name, then healthy services one line each in a packed grid."""
+    bad = _widget_svc("codex_db", state="down", running=0, crash_looping=True, failures_1h=7,
+               trouble_since=datetime.now(timezone.utc) - timedelta(minutes=9))
+    page = await _services_widget(
+        app, bad_services=[bad], services_total=3, services_healthy=2, running_count=4,
+        failures=[{"name": "codex_db", "host": "VDNT-Docker03", "count": 7, "at": NOW - timedelta(minutes=1),
+                   "detail": "exit 1"}],
+        healthy=[{"kind": "service", "s": _widget_svc("curator_app")},
+                 {"kind": "service", "s": _widget_svc("web", running=2, desired=2, hosts=("a", "b", "a"))},
+                 {"kind": "container", "c": _c("portainer_agent")}],
+        healthy_more=12)
+    assert "2/3 services healthy" in page and "4 other containers running" in page and "1 failed in 24h" in page
+    assert page.index("Needs a look") < page.index("Failed in the last 24h") < page.index("Healthy")
+    assert "crash-looping: 7 failed in the last hour" in page and "9m ago" in page
+    assert "7 times, last" in page and "exit 1" in page
+    assert 'class="w-grid"' in page and " · a, b</span>" in page and ">2/2<" in page
+    assert "+12 more healthy" in page
+    assert not hex_colours(page), hex_colours(page)
+
+
+async def test_the_services_widget_says_when_all_is_well(app):
+    page = await _services_widget(app, services_total=2, services_healthy=2, running_count=1,
+                                  healthy=[{"kind": "service", "s": _widget_svc("a")}])
+    assert "2/2 services healthy" in page and "nothing failed in 24h" in page and "tone-ok" in page
+    assert "Needs a look" not in page and "Failed in the last 24h" not in page