feat(docker): readable names, CPU in cores and memory in bytes, failures listed with since-times (#5209)

#5209 "Docker views: readable names, CPU/memory in cores and bytes, failures listed with since-time" (milestone 477 "UI problems first"). - A swarm task reads as its service and slot ("codex_db · 1", or the node for a global service), with the raw name in the title: Docker page, host panel, busiest widget, container detail. - CPU reads in cores (Docker's 100% = one core), toned by its share of the host's CPUs from the agent's per-core series. Memory reads in bytes, against a limit only when one is really set (Docker reports host RAM otherwise). - Exit 137/143 without an OOM kill is a stop, not a failure: no exited_nonzero, no crash-loop death, no "failed (24h)", dim exit code. - "Failed (24h)" is a list (Docker page #failures, linked from the widget and the tile): plain containers by their deaths, swarm services by Swarm's failed tasks; a second manager's copy of a task death counts once. - Failing swarm services lead Needs attention; services sort problems first with since-times on the page, the widget and the Swarm page; nodes that are down or drained sort first. - Busiest widget orders by a 5-minute CPU average so the list holds still. - Host panel: a table, problems first with what is wrong and since when; replaced swarm tasks hidden and counted. - Container detail: node hostname, I/O labelled "since start", Restarts hidden for swarm tasks with a note on where task failures are. - Widget names: "Docker — Services & failures", "Docker — Busiest containers". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 23:41:14 -04:00
parent 8941ed0372
commit 61b22056d5
16 changed files with 713 additions and 177 deletions
@@ -0,0 +1,26 @@
+{#- A container's CPU and memory as service.usage() describes them (#5209).
+
+    Docker counts CPU per core (100% = one core busy), so CPU reads as cores,
+    toned by its share of the host's CPUs when the host's core count is known.
+    Memory reads in bytes, against a limit only when one is actually set;
+    mem_pct is then a share of that limit, otherwise of the host's memory. -#}
+
+{% macro cores(n) -%}
+{{ "%.2f"|format(n) if n < 1 else "%.1f"|format(n) }} {{ "core" if n == 1 else "cores" }}
+{%- endmacro %}
+
+{% macro cpu(u) -%}
+{%- if u.cores is none -%}<span class="tone-dim">—</span>
+{%- else -%}
+<span class="tone-{{ status_class(u.cpu_share, 'cpu') }}"
+      title="Docker counts CPU per core: {{ '%.0f'|format(u.cores * 100) }}% of one core{% if u.cpus %}, {{ '%.1f'|format(u.cpu_share) }}% of this host's {{ u.cpus }} core{{ 's' if u.cpus != 1 }}{% endif %}">{{ cores(u.cores) }}</span>
+{%- endif -%}
+{%- endmacro %}
+
+{% macro mem(u) -%}
+{%- if u.mem_bytes is none -%}<span class="tone-dim">—</span>
+{%- else -%}
+<span class="tone-{{ status_class(u.mem_pct, 'mem') if u.mem_of else 'dim' }}"
+      title="{% if u.mem_pct is not none and u.mem_of %}{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% else %}No memory limit set{% endif %}">{{ u.mem_bytes|bytes }}{% if u.mem_limit %} / {{ u.mem_limit|bytes }}{% endif %}</span>
+{%- endif -%}
+{%- endmacro %}
@@ -1,8 +1,11 @@
-{# docker/container_detail.html — full detail page for one container #}
+{# docker/container_detail.html — full detail page for one container. A swarm
+   task is titled by its service and slot, with the raw container name under
+   it; CPU reads in cores and memory in bytes (see _usage.html). #}
 {% extends "base.html" %}
 {% from "_macros.html" import crumbs %}
-{% block title %}{{ name }} — Docker — Steward{% endblock %}
-{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %}
+{% from "docker/_usage.html" import cpu, mem %}
+{% block title %}{{ label or name }} — Docker — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (label or name, "")]) }}{% endblock %}
 {% block content %}

 {% if container is none %}
@@ -18,11 +21,14 @@
 {# ── Header ──────────────────────────────────────────────────────────────── #}
 <div style="display:flex;align-items:center;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
  <span class="dot {% if container.status == 'running' %}dot-up{% elif container.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
-  <h1 class="page-title" style="margin-bottom:0;">{{ container.name }}</h1>
+  <h1 class="page-title" style="margin-bottom:0;">{{ label }}</h1>
  {% if container.health == 'healthy' %}<span style="font-size:0.72rem;color:var(--green);border:1px solid var(--green);border-radius:3px;padding:0.05rem 0.4rem;">healthy</span>
  {% elif container.health == 'unhealthy' %}<span style="font-size:0.72rem;color:var(--red);border:1px solid var(--red);border-radius:3px;padding:0.05rem 0.4rem;">unhealthy</span>
  {% elif container.health == 'starting' %}<span style="font-size:0.72rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0.05rem 0.4rem;">starting</span>{% endif %}
 </div>
+{% if label != container.name %}
+<div style="font-size:0.75rem;color:var(--text-dim);font-family:ui-monospace,monospace;margin-bottom:0.25rem;overflow-wrap:anywhere;">{{ container.name }}</div>
+{% endif %}
 <div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
  {{ container.status }}{% if uptime %} · up {{ uptime }}{% endif %}
  {% if host %} · on <a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %}
@@ -38,16 +44,27 @@
 {% endmacro %}

 <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
-  {{ fact("CPU", "%.1f%%" | format(container.cpu_pct) if container.cpu_pct is not none else "—") }}
-  {{ fact("Memory", "%.1f%%" | format(container.mem_pct) if container.mem_pct is not none else "—") }}
+  {{ fact("CPU", cpu(usage) if container.status == 'running' else "—") }}
+  {{ fact("Memory", mem(usage) if container.status == 'running' else "—") }}
+  {#- Swarm never restarts a task's container: it replaces the task with a new
+      one, so restart_count is always 0 there and the service's task failures
+      (Swarm page) are the count that means something. #}
+  {% if not container.service_name %}
  {{ fact("Restarts", container.restart_count, "var(--orange)" if container.restart_count else "") }}
+  {% endif %}
  {{ fact("Last exit", (container.exit_code ~ (" (OOM)" if container.oom_killed else "")) if container.exit_code is not none else "—",
-          "var(--red)" if (container.exit_code is not none and container.exit_code != 0) else "") }}
-  {{ fact("Net in", net_rx) }}
-  {{ fact("Net out", net_tx) }}
-  {{ fact("Block read", blk_read) }}
-  {{ fact("Block write", blk_write) }}
+          "var(--red)" if exit_failed else "") }}
+  {{ fact("Net in since start", net_rx) }}
+  {{ fact("Net out since start", net_tx) }}
+  {{ fact("Disk read since start", blk_read) }}
+  {{ fact("Disk written since start", blk_write) }}
 </div>
+{% if container.service_name %}
+<div style="font-size:0.78rem;color:var(--text-muted);margin:-0.75rem 0 1.5rem;">
+  Swarm replaces a failed task with a new container rather than restarting it; the
+  service's failed tasks are on the <a href="/plugins/docker/swarm">Swarm page</a>.
+</div>
+{% endif %}

 {# ── Image / placement ───────────────────────────────────────────────────── #}
 <div class="card" style="margin-bottom:1.5rem;">
@@ -61,7 +78,9 @@
    <span style="color:var(--text-muted);">Swarm service</span><span>{{ container.service_name }}</span>
    {% endif %}
    {% if container.node_id %}
-    <span style="color:var(--text-muted);">Node</span><span style="font-family:ui-monospace,monospace;">{{ container.node_id }}</span>
+    <span style="color:var(--text-muted);">Node</span>
+    {% if node_name %}<span title="node id {{ container.node_id }}">{{ node_name }}</span>
+    {% else %}<span style="font-family:ui-monospace,monospace;">{{ container.node_id }}</span>{% endif %}
    {% endif %}
    {% if ports %}
    <span style="color:var(--text-muted);">Ports</span>
@@ -1,22 +1,58 @@
-{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub #}
+{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub:
+   problems first with what is wrong and since when, then running, then
+   stopped. Tasks Swarm has replaced are left out (counted in the header). #}
+{% from "_macros.html" import status_badge, status_dot %}
+{% from "docker/_usage.html" import cpu, mem %}
 <div class="card">
-  <div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;">
+  <div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;flex-wrap:wrap;">
    <h3 class="section-title" style="margin-bottom:0;">Docker</h3>
    <span style="font-size:0.78rem;color:var(--text-muted);">
+      {% if problems %}<span class="tone-crit">{{ problems }} with problems</span> · {% endif %}
      {{ running }} running{% if stopped %} · {{ stopped }} stopped{% endif %}
+      {% if replaced %}<span title="Old swarm tasks Swarm has replaced with new ones: history, not containers that need anything"> · {{ replaced }} replaced task{{ "s" if replaced != 1 }} hidden</span>{% endif %}
      <a href="/plugins/docker/" style="margin-left:0.6rem;color:var(--text-muted);">All →</a>
    </span>
  </div>
-  <div style="display:grid;gap:2px;">
-    {% for c in containers %}
-    <div style="display:flex;align-items:center;gap:0.5rem;font-size:0.85rem;padding:0.2rem 0;">
-      <span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
-      <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;" title="{{ c.image }}">{{ c.name }}</a>
-      <span style="font-size:0.74rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
-        {% if c.cpu_pct is not none %}{{ "%.1f" | format(c.cpu_pct) }}%{% endif %}
-        {% if c.mem_pct is not none %} · {{ "%.1f" | format(c.mem_pct) }}%{% endif %}
-      </span>
-    </div>
-    {% endfor %}
-  </div>
+  <table class="table">
+    <thead>
+      <tr>
+        <th>Container</th>
+        <th title="Docker counts CPU per core: 1 core = one core fully busy">CPU</th>
+        <th title="Used, and the limit when one is set">Memory</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for r in rows %}
+      {% set c = r.c %}
+      <tr>
+        <td>
+          <div style="display:flex;align-items:center;gap:0.5rem;min-width:0;">
+            {{ status_dot('crit' if r.problems else ('up' if c.status == 'running' else 'inactive'), c.status|capitalize) }}
+            <div style="min-width:0;">
+              <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" title="{{ c.name }} · {{ c.image }}" style="font-weight:500;color:inherit;text-decoration:none;overflow-wrap:anywhere;">{{ r.label }}</a>
+              {% if r.problems %}
+              {% set worst = r.problems[0] %}
+              <div style="font-size:0.75rem;margin-top:0.1rem;">
+                {{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }}
+                <span style="color:var(--text-muted);">
+                  {%- for p in r.problems %} {{ says[p] }}{{ " and" if not loop.last }}{% endfor %}
+                  {%- if c.exit_code %} · exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}{% endif %}
+                  {%- if r.since %} · since {{ r.since|ago }}{% endif -%}
+                </span>
+              </div>
+              {% else %}
+              <div style="font-size:0.72rem;color:var(--text-muted);">
+                {{ c.status }}{% if r.uptime %} · up {{ r.uptime }}{% endif %}
+                {%- if c.status != 'running' and c.exit_code %} · exit {{ c.exit_code }}{% endif %}
+              </div>
+              {% endif %}
+            </div>
+          </div>
+        </td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">{% if c.status == 'running' %}{{ cpu(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">{% if c.status == 'running' %}{{ mem(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
 </div>
@@ -13,4 +13,17 @@
     hx-swap="innerHTML">
  <div class="card"><p class="field-hint">Loading…</p></div>
 </div>
+{# A link to a section (the dashboard's "failed (24h)" → #failures) lands
+   before the rows exist; scroll to it once, when they first arrive. #}
+<script>
+(function () {
+  var rows = document.getElementById('docker-rows');
+  if (!location.hash || !rows) return;
+  rows.addEventListener('htmx:afterSwap', function once() {
+    rows.removeEventListener('htmx:afterSwap', once);
+    var target = document.getElementById(location.hash.slice(1));
+    if (target) target.scrollIntoView({ block: 'start' });
+  });
+})();
+</script>
 {% endblock %}
@@ -2,18 +2,36 @@
   first (service.needs_attention, the ranking the MCP tools use), the counts,
   then swarm services and each host's containers. #}
 {% from "_macros.html" import stat_tile, status_badge, status_dot %}
+{% from "docker/_usage.html" import cpu, mem, cores %}

 {# ── Needs attention ──────────────────────────────────────────────────────── #}
-{% if attention %}
+{% if attention or failing_services %}
 <section class="card problem-card" aria-labelledby="docker-problems">
  <h2 id="docker-problems" class="section-title" style="margin-bottom:0.6rem;">Needs attention</h2>
  <ul class="problem-list">
+    {# A swarm service in trouble: Swarm replaces its failed tasks, so the
+       service, not any one container, is what is failing. #}
+    {% for s in failing_services %}
+    {% set e = s.last_error %}
+    <li>
+      {% if s.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}
+      {% elif s.state == 'down' %}{{ status_badge('crit', 'Down') }}
+      {% else %}{{ status_badge('warn', 'Degraded') }}{% endif %}
+      <a href="/plugins/docker/swarm">{{ s.name }}</a>
+      <span class="problem-detail">
+        {{- s.running }} of {{ s.desired }} running
+        {%- if s.crash_looping %} · {{ s.failures_1h }} tasks failed in the last hour{% endif %}
+        {%- if e %} · last task {{ e.state }}: {{ e.error or e.message or e.state }}{% if e.exit_code %} (exit {{ e.exit_code }}){% endif %}{% if e.node %} on {{ e.node }}{% endif %}{% endif %}
+        {%- if s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif -%}
+      </span>
+    </li>
+    {% endfor %}
    {% for r in attention %}
    {% set c = r.container %}
    {% set worst = r.problems[0] %}
    <li>
      {{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }}
-      <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}">{{ c.name }}</a>
+      <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" title="{{ c.name }}">{{ r.label }}</a>
      <span class="problem-detail">
        {%- for p in r.problems %}{{ says[p] }}{{ " and " if not loop.last }}{% endfor %}
        {%- if c.exit_code and worst in ('exited_nonzero', 'crash_looping', 'restarting') %} · exit {{ c.exit_code }}{% endif %}
@@ -42,18 +60,45 @@
 </section>
 {% endif %}

-{# ── Counts ───────────────────────────────────────────────────────────────── #}
+{# ── Counts ──────────────────────────────────────────────────────────────
+   Swarm services are counted as services (a task's container comes and goes);
+   the container counts are the containers outside any swarm service. ────── #}
 <div class="stat-strip">
-  {{ stat_tile("Needs attention", attention|length, state="crit" if attention else "inactive") }}
+  {{ stat_tile("Needs attention", attention|length + failing_services|length, state="crit" if attention or failing_services else "inactive") }}
  {{ stat_tile("Unhealthy", unhealthy, state="crit" if unhealthy else "inactive") }}
  {{ stat_tile("Restarting", restarting, state="crit" if restarting else "inactive", hint="restarting or crash-looping") }}
-  {{ stat_tile("Running", running, state="up" if running else "inactive") }}
-  {{ stat_tile("Stopped", stopped, state="inactive") }}
+  {{ stat_tile("Failed (24h)", failures|length, state="warn" if failures else "inactive", hint="containers and services; listed below", href="#failures" if failures else none) }}
+  {% if swarm_services %}{{ stat_tile("Swarm services", services_healthy ~ "/" ~ swarm_services|length, state="up" if services_healthy == swarm_services|length else "warn", hint="healthy", href="/plugins/docker/swarm") }}{% endif %}
+  {{ stat_tile("Containers running", running, state="up" if running else "inactive", hint="outside swarm services" if swarm_services else none) }}
+  {{ stat_tile("Containers stopped", stopped, state="inactive") }}
  {% if stale %}{{ stat_tile("Stale", stale, state="stale", hint="no recent report") }}{% endif %}
-  {% if swarm_services %}{{ stat_tile("Services", swarm_services|length, href="/plugins/docker/swarm") }}{% endif %}
-  {{ stat_tile("Hosts", host_groups|length) }}
+  {{ stat_tile("Docker hosts", docker_hosts) }}
 </div>

+{# ── What failed in the last 24h: what the dashboard's "failed (24h)" counts.
+   A plain container's death, or a swarm service's failed tasks; a container
+   someone stopped (exit 137/143) is not a failure. ───────────────────────── #}
+{% if failures %}
+<section id="failures" class="card" aria-labelledby="docker-failures" style="margin-bottom:1.5rem;">
+  <h2 id="docker-failures" class="section-title" style="margin-bottom:0.6rem;">Failed in the last 24h</h2>
+  <ul class="problem-list">
+    {% for f in failures %}
+    <li>
+      {{ status_badge('warn', 'Service' if f.kind == 'service' else 'Container') }}
+      {% if f.kind == 'service' %}<a href="/plugins/docker/swarm">{{ f.name }}</a>
+      {% else %}<a href="/plugins/docker/container/{{ f.host_id }}/{{ f.name }}">{{ f.name }}</a>{% endif %}
+      <span class="problem-detail">
+        {{- f.count }} {{ "failure" if f.count == 1 else "failures" }}
+        {%- if f.detail %} · last: {{ f.detail }}{% endif %}
+        {%- if f.host %} · {{ f.host }}{% endif %}
+        {%- if f.at %} · {{ f.at|ago }}{% endif -%}
+      </span>
+    </li>
+    {% endfor %}
+  </ul>
+</section>
+{% endif %}
+
 {# ── Swarm services: collapsed per-service, each current replica labelled with
   its host (tasks Swarm replaced are left out). Ghost replicas (tasks on nodes
   with no Steward agent) come from the managers' placement data so the picture
@@ -72,7 +117,7 @@
        <span style="font-weight:500;font-size:0.9rem;">{{ s.name }}</span>
        <span style="font-size:0.72rem;color:var(--text-dim);">{{ s.mode }}</span>
        <span class="tone-{{ 'ok' if s.state == 'healthy' else ('warn' if s.state == 'degraded' else 'crit') }}" style="font-size:0.78rem;font-variant-numeric:tabular-nums;">
-          {{ s.running }}/{{ s.desired }} running
+          {{ s.running }}/{{ s.desired }} running{% if s.state != 'healthy' and s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif %}
        </span>
      </div>
      {% with style="padding-left:1.1rem;margin-bottom:0.35rem;" %}{% include "docker/_swarm_failures.html" %}{% endwith %}
@@ -88,7 +133,7 @@
        <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px solid var(--border);border-radius:5px;padding:0.3rem 0.55rem;">
          <span class="dot {% if r.status == 'running' %}dot-up{% elif r.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
          <a href="/plugins/docker/container/{{ r.host_id }}/{{ r.name }}" style="font-weight:500;color:inherit;text-decoration:none;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;" title="{{ r.name }}">{{ r.host }}</a>
-          {% if r.cpu_pct is not none %}<span style="color:var(--text-muted);font-family:ui-monospace,monospace;margin-left:auto;">{{ "%.0f" | format(r.cpu_pct) }}%</span>{% endif %}
+          {% if r.cpu_pct is not none %}<span style="color:var(--text-muted);font-family:ui-monospace,monospace;margin-left:auto;" title="Docker counts CPU per core">{{ cores(r.cpu_pct / 100) }}</span>{% endif %}
        </div>
        {% endif %}
        {% endfor %}
@@ -119,8 +164,8 @@
        <th>Container</th>
        <th>Image</th>
        <th>Ports</th>
-        <th>CPU %</th>
-        <th>Mem %</th>
+        <th title="Docker counts CPU per core: 1 core = one core fully busy">CPU</th>
+        <th title="Used, and the limit when one is set">Memory</th>
      </tr>
    </thead>
    <tbody>
@@ -144,7 +189,7 @@
              <div style="font-size:0.73rem;color:var(--text-muted);">
                {{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %}
                {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
-                  · <span class="tone-crit">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
+                  · <span class="tone-{{ 'crit' if item.exit_failed else 'dim' }}" title="{{ 'last exit code' if item.exit_failed else 'stopped by a signal; not a failure' }}">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
                {% endif %}
                {% if c.restart_count %} · <span class="tone-warn" title="Restarted {{ c.restart_count }} times">⟳{{ c.restart_count }}</span>{% endif %}
              </div>
@@ -164,20 +209,8 @@
          <div style="color:var(--text-muted);">{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}</div>
          {% endfor %}
        </td>
-        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
-          {% if c.cpu_pct is not none %}
-          <span class="tone-{{ status_class(c.cpu_pct, 'cpu') }}">{{ "%.1f" | format(c.cpu_pct) }}%</span>
-          {% else %}
-          <span style="color:var(--text-dim);">—</span>
-          {% endif %}
-        </td>
-        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
-          {% if c.mem_pct is not none %}
-          <span class="tone-{{ status_class(c.mem_pct, 'mem') }}">{{ "%.1f" | format(c.mem_pct) }}%</span>
-          {% else %}
-          <span style="color:var(--text-dim);">—</span>
-          {% endif %}
-        </td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;white-space:nowrap;">{{ cpu(item.usage) }}</td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;white-space:nowrap;">{{ mem(item.usage) }}</td>
      </tr>
      {% endfor %}
      {% endfor %}
@@ -1,4 +1,6 @@
-{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement #}
+{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement.
+   Services in trouble come first (crash loop, down, short), with since-times;
+   nodes that are down or not taking work come first. #}
 {% extends "base.html" %}
 {% from "_macros.html" import crumbs %}
 {% block title %}Swarm — Docker — Steward{% endblock %}
@@ -34,6 +36,7 @@
            <span style="color:{% if s.healthy %}var(--green){% elif s.down %}var(--red){% elif s.degraded %}var(--orange){% else %}var(--text-muted){% endif %};">
              {{ s.running }}/{{ s.desired }}
            </span>
+            {% if s.state != 'healthy' and s.trouble_since %}<div style="font-family:inherit;font-size:0.72rem;color:var(--text-dim);" title="{{ s.trouble_since }}">since {{ s.trouble_since|ago }}</div>{% endif %}
          </td>
          <td style="font-size:0.8rem;color:var(--text-muted);max-width:220px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.image or "—" }}</td>
          <td style="font-size:0.78rem;color:var(--text-muted);">
@@ -1,21 +1,32 @@
-{# docker/widget.html — dashboard widget: container status overview, by host #}
+{# docker/widget.html — dashboard widget: what Docker is running and what has
+   failed. Swarm services first, troubled ones at the top with since-times; then
+   the containers outside any service, by host, problems first. The failed count
+   links to the Docker page's list of exactly what it counts. #}
+{% from "docker/_usage.html" import cpu %}
 {% if total_count == 0 %}
 <p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No containers reported yet.</p>
 {% else %}

-<div style="display:flex;gap:1.25rem;margin-bottom:0.65rem;">
-  <div>
+<div style="display:flex;flex-wrap:wrap;gap:0.4rem 1.25rem;margin-bottom:0.65rem;">
+  {% if swarm_services %}
+  <div title="Swarm services with every replica running">
+    <span class="tone-{{ 'ok' if services_healthy == swarm_services|length else 'warn' }}" style="font-size:1.4rem;font-weight:500;line-height:1;">{{ services_healthy }}/{{ swarm_services|length }}</span>
+    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">services healthy</span>
+  </div>
+  {% endif %}
+  <div title="Containers running outside any swarm service">
    <span style="font-size:1.4rem;font-weight:500;line-height:1;color:var(--green);">{{ running_count }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">running</span>
-  </div>
-  <div title="Containers that died or were OOM-killed in the last 24h, plus swarm tasks that failed (a task Swarm replaced is not a failure)">
-    <span style="font-size:1.4rem;font-weight:500;line-height:1;color:{% if failed_24h %}var(--red){% else %}var(--text-muted){% endif %};">{{ failed_24h }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">failed (24h)</span>
+    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">containers running</span>
  </div>
+  <a href="/plugins/docker/#failures" style="color:inherit;text-decoration:none;"
+     title="Containers that died or were OOM-killed in the last 24h, and swarm services with failed tasks. A container someone stopped is not counted. Click for the list.">
+    <span class="tone-{{ 'crit' if failures else 'dim' }}" style="font-size:1.4rem;font-weight:500;line-height:1;">{{ failures|length }}</span>
+    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;border-bottom:1px dotted var(--border-mid);">failed in 24h</span>
+  </a>
 </div>

-{# Swarm services — collapsed, each with its replicas' host chips (dashed = a
-   node with no Steward agent, count-only from placement). #}
+{# Swarm services — troubled first, each with its replicas' host chips
+   (dashed = a node with no Steward agent, count-only from placement). #}
 {% if swarm_services %}
 <div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">Swarm services</div>
 <div style="display:grid;gap:0.4rem;margin-bottom:0.4rem;">
@@ -24,8 +35,9 @@
    <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.82rem;">
      <span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
      <span style="font-weight:500;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.name }}</span>
-      {% if s.crash_looping %}<span style="color:var(--red);font-size:0.7rem;font-weight:500;flex-shrink:0;" title="{{ s.failures_1h }} tasks failed in the last hour{% if s.last_error %}: {{ s.last_error.error or s.last_error.message or s.last_error.state }}{% endif %}">crash-looping</span>{% endif %}
-      <span style="color:var(--text-muted);font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;">{{ s.running }}/{{ s.desired }}</span>
+      {% if s.crash_looping %}<span class="tone-crit" style="font-size:0.7rem;font-weight:500;flex-shrink:0;" title="{{ s.failures_1h }} tasks failed in the last hour{% if s.last_error %}: {{ s.last_error.error or s.last_error.message or s.last_error.state }}{% endif %}">crash-looping</span>{% endif %}
+      {% if s.state != 'healthy' and s.trouble_since %}<span class="tone-dim" style="font-size:0.7rem;flex-shrink:0;">since {{ s.trouble_since|ago }}</span>{% endif %}
+      <span class="tone-{{ 'ok' if s.state == 'healthy' else ('warn' if s.state == 'degraded' else 'crit') }}" style="font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
    </div>
    <div style="display:flex;flex-wrap:wrap;gap:0.25rem;margin:0.2rem 0 0 1rem;">
      {% for r in s.replicas %}
@@ -41,25 +53,25 @@
 </div>
 {% endif %}

+{% if host_groups and swarm_services %}
+<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.6rem 0 0.2rem;">Other containers</div>
+{% endif %}
 {% for g in host_groups %}
 {% if multi_host %}
-<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">{{ g.host_name }}</div>
+<div style="font-size:0.7rem;color:var(--text-dim);margin:0.4rem 0 0.15rem;">{{ g.host_name }}</div>
 {% endif %}
 <div style="display:grid;gap:2px;">
-  {% for c in g.containers %}
+  {% for item in g.containers %}
+  {% set c = item.c %}
  <div style="display:flex;align-items:center;gap:0.5rem;font-size:0.82rem;padding:0.15rem 0;">
-    <span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
+    <span class="dot {% if c.status == 'running' and c.health != 'unhealthy' %}dot-up{% elif c.status in ('paused', 'restarting') %}dot-warn{% else %}dot-down{% endif %}"></span>
    <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;">
-      {{ c.name }}{% if c.health == 'unhealthy' %}<span title="unhealthy" style="color:var(--red);"> ●</span>{% elif c.health == 'healthy' %}<span title="healthy" style="color:var(--green);"> ●</span>{% endif %}{% if c.restart_count %}<span title="restarts" style="color:var(--orange);font-size:0.7rem;"> ⟳{{ c.restart_count }}</span>{% endif %}
+      {{ c.name }}{% if c.health == 'unhealthy' %} <span class="tone-crit" style="font-size:0.7rem;">unhealthy</span>{% endif %}{% if c.status == 'restarting' %} <span class="tone-warn" style="font-size:0.7rem;">restarting</span>{% endif %}
    </a>
    {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
-    <span title="last exit code" style="font-size:0.7rem;color:var(--red);flex-shrink:0;">exit {{ c.exit_code }}</span>
-    {% endif %}
-    {% if c.cpu_pct is not none %}
-    <span style="font-size:0.72rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
-      {{ "%.1f" | format(c.cpu_pct) }}%
-    </span>
+    <span title="{{ 'last exit code' if item.exit_failed else 'stopped by a signal; not a failure' }}" class="tone-{{ 'crit' if item.exit_failed else 'dim' }}" style="font-size:0.7rem;flex-shrink:0;">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
    {% endif %}
+    {% if c.status == 'running' %}<span style="font-size:0.72rem;font-family:ui-monospace,monospace;flex-shrink:0;">{{ cpu(item.usage) }}</span>{% endif %}
  </div>
  {% endfor %}
 </div>
@@ -1,28 +1,41 @@
-{# docker/widget_resources.html — dashboard widget: the busiest running containers
-   by CPU, each with labeled CPU + memory utilisation bars. #}
+{# docker/widget_resources.html — dashboard widget: the containers using the
+   most CPU (averaged over the last few minutes, so the order holds still),
+   each with its CPU in cores and memory in bytes. The CPU bar is a share of
+   the host's CPUs; the memory bar a share of the container's limit, or of
+   the host's memory when no limit is set. #}
+{% from "docker/_usage.html" import cpu, mem %}
 {% if not rows %}
 <div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No running containers.</div>
 {% else %}
-{% macro bar(label, pct, warn, crit) %}
-<div style="display:flex;align-items:center;gap:0.45rem;margin-bottom:2px;">
+{% macro bar(label, pct, tone, text, title) %}
+<div style="display:flex;align-items:center;gap:0.45rem;margin-bottom:2px;" title="{{ title }}">
  <span style="font-size:0.62rem;color:var(--text-dim);width:1.9rem;flex-shrink:0;letter-spacing:0.03em;">{{ label }}</span>
  <div style="flex:1;height:6px;background:var(--bg-elevated);border-radius:3px;overflow:hidden;">
-    <div style="height:100%;border-radius:3px;width:{{ [pct, 100] | min }}%;
-                background:{% if pct > crit %}var(--red){% elif pct > warn %}var(--orange){% else %}var(--accent){% endif %};
+    {% if pct is not none %}
+    <div style="height:100%;border-radius:3px;width:{{ [[pct, 0]|max, 100]|min }}%;
+                background:{% if tone == 'crit' %}var(--red){% elif tone == 'warn' %}var(--orange){% else %}var(--accent){% endif %};
                transition:width 0.4s;"></div>
+    {% endif %}
  </div>
-  <span style="font-size:0.7rem;color:var(--text-muted);font-family:ui-monospace,monospace;width:3rem;text-align:right;flex-shrink:0;">{{ "%.1f" | format(pct) }}%</span>
+  <span style="font-size:0.7rem;color:var(--text-muted);font-family:ui-monospace,monospace;min-width:4.5rem;text-align:right;flex-shrink:0;white-space:nowrap;">{{ text }}</span>
 </div>
 {% endmacro %}
+<div style="font-size:0.7rem;color:var(--text-dim);margin-bottom:0.45rem;">Busiest by CPU, {{ minutes }}-minute average</div>
 <div style="display:grid;gap:0.6rem;">
  {% for r in rows %}
-  {% set c = r.c %}
+  {% set u = r.usage %}
  <div>
-    <div style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-bottom:0.25rem;">
-      {{ c.name }}{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
+    <div style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-bottom:0.25rem;" title="{{ r.c.name }}">
+      <a href="/plugins/docker/container/{{ r.c.host_id }}/{{ r.c.name }}" style="color:inherit;text-decoration:none;">{{ r.label }}</a>{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
    </div>
-    {% if c.cpu_pct is not none %}{{ bar("CPU", c.cpu_pct, 50, 80) }}{% endif %}
-    {% if c.mem_pct is not none %}{{ bar("MEM", c.mem_pct, 70, 90) }}{% endif %}
+    {% if u.cores is not none %}
+    {{ bar("CPU", u.cpu_share, status_class(u.cpu_share, 'cpu'), cpu(u),
+           ("%.1f%% of the host's %d core%s" % (u.cpu_share, u.cpus, "" if u.cpus == 1 else "s")) if u.cpus else "Host core count unknown") }}
+    {% endif %}
+    {% if u.mem_bytes is not none %}
+    {{ bar("MEM", u.mem_pct if u.mem_of else none, status_class(u.mem_pct, 'mem') if u.mem_of else 'unknown', mem(u),
+           ("%.1f%% of %s" % (u.mem_pct, u.mem_of)) if u.mem_of and u.mem_pct is not none else "") }}
+    {% endif %}
  </div>
  {% endfor %}
 </div>