fix(docker): reap vanished containers; clarify the resources widget

The container persister upserted current state keyed (host_id, name) but — unlike
the image/swarm/disk persisters — never deleted containers that vanished from the
host's latest listing. Every removed one-shot container (CI job runners, buildkit
builders, codex jobs) left a permanent "stopped" row, so the dashboard counts
ballooned (e.g. 856 stopped) and read as "not dedup'd". It wasn't dedup — it was
a missing reaper.

- ingest.py: after the upsert loop, delete this host's containers whose name is
  notin the newest snapshot (the listing is authoritative — event derivation
  already treats absence as removal). Mirrors images/swarm/disk. The existing
  `if not snapshots: return` keeps swarm/disk-only samples from touching
  containers; an empty container snapshot legitimately means none exist.
- widget_resources.html: this widget shows the busiest running containers by CPU.
  Replaced the two unlabeled hairline bars with labeled CPU/MEM bars (label · bar ·
  % value, warn/crit colors) so it's self-explanatory.
- Regression test: a container present then absent across snapshots is reaped.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC

plugins/docker/ingest.py

@@ -321,3 +321,16 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None) -
                 resource_name=resource, metric_name="is_healthy",
                 value=1.0 if health == "healthy" else 0.0,
             )
+
+    # Reap containers that vanished from the host's latest listing. Without this,
+    # removed one-shot containers (CI job runners, buildkit builders, codex jobs)
+    # accumulate forever as permanent "stopped" rows and inflate every count.
+    # The listing is authoritative (event derivation already treats absence as
+    # removal), so anything not in the newest snapshot no longer exists on the
+    # host. Mirrors the image/swarm/disk persisters. An empty snapshot legitimately
+    # means the host has no containers, so the unfiltered delete is correct.
+    seen_names = {c["name"] for c in latest_containers if c.get("name")}
+    reap = delete(DockerContainer).where(DockerContainer.host_id == host.id)
+    if seen_names:
+        reap = reap.where(DockerContainer.name.notin_(seen_names))
+    await session.execute(reap)

plugins/docker/templates/docker/widget_resources.html

@@ -1,36 +1,28 @@
-{# docker/widget_resources.html — dashboard widget: CPU + memory bars, busiest containers #}
+{# docker/widget_resources.html — dashboard widget: the busiest running containers
+   by CPU, each with labeled CPU + memory utilisation bars. #}
 {% if not rows %}
 <div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No running containers.</div>
 {% else %}
-<div style="display:grid;gap:0.5rem;">
+{% macro bar(label, pct, warn, crit) %}
+<div style="display:flex;align-items:center;gap:0.45rem;margin-bottom:2px;">
+  <span style="font-size:0.62rem;color:var(--text-dim);width:1.9rem;flex-shrink:0;letter-spacing:0.03em;">{{ label }}</span>
+  <div style="flex:1;height:6px;background:var(--bg-elevated);border-radius:3px;overflow:hidden;">
+    <div style="height:100%;border-radius:3px;width:{{ [pct, 100] | min }}%;
+                background:{% if pct > crit %}var(--red){% elif pct > warn %}var(--orange){% else %}var(--accent){% endif %};
+                transition:width 0.4s;"></div>
+  </div>
+  <span style="font-size:0.7rem;color:var(--text-muted);font-family:ui-monospace,monospace;width:3rem;text-align:right;flex-shrink:0;">{{ "%.1f" | format(pct) }}%</span>
+</div>
+{% endmacro %}
+<div style="display:grid;gap:0.6rem;">
   {% for r in rows %}
   {% set c = r.c %}
   <div>
-    <div style="display:flex;justify-content:space-between;align-items:baseline;margin-bottom:0.15rem;">
-      <span style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;max-width:55%;">
-        {{ c.name }}{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
-      </span>
-      <span style="font-size:0.72rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;margin-left:0.5rem;">
-        {% if c.cpu_pct is not none %}CPU {{ "%.1f" | format(c.cpu_pct) }}%{% endif %}
-        {% if c.mem_pct is not none %} · Mem {{ "%.1f" | format(c.mem_pct) }}%{% endif %}
-      </span>
+    <div style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-bottom:0.25rem;">
+      {{ c.name }}{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
     </div>
-    {% if c.cpu_pct is not none %}
-    <div style="height:3px;background:var(--bg-elevated);border-radius:2px;margin-bottom:2px;">
-      <div style="height:100%;border-radius:2px;width:{{ [c.cpu_pct, 100] | min }}%;
-                  background:{% if c.cpu_pct > 80 %}var(--red){% elif c.cpu_pct > 50 %}var(--orange){% else %}var(--accent){% endif %};
-                  transition:width 0.4s;">
-      </div>
-    </div>
-    {% endif %}
-    {% if c.mem_pct is not none %}
-    <div style="height:3px;background:var(--bg-elevated);border-radius:2px;">
-      <div style="height:100%;border-radius:2px;width:{{ [c.mem_pct, 100] | min }}%;
-                  background:{% if c.mem_pct > 90 %}var(--red){% elif c.mem_pct > 70 %}var(--orange){% else %}var(--green){% endif %};
-                  transition:width 0.4s;">
-      </div>
-    </div>
-    {% endif %}
+    {% if c.cpu_pct is not none %}{{ bar("CPU", c.cpu_pct, 50, 80) }}{% endif %}
+    {% if c.mem_pct is not none %}{{ bar("MEM", c.mem_pct, 70, 90) }}{% endif %}
   </div>
   {% endfor %}
 </div>

tests/integration/test_docker.py

@@ -231,6 +231,44 @@ def test_lifecycle_events_derived_across_snapshots(app):
     assert not any(e[0] == "start" for e in events)
 
 
+@_NEEDS_DB
+def test_vanished_container_is_reaped(app):
+    """A container absent from the latest snapshot is deleted, not left behind as
+    a permanent stopped row. Regression: removed one-shot containers (CI jobs,
+    buildkit builders) accumulated forever and inflated the dashboard counts."""
+    from sqlalchemy import text
+    from steward.models.hosts import Host
+
+    persist = _persist_fn(app)
+    t1 = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc)
+    t2 = datetime(2026, 6, 19, 12, 0, 30, tzinfo=timezone.utc)
+    base = {"status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
+            "restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
+    keep = {**base, "name": "keep"}
+    gone = {**base, "name": "ci-job-123"}
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await s.execute(text("DELETE FROM docker_containers"))
+                h = Host(id=str(uuid.uuid4()), name="reap", address="10.9.9.8")
+                s.add(h)
+                await s.flush()
+                await persist(s, h, [(t1, [keep, gone])])   # both present
+                before = [r[0] for r in (await s.execute(text(
+                    "SELECT name FROM docker_containers WHERE host_id = :h ORDER BY name"),
+                    {"h": h.id})).all()]
+                await persist(s, h, [(t2, [keep])])         # ci-job-123 vanished
+                after = [r[0] for r in (await s.execute(text(
+                    "SELECT name FROM docker_containers WHERE host_id = :h ORDER BY name"),
+                    {"h": h.id})).all()]
+                return before, after
+
+    before, after = asyncio.run(_go())
+    assert before == ["ci-job-123", "keep"]
+    assert after == ["keep"]   # the vanished container was reaped
+
+
 @_NEEDS_DB
 def test_swarm_topology_persisted(app):
     from sqlalchemy import text