diff --git a/plugins/docker/ingest.py b/plugins/docker/ingest.py index bf6726f..b67a404 100644 --- a/plugins/docker/ingest.py +++ b/plugins/docker/ingest.py @@ -321,3 +321,16 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None) - resource_name=resource, metric_name="is_healthy", value=1.0 if health == "healthy" else 0.0, ) + + # Reap containers that vanished from the host's latest listing. Without this, + # removed one-shot containers (CI job runners, buildkit builders, codex jobs) + # accumulate forever as permanent "stopped" rows and inflate every count. + # The listing is authoritative (event derivation already treats absence as + # removal), so anything not in the newest snapshot no longer exists on the + # host. Mirrors the image/swarm/disk persisters. An empty snapshot legitimately + # means the host has no containers, so the unfiltered delete is correct. + seen_names = {c["name"] for c in latest_containers if c.get("name")} + reap = delete(DockerContainer).where(DockerContainer.host_id == host.id) + if seen_names: + reap = reap.where(DockerContainer.name.notin_(seen_names)) + await session.execute(reap) diff --git a/plugins/docker/templates/docker/widget_resources.html b/plugins/docker/templates/docker/widget_resources.html index 3358bc3..c4605fa 100644 --- a/plugins/docker/templates/docker/widget_resources.html +++ b/plugins/docker/templates/docker/widget_resources.html @@ -1,36 +1,28 @@ -{# docker/widget_resources.html — dashboard widget: CPU + memory bars, busiest containers #} +{# docker/widget_resources.html — dashboard widget: the busiest running containers + by CPU, each with labeled CPU + memory utilisation bars. #} {% if not rows %}
No running containers.
{% else %} -
+{% macro bar(label, pct, warn, crit) %} +
+ {{ label }} +
+
+
+ {{ "%.1f" | format(pct) }}% +
+{% endmacro %} +
{% for r in rows %} {% set c = r.c %}
-
- - {{ c.name }}{% if multi_host %} · {{ r.host_name }}{% endif %} - - - {% if c.cpu_pct is not none %}CPU {{ "%.1f" | format(c.cpu_pct) }}%{% endif %} - {% if c.mem_pct is not none %} · Mem {{ "%.1f" | format(c.mem_pct) }}%{% endif %} - +
+ {{ c.name }}{% if multi_host %} · {{ r.host_name }}{% endif %}
- {% if c.cpu_pct is not none %} -
-
-
-
- {% endif %} - {% if c.mem_pct is not none %} -
-
-
-
- {% endif %} + {% if c.cpu_pct is not none %}{{ bar("CPU", c.cpu_pct, 50, 80) }}{% endif %} + {% if c.mem_pct is not none %}{{ bar("MEM", c.mem_pct, 70, 90) }}{% endif %}
{% endfor %}
diff --git a/tests/integration/test_docker.py b/tests/integration/test_docker.py index fa1fcac..408ae6e 100644 --- a/tests/integration/test_docker.py +++ b/tests/integration/test_docker.py @@ -231,6 +231,44 @@ def test_lifecycle_events_derived_across_snapshots(app): assert not any(e[0] == "start" for e in events) +@_NEEDS_DB +def test_vanished_container_is_reaped(app): + """A container absent from the latest snapshot is deleted, not left behind as + a permanent stopped row. Regression: removed one-shot containers (CI jobs, + buildkit builders) accumulated forever and inflated the dashboard counts.""" + from sqlalchemy import text + from steward.models.hosts import Host + + persist = _persist_fn(app) + t1 = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc) + t2 = datetime(2026, 6, 19, 12, 0, 30, tzinfo=timezone.utc) + base = {"status": "running", "cpu_pct": 1.0, "mem_pct": 1.0, + "restart_count": 0, "exit_code": None, "oom_killed": False, "health": None} + keep = {**base, "name": "keep"} + gone = {**base, "name": "ci-job-123"} + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + await s.execute(text("DELETE FROM docker_containers")) + h = Host(id=str(uuid.uuid4()), name="reap", address="10.9.9.8") + s.add(h) + await s.flush() + await persist(s, h, [(t1, [keep, gone])]) # both present + before = [r[0] for r in (await s.execute(text( + "SELECT name FROM docker_containers WHERE host_id = :h ORDER BY name"), + {"h": h.id})).all()] + await persist(s, h, [(t2, [keep])]) # ci-job-123 vanished + after = [r[0] for r in (await s.execute(text( + "SELECT name FROM docker_containers WHERE host_id = :h ORDER BY name"), + {"h": h.id})).all()] + return before, after + + before, after = asyncio.run(_go()) + assert before == ["ci-job-123", "keep"] + assert after == ["keep"] # the vanished container was reaped + + @_NEEDS_DB def test_swarm_topology_persisted(app): from sqlalchemy import text