From e289b6c49ffd3b95aeaef0bd22ff356dad36a034 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 20 Jun 2026 23:02:19 -0400 Subject: [PATCH] fix(docker): dedup widget counts across swarm managers; show failures not stopped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The running count was inflated: swarm-aware agents on every manager list cluster-wide tasks, so the same container (identical container_id) was reported once per manager and the dashboard widgets summed them. The earlier swarm dedup only covered the Swarm topology page — the widgets read raw rows. - _dedup_by_container_id(): collapse rows sharing a non-empty container_id (globally unique, so only true duplicates merge); first occurrence wins under the running-first ordering. Applied in both the containers and resources widgets before counting/limiting. Rows without a container_id (older agents) are kept as-is. - Replaced the static "stopped" tally — not actionable — with "failed (24h)": distinct containers with a die/oom event in the last 24h (DISTINCT so the managers' duplicate events don't double it), rendered red when non-zero. - widget.html empty-state now keys off total_count. - Regression test: same container_id from two managers counts once; id-less rows all kept. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- plugins/docker/routes.py | 45 +++++++++++++++++---- plugins/docker/templates/docker/widget.html | 12 +++--- tests/integration/test_docker.py | 20 +++++++++ 3 files changed, 62 insertions(+), 15 deletions(-) diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index 1cfc455..6b1b9ee 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -1,10 +1,10 @@ # plugins/docker/routes.py from __future__ import annotations import json -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from quart import Blueprint, current_app, render_template, request -from sqlalchemy import select +from sqlalchemy import func, select from steward.auth.middleware import require_role from steward.models.users import UserRole @@ -205,24 +205,50 @@ async def rows(): ) +def _dedup_by_container_id(containers: list) -> list: + """Collapse containers double-reported across hosts. Swarm-aware agents on + every manager list cluster-wide tasks, so the same container (identical + container_id) arrives once per manager and would otherwise be counted N times. + Keep the first occurrence per non-empty container_id — callers order + running-first, so a running report wins over a stale stopped one. Rows without + a container_id (older agents) can't be matched, so they're kept as-is.""" + seen: set[str] = set() + out = [] + for c in containers: + cid = (c.container_id or "").strip() + if cid: + if cid in seen: + continue + seen.add(cid) + out.append(c) + return out + + @docker_bp.get("/widget") @require_role(UserRole.viewer) async def widget(): """HTMX dashboard widget: container status overview, grouped by host.""" show_stopped = request.args.get("show_stopped", "no") == "yes" widget_id = request.args.get("wid", "0") + since_24h = datetime.now(timezone.utc) - timedelta(hours=24) async with current_app.db_sessionmaker() as db: - all_containers = list((await db.execute( + all_containers = _dedup_by_container_id(list((await db.execute( select(DockerContainer).order_by( (DockerContainer.status == "running").desc(), DockerContainer.name, ) - )).scalars()) + )).scalars())) + # Recent failures are more actionable than a static "stopped" tally. Count + # DISTINCT container names so the two managers' duplicate die/oom events + # don't double it. + failed_24h = (await db.execute( + select(func.count(func.distinct(DockerEvent.container_name))) + .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since_24h) + )).scalar() or 0 hosts = await _host_map(db, {c.host_id for c in all_containers}) running = [c for c in all_containers if c.status == "running"] - stopped = [c for c in all_containers if c.status != "running"] display = all_containers if show_stopped else running # Group for display so multi-host fleets read clearly; single-host stays flat. @@ -241,7 +267,8 @@ async def widget(): host_groups=host_groups, multi_host=len(host_groups) > 1, running_count=len(running), - stopped_count=len(stopped), + failed_24h=failed_24h, + total_count=len(all_containers), show_stopped=show_stopped, widget_id=widget_id, ) @@ -255,11 +282,13 @@ async def widget_resources(): widget_id = request.args.get("wid", "0") async with current_app.db_sessionmaker() as db: - containers = list((await db.execute( + # Dedup before the limit, else swarm tasks reported by both managers can + # fill the list with duplicates of the same container. + containers = _dedup_by_container_id(list((await db.execute( select(DockerContainer) .where(DockerContainer.status == "running") .order_by(DockerContainer.cpu_pct.desc().nullslast()) - )).scalars())[:limit] + )).scalars()))[:limit] hosts = await _host_map(db, {c.host_id for c in containers}) rows_data = [ diff --git a/plugins/docker/templates/docker/widget.html b/plugins/docker/templates/docker/widget.html index cd66b22..01c4e80 100644 --- a/plugins/docker/templates/docker/widget.html +++ b/plugins/docker/templates/docker/widget.html @@ -1,19 +1,17 @@ {# docker/widget.html — dashboard widget: container status overview, by host #} -{% if running_count == 0 and stopped_count == 0 %} +{% if total_count == 0 %}

No containers reported yet.

{% else %} -
+
{{ running_count }} running
- {% if stopped_count %} -
- {{ stopped_count }} - stopped +
+ {{ failed_24h }} + failed (24h)
- {% endif %}
{% for g in host_groups %} diff --git a/tests/integration/test_docker.py b/tests/integration/test_docker.py index 408ae6e..b54da02 100644 --- a/tests/integration/test_docker.py +++ b/tests/integration/test_docker.py @@ -449,3 +449,23 @@ def test_retention_rollup_and_prune(app): assert events_left == 1 # only the in-window event survives assert counts["buckets_rolled"] == 1 and counts["raw_rows_rolled"] == 3 assert counts["events_pruned"] == 1 and counts["rollup_pruned"] == 1 + + +def test_widget_dedup_collapses_cross_manager_duplicates(): + """The same swarm task is reported by every manager (identical container_id); + the dashboard widget must count it once. Older agents send no container_id, + so those rows can't be matched and are all kept.""" + from types import SimpleNamespace + from plugins.docker.routes import _dedup_by_container_id + + def c(cid, name, status="running"): + return SimpleNamespace(container_id=cid, name=name, status=status) + + rows = [c("abc", "web@m1"), c("abc", "web@m2"), # same task, two managers + c("def", "db"), c("", "noid-1"), c("", "noid-2")] + out = _dedup_by_container_id(rows) + + assert len(out) == 4 # one "abc" dropped + abc = [r for r in out if r.container_id == "abc"] + assert len(abc) == 1 and abc[0].name == "web@m1" # first occurrence wins + assert sum(1 for r in out if r.container_id == "") == 2 # id-less rows all kept