From 213a28c23fd36855a9aa3f19652bdd2cfe428e40 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 7 Oct 2026 13:17:09 -0400 Subject: [PATCH] feat(docker): Services & Failures widget lists what is wrong, then packs the healthy (#5252) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A verdict line (services healthy · other containers running · failed in 24h), then "Needs a look": troubled services with why (crash-looping, N of M running), since when and the last error, and troubled containers (unhealthy, restarting, exited with an error, out of memory). Then "Failed in the last 24h" by name, with how often, when and where. Then the healthy rest one line each (name · hosts · replicas) in as many columns as the widget is wide, collapsing after 40 to "+N more healthy". Per-container CPU is gone from this widget: Busiest Containers owns it. Co-Authored-By: Claude Opus 5.5 --- plugins/docker/routes.py | 46 +++---- plugins/docker/templates/docker/widget.html | 142 ++++++++++---------- steward/static/steward.css | 8 ++ tests/plugins/docker/test_docker_page.py | 46 +++++++ 4 files changed, 147 insertions(+), 95 deletions(-) diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index c426459..d8c8bae 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -214,7 +214,6 @@ async def widget(): {c.host_id for c in all_containers} | {n.host_id for n in nodes} | {s.host_id for s in services}, ) - capacity = await docker_svc.host_capacity(db, {c.host_id for c in all_containers}) # Swarm tasks collapse into per-service rows (each replica tagged with its # host); non-swarm containers stay grouped by host. @@ -227,42 +226,41 @@ async def widget(): standalone = [c for c in all_containers if not c.service_name] running = [c for c in standalone if c.status == "running"] - display = standalone if show_stopped else running - def trouble(c) -> int: - if (c.status or "") == "restarting" or c.health == "unhealthy": - return 0 - if c.status != "running" and docker_svc.exit_is_failure(c.exit_code, c.oom_killed): - return 1 - return 2 if c.status == "running" else 3 + def troubled(c) -> bool: + return ((c.status or "") == "restarting" or c.health == "unhealthy" + or (c.status != "running" and docker_svc.exit_is_failure(c.exit_code, c.oom_killed))) - # Group for display so multi-host fleets read clearly; single-host stays flat. - groups: dict[str, dict] = {} - for c in sorted(display, key=lambda c: (trouble(c), c.name)): - g = groups.setdefault(c.host_id, { - "host_id": c.host_id, - "host_name": host_name.get(c.host_id, c.host_id), - "containers": [], - }) - g["containers"].append({"c": c, "usage": docker_svc.usage( - c, *capacity.get(c.host_id, (None, None))), - "exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed)}) - host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower()) + # The widget-shape order (UI & Design ruling): what is wrong first, then + # the healthy rest one line each, collapsing past HEALTHY_SHOWN. + bad_containers = sorted((c for c in standalone if troubled(c)), key=lambda c: c.name) + healthy = ([{"kind": "service", "s": s} for s in swarm_services if s["state"] == "healthy"] + + [{"kind": "container", "c": c} for c in sorted(running, key=lambda c: c.name) + if not troubled(c)] + + ([{"kind": "stopped", "c": c} for c in sorted(standalone, key=lambda c: c.name) + if c.status != "running" and not troubled(c)] if show_stopped else [])) return await render_template( "docker/widget.html", - host_groups=host_groups, - multi_host=len(host_groups) > 1, - swarm_services=swarm_services, + bad_services=[s for s in swarm_services if s["state"] != "healthy"], + bad_containers=bad_containers, + healthy=healthy[:HEALTHY_SHOWN], healthy_more=max(0, len(healthy) - HEALTHY_SHOWN), + services_total=len(swarm_services), services_healthy=sum(s["state"] == "healthy" for s in swarm_services), running_count=len(running), failures=failures, + host_name=host_name, total_count=len(all_containers) + len(swarm_services), - show_stopped=show_stopped, widget_id=widget_id, ) +# Healthy services and containers listed on the dashboard widget before the +# rest collapse to "+N healthy"; the Docker page lists them all. +HEALTHY_SHOWN = 40 +FAILURES_SHOWN = 5 + + CPU_AVERAGE_MINUTES = 5 diff --git a/plugins/docker/templates/docker/widget.html b/plugins/docker/templates/docker/widget.html index 9d0ce9a..395ca89 100644 --- a/plugins/docker/templates/docker/widget.html +++ b/plugins/docker/templates/docker/widget.html @@ -1,80 +1,80 @@ -{# docker/widget.html — dashboard widget: what Docker is running and what has - failed. Swarm services first, troubled ones at the top with since-times; then - the containers outside any service, by host, problems first. The failed count - links to the Docker page's list of exactly what it counts. #} -{% from "docker/_usage.html" import cpu %} +{# docker/widget.html — the dashboard Docker widget (#5252), in the widget-shape + order: a verdict line; the services and containers in trouble; what failed + in the last 24h, by name; then everything healthy, one line each, packed + into as many columns as the widget is wide. Resource use is the Busiest + Containers widget's, so it isn't repeated here. #} +{% from "_macros.html" import status_dot %} {% if total_count == 0 %} -

No containers reported yet.

+
No containers reported yet.
{% else %} - -
- {% if swarm_services %} -
- {{ services_healthy }}/{{ swarm_services|length }} - services healthy -
- {% endif %} -
- {{ running_count }} - containers running -
- - {{ failures|length }} - failed in 24h - +{% set wrong = bad_services|length + bad_containers|length %} +
+ {%- if services_total %}{{ services_healthy }}/{{ services_total }} services healthy · {% endif -%} + {{ running_count }} other container{{ 's' if running_count != 1 }} running · + {% if failures %}{{ failures|length }} failed in 24h{% else %}nothing failed in 24h{% endif -%}
-{# Swarm services — troubled first, each with its replicas' host chips - (dashed = a node with no Steward agent, count-only from placement). #} -{% if swarm_services %} -
Swarm services
-
- {% for s in swarm_services %} -
-
- - {{ s.name }} - {% if s.crash_looping %}crash-looping{% endif %} - {% if s.state != 'healthy' and s.trouble_since %}since {{ s.trouble_since|ago }}{% endif %} - {{ s.running }}/{{ s.desired }} -
-
- {% for r in s.replicas %} - {% if r.ghost %} - {{ r.host }} ×{{ r.count }} - {% else %} - {{ r.host }} - {% endif %} - {% endfor %} -
-
- {% endfor %} +{% if wrong %} +
Needs a look
+{% for s in bad_services %} +
+ {{ status_dot('degraded' if s.state == 'degraded' else 'down') }} + {{ s.name }} + + {%- if s.crash_looping %}crash-looping: {{ s.failures_1h }} failed in the last hour{% else %}{{ s.running }} of {{ s.desired }} running{% endif %} + {%- if s.trouble_since %}, since {{ s.trouble_since|ago }}{% endif %} + {%- if s.last_error and (s.last_error.error or s.last_error.message) %} · {{ s.last_error.error or s.last_error.message }}{% endif -%} + + + {{ s.running }}/{{ s.desired }}
-{% endif %} - -{% if host_groups and swarm_services %} -
Other containers
-{% endif %} -{% for g in host_groups %} -{% if multi_host %} -
{{ g.host_name }}
-{% endif %} -
- {% for item in g.containers %} - {% set c = item.c %} -
- - - {{ c.name }}{% if c.health == 'unhealthy' %} unhealthy{% endif %}{% if c.status == 'restarting' %} restarting{% endif %} - - {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %} - exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %} - {% endif %} - {% if c.status == 'running' %}{{ cpu(item.usage) }}{% endif %} -
- {% endfor %} +{% endfor %} +{% for c in bad_containers %} +
+ {{ status_dot('warn' if c.status == 'restarting' else 'down') }} + + {%- if session.user_id %}{{ c.name }}{% else %}{{ c.name }}{% endif %} + + {%- if c.health == 'unhealthy' %}unhealthy{% elif c.status == 'restarting' %}restarting{% else %}exited {{ c.exit_code }}{% if c.oom_killed %}, out of memory{% endif %}{% endif %} · {{ host_name.get(c.host_id, c.host_id) }} + +
{% endfor %} +{% endif %} +{% if failures %} +
Failed in the last 24h
+{% for f in failures[:5] %} +
+ {{ status_dot('failed') }} + {{ f.name }} + {% if f.count > 1 %}{{ f.count }} times, last {% endif %}{{ f.at|ago }}{% if f.detail %} · {{ f.detail }}{% endif %} · {{ f.host }} + +
+{% endfor %} +{% if failures|length > 5 %}
+{{ failures|length - 5 }} more · the full list
{% endif %} +{% endif %} + +{% if healthy %} +
Healthy
+
+ {% for h in healthy %} +
+ {% if h.kind == 'service' %}{% set s = h.s %} + {{ status_dot('up') }} + {{ s.name }} + · {{ s.replicas|map(attribute='host')|unique|join(', ') }} + {{ s.running }}/{{ s.desired }} + {% else %}{% set c = h.c %} + {{ status_dot('up' if h.kind == 'container' else 'unknown') }} + + {%- if session.user_id %}{{ c.name }}{% else %}{{ c.name }}{% endif %} + · {{ host_name.get(c.host_id, c.host_id) }} + {% if h.kind == 'stopped' %}stopped{% endif %} + {% endif %} +
+ {% endfor %} +
+{% if healthy_more %}
+{{ healthy_more }} more healthy · all of Docker
{% endif %} +{% endif %} {% endif %} diff --git a/steward/static/steward.css b/steward/static/steward.css index b4290c9..a4461d3 100644 --- a/steward/static/steward.css +++ b/steward/static/steward.css @@ -396,6 +396,14 @@ body.dash-editing .widget-drawer { transform:translateY(0); } .w-table .dot { margin-right:0.4rem; vertical-align:middle; } .w-sub { display:block; font-size:0.72rem; color:var(--text-muted); white-space:nowrap; overflow:hidden; text-overflow:ellipsis; } .w-more { font-size:0.75rem; color:var(--text-dim); padding-top:0.3rem; } +.w-section { font-size:var(--fs-size-tiny); color:var(--text-dim); text-transform:uppercase; letter-spacing:var(--fs-tracking-tiny); margin:0.5rem 0 0.2rem; } +/* The healthy rest: one line per item, as many columns as the widget's width + allows (it is a size container, so this follows the widget, not the window). */ +.w-grid { display:grid; grid-template-columns:repeat(auto-fill, minmax(15rem, 1fr)); gap:0 1.25rem; } +.w-item { display:flex; align-items:center; gap:0.4rem; min-width:0; font-size:0.82rem; padding:0.12rem 0; } +.w-item .w-label { flex:1; min-width:0; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; } +.w-item .w-label a { color:inherit; text-decoration:none; } +.w-item .w-meta { flex-shrink:0; font-size:0.72rem; color:var(--text-muted); font-variant-numeric:tabular-nums; } /* Time range selector */ diff --git a/tests/plugins/docker/test_docker_page.py b/tests/plugins/docker/test_docker_page.py index b016988..19614c0 100644 --- a/tests/plugins/docker/test_docker_page.py +++ b/tests/plugins/docker/test_docker_page.py @@ -162,3 +162,49 @@ async def test_the_disk_page_explains_its_terms_and_names_untagged_images(app): assert "held by images no container uses" in page and "safe to clear" in page assert "Shared with other images" in page and "2 containers" in page and ">nothing<" in page assert "dangling" not in page + + +def _widget_svc(name, state="healthy", running=1, desired=1, hosts=("VDNT-Docker02",), **kw): + base = dict(name=name, state=state, running=running, desired=desired, crash_looping=False, + failures_1h=0, last_error=None, trouble_since=None, + replicas=[{"host": h, "ghost": False} for h in hosts]) + base.update(kw) + return base + + +async def _services_widget(app, **ctx): + base = dict(bad_services=[], bad_containers=[], healthy=[], healthy_more=0, services_total=0, + services_healthy=0, running_count=0, failures=[], host_name={"h1": "VDNT-Docker02"}, + total_count=1, widget_id="1") + base.update(ctx) + async with app.test_request_context("/plugins/docker/widget"): + return await render_plugin_template(app, "docker", "docker/widget.html", **base) + + +async def test_the_services_widget_lists_what_is_wrong_then_packs_the_healthy(app): + """#5252: a verdict, the troubled service with why and since when, the + failures by name, then healthy services one line each in a packed grid.""" + bad = _widget_svc("codex_db", state="down", running=0, crash_looping=True, failures_1h=7, + trouble_since=datetime.now(timezone.utc) - timedelta(minutes=9)) + page = await _services_widget( + app, bad_services=[bad], services_total=3, services_healthy=2, running_count=4, + failures=[{"name": "codex_db", "host": "VDNT-Docker03", "count": 7, "at": NOW - timedelta(minutes=1), + "detail": "exit 1"}], + healthy=[{"kind": "service", "s": _widget_svc("curator_app")}, + {"kind": "service", "s": _widget_svc("web", running=2, desired=2, hosts=("a", "b", "a"))}, + {"kind": "container", "c": _c("portainer_agent")}], + healthy_more=12) + assert "2/3 services healthy" in page and "4 other containers running" in page and "1 failed in 24h" in page + assert page.index("Needs a look") < page.index("Failed in the last 24h") < page.index("Healthy") + assert "crash-looping: 7 failed in the last hour" in page and "9m ago" in page + assert "7 times, last" in page and "exit 1" in page + assert 'class="w-grid"' in page and " · a, b" in page and ">2/2<" in page + assert "+12 more healthy" in page + assert not hex_colours(page), hex_colours(page) + + +async def test_the_services_widget_says_when_all_is_well(app): + page = await _services_widget(app, services_total=2, services_healthy=2, running_count=1, + healthy=[{"kind": "service", "s": _widget_svc("a")}]) + assert "2/2 services healthy" in page and "nothing failed in 24h" in page and "tone-ok" in page + assert "Needs a look" not in page and "Failed in the last 24h" not in page