diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index c6d8007..b74212e 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -269,10 +269,16 @@ CPU_AVERAGE_MINUTES = 5 async def widget_resources(): """HTMX dashboard widget: the containers using the most CPU, averaged over the last few minutes so the order holds still between polls, with their - CPU in cores and memory in bytes.""" - limit = max(1, min(20, int(request.args.get("limit", 10) or 10))) + CPU in cores and memory in bytes, each beside bars of the last few hours + scaled to the row's own peak (never below a floor, #5214).""" + limit = max(1, min(20, _arg_number("limit", 10, int))) + hours = max(1, min(24, _arg_number("hours", 6, int))) + cpu_floor = max(0.1, _arg_number("cpu_floor", 1.0, float)) + mem_floor = max(0.1, _arg_number("mem_floor", 1.0, float)) widget_id = request.args.get("wid", "0") - since = datetime.now(timezone.utc) - timedelta(minutes=CPU_AVERAGE_MINUTES) + now = datetime.now(timezone.utc) + since = now - timedelta(minutes=CPU_AVERAGE_MINUTES) + slots, width = docker_svc.bar_slots(now, hours) async with current_app.db_sessionmaker() as db: # Dedup before the limit, else swarm tasks reported by both managers can @@ -284,28 +290,48 @@ async def widget_resources(): .where(DockerMetric.scraped_at >= since) .group_by(DockerMetric.host_id, DockerMetric.container_name) )).all()} - hosts = await docker_svc.host_map(db, {c.host_id for c in running}) - capacity = await docker_svc.host_capacity(db, {c.host_id for c in running}) - def cpu_of(c): - v = average.get((c.host_id, c.name)) - return c.cpu_pct if v is None else float(v) + def cpu_of(c): + v = average.get((c.host_id, c.name)) + return c.cpu_pct if v is None else float(v) + + busiest = sorted(running, key=lambda c: (-(cpu_of(c) or 0), c.name))[:limit] + history = await docker_svc.recent_usage( + db, {(c.host_id, c.name) for c in busiest}, slots[0], width) + hosts = await docker_svc.host_map(db, {c.host_id for c in busiest}) + capacity = await docker_svc.host_capacity(db, {c.host_id for c in busiest}) + + def bars(c, i, scale, floor): + seen = history.get((c.host_id, c.name), {}) + values = [None if seen.get(t) is None or seen[t][i] is None else seen[t][i] * scale + for t in slots] + return docker_svc.scaled_bars(values, floor) - busiest = sorted(running, key=lambda c: (-(cpu_of(c) or 0), c.name))[:limit] rows_data = [{ "c": c, "label": docker_svc.display_name(c), "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id, "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c)), + "cpu_bars": bars(c, 0, 0.01, cpu_floor), # cpu_pct → cores + "mem_bars": bars(c, 1, 1.0, mem_floor * 2**30), # bytes; floor in GiB } for c in busiest] return await render_template( "docker/widget_resources.html", rows=rows_data, minutes=CPU_AVERAGE_MINUTES, + hours=hours, cpu_floor=cpu_floor, mem_floor=mem_floor * 2**30, widget_id=widget_id, ) +def _arg_number(key, default, kind): + """A numeric query argument, or `default` when missing or not a number.""" + try: + return kind(request.args.get(key, default)) + except (TypeError, ValueError): + return default + + @docker_bp.get("/host/") @require_role(UserRole.viewer) async def host_panel(host_id: str): diff --git a/plugins/docker/service.py b/plugins/docker/service.py index d4d03c9..97d24a8 100644 --- a/plugins/docker/service.py +++ b/plugins/docker/service.py @@ -328,6 +328,56 @@ async def container_points(db, host_id: str, name: str, metric: str, since, return [(t, float(v)) for t, v in hourly] + [(t, float(v)) for t, v in raw] +# Busiest Containers bars (#5214): the recent past in BARS equal slices. +BARS = 24 + + +def bar_slots(now: datetime, hours: int, bars: int = BARS) -> tuple[list[datetime], int]: + """The start of each of the last `bars` buckets covering `hours`, oldest + first, aligned as date_bin aligns them (to the epoch), and their width in + seconds. The newest bucket is the one `now` falls in.""" + width = max(60, hours * 3600 // bars) + last = int(now.timestamp()) // width * width + return [datetime.fromtimestamp(last - (bars - 1 - i) * width, timezone.utc) + for i in range(bars)], width + + +async def recent_usage(db, keys, since: datetime, width: int) -> dict: + """{(host_id, container_name): {bucket start: (cpu_pct, mem_usage_bytes)}} + averaged per bucket since `since`, for the given containers, in one query.""" + keys = set(keys) + if not keys: + return {} + bucket = func.date_bin( + func.make_interval(0, 0, 0, 0, 0, 0, int(width)), DockerMetric.scraped_at, func.to_timestamp(0), + ).label("bucket") + rows = (await db.execute( + select(DockerMetric.host_id, DockerMetric.container_name, bucket, + func.avg(DockerMetric.cpu_pct), func.avg(DockerMetric.mem_usage_bytes)) + .where(DockerMetric.scraped_at >= since, + DockerMetric.host_id.in_({h for h, _ in keys}), + DockerMetric.container_name.in_({n for _, n in keys})) + .group_by(DockerMetric.host_id, DockerMetric.container_name, bucket) + )).all() + out: dict = {} + for host_id, name, at, cpu, mem in rows: + if (host_id, name) in keys: + out.setdefault((host_id, name), {})[at] = ( + None if cpu is None else float(cpu), None if mem is None else float(mem)) + return out + + +def scaled_bars(values: list, floor: float) -> dict: + """Bar heights in % of a roof: the row's own peak, but never below `floor`, + so a container idling at 0.05 cores draws flat and its 0.05 → 0.10 wobble + doesn't look like a 1.5 → 3 core spike. None (no samples) stays a gap.""" + present = [v for v in values if v is not None] + peak = max(present) if present else None + roof = max(peak or 0.0, floor) + return {"bars": [None if v is None else round(100.0 * v / roof, 1) for v in values], + "peak": peak, "roof": roof} + + async def search_logs(db, *, since, host_id: str | None = None, name: str | None = None, text: str = "", stream: str = "", before=None, limit: int = 50) -> list: """Retained log lines matching a filter, newest first (limit + 1 fetched by callers diff --git a/plugins/docker/templates/docker/widget_resources.html b/plugins/docker/templates/docker/widget_resources.html index 96f87e7..fa086c7 100644 --- a/plugins/docker/templates/docker/widget_resources.html +++ b/plugins/docker/templates/docker/widget_resources.html @@ -1,20 +1,29 @@ {# docker/widget_resources.html — the dashboard Busiest Containers widget - (#5253): a verdict naming the busiest, then a table of the top containers - by CPU (averaged over the last few minutes, so the order holds still). Each - figure carries its unit; the shaded bar behind it is its share of the - host's cores, or of the memory limit (the host's memory when none is set). #} + (#5253, #5214): a verdict naming the busiest, then the top containers by CPU + (averaged over the last few minutes, so the order holds still). Each figure + carries its unit, beside bars of the last few hours. A row's bars are scaled + to its own peak, but never below a floor (1 core, 1 GiB by default), so a + container idling at 0.05 cores reads flat and a spike means real load. #} +{% macro cores(v) -%}{{ "%.2f"|format(v) if v < 1 else "%.1f"|format(v) }}{%- endmacro %} +{% macro bars(b, peak_text, roof_text) -%} + +{%- endmacro %} {% if not rows %}
No running containers.
{% else %} {% set top = rows[0] %}
Busiest: {{ top.label }} on {{ top.host_name }}, - {% if top.usage.cores is not none %}{{ "%.2f"|format(top.usage.cores) if top.usage.cores < 1 else "%.1f"|format(top.usage.cores) }} + {% if top.usage.cores is not none %}{{ cores(top.usage.cores) }} {%- if top.usage.cpus %} of {{ top.usage.cpus }} cores{% else %} cores{% endif %}{% else %}no CPU reading yet{% endif %}.
- - + + + {% for r in rows %} {% set u = r.usage %} @@ -23,19 +32,19 @@ {%- if session.user_id %}{{ r.label }}{% else %}{{ r.label }}{% endif -%} - {% if u.cores is none %}{% else %} - - {% endif %} - {% if u.mem_bytes is none %}{% else %} - - {% endif %} + + {% endfor %}
ContainerHostCPUMemory
ContainerHostCPUMemory
{{ r.host_name }}— - {{- "%.2f"|format(u.cores) if u.cores < 1 else "%.1f"|format(u.cores) }} cores— - {{- u.mem_bytes|bytes }} + {{- bars(r.cpu_bars, (cores(r.cpu_bars.peak) ~ " cores") if r.cpu_bars.peak is not none else "", cores(r.cpu_bars.roof) ~ " cores") -}} + {%- if u.cores is none %}—{% else %}{{ cores(u.cores) }} cores{% endif -%} + + {{- bars(r.mem_bars, r.mem_bars.peak|bytes if r.mem_bars.peak is not none else "", r.mem_bars.roof|bytes) -}} + {%- if u.mem_bytes is none %}—{% else %}{{ u.mem_bytes|bytes }}{% endif -%} +
-
CPU over the last {{ minutes }} min; shading is the share of the host.
+
Bars: last {{ hours }}h, each row up to its own peak, at least {{ cores(cpu_floor) }} core{{ 's' if cpu_floor != 1 }} and {{ mem_floor|bytes }}.
{% endif %} diff --git a/steward/core/widgets.py b/steward/core/widgets.py index 57c2c60..8fa3fec 100644 --- a/steward/core/widgets.py +++ b/steward/core/widgets.py @@ -201,7 +201,7 @@ WIDGET_REGISTRY: dict[str, dict] = { "docker_resources": { "key": "docker_resources", "label": "Docker — Busiest containers", - "description": "The containers using the most CPU (cores, 5-minute average) and their memory", + "description": "The containers using the most CPU and memory, with bars of the last few hours", "hx_url": "/plugins/docker/widget/resources", "detail_url": "/plugins/docker/", "plugin": "docker", @@ -218,6 +218,41 @@ WIDGET_REGISTRY: dict[str, dict] = { {"value": 20, "label": "20 containers"}, ], }, + { + "key": "hours", + "label": "Bars cover", + "type": "select", + "default": 6, + "options": [ + {"value": 1, "label": "Last hour"}, + {"value": 6, "label": "Last 6 hours"}, + {"value": 24, "label": "Last 24 hours"}, + ], + }, + { + "key": "cpu_floor", + "label": "CPU bars scale to at least", + "type": "select", + "default": 1.0, # a float: the add form keeps fractions + "options": [ + {"value": 0.5, "label": "0.5 cores"}, + {"value": 1.0, "label": "1 core"}, + {"value": 2.0, "label": "2 cores"}, + {"value": 4.0, "label": "4 cores"}, + ], + }, + { + "key": "mem_floor", + "label": "Memory bars scale to at least", + "type": "select", + "default": 1.0, # GiB + "options": [ + {"value": 0.25, "label": "256 MiB"}, + {"value": 1.0, "label": "1 GiB"}, + {"value": 4.0, "label": "4 GiB"}, + {"value": 16.0, "label": "16 GiB"}, + ], + }, ], }, "snmp_devices": { diff --git a/steward/static/steward.css b/steward/static/steward.css index 24bce80..9507383 100644 --- a/steward/static/steward.css +++ b/steward/static/steward.css @@ -409,7 +409,15 @@ body.dash-editing .widget-drawer { transform:translateY(0); } tone: the bar never stands without the figure it measures. */ .w-meter { background:linear-gradient(90deg, color-mix(in srgb, currentColor 20%, transparent) var(--pct, 0%), transparent var(--pct, 0%)); border-radius:3px; } .w-note { font-size:0.7rem; color:var(--text-dim); margin-top:0.3rem; } -@container (max-width: 26rem) { .w-wide-only { display:none; } } +.w-table col.w-barcol { width:9.5rem; } +/* Bars of a figure's recent past, inline before it (Busiest Containers, #5214): + the row's tone, faint, with the newest bar solid. A gap is a slice with no + samples. Heights are set per bar as a % of the row's roof. */ +.w-bars { display:inline-flex; align-items:flex-end; gap:1px; height:0.95rem; width:4.3rem; margin-right:0.45rem; vertical-align:-0.15rem; } +.w-bars i { flex:1; min-height:1px; background:currentColor; opacity:0.4; border-radius:1px 1px 0 0; } +.w-bars i:last-child { opacity:1; } +.w-bars i.gap { background:none; } +@container (max-width: 26rem) { .w-wide-only { display:none; } .w-table col.w-barcol { width:5.5rem; } } /* Time range selector */ diff --git a/tests/integration/test_docker_recent_usage.py b/tests/integration/test_docker_recent_usage.py new file mode 100644 index 0000000..d33e35c --- /dev/null +++ b/tests/integration/test_docker_recent_usage.py @@ -0,0 +1,49 @@ +"""Integration: Busiest Containers bars average each container's samples per +time bucket in one query (#5214). Two samples share a bucket and must merge +(lesson #5262). Requires STEWARD_DATABASE_URL.""" +from __future__ import annotations + +import asyncio +import os +import uuid +from datetime import datetime, timedelta, timezone + +import pytest + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +def test_recent_usage_averages_each_containers_bucket(app): + from plugins.docker.models import DockerMetric + from plugins.docker.service import bar_slots, recent_usage + from steward.models.hosts import Host + + tag = uuid.uuid4().hex[:8] + hid = str(uuid.uuid4()) + slots, width = bar_slots(datetime.now(timezone.utc), 6) + at = slots[-2] # a whole bucket, in the past + + def m(name, secs, cpu, mem): + return DockerMetric(host_id=hid, container_name=name, scraped_at=at + timedelta(seconds=secs), + cpu_pct=cpu, mem_pct=0.0, mem_usage_bytes=mem) + + async def go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add(Host(id=hid, name=f"dock-{tag}", address="10.9.9.4")) + async with s.begin(): + s.add_all([m(f"a-{tag}", 10, 100.0, 1000), m(f"a-{tag}", 70, 300.0, 3000), + m(f"b-{tag}", 10, 50.0, 500)]) + async with s.begin(): + return await recent_usage(s, {(hid, f"a-{tag}")}, slots[0], width) + + got = asyncio.run(go()) + assert got == {(hid, f"a-{tag}"): {at: (200.0, 2000.0)}} diff --git a/tests/plugins/docker/test_docker_page.py b/tests/plugins/docker/test_docker_page.py index bdaca9c..22ccef5 100644 --- a/tests/plugins/docker/test_docker_page.py +++ b/tests/plugins/docker/test_docker_page.py @@ -218,23 +218,52 @@ def test_a_container_reads_by_its_service_or_ci_job(): assert svc.display_name(_c("portainer_agent")) == "portainer_agent" -async def test_busiest_containers_show_the_number_with_its_scale(app): - """#5253: every bar has its figure; the widget names the busiest.""" +def _bars(values, floor): + return svc.scaled_bars(values, floor) + + +async def test_busiest_containers_show_the_number_beside_its_recent_bars(app): + """#5253, #5214: every figure has its unit; the bars beside it are the last + hours, scaled to the row's own peak; the widget names the busiest.""" rows = [ {"c": _c("GITEA-ACTIONS-TASK-1-WORKFLOW-Android-JOB-build-0123456789abcdef0"), "label": "CI: Android · build", "host_name": "VDNT-Docker03", "usage": {"cores": 1.62, "cpus": 8, "cpu_share": 20.25, "mem_bytes": 820 * 2**20, - "mem_pct": 5.0, "mem_of": "the host's memory", "mem_limit": None}}, + "mem_pct": 5.0, "mem_of": "the host's memory", "mem_limit": None}, + "cpu_bars": _bars([None, 0.8, 3.2, 1.6], 1.0), "mem_bars": _bars([None] * 4, 2**30)}, {"c": _c("tdarr"), "label": "tdarr", "host_name": "VDNT-Docker03", "usage": {"cores": 0.25, "cpus": 8, "cpu_share": 3.1, "mem_bytes": None, - "mem_pct": None, "mem_of": None, "mem_limit": None}}, + "mem_pct": None, "mem_of": None, "mem_limit": None}, + "cpu_bars": _bars([0.25] * 4, 1.0), "mem_bars": _bars([None] * 4, 2**30)}, ] async with app.test_request_context("/plugins/docker/widget/resources"): page = await render_plugin_template(app, "docker", "docker/widget_resources.html", - rows=rows, minutes=5, widget_id="1") + rows=rows, minutes=5, hours=6, cpu_floor=1.0, + mem_floor=2**30, widget_id="1") assert "Busiest: CI: Android · build on VDNT-Docker03" in page and "1.6 of 8 cores" in page - assert ">1.6 cores<" in page and ">0.25 cores<" in page and ">820.0 MiB<" in page - assert "--pct:20.25%" in page and ">—<" in page + assert ">1.6 cores<" in page and ">0.25 cores<" in page and ">820.0 MiB<" in page and ">—<" in page + # Android peaks at 3.2 cores, so 1.6 is half height; tdarr's 0.25 sits under the 1-core floor. + assert 'class="gap"' in page and "height:50.0%" in page and "height:25.0%" in page + assert "Peak 3.2 cores in the last 6h; bars reach the top at 3.2 cores" in page + assert "Peak 0.25 cores in the last 6h; bars reach the top at 1.0 cores" in page + assert "No samples in the last 6h" in page assert len(re.findall(r"]", page)) == 4 assert ">CI: Android · build<" in page # the long CI name only as the hover title assert not hex_colours(page), hex_colours(page) + + +def test_bars_reach_the_peak_but_never_below_the_floor(): + """#5214: a 0.05 → 0.10 wobble stays near the floor; a 1.5 → 3 spike fills.""" + idle = svc.scaled_bars([0.05, 0.10, None], 1.0) + assert idle == {"bars": [5.0, 10.0, None], "peak": 0.10, "roof": 1.0} + busy = svc.scaled_bars([1.5, 3.0], 1.0) + assert busy["bars"] == [50.0, 100.0] and busy["roof"] == 3.0 + assert svc.scaled_bars([None, None], 1.0) == {"bars": [None, None], "peak": None, "roof": 1.0} + + +def test_bar_slots_end_at_the_bucket_now_falls_in(): + now = datetime(2026, 10, 7, 12, 7, 30, tzinfo=timezone.utc) + slots, width = svc.bar_slots(now, 6) + assert width == 900 and len(slots) == 24 + assert slots[-1] == datetime(2026, 10, 7, 12, 0, tzinfo=timezone.utc) + assert slots[0] == datetime(2026, 10, 7, 6, 15, tzinfo=timezone.utc)