feat(docker): Busiest Containers shows bars of the last hours beside each figure, scaled to the row's peak with a floor (#5214)
The fill bars were a share of the host's capacity, a sliver on hosts with 80 cores. Each figure now sits beside 24 bars of the last 6 hours (1h/24h per widget), each row scaled to its own peak but never below 1 core or 1 GiB (configurable per widget), so a 0.05-core container reads flat and a bar's height means real load. One grouped date_bin query feeds them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -269,10 +269,16 @@ CPU_AVERAGE_MINUTES = 5
|
||||
async def widget_resources():
|
||||
"""HTMX dashboard widget: the containers using the most CPU, averaged over
|
||||
the last few minutes so the order holds still between polls, with their
|
||||
CPU in cores and memory in bytes."""
|
||||
limit = max(1, min(20, int(request.args.get("limit", 10) or 10)))
|
||||
CPU in cores and memory in bytes, each beside bars of the last few hours
|
||||
scaled to the row's own peak (never below a floor, #5214)."""
|
||||
limit = max(1, min(20, _arg_number("limit", 10, int)))
|
||||
hours = max(1, min(24, _arg_number("hours", 6, int)))
|
||||
cpu_floor = max(0.1, _arg_number("cpu_floor", 1.0, float))
|
||||
mem_floor = max(0.1, _arg_number("mem_floor", 1.0, float))
|
||||
widget_id = request.args.get("wid", "0")
|
||||
since = datetime.now(timezone.utc) - timedelta(minutes=CPU_AVERAGE_MINUTES)
|
||||
now = datetime.now(timezone.utc)
|
||||
since = now - timedelta(minutes=CPU_AVERAGE_MINUTES)
|
||||
slots, width = docker_svc.bar_slots(now, hours)
|
||||
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
# Dedup before the limit, else swarm tasks reported by both managers can
|
||||
@@ -284,28 +290,48 @@ async def widget_resources():
|
||||
.where(DockerMetric.scraped_at >= since)
|
||||
.group_by(DockerMetric.host_id, DockerMetric.container_name)
|
||||
)).all()}
|
||||
hosts = await docker_svc.host_map(db, {c.host_id for c in running})
|
||||
capacity = await docker_svc.host_capacity(db, {c.host_id for c in running})
|
||||
|
||||
def cpu_of(c):
|
||||
v = average.get((c.host_id, c.name))
|
||||
return c.cpu_pct if v is None else float(v)
|
||||
def cpu_of(c):
|
||||
v = average.get((c.host_id, c.name))
|
||||
return c.cpu_pct if v is None else float(v)
|
||||
|
||||
busiest = sorted(running, key=lambda c: (-(cpu_of(c) or 0), c.name))[:limit]
|
||||
history = await docker_svc.recent_usage(
|
||||
db, {(c.host_id, c.name) for c in busiest}, slots[0], width)
|
||||
hosts = await docker_svc.host_map(db, {c.host_id for c in busiest})
|
||||
capacity = await docker_svc.host_capacity(db, {c.host_id for c in busiest})
|
||||
|
||||
def bars(c, i, scale, floor):
|
||||
seen = history.get((c.host_id, c.name), {})
|
||||
values = [None if seen.get(t) is None or seen[t][i] is None else seen[t][i] * scale
|
||||
for t in slots]
|
||||
return docker_svc.scaled_bars(values, floor)
|
||||
|
||||
busiest = sorted(running, key=lambda c: (-(cpu_of(c) or 0), c.name))[:limit]
|
||||
rows_data = [{
|
||||
"c": c,
|
||||
"label": docker_svc.display_name(c),
|
||||
"host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id,
|
||||
"usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c)),
|
||||
"cpu_bars": bars(c, 0, 0.01, cpu_floor), # cpu_pct → cores
|
||||
"mem_bars": bars(c, 1, 1.0, mem_floor * 2**30), # bytes; floor in GiB
|
||||
} for c in busiest]
|
||||
return await render_template(
|
||||
"docker/widget_resources.html",
|
||||
rows=rows_data,
|
||||
minutes=CPU_AVERAGE_MINUTES,
|
||||
hours=hours, cpu_floor=cpu_floor, mem_floor=mem_floor * 2**30,
|
||||
widget_id=widget_id,
|
||||
)
|
||||
|
||||
|
||||
def _arg_number(key, default, kind):
|
||||
"""A numeric query argument, or `default` when missing or not a number."""
|
||||
try:
|
||||
return kind(request.args.get(key, default))
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
@docker_bp.get("/host/<host_id>")
|
||||
@require_role(UserRole.viewer)
|
||||
async def host_panel(host_id: str):
|
||||
|
||||
@@ -328,6 +328,56 @@ async def container_points(db, host_id: str, name: str, metric: str, since,
|
||||
return [(t, float(v)) for t, v in hourly] + [(t, float(v)) for t, v in raw]
|
||||
|
||||
|
||||
# Busiest Containers bars (#5214): the recent past in BARS equal slices.
|
||||
BARS = 24
|
||||
|
||||
|
||||
def bar_slots(now: datetime, hours: int, bars: int = BARS) -> tuple[list[datetime], int]:
|
||||
"""The start of each of the last `bars` buckets covering `hours`, oldest
|
||||
first, aligned as date_bin aligns them (to the epoch), and their width in
|
||||
seconds. The newest bucket is the one `now` falls in."""
|
||||
width = max(60, hours * 3600 // bars)
|
||||
last = int(now.timestamp()) // width * width
|
||||
return [datetime.fromtimestamp(last - (bars - 1 - i) * width, timezone.utc)
|
||||
for i in range(bars)], width
|
||||
|
||||
|
||||
async def recent_usage(db, keys, since: datetime, width: int) -> dict:
|
||||
"""{(host_id, container_name): {bucket start: (cpu_pct, mem_usage_bytes)}}
|
||||
averaged per bucket since `since`, for the given containers, in one query."""
|
||||
keys = set(keys)
|
||||
if not keys:
|
||||
return {}
|
||||
bucket = func.date_bin(
|
||||
func.make_interval(0, 0, 0, 0, 0, 0, int(width)), DockerMetric.scraped_at, func.to_timestamp(0),
|
||||
).label("bucket")
|
||||
rows = (await db.execute(
|
||||
select(DockerMetric.host_id, DockerMetric.container_name, bucket,
|
||||
func.avg(DockerMetric.cpu_pct), func.avg(DockerMetric.mem_usage_bytes))
|
||||
.where(DockerMetric.scraped_at >= since,
|
||||
DockerMetric.host_id.in_({h for h, _ in keys}),
|
||||
DockerMetric.container_name.in_({n for _, n in keys}))
|
||||
.group_by(DockerMetric.host_id, DockerMetric.container_name, bucket)
|
||||
)).all()
|
||||
out: dict = {}
|
||||
for host_id, name, at, cpu, mem in rows:
|
||||
if (host_id, name) in keys:
|
||||
out.setdefault((host_id, name), {})[at] = (
|
||||
None if cpu is None else float(cpu), None if mem is None else float(mem))
|
||||
return out
|
||||
|
||||
|
||||
def scaled_bars(values: list, floor: float) -> dict:
|
||||
"""Bar heights in % of a roof: the row's own peak, but never below `floor`,
|
||||
so a container idling at 0.05 cores draws flat and its 0.05 → 0.10 wobble
|
||||
doesn't look like a 1.5 → 3 core spike. None (no samples) stays a gap."""
|
||||
present = [v for v in values if v is not None]
|
||||
peak = max(present) if present else None
|
||||
roof = max(peak or 0.0, floor)
|
||||
return {"bars": [None if v is None else round(100.0 * v / roof, 1) for v in values],
|
||||
"peak": peak, "roof": roof}
|
||||
|
||||
|
||||
async def search_logs(db, *, since, host_id: str | None = None, name: str | None = None,
|
||||
text: str = "", stream: str = "", before=None, limit: int = 50) -> list:
|
||||
"""Retained log lines matching a filter, newest first (limit + 1 fetched by callers
|
||||
|
||||
@@ -1,20 +1,29 @@
|
||||
{# docker/widget_resources.html — the dashboard Busiest Containers widget
|
||||
(#5253): a verdict naming the busiest, then a table of the top containers
|
||||
by CPU (averaged over the last few minutes, so the order holds still). Each
|
||||
figure carries its unit; the shaded bar behind it is its share of the
|
||||
host's cores, or of the memory limit (the host's memory when none is set). #}
|
||||
(#5253, #5214): a verdict naming the busiest, then the top containers by CPU
|
||||
(averaged over the last few minutes, so the order holds still). Each figure
|
||||
carries its unit, beside bars of the last few hours. A row's bars are scaled
|
||||
to its own peak, but never below a floor (1 core, 1 GiB by default), so a
|
||||
container idling at 0.05 cores reads flat and a spike means real load. #}
|
||||
{% macro cores(v) -%}{{ "%.2f"|format(v) if v < 1 else "%.1f"|format(v) }}{%- endmacro %}
|
||||
{% macro bars(b, peak_text, roof_text) -%}
|
||||
<span class="w-bars w-wide-only" aria-hidden="true"
|
||||
title="{% if b.peak is none %}No samples in the last {{ hours }}h{% else %}Peak {{ peak_text }} in the last {{ hours }}h; bars reach the top at {{ roof_text }}{% endif %}">
|
||||
{%- for h in b.bars %}<i{% if h is none %} class="gap"{% else %} style="height:{{ [h, 4]|max }}%"{% endif %}></i>{% endfor -%}
|
||||
</span>
|
||||
{%- endmacro %}
|
||||
{% if not rows %}
|
||||
<div class="w-verdict" style="color:var(--text-muted);">No running containers.</div>
|
||||
{% else %}
|
||||
{% set top = rows[0] %}
|
||||
<div class="w-verdict">
|
||||
Busiest: <strong>{{ top.label }}</strong> on {{ top.host_name }},
|
||||
{% if top.usage.cores is not none %}{{ "%.2f"|format(top.usage.cores) if top.usage.cores < 1 else "%.1f"|format(top.usage.cores) }}
|
||||
{% if top.usage.cores is not none %}{{ cores(top.usage.cores) }}
|
||||
{%- if top.usage.cpus %} of {{ top.usage.cpus }} cores{% else %} cores{% endif %}{% else %}no CPU reading yet{% endif %}.
|
||||
</div>
|
||||
<table class="w-table">
|
||||
<colgroup><col><col class="w-wide-only" style="width:7.5rem"><col style="width:5.5rem"><col style="width:6.5rem"></colgroup>
|
||||
<thead><tr><th>Container</th><th class="w-wide-only w-text">Host</th><th>CPU</th><th>Memory</th></tr></thead>
|
||||
<colgroup><col><col class="w-wide-only" style="width:7.5rem"><col class="w-barcol"><col class="w-barcol"></colgroup>
|
||||
<thead><tr><th>Container</th><th class="w-wide-only w-text">Host</th>
|
||||
<th title="Cores, averaged over the last {{ minutes }} minutes">CPU</th><th title="In use now">Memory</th></tr></thead>
|
||||
<tbody>
|
||||
{% for r in rows %}
|
||||
{% set u = r.usage %}
|
||||
@@ -23,19 +32,19 @@
|
||||
{%- if session.user_id %}<a href="/plugins/docker/container/{{ r.c.host_id }}/{{ r.c.name }}">{{ r.label }}</a>{% else %}{{ r.label }}{% endif -%}
|
||||
</td>
|
||||
<td class="w-wide-only w-text" title="{{ r.host_name }}">{{ r.host_name }}</td>
|
||||
{% if u.cores is none %}<td>—</td>{% else %}
|
||||
<td class="w-meter tone-{{ status_class(u.cpu_share, 'cpu') if u.cpus else 'dim' }}" style="--pct:{{ [[u.cpu_share or 0, 0]|max, 100]|min }}%"
|
||||
title="{% if u.cpus %}{{ '%.1f'|format(u.cpu_share) }}% of {{ r.host_name }}'s {{ u.cpus }} cores{% else %}The host's core count isn't known{% endif %}">
|
||||
{{- "%.2f"|format(u.cores) if u.cores < 1 else "%.1f"|format(u.cores) }} cores</td>
|
||||
{% endif %}
|
||||
{% if u.mem_bytes is none %}<td>—</td>{% else %}
|
||||
<td class="w-meter tone-{{ status_class(u.mem_pct, 'mem') if u.mem_of else 'dim' }}" style="--pct:{{ [[u.mem_pct or 0, 0]|max, 100]|min }}%"
|
||||
title="{% if u.mem_pct is not none and u.mem_of %}{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% else %}No memory limit set{% endif %}">
|
||||
{{- u.mem_bytes|bytes }}</td>
|
||||
{% endif %}
|
||||
<td class="tone-{{ status_class(u.cpu_share, 'cpu') if u.cpus and u.cores is not none else 'dim' }}"
|
||||
{% if u.cpus and u.cores is not none %}title="{{ '%.1f'|format(u.cpu_share) }}% of {{ r.host_name }}'s {{ u.cpus }} cores"{% endif %}>
|
||||
{{- bars(r.cpu_bars, (cores(r.cpu_bars.peak) ~ " cores") if r.cpu_bars.peak is not none else "", cores(r.cpu_bars.roof) ~ " cores") -}}
|
||||
{%- if u.cores is none %}—{% else %}{{ cores(u.cores) }} cores{% endif -%}
|
||||
</td>
|
||||
<td class="tone-{{ status_class(u.mem_pct, 'mem') if u.mem_of and u.mem_bytes is not none else 'dim' }}"
|
||||
{% if u.mem_pct is not none and u.mem_of and u.mem_bytes is not none %}title="{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}"{% endif %}>
|
||||
{{- bars(r.mem_bars, r.mem_bars.peak|bytes if r.mem_bars.peak is not none else "", r.mem_bars.roof|bytes) -}}
|
||||
{%- if u.mem_bytes is none %}—{% else %}{{ u.mem_bytes|bytes }}{% endif -%}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
<div class="w-note" title="Shading: share of the host's cores, and of the memory limit (or the host's memory when no limit is set).">CPU over the last {{ minutes }} min; shading is the share of the host.</div>
|
||||
<div class="w-note w-wide-only">Bars: last {{ hours }}h, each row up to its own peak, at least {{ cores(cpu_floor) }} core{{ 's' if cpu_floor != 1 }} and {{ mem_floor|bytes }}.</div>
|
||||
{% endif %}
|
||||
|
||||
Reference in New Issue
Block a user