bvandeusen/FabledSteward
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(docker): readable names, CPU in cores and memory in bytes, failures listed with since-times (#5209)
CI / lint (push) Successful in 2s
Details
CI / integration (push) Successful in 1m3s
Details
CI / unit (push) Failing after 1m28s
Details
CI / publish (push) Skipped
Details

Browse Source 
#5209 "Docker views: readable names, CPU/memory in cores and bytes, failures
listed with since-time" (milestone 477 "UI problems first").

- A swarm task reads as its service and slot ("codex_db · 1", or the node
  for a global service), with the raw name in the title: Docker page, host
  panel, busiest widget, container detail.
- CPU reads in cores (Docker's 100% = one core), toned by its share of the
  host's CPUs from the agent's per-core series. Memory reads in bytes, against
  a limit only when one is really set (Docker reports host RAM otherwise).
- Exit 137/143 without an OOM kill is a stop, not a failure: no
  exited_nonzero, no crash-loop death, no "failed (24h)", dim exit code.
- "Failed (24h)" is a list (Docker page #failures, linked from the widget
  and the tile): plain containers by their deaths, swarm services by Swarm's
  failed tasks; a second manager's copy of a task death counts once.
- Failing swarm services lead Needs attention; services sort problems first
  with since-times on the page, the widget and the Swarm page; nodes that
  are down or drained sort first.
- Busiest widget orders by a 5-minute CPU average so the list holds still.
- Host panel: a table, problems first with what is wrong and since when;
  replaced swarm tasks hidden and counted.
- Container detail: node hostname, I/O labelled "since start", Restarts
  hidden for swarm tasks with a note on where task failures are.
- Widget names: "Docker — Services & failures", "Docker — Busiest containers".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-10-06 23:41:14 -04:00
co-authored by Claude Opus 5.5
parent 8941ed0372
commit 61b22056d5
16 changed files with 713 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/docker/routes.py
+135 -54
View File
@@ -16,9 +16,12 @@ from steward.core.formatting import human_bytes, human_duration, seconds_since
from steward.core.time_range import parse_range, DEFAULT_RANGE
from . import service as docker_svc
from .dedup import dedup_by_container_id
from .swarm_view import build_swarm_services, parse_tasks, task_summary
from .swarm_view import (
build_swarm_services, failing_services, parse_tasks, replaced_tasks, services_problems_first,
task_summary, trouble_since,
)
from .models import (
DockerContainer, DockerDiskUsage, DockerEvent, DockerHostStatus, DockerImage,
DockerContainer, DockerDiskUsage, DockerHostStatus, DockerImage, DockerMetric,
DockerSwarmNode, DockerSwarmService,
)
@@ -109,6 +112,10 @@ async def rows():
swarm_view = build_swarm_services(
containers, services, node_hostname, host_name,
crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))
capacity = await docker_svc.host_capacity(db, {c.host_id for c in containers})
failures = docker_svc.failures_24h(
await docker_svc.failure_events(db, now - timedelta(hours=24)),
swarm_view["services"], host_name)
standalone = [c for c in containers if not c.service_name]
groups: dict[str, dict] = {}
@@ -127,6 +134,8 @@ async def rows():
"ports": json.loads(c.ports_json) if c.ports_json else [],
"uptime": _uptime(c.started_at) if c.status == "running" else None,
"stale": _is_stale(c, now, stale_after, docker_down),
"usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None))),
"exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed),
})
if c.status == "running":
g["running"] += 1
@@ -151,15 +160,20 @@ async def rows():
g["grouped"] = any(s["label"] for s in g["subgroups"])
host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
swarm_services = swarm_view["services"]
swarm_services = services_problems_first(swarm_view["services"])
standalone_running = sum(g["running"] for g in host_groups)
ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"])
attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down))
attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down),
label=docker_svc.task_label(r["container"], node_hostname))
for r in docker_svc.needs_attention(ranked)]
failing = failing_services(swarm_services)
return await render_template(
"docker/rows.html",
host_groups=host_groups,
swarm_services=swarm_services,
failing_services=failing,
failures=failures,
services_healthy=sum(s["state"] == "healthy" for s in swarm_services),
docker_hosts=len({c.host_id for c in containers}),
docker_down=sorted(
({"host_id": h, "host_name": hosts[h].name if h in hosts else h, "status": d.status,
"detail": d.detail, "since": d.since} for h, d in docker_down.items()),
@@ -168,9 +182,8 @@ async def rows():
unhealthy=sum("unhealthy" in r["problems"] for r in attention),
restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention),
stale=sum(1 for g in host_groups for i in g["containers"] if i["stale"]),
running=standalone_running + sum(s["running"] for s in swarm_services),
running=standalone_running,
stopped=len(standalone) - standalone_running,
total=len(containers) + ghost_total,
)
@@ -185,55 +198,55 @@ def _is_stale(c, now: datetime, stale_after: int, docker_down=()) -> bool:
@docker_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
"""HTMX dashboard widget: container status overview, grouped by host."""
"""HTMX dashboard widget: swarm services and containers, problems first,
with how many things failed in the last 24h (linked to the list)."""
show_stopped = request.args.get("show_stopped", "no") == "yes"
widget_id = request.args.get("wid", "0")
since_24h = datetime.now(timezone.utc) - timedelta(hours=24)
now = datetime.now(timezone.utc)
async with current_app.db_sessionmaker() as db:
all_containers = dedup_by_container_id(list((await db.execute(
select(DockerContainer).order_by(
(DockerContainer.status == "running").desc(),
DockerContainer.name,
)
)).scalars()))
all_containers = dedup_by_container_id(await docker_svc.list_containers(db))
services = list((await db.execute(select(DockerSwarmService))).scalars())
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
# Recent failures are more actionable than a static "stopped" tally. Read
# DISTINCT container names so the two managers' duplicate die/oom events
# don't double it; swarm tasks are counted on their service below.
dead_names = list((await db.execute(
select(func.distinct(DockerEvent.container_name))
.where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since_24h)
)).scalars())
events = await docker_svc.failure_events(db, now - timedelta(hours=24))
hosts = await docker_svc.host_map(
db,
{c.host_id for c in all_containers}
| {n.host_id for n in nodes} | {s.host_id for s in services},
)
capacity = await docker_svc.host_capacity(db, {c.host_id for c in all_containers})
# Swarm tasks collapse into per-service rows (each replica tagged with its
# host); non-swarm containers stay grouped by host.
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
host_name = {h.id: h.name for h in hosts.values()}
swarm_services = build_swarm_services(
swarm_services = services_problems_first(build_swarm_services(
all_containers, services, node_hostname, host_name,
crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"]
failed_24h = docker_svc.failed_24h(dead_names, swarm_services)
crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"])
failures = docker_svc.failures_24h(events, swarm_services, host_name)
standalone = [c for c in all_containers if not c.service_name]
running = [c for c in standalone if c.status == "running"]
display = standalone if show_stopped else running
def trouble(c) -> int:
if (c.status or "") == "restarting" or c.health == "unhealthy":
return 0
if c.status != "running" and docker_svc.exit_is_failure(c.exit_code, c.oom_killed):
return 1
return 2 if c.status == "running" else 3
# Group for display so multi-host fleets read clearly; single-host stays flat.
groups: dict[str, dict] = {}
for c in display:
for c in sorted(display, key=lambda c: (trouble(c), c.name)):
g = groups.setdefault(c.host_id, {
"host_id": c.host_id,
"host_name": host_name.get(c.host_id, c.host_id),
"containers": [],
})
g["containers"].append(c)
g["containers"].append({"c": c, "usage": docker_svc.usage(
c, *capacity.get(c.host_id, (None, None))),
"exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed)})
host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
return await render_template(
@@ -241,39 +254,59 @@ async def widget():
host_groups=host_groups,
multi_host=len(host_groups) > 1,
swarm_services=swarm_services,
running_count=len(running) + sum(s["running"] for s in swarm_services),
failed_24h=failed_24h,
services_healthy=sum(s["state"] == "healthy" for s in swarm_services),
running_count=len(running),
failures=failures,
total_count=len(all_containers) + len(swarm_services),
show_stopped=show_stopped,
widget_id=widget_id,
)
CPU_AVERAGE_MINUTES = 5
@docker_bp.get("/widget/resources")
@require_role(UserRole.viewer)
async def widget_resources():
"""HTMX dashboard widget: CPU + memory usage for the busiest containers."""
"""HTMX dashboard widget: the containers using the most CPU, averaged over
the last few minutes so the order holds still between polls, with their
CPU in cores and memory in bytes."""
limit = max(1, min(20, int(request.args.get("limit", 10) or 10)))
widget_id = request.args.get("wid", "0")
since = datetime.now(timezone.utc) - timedelta(minutes=CPU_AVERAGE_MINUTES)
async with current_app.db_sessionmaker() as db:
# Dedup before the limit, else swarm tasks reported by both managers can
# fill the list with duplicates of the same container.
containers = dedup_by_container_id(list((await db.execute(
select(DockerContainer)
.where(DockerContainer.status == "running")
.order_by(DockerContainer.cpu_pct.desc().nullslast())
)).scalars()))[:limit]
hosts = await docker_svc.host_map(db, {c.host_id for c in containers})
running = dedup_by_container_id([
c for c in await docker_svc.list_containers(db) if c.status == "running"])
average = {(h, n): v for h, n, v in (await db.execute(
select(DockerMetric.host_id, DockerMetric.container_name, func.avg(DockerMetric.cpu_pct))
.where(DockerMetric.scraped_at >= since)
.group_by(DockerMetric.host_id, DockerMetric.container_name)
)).all()}
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
hosts = await docker_svc.host_map(db, {c.host_id for c in running})
capacity = await docker_svc.host_capacity(db, {c.host_id for c in running})
rows_data = [
{"c": c, "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id}
for c in containers
]
def cpu_of(c):
v = average.get((c.host_id, c.name))
return c.cpu_pct if v is None else float(v)
busiest = sorted(running, key=lambda c: (-(cpu_of(c) or 0), c.name))[:limit]
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
rows_data = [{
"c": c,
"label": docker_svc.task_label(c, node_hostname),
"host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id,
"usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c)),
} for c in busiest]
return await render_template(
"docker/widget_resources.html",
rows=rows_data,
multi_host=len({c.host_id for c in containers}) > 1,
minutes=CPU_AVERAGE_MINUTES,
multi_host=len({c.host_id for c in busiest}) > 1,
widget_id=widget_id,
)
@@ -281,22 +314,50 @@ async def widget_resources():
@docker_bp.get("/host/<host_id>")
@require_role(UserRole.viewer)
async def host_panel(host_id: str):
"""Per-host Docker fragment embedded on the core Hosts hub via HTMX.
"""Per-host Docker fragment embedded on the core Hosts hub via HTMX:
problems first (with exit code and since when), then running, then
stopped. Swarm tasks Swarm has already replaced are history and left out.
Renders nothing when the host reports no containers, so hosts without Docker
don't carry an empty card.
"""
now = datetime.now(timezone.utc)
async with current_app.db_sessionmaker() as db:
containers = await docker_svc.list_containers(db, host_id)
if not containers:
return ""
services = list((await db.execute(select(DockerSwarmService))).scalars())
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
capacity = await docker_svc.host_capacity(db, {host_id})
if not containers:
return ""
running = sum(1 for c in containers if c.status == "running")
replaced = replaced_tasks(containers, services)
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
hour_ago = now - timedelta(hours=1)
crash = docker_svc.crash_loop_deaths(current_app.config)
rows = []
for c in containers:
if (c.host_id, c.name) in replaced:
continue
events = failures.get((c.host_id, c.name), [])
recent = sum(1 for e in events if e.at >= hour_ago)
problems = docker_svc.container_problems(c, recent, bool(events), crash)
rows.append({
"c": c, "label": docker_svc.task_label(c, node_hostname), "problems": problems,
"since": docker_svc.problem_since(problems, events, hour_ago),
"usage": docker_svc.usage(c, *capacity.get(host_id, (None, None))),
"uptime": _uptime(c.started_at) if c.status == "running" else None,
})
rows.sort(key=lambda r: (
docker_svc.PROBLEMS.index(r["problems"][0]) if r["problems"] else len(docker_svc.PROBLEMS),
r["c"].status != "running", r["label"].lower()))
running = sum(1 for r in rows if r["c"].status == "running")
return await render_template(
"docker/host_panel.html",
containers=containers,
running=running,
stopped=len(containers) - running,
rows=rows, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
running=running, stopped=len(rows) - running,
problems=sum(1 for r in rows if r["problems"]),
replaced=len(replaced),
)
@@ -319,6 +380,15 @@ async def container_detail(host_id: str, name: str):
c = await db.get(DockerContainer, (host_id, name))
host = await db.get(Host, host_id)
events = await docker_svc.container_events(db, host_id, name) if c is not None else []
node_name = None
if c is not None and c.node_id:
node_name = (await db.execute(
select(DockerSwarmNode.hostname)
.where(DockerSwarmNode.node_id == c.node_id, DockerSwarmNode.hostname != "")
.limit(1)
)).scalar()
cpus, mem_total = (await docker_svc.host_capacity(db, [host_id])).get(host_id, (None, None)) \
if c is not None else (None, None)
if c is None:
return await render_template(
@@ -332,6 +402,10 @@ async def container_detail(host_id: str, name: str):
host=host,
host_id=host_id,
name=name,
label=docker_svc.task_label(c, {c.node_id: node_name} if node_name else None),
node_name=node_name,
usage=docker_svc.usage(c, cpus, mem_total),
exit_failed=docker_svc.exit_is_failure(c.exit_code, c.oom_killed),
ports=json.loads(c.ports_json) if c.ports_json else [],
uptime=_uptime(c.started_at) if c.status == "running" else None,
net_rx=human_bytes(c.net_rx_bytes), net_tx=human_bytes(c.net_tx_bytes),
@@ -415,7 +489,11 @@ async def swarm():
now = datetime.now(timezone.utc)
deaths = docker_svc.crash_loop_deaths(current_app.config)
svc_dicts = []
for s in sorted(svc_by_name.values(), key=lambda s: s.service_name):
for s in svc_by_name.values():
tasks = parse_tasks(s)
summary = task_summary(tasks, now, deaths, node_names)
state = ("healthy" if s.running >= s.desired and s.desired > 0
else "degraded" if s.running > 0 else "down")
placement = []
for p in (json.loads(s.placement_json) if s.placement_json else []):
nid = p.get("node_id", "")
@@ -429,18 +507,21 @@ async def swarm():
"healthy": s.running >= s.desired and s.desired > 0,
"degraded": 0 < s.running < s.desired,
"down": s.running == 0 and s.desired > 0,
"state": ("healthy" if s.running >= s.desired and s.desired > 0
else "degraded" if s.running > 0 else "down"),
"state": state,
"placement": placement,
**task_summary(parse_tasks(s), now, deaths, node_names),
**summary,
"trouble_since": trouble_since(tasks, state, summary["crash_looping"], now),
})
# Problems first: a node that is down or not taking work, then by role.
node_rows = sorted(
node_by_id.values(), key=lambda n: (n.role, (n.hostname or n.node_id).lower())
node_by_id.values(), key=lambda n: (
n.status == "ready" and n.availability == "active",
n.role, (n.hostname or n.node_id).lower())
)
manager_names = sorted(hosts[m].name if m in hosts else m for m in members)
swarms.append({
"managers": manager_names,
"services": svc_dicts,
"services": services_problems_first(svc_dicts),
"nodes": node_rows,
})