bvandeusen/FabledSteward
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(docker): the Docker page leads with a verdict, lists services and containers as filterable tables, and each service has its own page with logs across its tasks (#5298–#5302)
CI / lint (push) Successful in 2s
Details
CI / integration (push) Failing after 1m4s
Details
CI / unit (push) Failing after 1m28s
Details
CI / publish (push) Skipped
Details

Browse Source 
- #5298 "Docker overview: verdict line; only the problem tiles that aren't zero"
- #5299 "Docker overview: swarm services as a filterable, sortable table with a Stack column"
- #5300 "Docker overview: failures listed once, each with a link to the failed task's logs"
- #5301 "Docker overview: standalone containers in one table with a Host column"
- #5302 "Service page: replicas, image, task history, resource history and logs across all its tasks"

overview.py holds the rows, filters and sort as plain functions. The filter
bar sits outside the polled fragment and the URL keeps its state. A service's
page reads every task container Steward still holds anything for, the live
ones and removed ones known by their events, so a failed task's logs are
reachable after Docker has removed it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-10-07 20:08:38 -04:00
co-authored by Claude Opus 5.5
parent 7dccb2f303
commit 2093f399e6
13 changed files with 1339 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/docker/routes.py
+216 -72
View File
@@ -20,6 +20,10 @@ from .swarm_view import (
build_swarm_services, failing_services, parse_tasks, replaced_tasks, services_problems_first,
task_summary, trouble_since,
)
from .overview import (
DOCKER_SORTS, DOCKER_STATES, NO_STACK, container_row, filter_rows, service_row, sort_rows,
stack_of,
)
from .models import (
DockerContainer, DockerDiskUsage, DockerHostStatus, DockerImage, DockerMetric,
DockerSwarmNode, DockerSwarmService,
@@ -62,6 +66,37 @@ def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
)
def _filters(args) -> dict:
"""The Docker page's search, filters and sort from the query string
(#5299): q, stack, host, state, sort, dir; anything unknown is dropped."""
sort = args.get("sort") if args.get("sort") in DOCKER_SORTS else "status"
desc = {"desc": True, "asc": False}.get(args.get("dir"), DOCKER_SORTS[sort])
return {"q": (args.get("q") or "").strip(), "stack": args.get("stack", ""),
"host": args.get("host", ""),
"state": args.get("state") if args.get("state") in DOCKER_STATES else "",
"sort": sort, "dir": "desc" if desc else "asc"}
def _page_url(f: dict, **change) -> str:
"""The Docker page with some filters changed; defaults are left out."""
from urllib.parse import urlencode
want = {**f, **change}
if want["dir"] == ("desc" if DOCKER_SORTS[want["sort"]] else "asc"):
want["dir"] = ""
if want["sort"] == "status":
want["sort"] = ""
query = urlencode({k: v for k, v in want.items() if v})
return "/plugins/docker/" + ("?" + query if query else "")
def _sort_url(f: dict, key: str) -> str:
"""A column header's link: its own default direction, or the other way
when it is already the sort."""
if key == f["sort"]:
return _page_url(f, sort=key, dir="asc" if f["dir"] == "desc" else "desc")
return _page_url(f, sort=key, dir="desc" if DOCKER_SORTS[key] else "asc")
@docker_bp.get("/")
@require_role(UserRole.viewer)
async def index():
@@ -73,21 +108,60 @@ async def index():
select(DockerSwarmService.host_id).limit(1))).first() is not None
has_disk = (await db.execute(
select(DockerDiskUsage.host_id).limit(1))).first() is not None
choices = await _filter_choices(db)
return await render_template(
"docker/index.html",
poll_interval=poll_interval,
has_swarm=has_swarm,
has_disk=has_disk,
f=_filters(request.args), choices=choices, states=DOCKER_STATES, no_stack=NO_STACK,
)
async def _filter_choices(db) -> dict:
"""The stacks and hosts the filter bar offers: every swarm service's stack
and compose project, and every host or node that runs something."""
names = set((await db.execute(select(DockerSwarmService.service_name).distinct())).scalars())
names |= {n for n in (await db.execute(
select(DockerContainer.service_name).distinct())).scalars() if n}
stacks = {stack_of(n) for n in names} | {p for p in (await db.execute(
select(DockerContainer.compose_project).distinct())).scalars() if p}
host_ids = set((await db.execute(select(DockerContainer.host_id).distinct())).scalars())
hosts = {h.name for h in (await docker_svc.host_map(db, host_ids)).values()}
hosts |= {n for n in (await db.execute(select(DockerSwarmNode.hostname).distinct())).scalars() if n}
return {"stacks": sorted((s for s in stacks if s), key=str.lower),
"hosts": sorted(hosts, key=str.lower)}
async def _swarm_context(db, now: datetime) -> dict:
"""What the Docker page and a service's page both build from: containers,
swarm services and nodes, host names, and the swarm view (each service with
its current replicas, Swarm's task records and their failures)."""
containers = await docker_svc.list_containers(db)
services = list((await db.execute(select(DockerSwarmService))).scalars())
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
hosts = await docker_svc.host_map(
db, {c.host_id for c in containers} | {n.host_id for n in nodes}
| {s.host_id for s in services})
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
host_name = {h.id: h.name for h in hosts.values()}
swarm_view = build_swarm_services(
containers, services, node_hostname, host_name, now=now,
crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))
return {"containers": containers, "services": services, "nodes": nodes, "hosts": hosts,
"node_hostname": node_hostname, "host_name": host_name, "swarm_view": swarm_view}
@docker_bp.get("/rows")
@require_role(UserRole.viewer)
async def rows():
"""HTMX fragment: what needs attention first, then swarm services
(collapsed, cluster-complete), then non-swarm containers grouped by host."""
"""HTMX fragment: a verdict line and the problems that aren't zero, what
needs attention, what failed in 24h (each with its logs), then the swarm
services and the standalone containers as two tables that share the
filter bar's search, filters and sort (#5298–#5301)."""
now = datetime.now(timezone.utc)
stale_after = docker_svc.stale_after_seconds(current_app.config)
f = _filters(request.args)
async with current_app.db_sessionmaker() as db:
# Hosts whose Docker is not answering: their rows are what it last said.
@@ -95,84 +169,43 @@ async def rows():
select(DockerHostStatus).where(DockerHostStatus.status != "ok"))).scalars()}
ranked = await docker_svc.ranked_containers(
db, now, docker_svc.crash_loop_deaths(current_app.config))
containers = await docker_svc.list_containers(db)
services = list((await db.execute(select(DockerSwarmService))).scalars())
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
hosts = await docker_svc.host_map(
db,
{c.host_id for c in containers}
| {n.host_id for n in nodes} | {s.host_id for s in services} | set(docker_down),
)
# Swarm tasks collapse into per-service rows (each replica labelled with
# its host, plus "ghost" replicas for tasks on agent-less nodes, drawn
# from the managers' placement). Non-swarm containers keep the host view.
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
ctx = await _swarm_context(db, now)
containers, hosts = ctx["containers"], ctx["hosts"]
hosts.update(await docker_svc.host_map(db, set(docker_down) - set(hosts)))
host_name = {h.id: h.name for h in hosts.values()}
swarm_view = build_swarm_services(
containers, services, node_hostname, host_name,
crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))
swarm_view = ctx["swarm_view"]
capacity = await docker_svc.host_capacity(db, {c.host_id for c in containers})
failures = docker_svc.failures_24h(
await docker_svc.failure_events(db, now - timedelta(hours=24)),
swarm_view["services"], host_name)
standalone = [c for c in containers if not c.service_name]
groups: dict[str, dict] = {}
for c in standalone:
g = groups.get(c.host_id)
if g is None:
host = hosts.get(c.host_id)
g = groups[c.host_id] = {
"host": host,
"host_id": c.host_id,
"host_name": host.name if host else c.host_id,
"containers": [], "running": 0, "stopped": 0,
}
g["containers"].append({
"container": c,
"ports": json.loads(c.ports_json) if c.ports_json else [],
"uptime": _uptime(c.started_at) if c.status == "running" else None,
"stale": _is_stale(c, now, stale_after, docker_down),
"usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None))),
"exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed),
})
if c.status == "running":
g["running"] += 1
else:
g["stopped"] += 1
service_rows = [service_row(s, now) for s in swarm_view["services"]]
standalone = [c for c in containers if not c.service_name]
container_rows = [
container_row(
c, label=docker_svc.display_name(c), host=host_name.get(c.host_id, c.host_id),
usage=docker_svc.usage(c, *capacity.get(c.host_id, (None, None))),
ports=json.loads(c.ports_json) if c.ports_json else [],
stale=_is_stale(c, now, stale_after, docker_down),
exit_failed=docker_svc.exit_is_failure(c.exit_code, c.oom_killed), now=now)
for c in standalone]
desc = f["dir"] == "desc"
shown_services = sort_rows(filter_rows(service_rows, q=f["q"], stack=f["stack"], host=f["host"],
state=f["state"]), f["sort"], desc)
shown_containers = sort_rows(filter_rows(container_rows, q=f["q"], stack=f["stack"],
host=f["host"], state=f["state"]), f["sort"], desc)
# Sub-group each host's containers by compose project, preserving the
# running-first ordering. Containers with no project fall into an unlabelled
# bucket rendered flat, so plain hosts look unchanged.
for g in groups.values():
subs: dict[str, dict] = {}
order: list[str] = []
for item in g["containers"]:
c = item["container"]
label = c.compose_project or None
key = label or ""
if key not in subs:
subs[key] = {"label": label, "containers": []}
order.append(key)
subs[key]["containers"].append(item)
g["subgroups"] = [subs[k] for k in order]
g["grouped"] = any(s["label"] for s in g["subgroups"])
host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
swarm_services = services_problems_first(swarm_view["services"])
standalone_running = sum(g["running"] for g in host_groups)
failing = failing_services(swarm_view["services"])
attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down),
label=docker_svc.task_label(r["container"], node_hostname))
label=docker_svc.task_label(r["container"], ctx["node_hostname"]))
for r in docker_svc.needs_attention(ranked)]
failing = failing_services(swarm_services)
running = sum(r["running"] for r in container_rows)
return await render_template(
"docker/rows.html",
host_groups=host_groups,
swarm_services=swarm_services,
failing_services=failing,
failures=failures,
services_healthy=sum(s["state"] == "healthy" for s in swarm_services),
services=shown_services, containers=shown_containers,
services_total=len(service_rows), containers_total=len(container_rows),
services_healthy=sum(s["state"] == "healthy" for s in service_rows),
failing_services=failing, failures=failures,
docker_hosts=len({c.host_id for c in containers}),
docker_down=sorted(
({"host_id": h, "host_name": hosts[h].name if h in hosts else h, "status": d.status,
@@ -181,9 +214,10 @@ async def rows():
attention=attention, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
unhealthy=sum("unhealthy" in r["problems"] for r in attention),
restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention),
stale=sum(1 for g in host_groups for i in g["containers"] if i["stale"]),
running=standalone_running,
stopped=len(standalone) - standalone_running,
stale=sum(r["stale"] for r in container_rows),
running=running, stopped=len(container_rows) - running,
f=f, sort_url=lambda key: _sort_url(f, key), page_url=lambda **kw: _page_url(f, **kw),
filtered=any(f[k] for k in ("q", "stack", "host", "state")),
)
@@ -743,3 +777,113 @@ async def container_logs_lines(host_id: str, name: str):
name=name, tail=_LOG_TAIL_DEFAULT,
filtered=bool(query) or stream in ("stdout", "stderr"),
)
# ── A swarm service's own page (#5302) ───────────────────────────────────────
def _task_label(slot: str, task_id: str) -> str:
"""How a task reads beside its log lines: its slot and the start of its id."""
return f"{slot} · {task_id[:10]}"
def _task_rows(tasks: list[dict], node_hostname: dict) -> list[dict]:
"""Swarm's records of a service's newest tasks, newest first, as the page
lists them: where each ran, what state it reached, and why it ended."""
from .swarm_view import FAILED_TASK_STATES, task_time
out = []
for t in tasks:
nid = t.get("node_id") or ""
out.append({
"task_id": t.get("task_id") or "", "slot": str(t.get("slot") or ""),
"node": node_hostname.get(nid) or (nid[:12] if nid else "—"),
"state": t.get("state") or "unknown", "desired": t.get("desired_state") or "",
"exit_code": t.get("exit_code"), "why": t.get("error") or t.get("message") or "",
"at": task_time(t), "failed": t.get("state") in FAILED_TASK_STATES,
"current": t.get("state") == "running" and t.get("desired_state") == "running",
})
return out
@docker_bp.get("/service/<name>")
@require_role(UserRole.viewer)
async def service_detail(name: str):
"""One swarm service: where its replicas run and what they use, its image,
Swarm's record of its recent tasks, its CPU and memory over time across
every replica, and the logs of every task it has run — newest first, each
line marked with its task. `?task=` opens the logs on one task, which is
where a failure's "Logs →" lands."""
now = datetime.now(timezone.utc)
async with current_app.db_sessionmaker() as db:
ctx = await _swarm_context(db, now)
task_containers = await docker_svc.service_containers(db, name)
s = next((s for s in ctx["swarm_view"]["services"] if s["name"] == name), None)
capacity = await docker_svc.host_capacity(
db, {r["host_id"] for r in (s["replicas"] if s else []) if not r["ghost"]})
if s is None and not task_containers:
return await render_template("docker/service.html", name=name, service=None), 404
newest = max((v for v in ctx["services"] if v.service_name == name),
key=lambda v: v.updated_at, default=None)
tasks = _task_rows(parse_tasks(newest) if newest is not None else [], ctx["node_hostname"])
by_key = {(c.host_id, c.name): c for c in ctx["containers"]}
replicas = []
for r in (s["replicas"] if s else []):
c = None if r["ghost"] else by_key.get((r["host_id"], r["name"]))
replicas.append(dict(r, usage=docker_svc.usage(c, *capacity.get(r["host_id"], (None, None)))
if c is not None else None,
uptime=_uptime(c.started_at) if c is not None and c.status == "running" else None))
# Every task Steward holds logs or events for, newest known first.
known = {t["task_id"]: _task_label(t["slot"], t["task_id"]) for t in tasks if t["task_id"]}
for _, cname in task_containers:
slot, tid = docker_svc.task_of(name, cname)
known.setdefault(tid, _task_label(slot, tid))
focus = request.args.get("task", "")
return await render_template(
"docker/service.html", name=name, service=s,
row=service_row(s, now) if s else None, svc=newest, tasks=tasks, replicas=replicas,
task_choices=list(known.items()), focus_task=focus if focus in known else "",
stack=stack_of(name), current_range=request.args.get("range", DEFAULT_RANGE),
)
@docker_bp.get("/service/<name>/history")
@require_role(UserRole.viewer)
async def service_history_fragment(name: str):
"""HTMX fragment: the service's CPU (cores) and memory, summed across its
replicas, over the selected range, with the latest and the peak."""
from steward.core.status import sparkline_svg
since, range_key = parse_range(request.args.get("range"))
async with current_app.db_sessionmaker() as db:
series = await docker_svc.service_history(
db, await docker_svc.service_containers(db, name), since)
cpu = [c / 100 for _, c, _ in series]
mem = [m for _, _, m in series]
return await render_template(
"docker/_service_history.html", have_data=len(series) >= 2, range_key=range_key,
cpu_now=cpu[-1] if cpu else None, cpu_peak=max(cpu) if cpu else None,
mem_now=mem[-1] if mem else None, mem_peak=max(mem) if mem else None,
spark_cpu=sparkline_svg(cpu, height=48, stroke="currentColor", fill_width=True),
spark_mem=sparkline_svg(mem, height=48, stroke="currentColor", fill_width=True),
)
@docker_bp.get("/service/<name>/logs/lines")
@require_role(UserRole.viewer)
async def service_logs_lines(name: str):
"""HTMX fragment: the newest retained lines across every task of the
service, stream, text and task filtered, each marked with its task."""
stream = request.args.get("stream", "all")
query = (request.args.get("q") or "").strip()
task = request.args.get("task") or None
async with current_app.db_sessionmaker() as db:
rows = await docker_svc.query_service_logs(
db, await docker_svc.service_containers(db, name), stream, query, _LOG_TAIL_DEFAULT,
task=task)
lines = []
for r in rows:
slot, tid = docker_svc.task_of(name, r.container_name) or ("?", r.container_name)
lines.append({"ts": r.ts, "stream": r.stream, "line": r.line, "task": _task_label(slot, tid)})
return await render_template(
"docker/_container_logs_lines.html", lines=lines, name=name, tail=_LOG_TAIL_DEFAULT,
filtered=bool(query) or bool(task) or stream in ("stdout", "stderr"), service=True,
)