feat(docker): the Docker page leads with a verdict, lists services and containers as filterable tables, and each service has its own page with logs across its tasks (#5298–#5302)
- #5298 "Docker overview: verdict line; only the problem tiles that aren't zero" - #5299 "Docker overview: swarm services as a filterable, sortable table with a Stack column" - #5300 "Docker overview: failures listed once, each with a link to the failed task's logs" - #5301 "Docker overview: standalone containers in one table with a Host column" - #5302 "Service page: replicas, image, task history, resource history and logs across all its tasks" overview.py holds the rows, filters and sort as plain functions. The filter bar sits outside the polled fragment and the URL keeps its state. A service's page reads every task container Steward still holds anything for, the live ones and removed ones known by their events, so a failed task's logs are reachable after Docker has removed it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,150 @@
|
||||
# plugins/docker/overview.py
|
||||
"""The Docker page's two tables, as plain rows (#5299, #5301): one row per
|
||||
swarm service and one per standalone container, with the search, stack, host
|
||||
and status filters and the column sort the page offers. Model-free, so it is
|
||||
unit-tested without a database; routes.rows() feeds it.
|
||||
|
||||
A stack is how the operator thinks of a group of services. Swarm names a
|
||||
stack's services "<stack>_<service>", so a service's stack is the part of its
|
||||
name before the first "_"; a standalone container's is its compose project.
|
||||
Stacks filter the list rather than section it, as host groups do on Hosts.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from .swarm_view import service_trouble
|
||||
|
||||
# Status filter values and their labels; NO_STACK picks rows with no stack.
|
||||
DOCKER_STATES = {"problem": "Needs attention", "healthy": "Healthy", "stopped": "Stopped"}
|
||||
NO_STACK = "__none__"
|
||||
# Each sort and whether it starts descending: the busiest first; "up" starts
|
||||
# with the shortest uptime, the thing that most recently restarted.
|
||||
DOCKER_SORTS = {"status": False, "name": False, "stack": False, "cpu": True, "mem": True,
|
||||
"up": False}
|
||||
|
||||
|
||||
def stack_of(service_name: str) -> str | None:
|
||||
"""The stack a swarm service was deployed in: "arr" for "arr_radarr";
|
||||
None for a name with no "_" (a service created on its own)."""
|
||||
stack, sep, rest = service_name.partition("_")
|
||||
return stack if sep and stack and rest else None
|
||||
|
||||
|
||||
def _seconds_up(started_at, now: datetime) -> float | None:
|
||||
return None if started_at is None else max((now - started_at).total_seconds(), 0.0)
|
||||
|
||||
|
||||
def service_row(s: dict, now: datetime) -> dict:
|
||||
"""A built swarm service (swarm_view.build_swarm_services) as a table row.
|
||||
CPU and memory add up the running replicas; "up" is the newest replica's
|
||||
start, so a service that just restarted a task reads as just restarted."""
|
||||
reals = [r for r in s["replicas"] if not r["ghost"]]
|
||||
running = [r for r in reals if (r["status"] or "").lower() == "running"]
|
||||
cpu = [r["cpu_pct"] for r in running if r.get("cpu_pct") is not None]
|
||||
mem = [r["mem_bytes"] for r in running if r.get("mem_bytes") is not None]
|
||||
starts = [r["started_at"] for r in running if r.get("started_at")]
|
||||
started = max(starts) if starts else None
|
||||
return {
|
||||
"kind": "service", "name": s["name"], "stack": stack_of(s["name"]),
|
||||
"mode": s["mode"],
|
||||
"image": s.get("image") or next((r["image"] for r in reals if r.get("image")), None),
|
||||
"running": s["running"], "desired": s["desired"], "state": s["state"],
|
||||
"trouble": service_trouble(s),
|
||||
"nodes": sorted({r["host"] for r in reals}, key=str.lower),
|
||||
"ghost_nodes": sorted({r["host"] for r in s["replicas"] if r["ghost"]}, key=str.lower),
|
||||
"cpu_cores": sum(cpu) / 100 if cpu else None,
|
||||
"mem_bytes": sum(mem) if mem else None,
|
||||
"started_at": started, "up_s": _seconds_up(started, now),
|
||||
"unhealthy": any(r.get("health") == "unhealthy" for r in reals),
|
||||
"failures_24h": s.get("failures_24h"), "crash_looping": s.get("crash_looping"),
|
||||
"last_error": s.get("last_error"), "trouble_since": s.get("trouble_since"),
|
||||
}
|
||||
|
||||
|
||||
def container_trouble(c, *, stale: bool, exit_failed: bool) -> int | None:
|
||||
"""How bad a standalone container is, on service_trouble's scale: 1
|
||||
unhealthy or restarting, 2 died with a failing exit, 3 its report is
|
||||
stale; None when nothing is wrong (a cleanly stopped one included)."""
|
||||
if c.health == "unhealthy" or c.status == "restarting":
|
||||
return 1
|
||||
if c.status != "running" and exit_failed:
|
||||
return 2
|
||||
if stale:
|
||||
return 3
|
||||
return None
|
||||
|
||||
|
||||
def container_row(c, *, label: str, host: str, usage: dict, ports: list, stale: bool,
|
||||
exit_failed: bool, now: datetime) -> dict:
|
||||
"""A standalone container as a table row; `usage` is service.usage()."""
|
||||
running = c.status == "running"
|
||||
return {
|
||||
"kind": "container", "name": label, "container_name": c.name, "host": host,
|
||||
"host_id": c.host_id, "stack": c.compose_project or None, "image": c.image,
|
||||
"status": c.status, "health": c.health, "running": running, "ports": ports,
|
||||
"usage": usage, "cpu_cores": usage.get("cores") if running else None,
|
||||
"mem_bytes": usage.get("mem_bytes") if running else None,
|
||||
"started_at": c.started_at if running else None,
|
||||
"up_s": _seconds_up(c.started_at, now) if running else None,
|
||||
"exit_code": c.exit_code, "oom_killed": c.oom_killed, "exit_failed": exit_failed,
|
||||
"restart_count": c.restart_count or 0, "stale": stale, "scraped_at": c.scraped_at,
|
||||
"trouble": container_trouble(c, stale=stale, exit_failed=exit_failed),
|
||||
}
|
||||
|
||||
|
||||
def _hosts(row: dict) -> list[str]:
|
||||
return row["nodes"] + row["ghost_nodes"] if row["kind"] == "service" else [row["host"]]
|
||||
|
||||
|
||||
def _stopped(row: dict) -> bool:
|
||||
return (row["running"] == 0) if row["kind"] == "service" else not row["running"]
|
||||
|
||||
|
||||
def filter_rows(rows: list[dict], *, q: str = "", stack: str = "", host: str = "",
|
||||
state: str = "") -> list[dict]:
|
||||
"""Rows matching a search (name, container name, image or stack, any
|
||||
case), a stack (or NO_STACK), a host or node, and a Status filter; an
|
||||
empty value doesn't filter."""
|
||||
q = (q or "").strip().lower()
|
||||
out = []
|
||||
for r in rows:
|
||||
if q and not any(q in (v or "").lower() for v in
|
||||
(r["name"], r.get("container_name"), r["image"], r["stack"])):
|
||||
continue
|
||||
if (stack == NO_STACK and r["stack"]) or (stack and stack != NO_STACK and r["stack"] != stack):
|
||||
continue
|
||||
if host and host not in _hosts(r):
|
||||
continue
|
||||
if state == "problem" and r["trouble"] is None:
|
||||
continue
|
||||
if state == "healthy" and (r["trouble"] is not None or _stopped(r)):
|
||||
continue
|
||||
if state == "stopped" and not _stopped(r):
|
||||
continue
|
||||
out.append(r)
|
||||
return out
|
||||
|
||||
|
||||
def sort_rows(rows: list[dict], sort: str, desc: bool) -> list[dict]:
|
||||
"""Rows in a column's order. Ties keep name order; a row with no value for
|
||||
the column goes last whichever way it runs. "status" is problems first,
|
||||
worst first, then running, then stopped."""
|
||||
by_name = sorted(rows, key=lambda r: r["name"].lower())
|
||||
if sort == "name":
|
||||
return list(reversed(by_name)) if desc else by_name
|
||||
if sort == "status":
|
||||
return sorted(by_name, key=lambda r: (4 if r["trouble"] is None else r["trouble"],
|
||||
_stopped(r)), reverse=desc)
|
||||
value = {"stack": lambda r: (r["stack"] or "").lower() or None,
|
||||
"cpu": lambda r: r["cpu_cores"], "mem": lambda r: r["mem_bytes"],
|
||||
"up": lambda r: r["up_s"]}[sort]
|
||||
have = [r for r in by_name if value(r) is not None]
|
||||
return sorted(have, key=value, reverse=desc) + [r for r in by_name if value(r) is None]
|
||||
|
||||
|
||||
def filter_options(services: list[dict], containers: list[dict]) -> dict:
|
||||
"""The stack and host choices the filter bar offers, from every row."""
|
||||
rows = services + containers
|
||||
return {"stacks": sorted({r["stack"] for r in rows if r["stack"]}, key=str.lower),
|
||||
"hosts": sorted({h for r in rows for h in _hosts(r)}, key=str.lower)}
|
||||
+216
-72
@@ -20,6 +20,10 @@ from .swarm_view import (
|
||||
build_swarm_services, failing_services, parse_tasks, replaced_tasks, services_problems_first,
|
||||
task_summary, trouble_since,
|
||||
)
|
||||
from .overview import (
|
||||
DOCKER_SORTS, DOCKER_STATES, NO_STACK, container_row, filter_rows, service_row, sort_rows,
|
||||
stack_of,
|
||||
)
|
||||
from .models import (
|
||||
DockerContainer, DockerDiskUsage, DockerHostStatus, DockerImage, DockerMetric,
|
||||
DockerSwarmNode, DockerSwarmService,
|
||||
@@ -62,6 +66,37 @@ def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
|
||||
)
|
||||
|
||||
|
||||
def _filters(args) -> dict:
|
||||
"""The Docker page's search, filters and sort from the query string
|
||||
(#5299): q, stack, host, state, sort, dir; anything unknown is dropped."""
|
||||
sort = args.get("sort") if args.get("sort") in DOCKER_SORTS else "status"
|
||||
desc = {"desc": True, "asc": False}.get(args.get("dir"), DOCKER_SORTS[sort])
|
||||
return {"q": (args.get("q") or "").strip(), "stack": args.get("stack", ""),
|
||||
"host": args.get("host", ""),
|
||||
"state": args.get("state") if args.get("state") in DOCKER_STATES else "",
|
||||
"sort": sort, "dir": "desc" if desc else "asc"}
|
||||
|
||||
|
||||
def _page_url(f: dict, **change) -> str:
|
||||
"""The Docker page with some filters changed; defaults are left out."""
|
||||
from urllib.parse import urlencode
|
||||
want = {**f, **change}
|
||||
if want["dir"] == ("desc" if DOCKER_SORTS[want["sort"]] else "asc"):
|
||||
want["dir"] = ""
|
||||
if want["sort"] == "status":
|
||||
want["sort"] = ""
|
||||
query = urlencode({k: v for k, v in want.items() if v})
|
||||
return "/plugins/docker/" + ("?" + query if query else "")
|
||||
|
||||
|
||||
def _sort_url(f: dict, key: str) -> str:
|
||||
"""A column header's link: its own default direction, or the other way
|
||||
when it is already the sort."""
|
||||
if key == f["sort"]:
|
||||
return _page_url(f, sort=key, dir="asc" if f["dir"] == "desc" else "desc")
|
||||
return _page_url(f, sort=key, dir="desc" if DOCKER_SORTS[key] else "asc")
|
||||
|
||||
|
||||
@docker_bp.get("/")
|
||||
@require_role(UserRole.viewer)
|
||||
async def index():
|
||||
@@ -73,21 +108,60 @@ async def index():
|
||||
select(DockerSwarmService.host_id).limit(1))).first() is not None
|
||||
has_disk = (await db.execute(
|
||||
select(DockerDiskUsage.host_id).limit(1))).first() is not None
|
||||
choices = await _filter_choices(db)
|
||||
return await render_template(
|
||||
"docker/index.html",
|
||||
poll_interval=poll_interval,
|
||||
has_swarm=has_swarm,
|
||||
has_disk=has_disk,
|
||||
f=_filters(request.args), choices=choices, states=DOCKER_STATES, no_stack=NO_STACK,
|
||||
)
|
||||
|
||||
|
||||
async def _filter_choices(db) -> dict:
|
||||
"""The stacks and hosts the filter bar offers: every swarm service's stack
|
||||
and compose project, and every host or node that runs something."""
|
||||
names = set((await db.execute(select(DockerSwarmService.service_name).distinct())).scalars())
|
||||
names |= {n for n in (await db.execute(
|
||||
select(DockerContainer.service_name).distinct())).scalars() if n}
|
||||
stacks = {stack_of(n) for n in names} | {p for p in (await db.execute(
|
||||
select(DockerContainer.compose_project).distinct())).scalars() if p}
|
||||
host_ids = set((await db.execute(select(DockerContainer.host_id).distinct())).scalars())
|
||||
hosts = {h.name for h in (await docker_svc.host_map(db, host_ids)).values()}
|
||||
hosts |= {n for n in (await db.execute(select(DockerSwarmNode.hostname).distinct())).scalars() if n}
|
||||
return {"stacks": sorted((s for s in stacks if s), key=str.lower),
|
||||
"hosts": sorted(hosts, key=str.lower)}
|
||||
|
||||
|
||||
async def _swarm_context(db, now: datetime) -> dict:
|
||||
"""What the Docker page and a service's page both build from: containers,
|
||||
swarm services and nodes, host names, and the swarm view (each service with
|
||||
its current replicas, Swarm's task records and their failures)."""
|
||||
containers = await docker_svc.list_containers(db)
|
||||
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
||||
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
|
||||
hosts = await docker_svc.host_map(
|
||||
db, {c.host_id for c in containers} | {n.host_id for n in nodes}
|
||||
| {s.host_id for s in services})
|
||||
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
|
||||
host_name = {h.id: h.name for h in hosts.values()}
|
||||
swarm_view = build_swarm_services(
|
||||
containers, services, node_hostname, host_name, now=now,
|
||||
crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))
|
||||
return {"containers": containers, "services": services, "nodes": nodes, "hosts": hosts,
|
||||
"node_hostname": node_hostname, "host_name": host_name, "swarm_view": swarm_view}
|
||||
|
||||
|
||||
@docker_bp.get("/rows")
|
||||
@require_role(UserRole.viewer)
|
||||
async def rows():
|
||||
"""HTMX fragment: what needs attention first, then swarm services
|
||||
(collapsed, cluster-complete), then non-swarm containers grouped by host."""
|
||||
"""HTMX fragment: a verdict line and the problems that aren't zero, what
|
||||
needs attention, what failed in 24h (each with its logs), then the swarm
|
||||
services and the standalone containers as two tables that share the
|
||||
filter bar's search, filters and sort (#5298–#5301)."""
|
||||
now = datetime.now(timezone.utc)
|
||||
stale_after = docker_svc.stale_after_seconds(current_app.config)
|
||||
f = _filters(request.args)
|
||||
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
# Hosts whose Docker is not answering: their rows are what it last said.
|
||||
@@ -95,84 +169,43 @@ async def rows():
|
||||
select(DockerHostStatus).where(DockerHostStatus.status != "ok"))).scalars()}
|
||||
ranked = await docker_svc.ranked_containers(
|
||||
db, now, docker_svc.crash_loop_deaths(current_app.config))
|
||||
containers = await docker_svc.list_containers(db)
|
||||
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
||||
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
|
||||
hosts = await docker_svc.host_map(
|
||||
db,
|
||||
{c.host_id for c in containers}
|
||||
| {n.host_id for n in nodes} | {s.host_id for s in services} | set(docker_down),
|
||||
)
|
||||
|
||||
# Swarm tasks collapse into per-service rows (each replica labelled with
|
||||
# its host, plus "ghost" replicas for tasks on agent-less nodes, drawn
|
||||
# from the managers' placement). Non-swarm containers keep the host view.
|
||||
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
|
||||
ctx = await _swarm_context(db, now)
|
||||
containers, hosts = ctx["containers"], ctx["hosts"]
|
||||
hosts.update(await docker_svc.host_map(db, set(docker_down) - set(hosts)))
|
||||
host_name = {h.id: h.name for h in hosts.values()}
|
||||
swarm_view = build_swarm_services(
|
||||
containers, services, node_hostname, host_name,
|
||||
crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))
|
||||
swarm_view = ctx["swarm_view"]
|
||||
capacity = await docker_svc.host_capacity(db, {c.host_id for c in containers})
|
||||
failures = docker_svc.failures_24h(
|
||||
await docker_svc.failure_events(db, now - timedelta(hours=24)),
|
||||
swarm_view["services"], host_name)
|
||||
|
||||
standalone = [c for c in containers if not c.service_name]
|
||||
groups: dict[str, dict] = {}
|
||||
for c in standalone:
|
||||
g = groups.get(c.host_id)
|
||||
if g is None:
|
||||
host = hosts.get(c.host_id)
|
||||
g = groups[c.host_id] = {
|
||||
"host": host,
|
||||
"host_id": c.host_id,
|
||||
"host_name": host.name if host else c.host_id,
|
||||
"containers": [], "running": 0, "stopped": 0,
|
||||
}
|
||||
g["containers"].append({
|
||||
"container": c,
|
||||
"ports": json.loads(c.ports_json) if c.ports_json else [],
|
||||
"uptime": _uptime(c.started_at) if c.status == "running" else None,
|
||||
"stale": _is_stale(c, now, stale_after, docker_down),
|
||||
"usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None))),
|
||||
"exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed),
|
||||
})
|
||||
if c.status == "running":
|
||||
g["running"] += 1
|
||||
else:
|
||||
g["stopped"] += 1
|
||||
service_rows = [service_row(s, now) for s in swarm_view["services"]]
|
||||
standalone = [c for c in containers if not c.service_name]
|
||||
container_rows = [
|
||||
container_row(
|
||||
c, label=docker_svc.display_name(c), host=host_name.get(c.host_id, c.host_id),
|
||||
usage=docker_svc.usage(c, *capacity.get(c.host_id, (None, None))),
|
||||
ports=json.loads(c.ports_json) if c.ports_json else [],
|
||||
stale=_is_stale(c, now, stale_after, docker_down),
|
||||
exit_failed=docker_svc.exit_is_failure(c.exit_code, c.oom_killed), now=now)
|
||||
for c in standalone]
|
||||
desc = f["dir"] == "desc"
|
||||
shown_services = sort_rows(filter_rows(service_rows, q=f["q"], stack=f["stack"], host=f["host"],
|
||||
state=f["state"]), f["sort"], desc)
|
||||
shown_containers = sort_rows(filter_rows(container_rows, q=f["q"], stack=f["stack"],
|
||||
host=f["host"], state=f["state"]), f["sort"], desc)
|
||||
|
||||
# Sub-group each host's containers by compose project, preserving the
|
||||
# running-first ordering. Containers with no project fall into an unlabelled
|
||||
# bucket rendered flat, so plain hosts look unchanged.
|
||||
for g in groups.values():
|
||||
subs: dict[str, dict] = {}
|
||||
order: list[str] = []
|
||||
for item in g["containers"]:
|
||||
c = item["container"]
|
||||
label = c.compose_project or None
|
||||
key = label or ""
|
||||
if key not in subs:
|
||||
subs[key] = {"label": label, "containers": []}
|
||||
order.append(key)
|
||||
subs[key]["containers"].append(item)
|
||||
g["subgroups"] = [subs[k] for k in order]
|
||||
g["grouped"] = any(s["label"] for s in g["subgroups"])
|
||||
|
||||
host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
|
||||
swarm_services = services_problems_first(swarm_view["services"])
|
||||
standalone_running = sum(g["running"] for g in host_groups)
|
||||
failing = failing_services(swarm_view["services"])
|
||||
attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down),
|
||||
label=docker_svc.task_label(r["container"], node_hostname))
|
||||
label=docker_svc.task_label(r["container"], ctx["node_hostname"]))
|
||||
for r in docker_svc.needs_attention(ranked)]
|
||||
failing = failing_services(swarm_services)
|
||||
running = sum(r["running"] for r in container_rows)
|
||||
return await render_template(
|
||||
"docker/rows.html",
|
||||
host_groups=host_groups,
|
||||
swarm_services=swarm_services,
|
||||
failing_services=failing,
|
||||
failures=failures,
|
||||
services_healthy=sum(s["state"] == "healthy" for s in swarm_services),
|
||||
services=shown_services, containers=shown_containers,
|
||||
services_total=len(service_rows), containers_total=len(container_rows),
|
||||
services_healthy=sum(s["state"] == "healthy" for s in service_rows),
|
||||
failing_services=failing, failures=failures,
|
||||
docker_hosts=len({c.host_id for c in containers}),
|
||||
docker_down=sorted(
|
||||
({"host_id": h, "host_name": hosts[h].name if h in hosts else h, "status": d.status,
|
||||
@@ -181,9 +214,10 @@ async def rows():
|
||||
attention=attention, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
|
||||
unhealthy=sum("unhealthy" in r["problems"] for r in attention),
|
||||
restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention),
|
||||
stale=sum(1 for g in host_groups for i in g["containers"] if i["stale"]),
|
||||
running=standalone_running,
|
||||
stopped=len(standalone) - standalone_running,
|
||||
stale=sum(r["stale"] for r in container_rows),
|
||||
running=running, stopped=len(container_rows) - running,
|
||||
f=f, sort_url=lambda key: _sort_url(f, key), page_url=lambda **kw: _page_url(f, **kw),
|
||||
filtered=any(f[k] for k in ("q", "stack", "host", "state")),
|
||||
)
|
||||
|
||||
|
||||
@@ -743,3 +777,113 @@ async def container_logs_lines(host_id: str, name: str):
|
||||
name=name, tail=_LOG_TAIL_DEFAULT,
|
||||
filtered=bool(query) or stream in ("stdout", "stderr"),
|
||||
)
|
||||
|
||||
|
||||
# ── A swarm service's own page (#5302) ───────────────────────────────────────
|
||||
|
||||
def _task_label(slot: str, task_id: str) -> str:
|
||||
"""How a task reads beside its log lines: its slot and the start of its id."""
|
||||
return f"{slot} · {task_id[:10]}"
|
||||
|
||||
|
||||
def _task_rows(tasks: list[dict], node_hostname: dict) -> list[dict]:
|
||||
"""Swarm's records of a service's newest tasks, newest first, as the page
|
||||
lists them: where each ran, what state it reached, and why it ended."""
|
||||
from .swarm_view import FAILED_TASK_STATES, task_time
|
||||
out = []
|
||||
for t in tasks:
|
||||
nid = t.get("node_id") or ""
|
||||
out.append({
|
||||
"task_id": t.get("task_id") or "", "slot": str(t.get("slot") or ""),
|
||||
"node": node_hostname.get(nid) or (nid[:12] if nid else "—"),
|
||||
"state": t.get("state") or "unknown", "desired": t.get("desired_state") or "",
|
||||
"exit_code": t.get("exit_code"), "why": t.get("error") or t.get("message") or "",
|
||||
"at": task_time(t), "failed": t.get("state") in FAILED_TASK_STATES,
|
||||
"current": t.get("state") == "running" and t.get("desired_state") == "running",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
@docker_bp.get("/service/<name>")
|
||||
@require_role(UserRole.viewer)
|
||||
async def service_detail(name: str):
|
||||
"""One swarm service: where its replicas run and what they use, its image,
|
||||
Swarm's record of its recent tasks, its CPU and memory over time across
|
||||
every replica, and the logs of every task it has run — newest first, each
|
||||
line marked with its task. `?task=` opens the logs on one task, which is
|
||||
where a failure's "Logs →" lands."""
|
||||
now = datetime.now(timezone.utc)
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
ctx = await _swarm_context(db, now)
|
||||
task_containers = await docker_svc.service_containers(db, name)
|
||||
s = next((s for s in ctx["swarm_view"]["services"] if s["name"] == name), None)
|
||||
capacity = await docker_svc.host_capacity(
|
||||
db, {r["host_id"] for r in (s["replicas"] if s else []) if not r["ghost"]})
|
||||
if s is None and not task_containers:
|
||||
return await render_template("docker/service.html", name=name, service=None), 404
|
||||
|
||||
newest = max((v for v in ctx["services"] if v.service_name == name),
|
||||
key=lambda v: v.updated_at, default=None)
|
||||
tasks = _task_rows(parse_tasks(newest) if newest is not None else [], ctx["node_hostname"])
|
||||
by_key = {(c.host_id, c.name): c for c in ctx["containers"]}
|
||||
replicas = []
|
||||
for r in (s["replicas"] if s else []):
|
||||
c = None if r["ghost"] else by_key.get((r["host_id"], r["name"]))
|
||||
replicas.append(dict(r, usage=docker_svc.usage(c, *capacity.get(r["host_id"], (None, None)))
|
||||
if c is not None else None,
|
||||
uptime=_uptime(c.started_at) if c is not None and c.status == "running" else None))
|
||||
# Every task Steward holds logs or events for, newest known first.
|
||||
known = {t["task_id"]: _task_label(t["slot"], t["task_id"]) for t in tasks if t["task_id"]}
|
||||
for _, cname in task_containers:
|
||||
slot, tid = docker_svc.task_of(name, cname)
|
||||
known.setdefault(tid, _task_label(slot, tid))
|
||||
focus = request.args.get("task", "")
|
||||
return await render_template(
|
||||
"docker/service.html", name=name, service=s,
|
||||
row=service_row(s, now) if s else None, svc=newest, tasks=tasks, replicas=replicas,
|
||||
task_choices=list(known.items()), focus_task=focus if focus in known else "",
|
||||
stack=stack_of(name), current_range=request.args.get("range", DEFAULT_RANGE),
|
||||
)
|
||||
|
||||
|
||||
@docker_bp.get("/service/<name>/history")
|
||||
@require_role(UserRole.viewer)
|
||||
async def service_history_fragment(name: str):
|
||||
"""HTMX fragment: the service's CPU (cores) and memory, summed across its
|
||||
replicas, over the selected range, with the latest and the peak."""
|
||||
from steward.core.status import sparkline_svg
|
||||
since, range_key = parse_range(request.args.get("range"))
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
series = await docker_svc.service_history(
|
||||
db, await docker_svc.service_containers(db, name), since)
|
||||
cpu = [c / 100 for _, c, _ in series]
|
||||
mem = [m for _, _, m in series]
|
||||
return await render_template(
|
||||
"docker/_service_history.html", have_data=len(series) >= 2, range_key=range_key,
|
||||
cpu_now=cpu[-1] if cpu else None, cpu_peak=max(cpu) if cpu else None,
|
||||
mem_now=mem[-1] if mem else None, mem_peak=max(mem) if mem else None,
|
||||
spark_cpu=sparkline_svg(cpu, height=48, stroke="currentColor", fill_width=True),
|
||||
spark_mem=sparkline_svg(mem, height=48, stroke="currentColor", fill_width=True),
|
||||
)
|
||||
|
||||
|
||||
@docker_bp.get("/service/<name>/logs/lines")
|
||||
@require_role(UserRole.viewer)
|
||||
async def service_logs_lines(name: str):
|
||||
"""HTMX fragment: the newest retained lines across every task of the
|
||||
service, stream, text and task filtered, each marked with its task."""
|
||||
stream = request.args.get("stream", "all")
|
||||
query = (request.args.get("q") or "").strip()
|
||||
task = request.args.get("task") or None
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
rows = await docker_svc.query_service_logs(
|
||||
db, await docker_svc.service_containers(db, name), stream, query, _LOG_TAIL_DEFAULT,
|
||||
task=task)
|
||||
lines = []
|
||||
for r in rows:
|
||||
slot, tid = docker_svc.task_of(name, r.container_name) or ("?", r.container_name)
|
||||
lines.append({"ts": r.ts, "stream": r.stream, "line": r.line, "task": _task_label(slot, tid)})
|
||||
return await render_template(
|
||||
"docker/_container_logs_lines.html", lines=lines, name=name, tail=_LOG_TAIL_DEFAULT,
|
||||
filtered=bool(query) or bool(task) or stream in ("stdout", "stderr"), service=True,
|
||||
)
|
||||
|
||||
@@ -73,7 +73,8 @@ def failures_24h(events, swarm_services, host_name: dict) -> list[dict]:
|
||||
(an agent older than 1.8.0) falls back to its tasks' deaths.
|
||||
|
||||
Each row: {kind: "container"|"service", name, host_id, host, count, at,
|
||||
detail}.
|
||||
detail, task_id}; task_id is the failed swarm task, so its logs can be
|
||||
found (None for a container, or a service without task records).
|
||||
"""
|
||||
by_prefix = {s["name"] + ".": s for s in swarm_services}
|
||||
rows: dict[tuple, dict] = {}
|
||||
@@ -97,7 +98,9 @@ def failures_24h(events, swarm_services, host_name: dict) -> list[dict]:
|
||||
if row is None:
|
||||
rows[key] = {"kind": kind, "name": name, "host_id": e.host_id,
|
||||
"host": host_name.get(e.host_id, e.host_id), "count": 1, "at": e.at,
|
||||
"detail": "OOM-killed" if e.event == "oom" else e.detail}
|
||||
"detail": "OOM-killed" if e.event == "oom" else e.detail,
|
||||
"task_id": (task_of(svc["name"], e.container_name) or (None, None))[1]
|
||||
if svc else None}
|
||||
else:
|
||||
row["count"] += 1
|
||||
from .swarm_view import task_time
|
||||
@@ -107,7 +110,8 @@ def failures_24h(events, swarm_services, host_name: dict) -> list[dict]:
|
||||
rows[("service", s["name"])] = {
|
||||
"kind": "service", "name": s["name"], "host_id": None, "host": err.get("node"),
|
||||
"count": s["failures_24h"], "at": task_time(err) if err else None,
|
||||
"detail": err.get("error") or err.get("message") or err.get("state")}
|
||||
"detail": err.get("error") or err.get("message") or err.get("state"),
|
||||
"task_id": err.get("task_id")}
|
||||
epoch = datetime.min.replace(tzinfo=timezone.utc)
|
||||
return sorted(rows.values(), key=lambda r: r["at"] or epoch, reverse=True)
|
||||
|
||||
@@ -470,6 +474,92 @@ async def retained_names(db, want: str, *, exact: bool, host_id: str | None = No
|
||||
return rows[:limit]
|
||||
|
||||
|
||||
# A swarm task's container is named "<service>.<slot>.<task id>": slot is the
|
||||
# replica number, or the node id for a global service; task ids are 25 chars.
|
||||
_TASK_SUFFIX = re.compile(r"^(?P<slot>[A-Za-z0-9]+)\.(?P<task>[a-z0-9]{25})$")
|
||||
|
||||
|
||||
def task_of(service: str, container_name: str) -> tuple[str, str] | None:
|
||||
"""(slot, task id) when `container_name` is one of `service`'s task
|
||||
containers, else None — so "arr" does not claim "arr_radarr.1.…"."""
|
||||
if not container_name.startswith(service + "."):
|
||||
return None
|
||||
m = _TASK_SUFFIX.match(container_name[len(service) + 1:])
|
||||
return (m["slot"], m["task"]) if m else None
|
||||
|
||||
|
||||
async def service_containers(db, service: str) -> list[tuple[str, str]]:
|
||||
"""(host_id, container_name) of every task container of a swarm service
|
||||
that Steward still holds anything for: the live ones, and the ones Docker
|
||||
removed that left events (each task container has a start event). The
|
||||
page's logs and history read across all of them (#5302)."""
|
||||
pattern = escape_like(service + ".") + "%"
|
||||
found: set[tuple[str, str]] = set()
|
||||
for model, col in ((DockerContainer, DockerContainer.name),
|
||||
(DockerEvent, DockerEvent.container_name)):
|
||||
rows = (await db.execute(
|
||||
select(model.host_id, col).where(col.like(pattern, escape="\\")).distinct())).all()
|
||||
found.update((h, n) for h, n in rows if task_of(service, n))
|
||||
return sorted(found)
|
||||
|
||||
|
||||
async def query_service_logs(db, containers, stream: str, query: str, limit: int,
|
||||
task: str | None = None) -> list:
|
||||
"""The newest retained lines across a service's task containers, newest
|
||||
first, each with its container name so the page can say which task wrote
|
||||
it. `task` keeps one task's lines (a failure's "Logs →")."""
|
||||
from sqlalchemy import tuple_
|
||||
pairs = [(h, n) for h, n in containers if task is None or n.endswith("." + task)]
|
||||
if not pairs:
|
||||
return []
|
||||
stmt = (select(DockerLog.ts, DockerLog.stream, DockerLog.line, DockerLog.container_name)
|
||||
.where(tuple_(DockerLog.host_id, DockerLog.container_name).in_(pairs)))
|
||||
if stream in ("stdout", "stderr"):
|
||||
stmt = stmt.where(DockerLog.stream == stream)
|
||||
if query:
|
||||
stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(query)}%", escape="\\"))
|
||||
stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
|
||||
return (await db.execute(stmt)).all()
|
||||
|
||||
|
||||
def sum_series(samples, since: datetime, until: datetime, points: int = 60) -> list:
|
||||
"""[(bucket start, cpu %, memory bytes)] summed across containers: each
|
||||
container's samples are averaged within a bucket, then the containers are
|
||||
added, so two replicas read as their combined load. `samples` are
|
||||
(container_name, at, cpu_pct, mem_bytes); buckets with no sample are left
|
||||
out rather than read as zero."""
|
||||
width = max((until - since).total_seconds() / points, 60.0)
|
||||
acc: dict[int, dict[str, list]] = {}
|
||||
for name, at, cpu, mem in samples:
|
||||
i = int((at - since).total_seconds() // width)
|
||||
if i < 0:
|
||||
continue
|
||||
per = acc.setdefault(i, {}).setdefault(name, [0.0, 0, 0])
|
||||
per[0] += cpu or 0.0
|
||||
per[1] += mem or 0
|
||||
per[2] += 1
|
||||
out = []
|
||||
for i in sorted(acc):
|
||||
cpu = sum(c / n for c, _, n in acc[i].values())
|
||||
mem = sum(m / n for _, m, n in acc[i].values())
|
||||
out.append((since + timedelta(seconds=i * width), cpu, int(mem)))
|
||||
return out
|
||||
|
||||
|
||||
async def service_history(db, containers, since: datetime) -> list:
|
||||
"""sum_series over a service's task containers' samples since `since`."""
|
||||
from sqlalchemy import tuple_
|
||||
if not containers:
|
||||
return []
|
||||
rows = (await db.execute(
|
||||
select(DockerMetric.container_name, DockerMetric.scraped_at,
|
||||
DockerMetric.cpu_pct, DockerMetric.mem_usage_bytes)
|
||||
.where(tuple_(DockerMetric.host_id, DockerMetric.container_name).in_(list(containers)),
|
||||
DockerMetric.scraped_at >= since)
|
||||
.order_by(DockerMetric.scraped_at))).all()
|
||||
return sum_series(rows, since, datetime.now(timezone.utc))
|
||||
|
||||
|
||||
async def unhealthy_since(db, keys) -> dict[tuple[str, str], datetime]:
|
||||
"""{(host_id, container_name): when it last turned unhealthy} for the given
|
||||
containers, from their health_change events; absent when none is retained."""
|
||||
|
||||
@@ -94,7 +94,8 @@ def last_task_error(tasks: list[dict]) -> dict | None:
|
||||
if t.get("state") in FAILED_TASK_STATES or t.get("error"):
|
||||
return {"state": t.get("state"), "error": t.get("error") or None,
|
||||
"message": t.get("message") or None, "exit_code": t.get("exit_code"),
|
||||
"node_id": t.get("node_id") or None, "at": t.get("at") or None}
|
||||
"node_id": t.get("node_id") or None, "at": t.get("at") or None,
|
||||
"task_id": t.get("task_id") or None, "slot": t.get("slot")}
|
||||
return None
|
||||
|
||||
|
||||
@@ -233,11 +234,12 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
|
||||
*, now: datetime | None = None, crash_loop_deaths: int = 3) -> dict:
|
||||
"""Return {"services": [...], "standalone": [...], "superseded": {(host_id, name)}}.
|
||||
|
||||
services[i] = {name, mode, desired, running, state, replicas:[...],
|
||||
services[i] = {name, mode, image, desired, running, state, replicas:[...],
|
||||
previous_tasks, tasks_known, failures_1h, failures_24h,
|
||||
crash_looping, last_error}
|
||||
real replica = {ghost:False, host, host_id, name, status, cpu_pct, mem_pct,
|
||||
health, restart_count} # current tasks only
|
||||
mem_bytes, started_at, image, health, restart_count}
|
||||
# current tasks only
|
||||
ghost replica = {ghost:True, host, count} # placement with no local row
|
||||
previous_tasks = task containers Swarm replaced, left out of replicas
|
||||
failures_* = tasks that failed or were rejected in the window (None
|
||||
@@ -288,6 +290,9 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
|
||||
"status": getattr(c, "status", "unknown"),
|
||||
"cpu_pct": getattr(c, "cpu_pct", None),
|
||||
"mem_pct": getattr(c, "mem_pct", None),
|
||||
"mem_bytes": getattr(c, "mem_usage_bytes", None),
|
||||
"started_at": getattr(c, "started_at", None),
|
||||
"image": getattr(c, "image", None) or None,
|
||||
"health": getattr(c, "health", None),
|
||||
"restart_count": getattr(c, "restart_count", 0) or 0,
|
||||
})
|
||||
@@ -329,6 +334,7 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
|
||||
out_services.append({
|
||||
"name": name,
|
||||
"mode": getattr(svc, "mode", "replicated") if svc is not None else "replicated",
|
||||
"image": getattr(svc, "image", None) if svc is not None else None,
|
||||
"running": running,
|
||||
"desired": desired,
|
||||
"state": state,
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
{# docker/_container_logs_lines.html — log-line fragment, HTMX-polled (m79).
|
||||
{{ l.line }} is auto-escaped by Jinja, so log content can't inject markup. #}
|
||||
{{ l.line }} is auto-escaped by Jinja, so log content can't inject markup.
|
||||
A service's lines (#5302) carry the task each came from. #}
|
||||
{% if lines %}
|
||||
{% for l in lines %}
|
||||
<div style="display:flex;gap:0.6rem;white-space:pre-wrap;word-break:break-word;">
|
||||
<span style="color:var(--text-dim);flex-shrink:0;" title="{{ l.ts }}">{{ l.ts.strftime("%m-%d %H:%M:%S") }}</span>
|
||||
{% if l.task %}<span class="tone-muted" style="flex-shrink:0;white-space:nowrap;" title="Slot and task id">{{ l.task }}</span>{% endif %}
|
||||
<span style="flex-shrink:0;width:3.2rem;color:{% if l.stream == 'stderr' %}var(--red){% else %}var(--text-muted){% endif %};">{{ l.stream }}</span>
|
||||
<span style="flex:1;min-width:0;{% if l.stream == 'stderr' %}color:var(--text);{% endif %}">{{ l.line }}</span>
|
||||
</div>
|
||||
@@ -14,7 +16,7 @@
|
||||
No lines match the current filter.
|
||||
{% else %}
|
||||
No logs collected yet for <code>{{ name }}</code>. Lines appear within a few
|
||||
seconds of the container writing to stdout/stderr — unless it's on the log
|
||||
seconds of {{ "one of its tasks" if service else "the container" }} writing to stdout/stderr — unless it's on the log
|
||||
exclude list or log collection is turned off in
|
||||
<a href="/settings/thresholds/">Settings → Thresholds & Retention</a>.
|
||||
{% endif %}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
{# docker/_service_history.html — a service's CPU and memory summed across its
|
||||
replicas over the selected range (#5302): the latest and the peak, over a
|
||||
line that spans its half. #}
|
||||
{% from "docker/_usage.html" import cores %}
|
||||
{% if have_data %}
|
||||
<div class="history-pair">
|
||||
<div>
|
||||
<div class="section-title">CPU</div>
|
||||
<div style="font-variant-numeric:tabular-nums;">{{ cores(cpu_now) }} <span class="tone-dim">now · peak {{ cores(cpu_peak) }}</span></div>
|
||||
<div class="spark">{{ spark_cpu|safe }}</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="section-title">Memory</div>
|
||||
<div style="font-variant-numeric:tabular-nums;">{{ mem_now|bytes }} <span class="tone-dim">now · peak {{ mem_peak|bytes }}</span></div>
|
||||
<div class="spark">{{ spark_mem|safe }}</div>
|
||||
</div>
|
||||
</div>
|
||||
{% else %}
|
||||
<p class="tone-muted" style="margin:0;font-size:0.85rem;">
|
||||
Not enough samples in the last {{ range_key }} yet. History fills in as the agents report its replicas.
|
||||
</p>
|
||||
{% endif %}
|
||||
@@ -1,5 +1,8 @@
|
||||
{% extends "base.html" %}
|
||||
{% from "_macros.html" import page_header %}
|
||||
{# The Docker page. The filter bar sits outside the polled fragment, so a poll
|
||||
never resets what is being typed; the fragment reads the bar on every poll,
|
||||
and a change re-fetches it and puts the filters in the URL (#5299). #}
|
||||
{% block title %}Docker — Steward{% endblock %}
|
||||
{% block content %}
|
||||
{% call page_header("Docker", "What needs attention first, then every service and container.") %}
|
||||
@@ -7,17 +10,53 @@
|
||||
{% if has_disk %}<a class="btn btn-sm btn-ghost" href="/plugins/docker/disk">Disk</a>{% endif %}
|
||||
{% endcall %}
|
||||
|
||||
<form id="docker-filters" class="filter-bar" method="get" action="/plugins/docker/" role="search">
|
||||
<input type="search" id="docker-q" name="q" value="{{ f.q }}" class="input"
|
||||
placeholder="Search name, image or stack" aria-label="Search services and containers" autocomplete="off">
|
||||
{% if choices.stacks %}
|
||||
<select name="stack" aria-label="Stack">
|
||||
<option value="">All stacks</option>
|
||||
{% for s in choices.stacks %}<option value="{{ s }}"{% if f.stack == s %} selected{% endif %}>{{ s }}</option>{% endfor %}
|
||||
<option value="{{ no_stack }}"{% if f.stack == no_stack %} selected{% endif %}>No stack</option>
|
||||
</select>
|
||||
{% endif %}
|
||||
{% if choices.hosts|length > 1 %}
|
||||
<select name="host" aria-label="Host or node">
|
||||
<option value="">All hosts</option>
|
||||
{% for h in choices.hosts %}<option value="{{ h }}"{% if f.host == h %} selected{% endif %}>{{ h }}</option>{% endfor %}
|
||||
</select>
|
||||
{% endif %}
|
||||
<select name="state" aria-label="Status">
|
||||
<option value="">Any status</option>
|
||||
{% for key, label in states.items() %}<option value="{{ key }}"{% if f.state == key %} selected{% endif %}>{{ label }}</option>{% endfor %}
|
||||
</select>
|
||||
{% if f.sort != 'status' %}<input type="hidden" name="sort" value="{{ f.sort }}"><input type="hidden" name="dir" value="{{ f.dir }}">{% endif %}
|
||||
<noscript><button type="submit" class="btn btn-sm">Filter</button></noscript>
|
||||
</form>
|
||||
|
||||
<div id="docker-rows"
|
||||
hx-get="/plugins/docker/rows"
|
||||
hx-trigger="load, every {{ poll_interval }}s"
|
||||
hx-include="#docker-filters"
|
||||
hx-trigger="load, every {{ poll_interval }}s, change from:#docker-filters, input changed delay:250ms from:#docker-q, search from:#docker-q"
|
||||
hx-swap="innerHTML">
|
||||
<div class="card"><p class="field-hint">Loading…</p></div>
|
||||
</div>
|
||||
{# A link to a section (the dashboard's "failed (24h)" → #failures) lands
|
||||
before the rows exist; scroll to it once, when they first arrive. #}
|
||||
<script>
|
||||
(function () {
|
||||
var rows = document.getElementById('docker-rows');
|
||||
var form = document.getElementById('docker-filters');
|
||||
// The filters live in the URL too, so a filtered view can be bookmarked and
|
||||
// Back returns to it.
|
||||
function keepInUrl() {
|
||||
var q = new URLSearchParams(new FormData(form));
|
||||
Array.from(q.keys()).forEach(function (k) { if (!q.get(k)) q.delete(k); });
|
||||
var s = q.toString();
|
||||
history.replaceState(null, '', location.pathname + (s ? '?' + s : '') + location.hash);
|
||||
}
|
||||
form.addEventListener('change', keepInUrl);
|
||||
form.addEventListener('input', keepInUrl);
|
||||
// A link to a section (the dashboard's "failed (24h)" → #failures) lands
|
||||
// before the rows exist; scroll to it once, when they first arrive.
|
||||
if (!location.hash || !rows) return;
|
||||
rows.addEventListener('htmx:afterSwap', function once() {
|
||||
rows.removeEventListener('htmx:afterSwap', once);
|
||||
|
||||
@@ -1,8 +1,36 @@
|
||||
{# docker/rows.html — HTMX fragment for the Docker page: what needs attention
|
||||
first (service.needs_attention, the ranking the MCP tools use), the counts,
|
||||
then swarm services and each host's containers. #}
|
||||
{% from "_macros.html" import stat_tile, status_badge, status_dot %}
|
||||
{# docker/rows.html — HTMX fragment for the Docker page (#5298–#5301):
|
||||
a verdict line with only the problem counts that aren't zero; what needs
|
||||
attention (service.needs_attention, the ranking the MCP tools use); hosts
|
||||
whose Docker is not answering; what failed in 24h, each with its logs; then
|
||||
the swarm services and the standalone containers as two tables that share
|
||||
the filter bar's search, filters and sort (overview.py). #}
|
||||
{% from "_macros.html" import stat_tile, status_badge, status_dot, sort_th %}
|
||||
{% from "docker/_usage.html" import cpu, mem, cores %}
|
||||
{% macro th(key, label, cls=none, title=none) -%}
|
||||
{{ sort_th(label, sort_url(key), active=f.sort == key, desc=f.dir == 'desc', cls=cls, title=title) }}
|
||||
{%- endmacro %}
|
||||
{% macro up(row) -%}
|
||||
{%- if row.started_at %}<span title="Started {{ row.started_at.strftime('%Y-%m-%d %H:%M UTC') }}">{{ row.started_at|ago|striptags|replace(" ago", "") }}</span>
|
||||
{%- else %}<span class="tone-dim">—</span>{% endif -%}
|
||||
{%- endmacro %}
|
||||
|
||||
{# ── Verdict ──────────────────────────────────────────────────────────────── #}
|
||||
{% set problems = attention|length + failing_services|length %}
|
||||
<p class="docker-verdict">
|
||||
{% if problems %}<strong class="tone-crit">{{ problems }} need{{ "s" if problems == 1 }} attention</strong>
|
||||
{% else %}<strong class="tone-ok">Nothing needs attention</strong>{% endif %}
|
||||
{%- if services_total %} · <span class="tone-{{ 'ok' if services_healthy == services_total else 'warn' }}">{{ services_healthy }}/{{ services_total }}</span> services healthy{% endif %}
|
||||
{%- if containers_total %} · {{ running }} container{{ "s" if running != 1 }} running{% if stopped %}, {{ stopped }} stopped{% endif %}{% if services_total %} outside services{% endif %}{% endif %}
|
||||
{%- if docker_hosts %} · {{ docker_hosts }} Docker host{{ "s" if docker_hosts != 1 }}{% endif %}
|
||||
</p>
|
||||
{% if unhealthy or restarting or failures or stale %}
|
||||
<div class="stat-strip">
|
||||
{% if unhealthy %}{{ stat_tile("Unhealthy", unhealthy, state="crit") }}{% endif %}
|
||||
{% if restarting %}{{ stat_tile("Restarting", restarting, state="crit", hint="restarting or crash-looping") }}{% endif %}
|
||||
{% if failures %}{{ stat_tile("Failed (24h)", failures|length, state="warn", hint="listed below, with their logs", href="#failures") }}{% endif %}
|
||||
{% if stale %}{{ stat_tile("Stale", stale, state="stale", hint="no recent report") }}{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{# ── Needs attention ──────────────────────────────────────────────────────── #}
|
||||
{% if attention or failing_services %}
|
||||
@@ -17,13 +45,14 @@
|
||||
{% if s.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}
|
||||
{% elif s.state == 'down' %}{{ status_badge('crit', 'Down') }}
|
||||
{% else %}{{ status_badge('warn', 'Degraded') }}{% endif %}
|
||||
<a href="/plugins/docker/swarm">{{ s.name }}</a>
|
||||
<a href="/plugins/docker/service/{{ s.name }}">{{ s.name }}</a>
|
||||
<span class="problem-detail">
|
||||
{{- s.running }} of {{ s.desired }} running
|
||||
{%- if s.crash_looping %} · {{ s.failures_1h }} tasks failed in the last hour{% endif %}
|
||||
{%- if e %} · last task {{ e.state }}: {{ e.error or e.message or e.state }}{% if e.exit_code %} (exit {{ e.exit_code }}){% endif %}{% if e.node %} on {{ e.node }}{% endif %}{% endif %}
|
||||
{%- if s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif -%}
|
||||
</span>
|
||||
<a class="field-hint" href="/plugins/docker/service/{{ s.name }}{% if e and e.task_id %}?task={{ e.task_id }}{% endif %}#logs" style="margin-left:0.4rem;">Logs →</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
{% for r in attention %}
|
||||
@@ -38,6 +67,7 @@
|
||||
{%- if r.since %} · since {{ r.since|ago }}{% endif %}
|
||||
{%- if r.stale %} · {{ status_badge('stale') }} last report {{ c.scraped_at|ago if c.scraped_at else "never" }}{% endif -%}
|
||||
</span>
|
||||
<a class="field-hint" href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}/logs" style="margin-left:0.4rem;">Logs →</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
@@ -60,170 +90,144 @@
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{# ── Counts ──────────────────────────────────────────────────────────────
|
||||
Swarm services are counted as services (a task's container comes and goes);
|
||||
the container counts are the containers outside any swarm service. ────── #}
|
||||
<div class="stat-strip">
|
||||
{{ stat_tile("Needs attention", attention|length + failing_services|length, state="crit" if attention or failing_services else "inactive") }}
|
||||
{{ stat_tile("Unhealthy", unhealthy, state="crit" if unhealthy else "inactive") }}
|
||||
{{ stat_tile("Restarting", restarting, state="crit" if restarting else "inactive", hint="restarting or crash-looping") }}
|
||||
{{ stat_tile("Failed (24h)", failures|length, state="warn" if failures else "inactive", hint="containers and services; listed below", href="#failures" if failures else none) }}
|
||||
{% if swarm_services %}{{ stat_tile("Swarm services", services_healthy ~ "/" ~ swarm_services|length, state="up" if services_healthy == swarm_services|length else "warn", hint="healthy", href="/plugins/docker/swarm") }}{% endif %}
|
||||
{{ stat_tile("Containers running", running, state="up" if running else "inactive", hint="outside swarm services" if swarm_services else none) }}
|
||||
{{ stat_tile("Containers stopped", stopped, state="inactive") }}
|
||||
{% if stale %}{{ stat_tile("Stale", stale, state="stale", hint="no recent report") }}{% endif %}
|
||||
{{ stat_tile("Docker hosts", docker_hosts) }}
|
||||
</div>
|
||||
|
||||
{# ── What failed in the last 24h: what the dashboard's "failed (24h)" counts.
|
||||
A plain container's death, or a swarm service's failed tasks; a container
|
||||
someone stopped (exit 137/143) is not a failure. ───────────────────────── #}
|
||||
someone stopped (exit 137/143) is not a failure. Each links to the logs of
|
||||
what died: a service's to its failed task's lines (#5300). ───────────── #}
|
||||
{% if failures %}
|
||||
<section id="failures" class="card" aria-labelledby="docker-failures" style="margin-bottom:1.5rem;">
|
||||
<section id="failures" class="card" aria-labelledby="docker-failures">
|
||||
<h2 id="docker-failures" class="section-title" style="margin-bottom:0.6rem;">Failed in the last 24h</h2>
|
||||
<ul class="problem-list">
|
||||
{% for f in failures %}
|
||||
{% for x in failures %}
|
||||
<li>
|
||||
{{ status_badge('warn', 'Service' if f.kind == 'service' else 'Container') }}
|
||||
{% if f.kind == 'service' %}<a href="/plugins/docker/swarm">{{ f.name }}</a>
|
||||
{% else %}<a href="/plugins/docker/container/{{ f.host_id }}/{{ f.name }}">{{ f.name }}</a>{% endif %}
|
||||
{{ status_badge('warn', 'Service' if x.kind == 'service' else 'Container') }}
|
||||
{% if x.kind == 'service' %}<a href="/plugins/docker/service/{{ x.name }}">{{ x.name }}</a>
|
||||
{% else %}<a href="/plugins/docker/container/{{ x.host_id }}/{{ x.name }}">{{ x.name }}</a>{% endif %}
|
||||
<span class="problem-detail">
|
||||
{{- f.count }} {{ "failure" if f.count == 1 else "failures" }}
|
||||
{%- if f.detail %} · last: {{ f.detail }}{% endif %}
|
||||
{%- if f.host %} · {{ f.host }}{% endif %}
|
||||
{%- if f.at %} · {{ f.at|ago }}{% endif -%}
|
||||
{{- x.count }} {{ "failure" if x.count == 1 else "failures" }}
|
||||
{%- if x.detail %} · last: {{ x.detail }}{% endif %}
|
||||
{%- if x.host %} · {{ x.host }}{% endif %}
|
||||
{%- if x.at %} · {{ x.at|ago }}{% endif -%}
|
||||
</span>
|
||||
{% if x.kind == 'service' %}
|
||||
<a class="field-hint" href="/plugins/docker/service/{{ x.name }}{% if x.task_id %}?task={{ x.task_id }}{% endif %}#logs" style="margin-left:0.4rem;">Logs{% if x.task_id %} of that task{% endif %} →</a>
|
||||
{% else %}
|
||||
<a class="field-hint" href="/plugins/docker/container/{{ x.host_id }}/{{ x.name }}/logs" style="margin-left:0.4rem;">Logs →</a>
|
||||
{% endif %}
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{# ── Swarm services: collapsed per-service, each current replica labelled with
|
||||
its host (tasks Swarm replaced are left out). Ghost replicas (tasks on nodes
|
||||
with no Steward agent) come from the managers' placement data so the picture
|
||||
is cluster-complete. A failing service names Swarm's last task error. ───── #}
|
||||
{% if swarm_services %}
|
||||
<div class="card-flush" style="margin-bottom:1.5rem;">
|
||||
<div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
|
||||
<span style="font-weight:500;font-size:0.95rem;">Swarm services</span>
|
||||
<a href="/plugins/docker/swarm" style="font-size:0.78rem;color:var(--text-muted);">Topology →</a>
|
||||
</div>
|
||||
<div style="padding:0.5rem 0.75rem;display:grid;gap:0.85rem;">
|
||||
{% for s in swarm_services %}
|
||||
<div>
|
||||
<div style="display:flex;align-items:center;gap:0.55rem;flex-wrap:wrap;margin-bottom:0.35rem;">
|
||||
<span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
|
||||
<span style="font-weight:500;font-size:0.9rem;">{{ s.name }}</span>
|
||||
<span style="font-size:0.72rem;color:var(--text-dim);">{{ s.mode }}</span>
|
||||
<span class="tone-{{ 'ok' if s.state == 'healthy' else ('warn' if s.state == 'degraded' else 'crit') }}" style="font-size:0.78rem;font-variant-numeric:tabular-nums;">
|
||||
{{ s.running }}/{{ s.desired }} running{% if s.state != 'healthy' and s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif %}
|
||||
</span>
|
||||
</div>
|
||||
{% with style="padding-left:1.1rem;margin-bottom:0.35rem;" %}{% include "docker/_swarm_failures.html" %}{% endwith %}
|
||||
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(210px,1fr));gap:0.4rem;padding-left:1.1rem;">
|
||||
{% for r in s.replicas %}
|
||||
{% if r.ghost %}
|
||||
<div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:5px;padding:0.3rem 0.55rem;" title="Running on a node with no Steward agent — detail unavailable">
|
||||
<span class="dot dot-warn" style="opacity:0.6;"></span>
|
||||
<span style="font-weight:500;">{{ r.host }}</span>
|
||||
<span style="color:var(--text-dim);margin-left:auto;">{{ r.count }} · no agent</span>
|
||||
</div>
|
||||
{% else %}
|
||||
<div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px solid var(--border);border-radius:5px;padding:0.3rem 0.55rem;">
|
||||
<span class="dot {% if r.status == 'running' %}dot-up{% elif r.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
|
||||
<a href="/plugins/docker/container/{{ r.host_id }}/{{ r.name }}" style="font-weight:500;color:inherit;text-decoration:none;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;" title="{{ r.name }}">{{ r.host }}</a>
|
||||
{% if r.cpu_pct is not none %}<span style="color:var(--text-muted);font-family:ui-monospace,monospace;margin-left:auto;" title="Docker counts CPU per core">{{ cores(r.cpu_pct / 100) }}</span>{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{# ── Container tables, one per host ───────────────────────────────────────── #}
|
||||
{% if host_groups %}
|
||||
{% for g in host_groups %}
|
||||
<div class="card-flush" style="margin-bottom:1.5rem;">
|
||||
<div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
|
||||
{% if g.host %}
|
||||
<a href="/hosts/{{ g.host.id }}" style="font-weight:500;font-size:0.95rem;color:inherit;text-decoration:none;">{{ g.host.name }}</a>
|
||||
{% else %}
|
||||
<span style="font-weight:500;font-size:0.95rem;">{{ g.host_name }}</span>
|
||||
{% endif %}
|
||||
<span style="font-size:0.78rem;color:var(--text-muted);">
|
||||
{{ g.running }} running{% if g.stopped %} · {{ g.stopped }} stopped{% endif %}
|
||||
</span>
|
||||
</div>
|
||||
{# ── Swarm services: one row each, problems first unless a header picked
|
||||
another order; CPU and memory add up the running replicas (#5299). ───── #}
|
||||
{% if services %}
|
||||
<section class="card-flush" aria-labelledby="docker-services">
|
||||
<h2 id="docker-services" class="table-caption">Swarm services <span class="tone-muted">{{ services|length }}{% if services|length != services_total %} of {{ services_total }}{% endif %}</span>
|
||||
<a href="/plugins/docker/swarm" class="field-hint">By node →</a></h2>
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Container</th>
|
||||
<th>Image</th>
|
||||
<th>Ports</th>
|
||||
<th title="Docker counts CPU per core: 1 core = one core fully busy">CPU</th>
|
||||
<th title="Used, and the limit when one is set">Memory</th>
|
||||
{{ th("name", "Service") }}
|
||||
{{ th("status", "Status", title="Problems first: a crash loop, then down, then short of replicas, then failed tasks in 24h") }}
|
||||
{{ th("stack", "Stack", title="The part of the service's name before the first _: the stack it was deployed in") }}
|
||||
<th title="The nodes running its replicas">Nodes</th>
|
||||
{{ th("cpu", "CPU", "num", "Across its running replicas; Docker counts CPU per core") }}
|
||||
{{ th("mem", "Memory", "num", "Across its running replicas") }}
|
||||
{{ th("up", "Up", "num", "Since its newest replica started") }}
|
||||
<th class="num" title="Tasks that failed in the last 24h; Swarm replaced them">Failed 24h</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for sub in g.subgroups %}
|
||||
{% if g.grouped and sub.label %}
|
||||
<tr><td colspan="5" style="padding:0.4rem 0.75rem 0.2rem;font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;font-weight:500;">{{ sub.label }}</td></tr>
|
||||
{% endif %}
|
||||
{% for item in sub.containers %}
|
||||
{% set c = item.container %}
|
||||
{% for r in services %}
|
||||
<tr>
|
||||
<td class="cell-name"><a href="/plugins/docker/service/{{ r.name }}" title="{{ r.name }}{% if r.image %} · {{ r.image }}{% endif %}">{{ r.name }}</a></td>
|
||||
<td>
|
||||
<div style="display:flex;align-items:center;gap:0.5rem;">
|
||||
{{ status_dot('stale' if item.stale else ('up' if c.status == 'running' else ('pending' if c.status == 'paused' else 'down')), 'Stale' if item.stale else c.status|capitalize) }}
|
||||
<div>
|
||||
<div style="font-weight:500;font-size:0.9rem;">
|
||||
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="color:inherit;text-decoration:none;border-bottom:1px dotted var(--border-mid);">{{ c.name }}</a>
|
||||
{% if c.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}
|
||||
{% elif c.health == 'starting' %} {{ status_badge('pending', 'Starting') }}{% endif %}
|
||||
{% if item.stale %} {{ status_badge('stale', title="Last report " ~ (c.scraped_at.strftime('%Y-%m-%d %H:%M UTC') if c.scraped_at else 'never') ~ "; what is shown may no longer be true") }}{% endif %}
|
||||
</div>
|
||||
<div style="font-size:0.73rem;color:var(--text-muted);">
|
||||
{{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %}
|
||||
{% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
|
||||
· <span class="tone-{{ 'crit' if item.exit_failed else 'dim' }}" title="{{ 'last exit code' if item.exit_failed else 'stopped by a signal; not a failure' }}">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
|
||||
{% endif %}
|
||||
{% if c.restart_count %} · <span class="tone-warn" title="Restarted {{ c.restart_count }} times">⟳{{ c.restart_count }}</span>{% endif %}
|
||||
</div>
|
||||
{% if c.service_name or c.compose_project %}
|
||||
<div style="font-size:0.68rem;color:var(--text-dim);margin-top:0.1rem;">
|
||||
{{ c.service_name or c.compose_project }}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{{ status_dot('up' if r.state == 'healthy' else ('warn' if r.state == 'degraded' else 'down')) }}
|
||||
<span class="tone-{{ 'ok' if r.state == 'healthy' else ('warn' if r.state == 'degraded' else 'crit') }}" style="font-variant-numeric:tabular-nums;">{{ r.running }}/{{ r.desired }}</span>
|
||||
{%- if r.crash_looping %} {{ status_badge('crit', 'Crash-looping') }}{% elif r.unhealthy %} {{ status_badge('crit', 'Unhealthy') }}{% endif %}
|
||||
{%- if r.mode == 'global' %} <span class="tone-dim">global</span>{% endif %}
|
||||
</td>
|
||||
<td style="font-size:0.82rem;color:var(--text-muted);max-width:200px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
|
||||
{{ c.image }}
|
||||
<td>{% if r.stack %}<a class="chip" href="{{ page_url(stack=r.stack) }}" title="Show the {{ r.stack }} stack">{{ r.stack }}</a>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
|
||||
<td class="tone-muted">
|
||||
{{- r.nodes|join(", ") or ("" if r.ghost_nodes else "—") -}}
|
||||
{%- if r.ghost_nodes %}{{ ", " if r.nodes }}<span title="Running on a node with no Steward agent: no figures for it">{{ r.ghost_nodes|join(", ") }} (no agent)</span>{% endif -%}
|
||||
</td>
|
||||
<td style="font-size:0.78rem;font-family:ui-monospace,monospace;">
|
||||
{% for p in item.ports %}
|
||||
<div style="color:var(--text-muted);">{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}</div>
|
||||
{% endfor %}
|
||||
<td class="num">{% if r.cpu_cores is not none %}{{ cores(r.cpu_cores) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
|
||||
<td class="num">{% if r.mem_bytes is not none %}{{ r.mem_bytes|bytes }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
|
||||
<td class="num">{{ up(r) }}</td>
|
||||
<td class="num">
|
||||
{%- if r.failures_24h %}<a class="tone-warn" href="/plugins/docker/service/{{ r.name }}#tasks"
|
||||
title="{% if r.last_error %}Last: {{ r.last_error.error or r.last_error.message or r.last_error.state }}{% if r.last_error.exit_code %} (exit {{ r.last_error.exit_code }}){% endif %}{% endif %}">{{ r.failures_24h }}</a>
|
||||
{%- else %}<span class="tone-dim">—</span>{% endif -%}
|
||||
</td>
|
||||
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;white-space:nowrap;">{{ cpu(item.usage) }}</td>
|
||||
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;white-space:nowrap;">{{ mem(item.usage) }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% elif not swarm_services %}
|
||||
<div class="card" style="text-align:center;padding:3rem;">
|
||||
<div style="color:var(--text-muted);font-size:0.9rem;">
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{# ── Containers outside any swarm service, every host in one table (#5301);
|
||||
a compose project is its stack. ─────────────────────────────────────── #}
|
||||
{% if containers %}
|
||||
<section class="card-flush" aria-labelledby="docker-containers">
|
||||
<h2 id="docker-containers" class="table-caption">{{ "Other containers" if services_total else "Containers" }} <span class="tone-muted">{{ containers|length }}{% if containers|length != containers_total %} of {{ containers_total }}{% endif %}</span></h2>
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
{{ th("name", "Container") }}
|
||||
{{ th("status", "Status", title="Problems first: unhealthy or restarting, then died with a failing exit, then stale; then running, then stopped") }}
|
||||
<th>Host</th>
|
||||
{{ th("stack", "Stack", title="Its compose project") }}
|
||||
<th>Image</th>
|
||||
{{ th("cpu", "CPU", "num", "Docker counts CPU per core: 1 core = one core fully busy") }}
|
||||
{{ th("mem", "Memory", "num", "Used, and the limit when one is set") }}
|
||||
{{ th("up", "Up", "num", "Since it started") }}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for r in containers %}
|
||||
<tr>
|
||||
<td class="cell-name">
|
||||
<a href="/plugins/docker/container/{{ r.host_id }}/{{ r.container_name }}" title="{{ r.container_name }}">{{ r.name }}</a>
|
||||
{%- if r.ports %}<div class="cell-sub mono">{% for p in r.ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{{ " " if not loop.last }}{% endfor %}</div>{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{{ status_dot('stale' if r.stale else ('up' if r.running else ('pending' if r.status == 'paused' else 'down'))) }}
|
||||
{%- if r.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}
|
||||
{%- elif r.health == 'starting' %} {{ status_badge('pending', 'Starting') }}
|
||||
{%- else %} <span class="tone-muted">{{ r.status }}</span>{% endif %}
|
||||
{%- if not r.running and r.exit_code is not none and r.exit_code != 0 %}
|
||||
<span class="tone-{{ 'crit' if r.exit_failed else 'dim' }}" title="{{ 'last exit code' if r.exit_failed else 'stopped by a signal; not a failure' }}">exit {{ r.exit_code }}{% if r.oom_killed %} (OOM){% endif %}</span>
|
||||
{%- endif %}
|
||||
{%- if r.restart_count %} <span class="tone-warn" title="Restarted {{ r.restart_count }} times">⟳{{ r.restart_count }}</span>{% endif %}
|
||||
{%- if r.stale %} {{ status_badge('stale', title="Last report " ~ (r.scraped_at.strftime('%Y-%m-%d %H:%M UTC') if r.scraped_at else 'never') ~ "; what is shown may no longer be true") }}{% endif %}
|
||||
</td>
|
||||
<td class="tone-muted">{{ r.host }}</td>
|
||||
<td>{% if r.stack %}<a class="chip" href="{{ page_url(stack=r.stack) }}" title="Show the {{ r.stack }} stack">{{ r.stack }}</a>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
|
||||
<td class="cell-image" title="{{ r.image }}">{{ r.image }}</td>
|
||||
<td class="num">{% if r.running %}{{ cpu(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
|
||||
<td class="num">{% if r.running %}{{ mem(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
|
||||
<td class="num">{{ up(r) }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{% if not services and not containers %}
|
||||
<div class="card">
|
||||
{% if services_total or containers_total %}
|
||||
<p class="tone-muted" style="margin:0;">Nothing matches these filters. <a href="/plugins/docker/">Show everything</a></p>
|
||||
{% else %}
|
||||
<p class="tone-muted" style="margin:0;">
|
||||
No containers reported yet. Containers are collected by the Steward host agent —
|
||||
deploy the agent to a host running Docker (Hosts → the host → agent panel) and
|
||||
its containers will appear here under that host.
|
||||
</div>
|
||||
its containers will appear here.
|
||||
</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
{# docker/service.html — one swarm service (#5302): its state and what it
|
||||
runs, where its replicas are and what they use, Swarm's record of its recent
|
||||
tasks, its CPU and memory over time, and the logs of every task it has run.
|
||||
A failure's "Logs →" lands on #logs with ?task= picked. #}
|
||||
{% extends "base.html" %}
|
||||
{% from "_macros.html" import crumbs, status_badge, status_dot, stat_tile %}
|
||||
{% from "docker/_usage.html" import cpu, mem, cores %}
|
||||
{% block title %}{{ name }} — Docker — Steward{% endblock %}
|
||||
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %}
|
||||
{% block content %}
|
||||
|
||||
{% if service is none %}
|
||||
<div class="card empty-state">
|
||||
<strong>{{ "No longer running" if task_choices else "Service not found" }}</strong>
|
||||
{% if task_choices %}
|
||||
<span>No swarm service named <code>{{ name }}</code> is reported now; it may have been removed.
|
||||
The logs Steward still keeps from its tasks are below.</span>
|
||||
{% else %}
|
||||
<span>No swarm service named <code>{{ name }}</code> is reported, and Steward holds no logs or
|
||||
events for one. It may have been removed. <a href="/plugins/docker/">Back to Docker</a></span>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% else %}
|
||||
|
||||
{# ── Header: is it well, then what it is ─────────────────────────────────── #}
|
||||
{% set tone = 'ok' if row.state == 'healthy' else ('warn' if row.state == 'degraded' else 'crit') %}
|
||||
<div class="page-header">
|
||||
<div>
|
||||
<h1 class="page-title" style="overflow-wrap:anywhere;">{{ name }}</h1>
|
||||
<p class="page-subtitle">
|
||||
{% if row.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}
|
||||
{% elif row.state == 'down' %}{{ status_badge('crit', 'Down') }}
|
||||
{% elif row.state == 'degraded' %}{{ status_badge('warn', 'Degraded') }}
|
||||
{% else %}{{ status_badge('ok', 'Healthy') }}{% endif %}
|
||||
<span class="tone-{{ tone }}">{{ row.running }} of {{ row.desired }} running</span>
|
||||
· {{ row.mode }}
|
||||
{%- if stack %} · stack <a class="chip" href="/plugins/docker/?stack={{ stack }}">{{ stack }}</a>{% endif %}
|
||||
{%- if row.trouble_since %} · in trouble since {{ row.trouble_since|ago }}{% endif %}
|
||||
</p>
|
||||
</div>
|
||||
<div class="page-actions"><a class="btn btn-sm btn-ghost" href="#logs">Logs</a></div>
|
||||
</div>
|
||||
|
||||
{% if row.last_error %}
|
||||
{% set e = row.last_error %}
|
||||
<section class="card problem-card" aria-labelledby="svc-last-error">
|
||||
<h2 id="svc-last-error" class="section-title" style="margin-bottom:0.4rem;">Last failed task</h2>
|
||||
<p style="margin:0;">
|
||||
{{ e.error or e.message or e.state }}{% if e.exit_code %} <span class="tone-crit">(exit {{ e.exit_code }})</span>{% endif %}
|
||||
{%- if e.node %} · on {{ e.node }}{% endif %}{% if e.age_s is not none %} · {{ e.age_s|duration }} ago{% endif %}
|
||||
{%- if e.task_id %} · <a href="?task={{ e.task_id }}#logs">its logs →</a>{% endif %}
|
||||
</p>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<div class="stat-strip">
|
||||
{{ stat_tile("CPU", cores(row.cpu_cores) if row.cpu_cores is not none else "—", hint="across running replicas") }}
|
||||
{{ stat_tile("Memory", row.mem_bytes|bytes if row.mem_bytes is not none else "—", hint="across running replicas") }}
|
||||
{{ stat_tile("Up", (row.started_at|ago|striptags|replace(" ago", "")) if row.started_at else "—", hint="since its newest replica started") }}
|
||||
{{ stat_tile("Failed (24h)", row.failures_24h or 0, state="warn" if row.failures_24h else none, hint="tasks Swarm replaced", href="#tasks" if row.failures_24h else none) }}
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<dl class="kv">
|
||||
<dt>Image</dt><dd class="mono" style="overflow-wrap:anywhere;">{{ (svc.image if svc and svc.image else row.image) or "—" }}</dd>
|
||||
<dt>Mode</dt><dd>{{ row.mode }}{% if row.mode != 'global' %}, {{ row.desired }} replica{{ "s" if row.desired != 1 }}{% endif %}</dd>
|
||||
{% if svc %}<dt>Reported</dt><dd>{{ svc.updated_at|ago }} by the manager's agent</dd>{% endif %}
|
||||
</dl>
|
||||
</div>
|
||||
|
||||
{# ── Replicas: where it runs and what each uses ──────────────────────────── #}
|
||||
<section class="card-flush" aria-labelledby="svc-replicas">
|
||||
<h2 id="svc-replicas" class="table-caption">Replicas <span class="tone-muted">{{ replicas|length }}</span></h2>
|
||||
{% if replicas %}
|
||||
<table class="table">
|
||||
<thead><tr>
|
||||
<th>Node</th><th>Container</th><th>Status</th>
|
||||
<th class="num" title="Docker counts CPU per core">CPU</th><th class="num">Memory</th><th class="num">Up</th>
|
||||
</tr></thead>
|
||||
<tbody>
|
||||
{% for r in replicas %}
|
||||
<tr>
|
||||
{% if r.ghost %}
|
||||
<td>{{ r.host }}</td>
|
||||
<td colspan="5" class="tone-dim">{{ r.count }} running on a node with no Steward agent: no figures for {{ "it" if r.count == 1 else "them" }}</td>
|
||||
{% else %}
|
||||
<td>{% if r.host_id %}<a href="/hosts/{{ r.host_id }}">{{ r.host }}</a>{% else %}{{ r.host }}{% endif %}</td>
|
||||
<td class="cell-name"><a href="/plugins/docker/container/{{ r.host_id }}/{{ r.name }}" title="{{ r.name }}">{{ r.name }}</a></td>
|
||||
<td>
|
||||
{{ status_dot('up' if r.status == 'running' else 'down') }} <span class="tone-muted">{{ r.status }}</span>
|
||||
{%- if r.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}{% elif r.health == 'starting' %} {{ status_badge('pending', 'Starting') }}{% endif %}
|
||||
</td>
|
||||
<td class="num">{% if r.usage %}{{ cpu(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
|
||||
<td class="num">{% if r.usage %}{{ mem(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
|
||||
<td class="num">{{ r.uptime or "—" }}</td>
|
||||
{% endif %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% else %}
|
||||
<p class="tone-muted" style="margin:0;padding:1rem;">No replica is running.</p>
|
||||
{% endif %}
|
||||
</section>
|
||||
|
||||
{# ── Resource history, summed across replicas ────────────────────────────── #}
|
||||
<section class="card" aria-labelledby="svc-history">
|
||||
<div style="display:flex;align-items:center;justify-content:space-between;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
|
||||
<h2 id="svc-history" class="section-title" style="margin:0;">Resource history</h2>
|
||||
{% include "_time_range.html" %}
|
||||
</div>
|
||||
<div hx-get="/plugins/docker/service/{{ name }}/history"
|
||||
hx-trigger="load, rangeChange from:body"
|
||||
hx-include="#time-range"
|
||||
hx-swap="innerHTML">
|
||||
<p class="field-hint">Loading…</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{# ── Swarm's record of its recent tasks ──────────────────────────────────── #}
|
||||
<section id="tasks" class="card-flush" aria-labelledby="svc-tasks">
|
||||
<h2 id="svc-tasks" class="table-caption">Tasks <span class="tone-muted">Swarm's most recent, newest first</span></h2>
|
||||
{% if tasks %}
|
||||
<table class="table">
|
||||
<thead><tr>
|
||||
<th title="The replica slot this task filled">Slot</th><th>Task</th><th>Node</th><th>State</th>
|
||||
<th class="num">Exit</th><th>Why</th><th class="num">When</th>
|
||||
</tr></thead>
|
||||
<tbody>
|
||||
{% for t in tasks %}
|
||||
<tr>
|
||||
<td>{{ t.slot or "—" }}</td>
|
||||
<td class="mono" title="{{ t.task_id }}">{% if t.task_id %}<a href="?task={{ t.task_id }}#logs">{{ t.task_id[:10] }}</a>{% else %}—{% endif %}</td>
|
||||
<td>{{ t.node }}</td>
|
||||
<td>
|
||||
{%- if t.failed %}{{ status_badge('crit', t.state|capitalize) }}
|
||||
{%- elif t.current %}{{ status_badge('ok', 'Running') }}
|
||||
{%- else %}<span class="tone-muted">{{ t.state }}</span>{% endif %}
|
||||
{%- if t.desired and t.desired != t.state and not t.current %} <span class="tone-dim" title="What Swarm wants of it">→ {{ t.desired }}</span>{% endif -%}
|
||||
</td>
|
||||
<td class="num">{% if t.exit_code %}<span class="tone-{{ 'crit' if t.failed else 'dim' }}">{{ t.exit_code }}</span>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
|
||||
<td style="overflow-wrap:anywhere;">{{ t.why or "" }}</td>
|
||||
<td class="num">{% if t.at %}{{ t.at|ago }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% else %}
|
||||
<p class="tone-muted" style="margin:0;padding:1rem;">No task record yet: the manager's agent reports them from version 1.8.0.</p>
|
||||
{% endif %}
|
||||
</section>
|
||||
|
||||
{% endif %}
|
||||
|
||||
{# ── Logs across every task, newest first ─────────────────────────────────
|
||||
Shown for a removed service too, while its lines are still kept. #}
|
||||
{% if service is not none or task_choices %}
|
||||
<section id="logs" class="card-flush" aria-labelledby="svc-logs">
|
||||
<h2 id="svc-logs" class="table-caption">Logs <span class="tone-muted">every task, newest first</span></h2>
|
||||
<form id="svc-log-controls" class="filter-bar" onsubmit="return false;" style="padding:0.75rem 1rem 0;">
|
||||
<input type="search" name="q" class="input" placeholder="Filter lines…" autocomplete="off" aria-label="Filter log lines">
|
||||
{% if task_choices|length > 1 %}
|
||||
<select name="task" aria-label="Task">
|
||||
<option value="">All tasks</option>
|
||||
{% for tid, label in task_choices %}<option value="{{ tid }}"{% if tid == focus_task %} selected{% endif %}>{{ label }}</option>{% endfor %}
|
||||
</select>
|
||||
{% elif focus_task %}<input type="hidden" name="task" value="{{ focus_task }}">{% endif %}
|
||||
<select name="stream" aria-label="Stream">
|
||||
<option value="all">stdout and stderr</option>
|
||||
<option value="stdout">stdout</option>
|
||||
<option value="stderr">stderr</option>
|
||||
</select>
|
||||
</form>
|
||||
<div id="svc-log-lines"
|
||||
hx-get="/plugins/docker/service/{{ name }}/logs/lines"
|
||||
hx-trigger="load, every 5s, change from:#svc-log-controls, keyup changed delay:400ms from:#svc-log-controls"
|
||||
hx-include="#svc-log-controls"
|
||||
hx-swap="innerHTML"
|
||||
role="log" aria-live="polite" tabindex="0"
|
||||
style="max-height:60vh;overflow:auto;padding:0.5rem 1rem 0.75rem;font-family:var(--fs-font-mono);font-size:0.8rem;line-height:1.5;">
|
||||
<p class="field-hint">Loading…</p>
|
||||
</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -197,6 +197,22 @@ strong, b, th { font-weight: 500; }
|
||||
.chip { display: inline-block; margin: 0.1rem 0.3rem 0.1rem 0; padding: 0.05rem 0.45rem; font-size: 0.75rem;
|
||||
color: var(--text-muted); border: 1px solid var(--border); border-radius: var(--fs-radius-sm); white-space: nowrap; }
|
||||
.chip:hover { color: var(--text); border-color: var(--border-mid); }
|
||||
/* A table's own heading inside its .card-flush: the name, a count and a link. */
|
||||
.table-caption { display: flex; align-items: baseline; gap: 0.6rem; flex-wrap: wrap; margin: 0; padding: 0.75rem 1rem;
|
||||
border-bottom: 1px solid var(--border-mid); font-family: var(--fs-font-body); font-size: 0.95rem; font-weight: 500; }
|
||||
.table-caption .field-hint { margin-left: auto; }
|
||||
/* A name cell cut short with "…", whole on hover (its title); a second line
|
||||
under it; an image reference, long by nature, cut the same way. */
|
||||
.cell-name { max-width: 18rem; }
|
||||
.cell-name > a { display: block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.cell-sub { font-size: 0.75rem; color: var(--text-dim); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.table td.cell-image { max-width: 14rem; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-size: 0.8rem; color: var(--text-muted); }
|
||||
/* The Docker page's one-line answer, above its tiles (#5298). */
|
||||
.docker-verdict { margin: 0 0 0.75rem; font-size: 0.95rem; }
|
||||
/* A service's resource history: CPU and memory side by side, each a figure
|
||||
over a line that spans its half (docker/_service_history.html). */
|
||||
.history-pair { display: grid; grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr)); gap: 1.25rem; }
|
||||
.history-pair .spark { color: var(--blue); margin-top: 0.4rem; }
|
||||
|
||||
/* Forms */
|
||||
.form-group { margin-bottom: 1rem; }
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Integration: a swarm service's own page (#5302) and the Docker overview's
|
||||
service table (#5299, #5300) against real Postgres. The page reads the logs
|
||||
and samples of every task the service ran, the live one and one Docker has
|
||||
already removed (known only by its events), and a failure's "Logs →" picks
|
||||
that task. Requires STEWARD_DATABASE_URL."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
if not os.environ.get("STEWARD_DATABASE_URL"):
|
||||
pytest.skip("needs Postgres")
|
||||
from steward.app import create_app
|
||||
return create_app(testing=False)
|
||||
|
||||
|
||||
def _docker(sub):
|
||||
"""The app's loaded docker plugin module (see test_docker_recent_usage)."""
|
||||
from steward.core.plugin_manager import get_loaded_plugin_modules
|
||||
loaded = get_loaded_plugin_modules().get("docker")
|
||||
base = loaded.__name__ if loaded is not None else "plugins.docker"
|
||||
return importlib.import_module(f"{base}.{sub}")
|
||||
|
||||
|
||||
def test_a_service_page_reads_every_task(app):
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.users import User, UserRole
|
||||
m = _docker("models")
|
||||
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
name = f"it{tag}_web"
|
||||
live_task, dead_task = "a" * 17 + tag, "b" * 17 + tag # 25 characters, as Swarm's
|
||||
live, dead = f"{name}.1.{live_task}", f"{name}.1.{dead_task}"
|
||||
hid, uid, node = str(uuid.uuid4()), str(uuid.uuid4()), f"n{tag}"
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
def iso(dt):
|
||||
return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
|
||||
tasks = [{"task_id": live_task, "slot": 1, "node_id": node, "desired_state": "running",
|
||||
"state": "running", "message": "started", "error": "", "exit_code": None,
|
||||
"at": iso(now - timedelta(minutes=9))},
|
||||
{"task_id": dead_task, "slot": 1, "node_id": node, "desired_state": "shutdown",
|
||||
"state": "failed", "message": "started", "error": "task: non-zero exit (1)",
|
||||
"exit_code": 1, "at": iso(now - timedelta(minutes=10))}]
|
||||
|
||||
def sample(cname, mins, cpu, mem):
|
||||
return m.DockerMetric(host_id=hid, container_name=cname, scraped_at=now - timedelta(minutes=mins),
|
||||
cpu_pct=cpu, mem_pct=0.0, mem_usage_bytes=mem)
|
||||
|
||||
async def go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
s.add(User(id=uid, username=f"sp-{tag}", email=f"sp-{tag}@test",
|
||||
password_hash="x", role=UserRole.viewer, is_active=True))
|
||||
s.add(Host(id=hid, name=f"dock-{tag}", address="10.9.7.1"))
|
||||
async with s.begin():
|
||||
s.add_all([
|
||||
m.DockerSwarmNode(host_id=hid, node_id=node, hostname=f"node-{tag}", role="manager",
|
||||
availability="active", status="ready", leader=True),
|
||||
m.DockerSwarmService(host_id=hid, service_name=name, mode="replicated", desired=1,
|
||||
running=1, image="nginx:1.27",
|
||||
placement_json=json.dumps([{"node_id": node, "running": 1}]),
|
||||
tasks_json=json.dumps(tasks), updated_at=now),
|
||||
m.DockerContainer(host_id=hid, name=live, image="nginx:1.27", status="running",
|
||||
cpu_pct=50.0, mem_usage_bytes=100 << 20, service_name=name,
|
||||
task_id=live_task, node_id=node, started_at=now - timedelta(minutes=9),
|
||||
scraped_at=now),
|
||||
# Docker has removed the failed task's container: only its events are left.
|
||||
m.DockerEvent(host_id=hid, container_name=dead, event="start", at=now - timedelta(minutes=11)),
|
||||
m.DockerEvent(host_id=hid, container_name=dead, event="die", detail="exit 1",
|
||||
at=now - timedelta(minutes=10)),
|
||||
m.DockerLog(host_id=hid, container_name=live, stream="stdout", line=f"serving {tag}",
|
||||
ts=now - timedelta(minutes=5)),
|
||||
m.DockerLog(host_id=hid, container_name=dead, stream="stderr", line=f"panic {tag}",
|
||||
ts=now - timedelta(minutes=10)),
|
||||
sample(dead, 40, 200.0, 300 << 20), sample(live, 30, 40.0, 90 << 20),
|
||||
sample(live, 20, 60.0, 100 << 20), sample(live, 1, 50.0, 100 << 20),
|
||||
])
|
||||
|
||||
client = app.test_client()
|
||||
async with client.session_transaction() as sess:
|
||||
sess.update(user_id=uid, user_role="viewer", username=f"sp-{tag}", login_at=time.time())
|
||||
|
||||
async def get(url):
|
||||
resp = await client.get(url)
|
||||
return resp.status_code, await resp.get_data(as_text=True)
|
||||
|
||||
out = {"page": await get(f"/plugins/docker/service/{name}"),
|
||||
"focus": await get(f"/plugins/docker/service/{name}?task={dead_task}"),
|
||||
"missing": await get(f"/plugins/docker/service/it{tag}_nothing"),
|
||||
"logs": await get(f"/plugins/docker/service/{name}/logs/lines"),
|
||||
"one": await get(f"/plugins/docker/service/{name}/logs/lines?task={dead_task}"),
|
||||
"history": await get(f"/plugins/docker/service/{name}/history?range=1h"),
|
||||
"rows": await get(f"/plugins/docker/rows?q=it{tag}"),
|
||||
"other": await get(f"/plugins/docker/rows?q=it{tag}&stack=someotherstack")}
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await s.delete(await s.get(Host, hid))
|
||||
await s.delete(await s.get(User, uid))
|
||||
return out
|
||||
|
||||
got = asyncio.run(go())
|
||||
|
||||
status, page = got["page"]
|
||||
assert status == 200
|
||||
assert "nginx:1.27" in page and f"node-{tag}" in page and "1 of 1 running" in page
|
||||
assert "task: non-zero exit (1)" in page # Swarm's task record
|
||||
assert f'<option value="{live_task}">1 · {live_task[:10]}</option>' in page
|
||||
assert f'<option value="{dead_task}">1 · {dead_task[:10]}</option>' in page
|
||||
assert f'href="/plugins/docker/?stack=it{tag}"' in page
|
||||
assert f'<option value="{dead_task}" selected>' in got["focus"][1]
|
||||
assert got["missing"][0] == 404 and "Service not found" in got["missing"][1]
|
||||
|
||||
logs = got["logs"][1]
|
||||
assert f"serving {tag}" in logs and f"panic {tag}" in logs # the removed task's lines too
|
||||
assert logs.index(f"serving {tag}") < logs.index(f"panic {tag}") # newest first
|
||||
assert f"1 · {dead_task[:10]}" in logs
|
||||
one = got["one"][1]
|
||||
assert f"panic {tag}" in one and f"serving {tag}" not in one
|
||||
|
||||
history = got["history"][1]
|
||||
assert "<polyline" in history and "peak 2.0 cores" in history and "now · peak" in history
|
||||
|
||||
rows = got["rows"][1]
|
||||
assert f'href="/plugins/docker/service/{name}"' in rows
|
||||
assert f'<a class="chip" href="/plugins/docker/?stack=it{tag}"' in rows
|
||||
assert f'/plugins/docker/service/{name}?task={dead_task}#logs' in rows # the failure's logs
|
||||
assert f'href="/plugins/docker/service/{name}"' not in got["other"][1]
|
||||
@@ -6,6 +6,7 @@ from types import SimpleNamespace as NS
|
||||
|
||||
|
||||
from plugins.docker import service as svc
|
||||
from plugins.docker.overview import DOCKER_STATES, NO_STACK, container_row, service_row
|
||||
from plugins.docker.routes import _is_stale
|
||||
from tests._templates import render_plugin_template
|
||||
from tests.test_style_tokens import hex_colours
|
||||
@@ -40,28 +41,46 @@ def test_stale_is_older_than_the_agents_window():
|
||||
assert svc.stale_after_seconds({}) == 180
|
||||
|
||||
|
||||
_F = {"q": "", "stack": "", "host": "", "state": "", "sort": "status", "dir": "asc"}
|
||||
|
||||
|
||||
async def _rows(app, **over):
|
||||
"""docker/rows.html with what routes.rows() hands it (#5298–#5301),
|
||||
nothing set unless given."""
|
||||
ctx = dict(services=[], containers=[], services_total=0, containers_total=0, services_healthy=0,
|
||||
failing_services=[], failures=[], docker_hosts=0, docker_down=[], attention=[],
|
||||
says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0,
|
||||
stale=0, running=0, stopped=0, f=dict(_F), filtered=False,
|
||||
sort_url=lambda key: f"/plugins/docker/?sort={key}",
|
||||
page_url=lambda **kw: "/plugins/docker/?" + "&".join(f"{k}={v}" for k, v in kw.items()))
|
||||
ctx.update(over)
|
||||
async with app.test_request_context("/plugins/docker/rows"):
|
||||
page = await render_plugin_template(app, "docker", "docker/rows.html", **ctx)
|
||||
return re.sub(r">\s+<", "><", " ".join(page.split()))
|
||||
|
||||
|
||||
def _crow(c, *, host="nas", stale=False, exit_failed=False, ports=()):
|
||||
return container_row(c, label=svc.display_name(c), host=host, usage=svc.usage(c, 1, None),
|
||||
ports=list(ports), stale=stale, exit_failed=exit_failed, now=NOW)
|
||||
|
||||
|
||||
async def test_a_crash_loop_comes_first_and_a_stale_row_says_so(app):
|
||||
loop = _c("api", status="restarting", exit_code=1, restart_count=7)
|
||||
old = _c("cache", scraped_at=NOW - timedelta(minutes=30), cpu_pct=95.0)
|
||||
attention = [{"container": loop, "label": "api", "problems": ["restarting", "crash_looping"], "replaced": False,
|
||||
"since": datetime.now(timezone.utc) - timedelta(minutes=8), "stale": False}]
|
||||
groups = [{"host": None, "host_id": "h1", "host_name": "nas", "running": 1, "stopped": 1, "grouped": False,
|
||||
"subgroups": [{"label": None, "containers": [
|
||||
{"container": loop, "ports": [], "uptime": None, "stale": False,
|
||||
"usage": svc.usage(loop, 1, None), "exit_failed": True},
|
||||
{"container": old, "ports": [], "uptime": "2h", "stale": True,
|
||||
"usage": svc.usage(old, 1, None), "exit_failed": False}]}]}]
|
||||
async with app.test_request_context("/plugins/docker/rows"):
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/rows.html", attention=attention, says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
|
||||
unhealthy=0, restarting=1, stale=1, host_groups=groups, swarm_services=[],
|
||||
running=1, stopped=1, docker_hosts=1, failures=[], failing_services=[])
|
||||
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
||||
page = await _rows(app, attention=attention, restarting=1, stale=1, running=1, stopped=1, docker_hosts=1,
|
||||
containers=[_crow(loop, exit_failed=True), _crow(old, stale=True)], containers_total=2)
|
||||
|
||||
# #5298: the verdict first; only the tiles that aren't zero.
|
||||
assert page.index('class="docker-verdict"') < page.index("Needs attention")
|
||||
assert "1 needs attention" in page and "1 container running, 1 stopped" in page
|
||||
assert "Unhealthy</span>" not in page and "Failed (24h)" not in page
|
||||
card = page[page.index("Needs attention"):page.index("</section>")]
|
||||
assert '<span class="badge badge-crit">Restarting</span>' in card
|
||||
assert 'href="/plugins/docker/container/h1/api" title="api">api</a>' in card
|
||||
assert "is restarting and keeps dying · exit 1 · since <time" in card and ">8m ago</time>" in card
|
||||
assert 'href="/plugins/docker/container/h1/api/logs"' in card
|
||||
assert "cache" not in card
|
||||
assert page.index("Needs attention") < page.index("<table")
|
||||
|
||||
@@ -71,17 +90,14 @@ async def test_a_crash_loop_comes_first_and_a_stale_row_says_so(app):
|
||||
# CPU in cores, toned by its share of the host's CPUs through Settings → Thresholds.
|
||||
assert ('<span class="tone-crit" title="Docker counts CPU per core: 95% of one core, '
|
||||
'95.0% of this host\'s 1 core">0.95 cores</span>') in page
|
||||
assert "history" not in page.lower() and "<svg" not in page # no sparklines on the overview
|
||||
assert "<svg" not in page # no sparklines on the overview
|
||||
assert not hex_colours(page), hex_colours(page)
|
||||
|
||||
|
||||
async def test_a_healthy_board_has_no_attention_card(app):
|
||||
async with app.test_request_context("/plugins/docker/rows"):
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/rows.html", attention=[], says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
|
||||
unhealthy=0, restarting=0, stale=0, host_groups=[], swarm_services=[], running=0, stopped=0,
|
||||
docker_hosts=0, failures=[], failing_services=[])
|
||||
page = await _rows(app)
|
||||
assert "Needs attention</h2>" not in page and "No containers reported yet" in page
|
||||
assert "Nothing needs attention" in page and "stat-strip" not in page
|
||||
|
||||
|
||||
async def test_a_host_whose_docker_stopped_answering_says_so(app):
|
||||
@@ -90,12 +106,7 @@ async def test_a_host_whose_docker_stopped_answering_says_so(app):
|
||||
"detail": "no Docker socket at /var/run/docker.sock", "since": now - timedelta(minutes=5)},
|
||||
{"host_id": "h2", "host_name": "web-1", "status": "error",
|
||||
"detail": "permission denied on /var/run/docker.sock", "since": now - timedelta(minutes=1)}]
|
||||
async with app.test_request_context("/plugins/docker/rows"):
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/rows.html", docker_down=down, attention=[], says=svc.PROBLEM_SAYS,
|
||||
severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
|
||||
swarm_services=[], running=0, stopped=0, docker_hosts=0, failures=[], failing_services=[])
|
||||
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
||||
page = await _rows(app, docker_down=down)
|
||||
card = page[page.index("Docker not answering"):page.index("</section>")]
|
||||
assert '<span class="badge badge-warn">No Docker</span><a href="/hosts/h1">nas</a>' in card
|
||||
assert "no Docker socket at /var/run/docker.sock · since <time" in card and ">5m " in card
|
||||
@@ -103,42 +114,96 @@ async def test_a_host_whose_docker_stopped_answering_says_so(app):
|
||||
|
||||
|
||||
async def test_no_card_while_every_docker_answers(app):
|
||||
async with app.test_request_context("/plugins/docker/rows"):
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS,
|
||||
severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
|
||||
swarm_services=[], running=0, stopped=0, docker_hosts=0, failures=[], failing_services=[])
|
||||
assert "Docker not answering" not in page
|
||||
assert "Docker not answering" not in await _rows(app)
|
||||
|
||||
|
||||
async def test_a_failing_service_leads_and_the_failures_are_listed(app):
|
||||
now = datetime.now(timezone.utc)
|
||||
sick = {"name": "codex_db", "mode": "replicated", "running": 0, "desired": 1, "state": "down",
|
||||
def _sick(now):
|
||||
return {"name": "codex_db", "mode": "replicated", "running": 0, "desired": 1, "state": "down",
|
||||
"crash_looping": True, "failures_1h": 7, "failures_24h": 9, "tasks_known": True,
|
||||
"trouble_since": now - timedelta(minutes=20), "replicas": [], "previous_tasks": 7,
|
||||
"last_error": {"state": "failed", "error": "task: non-zero exit (1)", "exit_code": 1,
|
||||
"node": "nas", "at": None, "age_s": 120}}
|
||||
"node": "nas", "at": None, "age_s": 120, "task_id": "t" * 25, "slot": 1}}
|
||||
|
||||
|
||||
async def test_a_failing_service_leads_and_the_failures_are_listed_once_with_their_logs(app):
|
||||
"""#5300: each failure links to the logs of what died; a service's to its
|
||||
failed task's lines on the service page."""
|
||||
now = datetime.now(timezone.utc)
|
||||
sick = _sick(now)
|
||||
failures = [{"kind": "service", "name": "codex_db", "host_id": None, "host": "nas", "count": 9,
|
||||
"at": now - timedelta(minutes=2), "detail": "task: non-zero exit (1)"},
|
||||
"at": now - timedelta(minutes=2), "detail": "task: non-zero exit (1)", "task_id": "t" * 25},
|
||||
{"kind": "container", "name": "backup", "host_id": "h1", "host": "nas", "count": 1,
|
||||
"at": now - timedelta(hours=3), "detail": "exit 2"}]
|
||||
async with app.test_request_context("/plugins/docker/rows"):
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS,
|
||||
severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
|
||||
swarm_services=[sick], services_healthy=0, failing_services=[sick], failures=failures,
|
||||
running=0, stopped=0, docker_hosts=1)
|
||||
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
||||
"at": now - timedelta(hours=3), "detail": "exit 2", "task_id": None}]
|
||||
page = await _rows(app, services=[service_row(sick, now)], services_total=1, failing_services=[sick],
|
||||
failures=failures, docker_hosts=1)
|
||||
card = page[page.index("Needs attention"):page.index("</section>")]
|
||||
assert '<span class="badge badge-crit">Crash-looping</span><a href="/plugins/docker/swarm">codex_db</a>' in card
|
||||
assert ('<span class="badge badge-crit">Crash-looping</span>'
|
||||
'<a href="/plugins/docker/service/codex_db">codex_db</a>') in card
|
||||
assert "0 of 1 running · 7 tasks failed in the last hour · last task failed: task: non-zero exit (1)" in card
|
||||
assert f'href="/plugins/docker/service/codex_db?task={"t" * 25}#logs"' in card
|
||||
assert "since <time" in card
|
||||
assert '<span class="stat-tile-label">Swarm services</span><span class="stat-tile-value">0/1</span>' in page
|
||||
assert '<span class="tone-warn">0/1</span> services healthy' in page
|
||||
listed = page[page.index('id="failures"'):]
|
||||
listed = listed[:listed.index("</section>")]
|
||||
assert listed.index("codex_db") < listed.index("backup") # newest first
|
||||
assert "9 failures · last: task: non-zero exit (1) · nas" in listed
|
||||
assert 'href="/plugins/docker/container/h1/backup">backup</a>' in listed
|
||||
assert f'/plugins/docker/service/codex_db?task={"t" * 25}#logs"' in listed and "Logs of that task" in listed
|
||||
assert 'href="/plugins/docker/container/h1/backup/logs"' in listed
|
||||
assert '<span class="stat-tile-label">Failed (24h)</span><span class="stat-tile-value">2</span>' in page
|
||||
|
||||
|
||||
async def test_services_and_containers_are_tables_that_sort_and_filter(app):
|
||||
"""#5299, #5301: one row per service with its stack; every standalone
|
||||
container in one table with its host; headers sort; a stack chip filters."""
|
||||
now = datetime.now(timezone.utc)
|
||||
web = {"name": "arr_radarr", "mode": "replicated", "running": 2, "desired": 2, "state": "healthy",
|
||||
"crash_looping": False, "failures_24h": 0, "last_error": None, "trouble_since": None,
|
||||
"image": "lscr.io/linuxserver/radarr:5", "replicas": [
|
||||
{"ghost": False, "host": "docker02", "host_id": "h2", "name": "arr_radarr.1.x", "status": "running",
|
||||
"cpu_pct": 25.0, "mem_bytes": 300 << 20, "started_at": now - timedelta(hours=3),
|
||||
"image": "lscr.io/linuxserver/radarr:5", "health": None},
|
||||
{"ghost": True, "host": "edge01", "count": 1}]}
|
||||
proxy = _c("traefik", host_id="h1", compose_project="edge", image="traefik:v3.1",
|
||||
ports_json=None, cpu_pct=3.0)
|
||||
page = await _rows(app, services=[service_row(web, now)], services_total=3, services_healthy=3,
|
||||
containers=[_crow(proxy, ports=[{"host_port": 443, "container_port": 443, "protocol": "tcp"}])],
|
||||
containers_total=1, running=1, docker_hosts=2,
|
||||
f=dict(_F, sort="cpu", dir="desc"))
|
||||
services = page[page.index('id="docker-services"'):]
|
||||
services = services[:services.index("</section>")]
|
||||
assert "1 of 3" in services and 'href="/plugins/docker/swarm"' in services
|
||||
assert 'href="/plugins/docker/service/arr_radarr" title="arr_radarr · lscr.io/linuxserver/radarr:5">arr_radarr</a>' in services
|
||||
assert '<a class="chip" href="/plugins/docker/?stack=arr"' in services
|
||||
assert "docker02, " in services and "edge01 (no agent)" in services
|
||||
assert ">0.25 cores<" in services and ">300.0 MiB<" in services and ">3h" in services
|
||||
assert 'aria-sort="descending"' in services and 'href="/plugins/docker/?sort=mem"' in services
|
||||
containers = page[page.index('id="docker-containers"'):]
|
||||
assert "Other containers" in containers and ">nas<" in containers
|
||||
assert '<a class="chip" href="/plugins/docker/?stack=edge"' in containers
|
||||
assert '<td class="cell-image" title="traefik:v3.1">traefik:v3.1</td>' in containers
|
||||
assert '<div class="cell-sub mono">443:443/tcp</div>' in containers
|
||||
assert not hex_colours(page), hex_colours(page)
|
||||
|
||||
|
||||
async def test_a_filter_that_matches_nothing_says_so(app):
|
||||
page = await _rows(app, services_total=4, containers_total=2, filtered=True, f=dict(_F, q="zzz"))
|
||||
assert "Nothing matches these filters" in page and "No containers reported yet" not in page
|
||||
|
||||
|
||||
async def test_the_docker_page_has_a_filter_bar_the_rows_read(app):
|
||||
from quart import session
|
||||
async with app.test_request_context("/plugins/docker/?stack=arr"):
|
||||
session["user_role"] = "viewer"
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/index.html", poll_interval=10, has_swarm=True, has_disk=False,
|
||||
f=dict(_F, stack="arr", sort="mem", dir="desc"), choices={"stacks": ["arr", "edge"], "hosts": ["a", "b"]},
|
||||
states=DOCKER_STATES, no_stack=NO_STACK)
|
||||
bar = re.search(r'<form id="docker-filters".*?</form>', page, re.S).group(0)
|
||||
assert 'type="search" id="docker-q" name="q"' in bar
|
||||
assert '<option value="arr" selected>arr</option>' in bar and f'<option value="{NO_STACK}"' in bar
|
||||
assert 'aria-label="Host or node"' in bar and 'aria-label="Status"' in bar
|
||||
assert '<input type="hidden" name="sort" value="mem">' in bar
|
||||
assert 'hx-include="#docker-filters"' in page and "change from:#docker-filters" in page
|
||||
|
||||
|
||||
async def test_the_disk_page_explains_its_terms_and_names_untagged_images(app):
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
"""The Docker page's tables (#5299, #5301) and a service's page (#5302): the
|
||||
rows, their search, filters and sort, and the helpers that find a service's
|
||||
tasks, logs and summed history."""
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace as NS
|
||||
|
||||
from quart import session
|
||||
|
||||
from plugins.docker import service as svc
|
||||
from plugins.docker.overview import (
|
||||
NO_STACK, container_row, filter_options, filter_rows, service_row, sort_rows, stack_of,
|
||||
)
|
||||
from plugins.docker.routes import _filters, _page_url, _sort_url, _task_rows
|
||||
from tests._templates import render_plugin_template
|
||||
|
||||
NOW = datetime(2026, 10, 7, 12, 0, tzinfo=timezone.utc)
|
||||
TASK = "x7q2m9k4p1r8s5t3v6w0y2z4a" # 25 characters, like Swarm's
|
||||
|
||||
|
||||
def _replica(host, cpu=10.0, mem=100, started=2, status="running", **kw):
|
||||
return dict(ghost=False, host=host, host_id=f"id-{host}", name=f"r.{host}", status=status,
|
||||
cpu_pct=cpu, mem_bytes=mem, started_at=NOW - timedelta(hours=started), image="img:1",
|
||||
health=kw.get("health"))
|
||||
|
||||
|
||||
def _service(name, state="healthy", running=1, desired=1, replicas=None, **kw):
|
||||
s = dict(name=name, mode="replicated", running=running, desired=desired, state=state,
|
||||
crash_looping=False, failures_24h=0, last_error=None, trouble_since=None,
|
||||
replicas=replicas if replicas is not None else [_replica("a")])
|
||||
s.update(kw)
|
||||
return s
|
||||
|
||||
|
||||
def _container(name, **kw):
|
||||
base = dict(name=name, host_id="h1", image="nginx:1", status="running", health=None, exit_code=None,
|
||||
oom_killed=False, restart_count=0, compose_project=None, started_at=NOW - timedelta(hours=1),
|
||||
scraped_at=NOW)
|
||||
base.update(kw)
|
||||
c = NS(**base)
|
||||
usage = {"cores": 0.5, "mem_bytes": 2048}
|
||||
return container_row(c, label=name, host="nas", usage=usage, ports=[], stale=False,
|
||||
exit_failed=bool(kw.get("exit_code")), now=NOW)
|
||||
|
||||
|
||||
def test_a_service_reads_its_stack_from_its_name():
|
||||
assert stack_of("arr_radarr") == "arr" and stack_of("steward_app_db") == "steward"
|
||||
assert stack_of("portainer") is None and stack_of("_x") is None and stack_of("x_") is None
|
||||
|
||||
|
||||
def test_a_service_row_adds_up_its_running_replicas():
|
||||
s = _service("arr_radarr", running=2, desired=3, state="degraded", replicas=[
|
||||
_replica("b", cpu=50.0, mem=300, started=5), _replica("a", cpu=25.0, mem=100, started=1),
|
||||
_replica("c", cpu=99.0, mem=999, status="exited"), {"ghost": True, "host": "edge", "count": 1}])
|
||||
row = service_row(s, NOW)
|
||||
assert row["stack"] == "arr" and row["trouble"] == 2
|
||||
assert row["cpu_cores"] == 0.75 and row["mem_bytes"] == 400 # the exited one is not counted
|
||||
assert row["started_at"] == NOW - timedelta(hours=1) and row["up_s"] == 3600
|
||||
assert row["nodes"] == ["a", "b", "c"] and row["ghost_nodes"] == ["edge"]
|
||||
assert row["image"] == "img:1"
|
||||
empty = service_row(_service("solo", state="down", running=0, replicas=[]), NOW)
|
||||
assert empty["cpu_cores"] is None and empty["mem_bytes"] is None and empty["up_s"] is None
|
||||
|
||||
|
||||
def test_rows_filter_by_search_stack_host_and_status():
|
||||
rows = [service_row(_service("arr_radarr"), NOW),
|
||||
service_row(_service("arr_sonarr", state="down", running=0, replicas=[]), NOW),
|
||||
service_row(_service("portainer", replicas=[_replica("b")]), NOW),
|
||||
_container("traefik", compose_project="edge"),
|
||||
_container("backup", status="exited", exit_code=0)]
|
||||
|
||||
def names(**kw):
|
||||
return [r["name"] for r in filter_rows(rows, **kw)]
|
||||
|
||||
assert names(q="SONARR") == ["arr_sonarr"] and names(q="nginx") == ["traefik", "backup"]
|
||||
assert names(stack="arr") == ["arr_radarr", "arr_sonarr"] and names(q="edge") == ["traefik"]
|
||||
assert names(stack=NO_STACK) == ["portainer", "backup"]
|
||||
assert names(host="b") == ["portainer"] and names(host="nas") == ["traefik", "backup"]
|
||||
assert names(state="problem") == ["arr_sonarr"]
|
||||
assert names(state="stopped") == ["arr_sonarr", "backup"]
|
||||
assert names(state="healthy") == ["arr_radarr", "portainer", "traefik"]
|
||||
assert filter_options(rows[:3], rows[3:]) == {"stacks": ["arr", "edge"], "hosts": ["a", "b", "nas"]}
|
||||
|
||||
|
||||
def test_rows_sort_by_column_with_missing_values_last():
|
||||
busy = service_row(_service("busy", replicas=[_replica("a", cpu=300.0)]), NOW)
|
||||
idle = service_row(_service("idle", replicas=[_replica("a", cpu=5.0)]), NOW)
|
||||
down = service_row(_service("down", state="down", running=0, replicas=[]), NOW)
|
||||
rows = [idle, down, busy]
|
||||
assert [r["name"] for r in sort_rows(rows, "cpu", True)] == ["busy", "idle", "down"]
|
||||
assert [r["name"] for r in sort_rows(rows, "cpu", False)] == ["idle", "busy", "down"]
|
||||
assert [r["name"] for r in sort_rows(rows, "status", False)] == ["down", "busy", "idle"]
|
||||
assert [r["name"] for r in sort_rows(rows, "name", True)] == ["idle", "down", "busy"]
|
||||
|
||||
|
||||
def test_the_filters_live_in_the_url_without_their_defaults():
|
||||
f = _filters({"q": " web ", "stack": "arr", "state": "bogus", "sort": "nope", "dir": "x"})
|
||||
assert f == {"q": "web", "stack": "arr", "host": "", "state": "", "sort": "status", "dir": "asc"}
|
||||
assert _page_url(f) == "/plugins/docker/?q=web&stack=arr"
|
||||
assert _page_url(f, stack="") == "/plugins/docker/?q=web"
|
||||
plain = _filters({})
|
||||
assert _sort_url(plain, "cpu") == "/plugins/docker/?sort=cpu" # busiest first by default
|
||||
assert _sort_url(plain, "name") == "/plugins/docker/?sort=name"
|
||||
cpu = _filters({"sort": "cpu"})
|
||||
assert cpu["dir"] == "desc" and _sort_url(cpu, "cpu") == "/plugins/docker/?sort=cpu&dir=asc"
|
||||
|
||||
|
||||
def test_a_task_container_name_gives_its_slot_and_task():
|
||||
assert svc.task_of("web", f"web.1.{TASK}") == ("1", TASK)
|
||||
assert svc.task_of("web", f"web.q8n2.{TASK}") == ("q8n2", TASK) # a global service's node slot
|
||||
assert svc.task_of("web", f"web_db.1.{TASK}") is None
|
||||
assert svc.task_of("arr", f"arr_radarr.1.{TASK}") is None
|
||||
assert svc.task_of("web", "web.1.short") is None
|
||||
|
||||
|
||||
def test_service_history_sums_replicas_after_averaging_each():
|
||||
since = NOW - timedelta(hours=1)
|
||||
t = since + timedelta(minutes=10)
|
||||
samples = [("a", t, 100.0, 1000), ("a", t + timedelta(seconds=20), 300.0, 3000),
|
||||
("b", t + timedelta(seconds=5), 50.0, 500),
|
||||
("a", t + timedelta(minutes=5), 10.0, 100)]
|
||||
out = svc.sum_series(samples, since, NOW, points=60)
|
||||
assert out == [(t, 250.0, 2500), (t + timedelta(minutes=5), 10.0, 100)]
|
||||
|
||||
|
||||
def test_a_failure_carries_the_task_whose_logs_tell_why():
|
||||
sick = {"name": "codex_db", "tasks_known": True, "failures_24h": 2,
|
||||
"last_error": {"state": "failed", "error": "boom", "task_id": TASK, "node": "nas", "at": None}}
|
||||
(row,) = svc.failures_24h([], [sick], {})
|
||||
assert row["kind"] == "service" and row["task_id"] == TASK
|
||||
old = {"name": "legacy", "tasks_known": False, "failures_24h": None, "last_error": None}
|
||||
died = NS(container_name=f"legacy.1.{TASK}", host_id="h1", at=NOW, event="die", detail="exit 1")
|
||||
(row,) = svc.failures_24h([died], [old], {"h1": "nas"})
|
||||
assert row["kind"] == "service" and row["task_id"] == TASK
|
||||
|
||||
|
||||
def test_task_rows_say_where_each_ran_and_why_it_ended():
|
||||
rows = _task_rows([
|
||||
{"task_id": TASK, "slot": 2, "node_id": "n1", "state": "failed", "desired_state": "shutdown",
|
||||
"error": "task: non-zero exit (1)", "exit_code": 1, "at": "2026-10-07T11:00:00Z"},
|
||||
{"task_id": "y" * 25, "slot": 2, "node_id": "n9" * 10, "state": "running", "desired_state": "running"}],
|
||||
{"n1": "docker02"})
|
||||
assert rows[0]["node"] == "docker02" and rows[0]["failed"] and rows[0]["why"] == "task: non-zero exit (1)"
|
||||
assert rows[1]["node"] == ("n9" * 10)[:12] and rows[1]["current"] and not rows[1]["failed"]
|
||||
|
||||
|
||||
async def test_the_service_page_puts_state_first_and_offers_each_tasks_logs(app):
|
||||
now = datetime.now(timezone.utc)
|
||||
s = _service("arr_radarr", running=1, desired=2, state="degraded", failures_24h=3, replicas=[
|
||||
dict(_replica("docker02"), name=f"arr_radarr.1.{TASK}"), {"ghost": True, "host": "edge01", "count": 1}],
|
||||
last_error={"state": "failed", "error": "task: non-zero exit (1)", "exit_code": 1, "node": "docker02",
|
||||
"age_s": 300, "task_id": TASK, "slot": 2})
|
||||
replicas = [dict(s["replicas"][0], usage={"cores": 0.1, "cpus": 4, "cpu_share": 2.5, "mem_bytes": 100,
|
||||
"mem_pct": None, "mem_of": None, "mem_limit": None},
|
||||
uptime="2h"), s["replicas"][1]]
|
||||
tasks = _task_rows([{"task_id": TASK, "slot": 2, "node_id": "n1", "state": "failed",
|
||||
"desired_state": "shutdown", "error": "task: non-zero exit (1)", "exit_code": 1}],
|
||||
{"n1": "docker02"})
|
||||
svc_row = NS(image="lscr.io/linuxserver/radarr:5@sha256:abc", updated_at=now)
|
||||
async with app.test_request_context(f"/plugins/docker/service/arr_radarr?task={TASK}"):
|
||||
session["user_role"] = "viewer"
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/service.html", name="arr_radarr", service=s, row=service_row(s, now),
|
||||
svc=svc_row, tasks=tasks, replicas=replicas, stack="arr", current_range="24h",
|
||||
task_choices=[(TASK, f"2 · {TASK[:10]}"), ("y" * 25, "1 · yyyyyyyyyy")], focus_task=TASK)
|
||||
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
||||
assert '<nav class="breadcrumb"' in page and 'href="/plugins/docker/">Docker</a>' in page
|
||||
assert page.index("Degraded") < page.index("Last failed task") < page.index('id="svc-replicas"')
|
||||
assert "1 of 2 running" in page and '<a class="chip" href="/plugins/docker/?stack=arr">arr</a>' in page
|
||||
assert f'href="?task={TASK}#logs">its logs →</a>' in page and "5m ago" in page
|
||||
assert "radarr:5@sha256:abc" in page
|
||||
assert "1 running on a node with no Steward agent" in page and '<a href="/hosts/id-docker02">docker02</a>' in page
|
||||
assert page.index('id="svc-replicas"') < page.index('id="svc-history"') < page.index('id="tasks"') < page.index('id="logs"')
|
||||
assert 'hx-get="/plugins/docker/service/arr_radarr/history"' in page and 'id="time-range"' in page
|
||||
assert '<span class="badge badge-crit">Failed</span>' in page
|
||||
assert f'<option value="{TASK}" selected>2 · {TASK[:10]}</option>' in page
|
||||
assert 'hx-get="/plugins/docker/service/arr_radarr/logs/lines"' in page
|
||||
|
||||
|
||||
async def test_a_service_nobody_reports_is_not_found(app):
|
||||
async with app.test_request_context("/plugins/docker/service/gone"):
|
||||
session["user_role"] = "viewer"
|
||||
page = await render_plugin_template(app, "docker", "docker/service.html", name="gone", service=None)
|
||||
assert "Service not found" in page and 'id="logs"' not in page
|
||||
|
||||
|
||||
async def test_a_service_history_shows_now_and_peak(app):
|
||||
async with app.test_request_context("/plugins/docker/service/web/history"):
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/_service_history.html", have_data=True, range_key="24h",
|
||||
cpu_now=0.5, cpu_peak=2.0, mem_now=100 << 20, mem_peak=300 << 20,
|
||||
spark_cpu="<svg></svg>", spark_mem="<svg></svg>")
|
||||
empty = await render_plugin_template(
|
||||
app, "docker", "docker/_service_history.html", have_data=False, range_key="1h")
|
||||
assert "0.50 cores" in page and "peak 2.0 cores" in page and "peak 300.0 MiB" in page
|
||||
assert "Not enough samples in the last 1h" in empty
|
||||
|
||||
|
||||
async def test_a_service_log_line_names_its_task(app):
|
||||
lines = [{"ts": NOW, "stream": "stderr", "line": "<b>boom</b>", "task": f"1 · {TASK[:10]}"}]
|
||||
async with app.test_request_context("/plugins/docker/service/web/logs/lines"):
|
||||
page = await render_plugin_template(app, "docker", "docker/_container_logs_lines.html",
|
||||
lines=lines, name="web", filtered=False, service=True)
|
||||
empty = await render_plugin_template(app, "docker", "docker/_container_logs_lines.html",
|
||||
lines=[], name="web", filtered=False, service=True)
|
||||
assert f"1 · {TASK[:10]}</span>" in page and "<b>boom</b>" in page
|
||||
assert "one of its tasks" in empty
|
||||
Reference in New Issue
Block a user