bvandeusenandClaude Opus 5.5
61b22056d5
#5209 "Docker views: readable names, CPU/memory in cores and bytes, failures listed with since-time" (milestone 477 "UI problems first"). - A swarm task reads as its service and slot ("codex_db · 1", or the node for a global service), with the raw name in the title: Docker page, host panel, busiest widget, container detail. - CPU reads in cores (Docker's 100% = one core), toned by its share of the host's CPUs from the agent's per-core series. Memory reads in bytes, against a limit only when one is really set (Docker reports host RAM otherwise). - Exit 137/143 without an OOM kill is a stop, not a failure: no exited_nonzero, no crash-loop death, no "failed (24h)", dim exit code. - "Failed (24h)" is a list (Docker page #failures, linked from the widget and the tile): plain containers by their deaths, swarm services by Swarm's failed tasks; a second manager's copy of a task death counts once. - Failing swarm services lead Needs attention; services sort problems first with since-times on the page, the widget and the Swarm page; nodes that are down or drained sort first. - Busiest widget orders by a 5-minute CPU average so the list holds still. - Host panel: a table, problems first with what is wrong and since when; replaced swarm tasks hidden and counted. - Container detail: node hostname, I/O labelled "since start", Restarts hidden for swarm tasks with a note on where task failures are. - Widget names: "Docker — Services & failures", "Docker — Busiest containers". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
381 lines
16 KiB
Python
381 lines
16 KiB
Python
"""Model-free builder for the swarm-aware container view.
|
|
|
|
Merges two sources into one service-grouped, cluster-complete picture:
|
|
• docker_containers — real container rows, collected LOCAL-per-node, so each
|
|
carries the host (= node) it runs on plus its swarm service/node labels.
|
|
• DockerSwarmService.placement_json — the managers' cluster-wide task→node
|
|
counts, which include tasks on nodes that run no Steward agent (and so have
|
|
no container row of their own).
|
|
|
|
For every swarm service we list the real replicas we have detail for, then add
|
|
"ghost" replicas for the remaining placement count on each node (agent-less
|
|
nodes). Non-swarm containers pass through grouped by host, unchanged.
|
|
|
|
A third source, DockerSwarmService.tasks_json (agent ≥ 1.8.0), is Swarm's own
|
|
record of each service's newest tasks. It decides which task containers are
|
|
*superseded* — replaced by Swarm, so neither a replica nor a crash — and gives
|
|
each service its failures across task retries and the error of its last
|
|
failed task. Swarm restarts a task by creating a new one (a new container), so
|
|
failures can only be counted per service, never per container.
|
|
|
|
Kept model-free (no ORM imports) so it's unit-testable and safe to import in
|
|
tests without the plugin-loader "table already defined" gotcha.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
# Task states that mean the task ended in error: `failed` (its container exited
|
|
# non-zero or never started) and `rejected` (Swarm could not start it at all,
|
|
# e.g. no such image, a bad mount).
|
|
FAILED_TASK_STATES = ("failed", "rejected")
|
|
# Desired states of a task Swarm still means to run. Anything else (shutdown,
|
|
# remove, complete, orphaned) is a task Swarm has finished with.
|
|
_CURRENT_DESIRED = ("running", "ready", "")
|
|
|
|
|
|
def _service_state(running: int, desired: int) -> str:
|
|
if desired > 0 and running >= desired:
|
|
return "healthy"
|
|
if running > 0:
|
|
return "degraded"
|
|
return "down"
|
|
|
|
|
|
def _placement(service) -> list[dict]:
|
|
raw = getattr(service, "placement_json", None) or "[]"
|
|
try:
|
|
data = json.loads(raw)
|
|
except (ValueError, TypeError):
|
|
return []
|
|
return [p for p in data if isinstance(p, dict)]
|
|
|
|
|
|
def parse_tasks(service) -> list[dict]:
|
|
"""A service row's task records, newest first; [] when its agent sent none."""
|
|
raw = getattr(service, "tasks_json", None) or "[]"
|
|
try:
|
|
data = json.loads(raw)
|
|
except (ValueError, TypeError):
|
|
return []
|
|
return [t for t in data if isinstance(t, dict)]
|
|
|
|
|
|
def task_time(task: dict) -> datetime | None:
|
|
"""A task's status timestamp. Docker writes nanoseconds; Python ≥ 3.11 keeps
|
|
the first six digits."""
|
|
raw = task.get("at") or ""
|
|
try:
|
|
at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
return at if at.tzinfo else at.replace(tzinfo=timezone.utc)
|
|
|
|
|
|
def task_failures(tasks: list[dict], since: datetime) -> int:
|
|
"""How many tasks ended in error at or after `since`."""
|
|
n = 0
|
|
for t in tasks:
|
|
at = task_time(t)
|
|
if t.get("state") in FAILED_TASK_STATES and at is not None and at >= since:
|
|
n += 1
|
|
return n
|
|
|
|
|
|
def last_task_error(tasks: list[dict]) -> dict | None:
|
|
"""Why the service's newest troubled task is in trouble, from Swarm itself.
|
|
|
|
The newest task that failed or was rejected, or that Swarm attached an error
|
|
to while still trying (a pending task with "no suitable node", say). None
|
|
when no retained task has a problem.
|
|
"""
|
|
for t in tasks: # newest first
|
|
if t.get("state") in FAILED_TASK_STATES or t.get("error"):
|
|
return {"state": t.get("state"), "error": t.get("error") or None,
|
|
"message": t.get("message") or None, "exit_code": t.get("exit_code"),
|
|
"node_id": t.get("node_id") or None, "at": t.get("at") or None}
|
|
return None
|
|
|
|
|
|
def task_summary(tasks: list[dict], now: datetime, crash_loop_deaths: int,
|
|
node_hostname: dict) -> dict:
|
|
"""What a service's task records say: tasks_known, failures_1h,
|
|
failures_24h (None without records), crash_looping, and last_error with
|
|
the node's hostname as `node`."""
|
|
known = bool(tasks)
|
|
failures_1h = task_failures(tasks, now - timedelta(hours=1)) if known else None
|
|
last_error = last_task_error(tasks)
|
|
if last_error is not None:
|
|
nid = last_error["node_id"]
|
|
last_error["node"] = node_hostname.get(nid) or (nid[:12] if nid else None)
|
|
at = task_time(last_error)
|
|
last_error["age_s"] = max(0, int((now - at).total_seconds())) if at else None
|
|
return {
|
|
"tasks_known": known,
|
|
"failures_1h": failures_1h,
|
|
"failures_24h": task_failures(tasks, now - timedelta(hours=24)) if known else None,
|
|
"crash_looping": bool(known and failures_1h >= crash_loop_deaths),
|
|
"last_error": last_error,
|
|
}
|
|
|
|
|
|
def _oldest_failure(tasks: list[dict], since: datetime) -> datetime | None:
|
|
times = [at for t in tasks if t.get("state") in FAILED_TASK_STATES
|
|
and (at := task_time(t)) is not None and at >= since]
|
|
return min(times, default=None)
|
|
|
|
|
|
def trouble_since(tasks: list[dict], state: str, crash_looping: bool,
|
|
now: datetime) -> datetime | None:
|
|
"""When a service's current trouble began, as far as its retained tasks
|
|
tell; None when it has none, or the tasks cannot say.
|
|
|
|
A crash loop dates from its first failure in the last hour, and a healthy
|
|
service that lost tasks from its first failure in the last 24h. A service
|
|
short of replicas dates from its troubled slots (replicated: the slot;
|
|
global: the node): in each slot whose newest task is not running, the
|
|
oldest of its newest troubled tasks (failed, rejected, or carrying an
|
|
error), walking back until a running task interrupts them. Swarm keeps few
|
|
old tasks, so a long outage reads as the oldest one retained: a lower
|
|
bound.
|
|
"""
|
|
if crash_looping:
|
|
return _oldest_failure(tasks, now - timedelta(hours=1))
|
|
if state == "healthy":
|
|
return _oldest_failure(tasks, now - timedelta(hours=24))
|
|
by_slot: dict = {}
|
|
for t in tasks: # newest first, so each slot's list is too
|
|
by_slot.setdefault(t.get("slot") or t.get("node_id"), []).append(t)
|
|
began = []
|
|
for slot_tasks in by_slot.values():
|
|
oldest = None
|
|
for t in slot_tasks:
|
|
if t.get("state") == "running":
|
|
break
|
|
if t.get("state") in FAILED_TASK_STATES or t.get("error"):
|
|
oldest = task_time(t) or oldest
|
|
if oldest is not None:
|
|
began.append(oldest)
|
|
return min(began, default=None)
|
|
|
|
|
|
def _slot_key(c) -> str:
|
|
"""A task container's slot within its service: "<service>.<slot>.<task>" for
|
|
replicated services, "<service>.<node id>.<task>" for global ones."""
|
|
svc, name = c.service_name, c.name or ""
|
|
rest = name[len(svc) + 1:] if name.startswith(svc + ".") else name
|
|
return rest.split(".", 1)[0]
|
|
|
|
|
|
def superseded_containers(swarm_cs, tasks_by_id: dict[str, dict]) -> set[tuple]:
|
|
"""(host_id, name) of task containers Swarm has replaced.
|
|
|
|
With the task on record, its desired state decides: Swarm sets a replaced
|
|
or failed task's desired state to shutdown. Without it (an older agent, or a
|
|
task older than the retained history), a container that is not running is
|
|
superseded when another task in the same slot is running or started later.
|
|
"""
|
|
out: set[tuple] = set()
|
|
by_slot: dict[tuple, list] = {}
|
|
for c in swarm_cs:
|
|
by_slot.setdefault((c.service_name, _slot_key(c)), []).append(c)
|
|
for c in swarm_cs:
|
|
task = tasks_by_id.get(getattr(c, "task_id", None) or "")
|
|
if task is not None:
|
|
if task.get("desired_state") not in _CURRENT_DESIRED:
|
|
out.add((c.host_id, c.name))
|
|
continue
|
|
if (getattr(c, "status", "") or "").lower() == "running":
|
|
continue
|
|
mine = getattr(c, "started_at", None)
|
|
for other in by_slot[(c.service_name, _slot_key(c))]:
|
|
if other is c:
|
|
continue
|
|
theirs = getattr(other, "started_at", None)
|
|
if ((getattr(other, "status", "") or "").lower() == "running"
|
|
or (theirs is not None and (mine is None or theirs > mine))):
|
|
out.add((c.host_id, c.name))
|
|
break
|
|
return out
|
|
|
|
|
|
def _newest_per_name(services) -> dict[str, object]:
|
|
"""Every manager reports the same cluster-global service set; keep the most
|
|
recently updated row per name, so a manager on an older agent (no tasks)
|
|
doesn't shadow one that reports them."""
|
|
by_name: dict[str, object] = {}
|
|
for s in services:
|
|
name = getattr(s, "service_name", None)
|
|
if not name:
|
|
continue
|
|
kept = by_name.get(name)
|
|
if kept is None:
|
|
by_name[name] = s
|
|
continue
|
|
a, b = getattr(s, "updated_at", None), getattr(kept, "updated_at", None)
|
|
if a is not None and (b is None or a > b):
|
|
by_name[name] = s
|
|
return by_name
|
|
|
|
|
|
def replaced_tasks(containers, services) -> set[tuple]:
|
|
"""(host_id, name) of the task containers Swarm has replaced, from the
|
|
container rows and the managers' service rows. What the container views use
|
|
to leave history out of their problem lists."""
|
|
swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
|
|
tasks_by_id = {t["task_id"]: t for s in _newest_per_name(services).values()
|
|
for t in parse_tasks(s) if t.get("task_id")}
|
|
return superseded_containers(swarm_cs, tasks_by_id)
|
|
|
|
|
|
def build_swarm_services(containers, services, node_hostname: dict, host_name: dict,
|
|
*, now: datetime | None = None, crash_loop_deaths: int = 3) -> dict:
|
|
"""Return {"services": [...], "standalone": [...], "superseded": {(host_id, name)}}.
|
|
|
|
services[i] = {name, mode, desired, running, state, replicas:[...],
|
|
previous_tasks, tasks_known, failures_1h, failures_24h,
|
|
crash_looping, last_error}
|
|
real replica = {ghost:False, host, host_id, name, status, cpu_pct, mem_pct,
|
|
health, restart_count} # current tasks only
|
|
ghost replica = {ghost:True, host, count} # placement with no local row
|
|
previous_tasks = task containers Swarm replaced, left out of replicas
|
|
failures_* = tasks that failed or were rejected in the window (None
|
|
without task data); crash_looping = failures_1h ≥ the
|
|
Settings → Thresholds crash-loop count
|
|
last_error = last_task_error(), plus `node` (the node's hostname)
|
|
trouble_since = trouble_since(): when the current trouble began, or None
|
|
standalone[i] = {host_id, host, containers:[...]} # non-swarm, by host
|
|
superseded = the task containers Swarm replaced, so callers can skip them
|
|
"""
|
|
now = now or datetime.now(timezone.utc)
|
|
containers = list(containers)
|
|
swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
|
|
standalone_cs = [c for c in containers if not getattr(c, "service_name", None)]
|
|
|
|
svc_by_name = _newest_per_name(services)
|
|
tasks_by_svc = {name: parse_tasks(s) for name, s in svc_by_name.items()}
|
|
tasks_by_id = {t["task_id"]: t for ts in tasks_by_svc.values() for t in ts
|
|
if t.get("task_id")}
|
|
superseded = superseded_containers(swarm_cs, tasks_by_id)
|
|
|
|
# Current task containers grouped by service name; replaced ones are counted.
|
|
cs_by_service: dict[str, list] = {}
|
|
previous: dict[str, int] = {}
|
|
for c in swarm_cs:
|
|
if (c.host_id, c.name) in superseded:
|
|
previous[c.service_name] = previous.get(c.service_name, 0) + 1
|
|
continue
|
|
cs_by_service.setdefault(c.service_name, []).append(c)
|
|
|
|
def _host_of(c):
|
|
return (host_name.get(getattr(c, "host_id", None))
|
|
or node_hostname.get(getattr(c, "node_id", None))
|
|
or getattr(c, "host_id", None) or "?")
|
|
|
|
out_services = []
|
|
for name in sorted(set(svc_by_name) | set(cs_by_service)):
|
|
svc = svc_by_name.get(name)
|
|
reals = cs_by_service.get(name, [])
|
|
|
|
replicas = []
|
|
for c in sorted(reals, key=lambda c: (_host_of(c).lower(), c.name)):
|
|
replicas.append({
|
|
"ghost": False,
|
|
"host": _host_of(c),
|
|
"host_id": getattr(c, "host_id", None),
|
|
"name": c.name,
|
|
"status": getattr(c, "status", "unknown"),
|
|
"cpu_pct": getattr(c, "cpu_pct", None),
|
|
"mem_pct": getattr(c, "mem_pct", None),
|
|
"health": getattr(c, "health", None),
|
|
"restart_count": getattr(c, "restart_count", 0) or 0,
|
|
})
|
|
|
|
# Real running replicas per node, to subtract from placement.
|
|
real_running_by_node: dict[str, int] = {}
|
|
for c in reals:
|
|
if (getattr(c, "status", "") or "").lower() == "running":
|
|
nid = getattr(c, "node_id", None)
|
|
if nid:
|
|
real_running_by_node[nid] = real_running_by_node.get(nid, 0) + 1
|
|
|
|
ghosts = []
|
|
for p in _placement(svc) if svc is not None else []:
|
|
nid = p.get("node_id", "")
|
|
placed = int(p.get("running", 0) or 0)
|
|
ghost = placed - real_running_by_node.get(nid, 0)
|
|
if ghost > 0:
|
|
ghosts.append({
|
|
"ghost": True,
|
|
"host": node_hostname.get(nid) or (nid[:12] if nid else "?"),
|
|
"count": ghost,
|
|
})
|
|
ghosts.sort(key=lambda g: g["host"].lower())
|
|
replicas.extend(ghosts)
|
|
|
|
running = getattr(svc, "running", None) if svc is not None else None
|
|
desired = getattr(svc, "desired", None) if svc is not None else None
|
|
# No service row (manager didn't report it): infer from what we can see.
|
|
if running is None:
|
|
running = sum(1 for r in replicas
|
|
if (not r["ghost"] and (r["status"] or "").lower() == "running"))
|
|
if desired is None:
|
|
desired = len(replicas)
|
|
|
|
tasks = tasks_by_svc.get(name, [])
|
|
state = _service_state(int(running or 0), int(desired or 0))
|
|
summary = task_summary(tasks, now, crash_loop_deaths, node_hostname)
|
|
out_services.append({
|
|
"name": name,
|
|
"mode": getattr(svc, "mode", "replicated") if svc is not None else "replicated",
|
|
"running": running,
|
|
"desired": desired,
|
|
"state": state,
|
|
"replicas": replicas,
|
|
"previous_tasks": previous.get(name, 0),
|
|
**summary,
|
|
"trouble_since": trouble_since(tasks, state, summary["crash_looping"], now),
|
|
})
|
|
|
|
# Non-swarm containers grouped by host, running-first order preserved.
|
|
groups: dict[str, dict] = {}
|
|
for c in standalone_cs:
|
|
hid = getattr(c, "host_id", None)
|
|
g = groups.get(hid)
|
|
if g is None:
|
|
g = groups[hid] = {"host_id": hid, "host": host_name.get(hid) or hid or "?",
|
|
"containers": []}
|
|
g["containers"].append(c)
|
|
standalone = sorted(groups.values(), key=lambda g: g["host"].lower())
|
|
|
|
return {"services": out_services, "standalone": standalone, "superseded": superseded}
|
|
|
|
|
|
def service_trouble(s: dict) -> int | None:
|
|
"""How bad a built service is, for sorting and for Needs attention: 0 a
|
|
crash loop, 1 no replica running, 2 short of replicas, 3 healthy but with
|
|
failed tasks in the last 24h; None when nothing is wrong."""
|
|
if s.get("crash_looping"):
|
|
return 0
|
|
if s.get("state") == "down":
|
|
return 1
|
|
if s.get("state") == "degraded":
|
|
return 2
|
|
if s.get("failures_24h"):
|
|
return 3
|
|
return None
|
|
|
|
|
|
def services_problems_first(services: list[dict]) -> list[dict]:
|
|
"""Services in trouble first, worst first, then the rest by name."""
|
|
return sorted(services, key=lambda s: (
|
|
4 if service_trouble(s) is None else service_trouble(s), s["name"].lower()))
|
|
|
|
|
|
def failing_services(services: list[dict]) -> list[dict]:
|
|
"""The services that belong in Needs attention: a crash loop, or short of
|
|
their desired replicas. A healthy service whose failed tasks Swarm has
|
|
already replaced is history, shown on the service, not a problem now."""
|
|
return [s for s in services_problems_first(services) if (service_trouble(s) or 9) <= 2]
|