bvandeusenandClaude Opus 5.5
56063f27a4
The agent sent Docker's working set (usage less inactive_file), which still counts recently used file cache. A file-heavy container with no limit read 191 GB while its processes held 2.2 GB, and the host's own figure, which already leaves cache out, disagreed with it. Agent 1.23.0 sends mem_used_bytes / mem_used_pct (anon + shmem, so Postgres's shared buffers count; v1: rss + shmem) and mem_cache_bytes apart. The new names keep the two meanings apart in history and in learned normals: docker_013 renames the columns, clears the old values and adds mem_cache_bytes. The Docker page, swarm by-node sums, Busiest Containers, container detail, normals and the MCP tools all read the new figure. File cache is shown only on the container's page and in get_container. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
389 lines
17 KiB
Python
389 lines
17 KiB
Python
"""Model-free builder for the swarm-aware container view.
|
|
|
|
Merges two sources into one service-grouped, cluster-complete picture:
|
|
• docker_containers — real container rows, collected LOCAL-per-node, so each
|
|
carries the host (= node) it runs on plus its swarm service/node labels.
|
|
• DockerSwarmService.placement_json — the managers' cluster-wide task→node
|
|
counts, which include tasks on nodes that run no Steward agent (and so have
|
|
no container row of their own).
|
|
|
|
For every swarm service we list the real replicas we have detail for, then add
|
|
"ghost" replicas for the remaining placement count on each node (agent-less
|
|
nodes). Non-swarm containers pass through grouped by host, unchanged.
|
|
|
|
A third source, DockerSwarmService.tasks_json (agent ≥ 1.8.0), is Swarm's own
|
|
record of each service's newest tasks. It decides which task containers are
|
|
*superseded* — replaced by Swarm, so neither a replica nor a crash — and gives
|
|
each service its failures across task retries and the error of its last
|
|
failed task. Swarm restarts a task by creating a new one (a new container), so
|
|
failures can only be counted per service, never per container.
|
|
|
|
Kept model-free (no ORM imports) so it's unit-testable and safe to import in
|
|
tests without the plugin-loader "table already defined" gotcha.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
# Task states that mean the task ended in error: `failed` (its container exited
|
|
# non-zero or never started) and `rejected` (Swarm could not start it at all,
|
|
# e.g. no such image, a bad mount).
|
|
FAILED_TASK_STATES = ("failed", "rejected")
|
|
# Desired states of a task Swarm still means to run. Anything else (shutdown,
|
|
# remove, complete, orphaned) is a task Swarm has finished with.
|
|
_CURRENT_DESIRED = ("running", "ready", "")
|
|
|
|
|
|
def _service_state(running: int, desired: int) -> str:
|
|
if desired > 0 and running >= desired:
|
|
return "healthy"
|
|
if running > 0:
|
|
return "degraded"
|
|
return "down"
|
|
|
|
|
|
def _placement(service) -> list[dict]:
|
|
raw = getattr(service, "placement_json", None) or "[]"
|
|
try:
|
|
data = json.loads(raw)
|
|
except (ValueError, TypeError):
|
|
return []
|
|
return [p for p in data if isinstance(p, dict)]
|
|
|
|
|
|
def parse_tasks(service) -> list[dict]:
|
|
"""A service row's task records, newest first; [] when its agent sent none."""
|
|
raw = getattr(service, "tasks_json", None) or "[]"
|
|
try:
|
|
data = json.loads(raw)
|
|
except (ValueError, TypeError):
|
|
return []
|
|
return [t for t in data if isinstance(t, dict)]
|
|
|
|
|
|
def task_time(task: dict) -> datetime | None:
|
|
"""A task's status timestamp. Docker writes nanoseconds; Python ≥ 3.11 keeps
|
|
the first six digits."""
|
|
raw = task.get("at") or ""
|
|
try:
|
|
at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
return at if at.tzinfo else at.replace(tzinfo=timezone.utc)
|
|
|
|
|
|
def task_failures(tasks: list[dict], since: datetime) -> int:
|
|
"""How many tasks ended in error at or after `since`."""
|
|
n = 0
|
|
for t in tasks:
|
|
at = task_time(t)
|
|
if t.get("state") in FAILED_TASK_STATES and at is not None and at >= since:
|
|
n += 1
|
|
return n
|
|
|
|
|
|
def last_task_error(tasks: list[dict]) -> dict | None:
|
|
"""Why the service's newest troubled task is in trouble, from Swarm itself.
|
|
|
|
The newest task that failed or was rejected, or that Swarm attached an error
|
|
to while still trying (a pending task with "no suitable node", say). None
|
|
when no retained task has a problem.
|
|
"""
|
|
for t in tasks: # newest first
|
|
if t.get("state") in FAILED_TASK_STATES or t.get("error"):
|
|
return {"state": t.get("state"), "error": t.get("error") or None,
|
|
"message": t.get("message") or None, "exit_code": t.get("exit_code"),
|
|
"node_id": t.get("node_id") or None, "at": t.get("at") or None,
|
|
"task_id": t.get("task_id") or None, "slot": t.get("slot")}
|
|
return None
|
|
|
|
|
|
def task_summary(tasks: list[dict], now: datetime, crash_loop_deaths: int,
|
|
node_hostname: dict) -> dict:
|
|
"""What a service's task records say: tasks_known, failures_1h,
|
|
failures_24h (None without records), crash_looping, and last_error with
|
|
the node's hostname as `node`."""
|
|
known = bool(tasks)
|
|
failures_1h = task_failures(tasks, now - timedelta(hours=1)) if known else None
|
|
last_error = last_task_error(tasks)
|
|
if last_error is not None:
|
|
nid = last_error["node_id"]
|
|
last_error["node"] = node_hostname.get(nid) or (nid[:12] if nid else None)
|
|
at = task_time(last_error)
|
|
last_error["age_s"] = max(0, int((now - at).total_seconds())) if at else None
|
|
return {
|
|
"tasks_known": known,
|
|
"failures_1h": failures_1h,
|
|
"failures_24h": task_failures(tasks, now - timedelta(hours=24)) if known else None,
|
|
"crash_looping": bool(known and failures_1h >= crash_loop_deaths),
|
|
"last_error": last_error,
|
|
}
|
|
|
|
|
|
def _oldest_failure(tasks: list[dict], since: datetime) -> datetime | None:
|
|
times = [at for t in tasks if t.get("state") in FAILED_TASK_STATES
|
|
and (at := task_time(t)) is not None and at >= since]
|
|
return min(times, default=None)
|
|
|
|
|
|
def trouble_since(tasks: list[dict], state: str, crash_looping: bool,
|
|
now: datetime) -> datetime | None:
|
|
"""When a service's current trouble began, as far as its retained tasks
|
|
tell; None when it has none, or the tasks cannot say.
|
|
|
|
A crash loop dates from its first failure in the last hour, and a healthy
|
|
service that lost tasks from its first failure in the last 24h. A service
|
|
short of replicas dates from its troubled slots (replicated: the slot;
|
|
global: the node): in each slot whose newest task is not running, the
|
|
oldest of its newest troubled tasks (failed, rejected, or carrying an
|
|
error), walking back until a running task interrupts them. Swarm keeps few
|
|
old tasks, so a long outage reads as the oldest one retained: a lower
|
|
bound.
|
|
"""
|
|
if crash_looping:
|
|
return _oldest_failure(tasks, now - timedelta(hours=1))
|
|
if state == "healthy":
|
|
return _oldest_failure(tasks, now - timedelta(hours=24))
|
|
by_slot: dict = {}
|
|
for t in tasks: # newest first, so each slot's list is too
|
|
by_slot.setdefault(t.get("slot") or t.get("node_id"), []).append(t)
|
|
began = []
|
|
for slot_tasks in by_slot.values():
|
|
oldest = None
|
|
for t in slot_tasks:
|
|
if t.get("state") == "running":
|
|
break
|
|
if t.get("state") in FAILED_TASK_STATES or t.get("error"):
|
|
oldest = task_time(t) or oldest
|
|
if oldest is not None:
|
|
began.append(oldest)
|
|
return min(began, default=None)
|
|
|
|
|
|
def _slot_key(c) -> str:
|
|
"""A task container's slot within its service: "<service>.<slot>.<task>" for
|
|
replicated services, "<service>.<node id>.<task>" for global ones."""
|
|
svc, name = c.service_name, c.name or ""
|
|
rest = name[len(svc) + 1:] if name.startswith(svc + ".") else name
|
|
return rest.split(".", 1)[0]
|
|
|
|
|
|
def superseded_containers(swarm_cs, tasks_by_id: dict[str, dict]) -> set[tuple]:
|
|
"""(host_id, name) of task containers Swarm has replaced.
|
|
|
|
With the task on record, its desired state decides: Swarm sets a replaced
|
|
or failed task's desired state to shutdown. Without it (an older agent, or a
|
|
task older than the retained history), a container that is not running is
|
|
superseded when another task in the same slot is running or started later.
|
|
"""
|
|
out: set[tuple] = set()
|
|
by_slot: dict[tuple, list] = {}
|
|
for c in swarm_cs:
|
|
by_slot.setdefault((c.service_name, _slot_key(c)), []).append(c)
|
|
for c in swarm_cs:
|
|
task = tasks_by_id.get(getattr(c, "task_id", None) or "")
|
|
if task is not None:
|
|
if task.get("desired_state") not in _CURRENT_DESIRED:
|
|
out.add((c.host_id, c.name))
|
|
continue
|
|
if (getattr(c, "status", "") or "").lower() == "running":
|
|
continue
|
|
mine = getattr(c, "started_at", None)
|
|
for other in by_slot[(c.service_name, _slot_key(c))]:
|
|
if other is c:
|
|
continue
|
|
theirs = getattr(other, "started_at", None)
|
|
if ((getattr(other, "status", "") or "").lower() == "running"
|
|
or (theirs is not None and (mine is None or theirs > mine))):
|
|
out.add((c.host_id, c.name))
|
|
break
|
|
return out
|
|
|
|
|
|
def _newest_per_name(services) -> dict[str, object]:
|
|
"""Every manager reports the same cluster-global service set; keep the most
|
|
recently updated row per name, so a manager on an older agent (no tasks)
|
|
doesn't shadow one that reports them."""
|
|
by_name: dict[str, object] = {}
|
|
for s in services:
|
|
name = getattr(s, "service_name", None)
|
|
if not name:
|
|
continue
|
|
kept = by_name.get(name)
|
|
if kept is None:
|
|
by_name[name] = s
|
|
continue
|
|
a, b = getattr(s, "updated_at", None), getattr(kept, "updated_at", None)
|
|
if a is not None and (b is None or a > b):
|
|
by_name[name] = s
|
|
return by_name
|
|
|
|
|
|
def replaced_tasks(containers, services) -> set[tuple]:
|
|
"""(host_id, name) of the task containers Swarm has replaced, from the
|
|
container rows and the managers' service rows. What the container views use
|
|
to leave history out of their problem lists."""
|
|
swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
|
|
tasks_by_id = {t["task_id"]: t for s in _newest_per_name(services).values()
|
|
for t in parse_tasks(s) if t.get("task_id")}
|
|
return superseded_containers(swarm_cs, tasks_by_id)
|
|
|
|
|
|
def build_swarm_services(containers, services, node_hostname: dict, host_name: dict,
|
|
*, now: datetime | None = None, crash_loop_deaths: int = 3) -> dict:
|
|
"""Return {"services": [...], "standalone": [...], "superseded": {(host_id, name)}}.
|
|
|
|
services[i] = {name, mode, image, desired, running, state, replicas:[...],
|
|
previous_tasks, tasks_known, failures_1h, failures_24h,
|
|
crash_looping, last_error}
|
|
real replica = {ghost:False, host, host_id, name, status, cpu_pct, mem_pct,
|
|
mem_bytes, started_at, image, health, restart_count}
|
|
# current tasks only
|
|
ghost replica = {ghost:True, host, count} # placement with no local row
|
|
previous_tasks = task containers Swarm replaced, left out of replicas
|
|
failures_* = tasks that failed or were rejected in the window (None
|
|
without task data); crash_looping = failures_1h ≥ the
|
|
Settings → Thresholds crash-loop count
|
|
last_error = last_task_error(), plus `node` (the node's hostname)
|
|
trouble_since = trouble_since(): when the current trouble began, or None
|
|
standalone[i] = {host_id, host, containers:[...]} # non-swarm, by host
|
|
superseded = the task containers Swarm replaced, so callers can skip them
|
|
"""
|
|
now = now or datetime.now(timezone.utc)
|
|
containers = list(containers)
|
|
swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
|
|
standalone_cs = [c for c in containers if not getattr(c, "service_name", None)]
|
|
|
|
svc_by_name = _newest_per_name(services)
|
|
tasks_by_svc = {name: parse_tasks(s) for name, s in svc_by_name.items()}
|
|
tasks_by_id = {t["task_id"]: t for ts in tasks_by_svc.values() for t in ts
|
|
if t.get("task_id")}
|
|
superseded = superseded_containers(swarm_cs, tasks_by_id)
|
|
|
|
# Current task containers grouped by service name; replaced ones are counted.
|
|
cs_by_service: dict[str, list] = {}
|
|
previous: dict[str, int] = {}
|
|
for c in swarm_cs:
|
|
if (c.host_id, c.name) in superseded:
|
|
previous[c.service_name] = previous.get(c.service_name, 0) + 1
|
|
continue
|
|
cs_by_service.setdefault(c.service_name, []).append(c)
|
|
|
|
def _host_of(c):
|
|
return (host_name.get(getattr(c, "host_id", None))
|
|
or node_hostname.get(getattr(c, "node_id", None))
|
|
or getattr(c, "host_id", None) or "?")
|
|
|
|
out_services = []
|
|
for name in sorted(set(svc_by_name) | set(cs_by_service)):
|
|
svc = svc_by_name.get(name)
|
|
reals = cs_by_service.get(name, [])
|
|
|
|
replicas = []
|
|
for c in sorted(reals, key=lambda c: (_host_of(c).lower(), c.name)):
|
|
replicas.append({
|
|
"ghost": False,
|
|
"host": _host_of(c),
|
|
"host_id": getattr(c, "host_id", None),
|
|
"name": c.name,
|
|
"status": getattr(c, "status", "unknown"),
|
|
"cpu_pct": getattr(c, "cpu_pct", None),
|
|
"mem_pct": getattr(c, "mem_used_pct", None),
|
|
"mem_bytes": getattr(c, "mem_used_bytes", None),
|
|
"started_at": getattr(c, "started_at", None),
|
|
"image": getattr(c, "image", None) or None,
|
|
"health": getattr(c, "health", None),
|
|
"restart_count": getattr(c, "restart_count", 0) or 0,
|
|
})
|
|
|
|
# Real running replicas per node, to subtract from placement.
|
|
real_running_by_node: dict[str, int] = {}
|
|
for c in reals:
|
|
if (getattr(c, "status", "") or "").lower() == "running":
|
|
nid = getattr(c, "node_id", None)
|
|
if nid:
|
|
real_running_by_node[nid] = real_running_by_node.get(nid, 0) + 1
|
|
|
|
ghosts = []
|
|
for p in _placement(svc) if svc is not None else []:
|
|
nid = p.get("node_id", "")
|
|
placed = int(p.get("running", 0) or 0)
|
|
ghost = placed - real_running_by_node.get(nid, 0)
|
|
if ghost > 0:
|
|
ghosts.append({
|
|
"ghost": True,
|
|
"host": node_hostname.get(nid) or (nid[:12] if nid else "?"),
|
|
"count": ghost,
|
|
})
|
|
ghosts.sort(key=lambda g: g["host"].lower())
|
|
replicas.extend(ghosts)
|
|
|
|
running = getattr(svc, "running", None) if svc is not None else None
|
|
desired = getattr(svc, "desired", None) if svc is not None else None
|
|
# No service row (manager didn't report it): infer from what we can see.
|
|
if running is None:
|
|
running = sum(1 for r in replicas
|
|
if (not r["ghost"] and (r["status"] or "").lower() == "running"))
|
|
if desired is None:
|
|
desired = len(replicas)
|
|
|
|
tasks = tasks_by_svc.get(name, [])
|
|
state = _service_state(int(running or 0), int(desired or 0))
|
|
summary = task_summary(tasks, now, crash_loop_deaths, node_hostname)
|
|
out_services.append({
|
|
"name": name,
|
|
"mode": getattr(svc, "mode", "replicated") if svc is not None else "replicated",
|
|
"image": getattr(svc, "image", None) if svc is not None else None,
|
|
"running": running,
|
|
"desired": desired,
|
|
"state": state,
|
|
"replicas": replicas,
|
|
"previous_tasks": previous.get(name, 0),
|
|
**summary,
|
|
"trouble_since": trouble_since(tasks, state, summary["crash_looping"], now),
|
|
})
|
|
|
|
# Non-swarm containers grouped by host, running-first order preserved.
|
|
groups: dict[str, dict] = {}
|
|
for c in standalone_cs:
|
|
hid = getattr(c, "host_id", None)
|
|
g = groups.get(hid)
|
|
if g is None:
|
|
g = groups[hid] = {"host_id": hid, "host": host_name.get(hid) or hid or "?",
|
|
"containers": []}
|
|
g["containers"].append(c)
|
|
standalone = sorted(groups.values(), key=lambda g: g["host"].lower())
|
|
|
|
return {"services": out_services, "standalone": standalone, "superseded": superseded}
|
|
|
|
|
|
def service_trouble(s: dict) -> int | None:
|
|
"""How bad a built service is, for sorting and for Needs attention: 0 a
|
|
crash loop, 1 no replica running, 2 short of replicas, 3 healthy but with
|
|
failed tasks in the last 24h; None when nothing is wrong."""
|
|
if s.get("crash_looping"):
|
|
return 0
|
|
if s.get("state") == "down":
|
|
return 1
|
|
if s.get("state") == "degraded":
|
|
return 2
|
|
if s.get("failures_24h"):
|
|
return 3
|
|
return None
|
|
|
|
|
|
def services_problems_first(services: list[dict]) -> list[dict]:
|
|
"""Services in trouble first, worst first, then the rest by name."""
|
|
return sorted(services, key=lambda s: (
|
|
4 if service_trouble(s) is None else service_trouble(s), s["name"].lower()))
|
|
|
|
|
|
def failing_services(services: list[dict]) -> list[dict]:
|
|
"""The services that belong in Needs attention: a crash loop, or short of
|
|
their desired replicas. A healthy service whose failed tasks Swarm has
|
|
already replaced is history, shown on the service, not a problem now."""
|
|
# A crash loop ranks 0, so test the rank against None, not for truth.
|
|
return [s for s in services_problems_first(services)
|
|
if (t := service_trouble(s)) is not None and t <= 2]
|