diff --git a/plugins/docker/mcp_tools.py b/plugins/docker/mcp_tools.py index d366802..df2533b 100644 --- a/plugins/docker/mcp_tools.py +++ b/plugins/docker/mcp_tools.py @@ -194,7 +194,8 @@ async def list_containers(host: str = "", status: str = "", problem: str = "", "restarting", "crash_looping" (died repeatedly within the last hour; the count is set under Settings → Thresholds, default 3), "unhealthy" (failing its healthcheck), "oom_killed", "exited_nonzero" (died with an - error in the last 24h); empty for no filter. + error in the last 24h; exit 137/143 without an OOM kill is a stop, not + an error); empty for no filter. compose_project: only containers from this Compose project. limit: default 100, max 500. diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index 6911b40..93d1a68 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -16,9 +16,12 @@ from steward.core.formatting import human_bytes, human_duration, seconds_since from steward.core.time_range import parse_range, DEFAULT_RANGE from . import service as docker_svc from .dedup import dedup_by_container_id -from .swarm_view import build_swarm_services, parse_tasks, task_summary +from .swarm_view import ( + build_swarm_services, failing_services, parse_tasks, replaced_tasks, services_problems_first, + task_summary, trouble_since, +) from .models import ( - DockerContainer, DockerDiskUsage, DockerEvent, DockerHostStatus, DockerImage, + DockerContainer, DockerDiskUsage, DockerHostStatus, DockerImage, DockerMetric, DockerSwarmNode, DockerSwarmService, ) @@ -109,6 +112,10 @@ async def rows(): swarm_view = build_swarm_services( containers, services, node_hostname, host_name, crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config)) + capacity = await docker_svc.host_capacity(db, {c.host_id for c in containers}) + failures = docker_svc.failures_24h( + await docker_svc.failure_events(db, now - timedelta(hours=24)), + swarm_view["services"], host_name) standalone = [c for c in containers if not c.service_name] groups: dict[str, dict] = {} @@ -127,6 +134,8 @@ async def rows(): "ports": json.loads(c.ports_json) if c.ports_json else [], "uptime": _uptime(c.started_at) if c.status == "running" else None, "stale": _is_stale(c, now, stale_after, docker_down), + "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None))), + "exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed), }) if c.status == "running": g["running"] += 1 @@ -151,15 +160,20 @@ async def rows(): g["grouped"] = any(s["label"] for s in g["subgroups"]) host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower()) - swarm_services = swarm_view["services"] + swarm_services = services_problems_first(swarm_view["services"]) standalone_running = sum(g["running"] for g in host_groups) - ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"]) - attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down)) + attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down), + label=docker_svc.task_label(r["container"], node_hostname)) for r in docker_svc.needs_attention(ranked)] + failing = failing_services(swarm_services) return await render_template( "docker/rows.html", host_groups=host_groups, swarm_services=swarm_services, + failing_services=failing, + failures=failures, + services_healthy=sum(s["state"] == "healthy" for s in swarm_services), + docker_hosts=len({c.host_id for c in containers}), docker_down=sorted( ({"host_id": h, "host_name": hosts[h].name if h in hosts else h, "status": d.status, "detail": d.detail, "since": d.since} for h, d in docker_down.items()), @@ -168,9 +182,8 @@ async def rows(): unhealthy=sum("unhealthy" in r["problems"] for r in attention), restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention), stale=sum(1 for g in host_groups for i in g["containers"] if i["stale"]), - running=standalone_running + sum(s["running"] for s in swarm_services), + running=standalone_running, stopped=len(standalone) - standalone_running, - total=len(containers) + ghost_total, ) @@ -185,55 +198,55 @@ def _is_stale(c, now: datetime, stale_after: int, docker_down=()) -> bool: @docker_bp.get("/widget") @require_role(UserRole.viewer) async def widget(): - """HTMX dashboard widget: container status overview, grouped by host.""" + """HTMX dashboard widget: swarm services and containers, problems first, + with how many things failed in the last 24h (linked to the list).""" show_stopped = request.args.get("show_stopped", "no") == "yes" widget_id = request.args.get("wid", "0") - since_24h = datetime.now(timezone.utc) - timedelta(hours=24) + now = datetime.now(timezone.utc) async with current_app.db_sessionmaker() as db: - all_containers = dedup_by_container_id(list((await db.execute( - select(DockerContainer).order_by( - (DockerContainer.status == "running").desc(), - DockerContainer.name, - ) - )).scalars())) + all_containers = dedup_by_container_id(await docker_svc.list_containers(db)) services = list((await db.execute(select(DockerSwarmService))).scalars()) nodes = list((await db.execute(select(DockerSwarmNode))).scalars()) - # Recent failures are more actionable than a static "stopped" tally. Read - # DISTINCT container names so the two managers' duplicate die/oom events - # don't double it; swarm tasks are counted on their service below. - dead_names = list((await db.execute( - select(func.distinct(DockerEvent.container_name)) - .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since_24h) - )).scalars()) + events = await docker_svc.failure_events(db, now - timedelta(hours=24)) hosts = await docker_svc.host_map( db, {c.host_id for c in all_containers} | {n.host_id for n in nodes} | {s.host_id for s in services}, ) + capacity = await docker_svc.host_capacity(db, {c.host_id for c in all_containers}) # Swarm tasks collapse into per-service rows (each replica tagged with its # host); non-swarm containers stay grouped by host. node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname} host_name = {h.id: h.name for h in hosts.values()} - swarm_services = build_swarm_services( + swarm_services = services_problems_first(build_swarm_services( all_containers, services, node_hostname, host_name, - crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"] - failed_24h = docker_svc.failed_24h(dead_names, swarm_services) + crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"]) + failures = docker_svc.failures_24h(events, swarm_services, host_name) standalone = [c for c in all_containers if not c.service_name] running = [c for c in standalone if c.status == "running"] display = standalone if show_stopped else running + def trouble(c) -> int: + if (c.status or "") == "restarting" or c.health == "unhealthy": + return 0 + if c.status != "running" and docker_svc.exit_is_failure(c.exit_code, c.oom_killed): + return 1 + return 2 if c.status == "running" else 3 + # Group for display so multi-host fleets read clearly; single-host stays flat. groups: dict[str, dict] = {} - for c in display: + for c in sorted(display, key=lambda c: (trouble(c), c.name)): g = groups.setdefault(c.host_id, { "host_id": c.host_id, "host_name": host_name.get(c.host_id, c.host_id), "containers": [], }) - g["containers"].append(c) + g["containers"].append({"c": c, "usage": docker_svc.usage( + c, *capacity.get(c.host_id, (None, None))), + "exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed)}) host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower()) return await render_template( @@ -241,39 +254,59 @@ async def widget(): host_groups=host_groups, multi_host=len(host_groups) > 1, swarm_services=swarm_services, - running_count=len(running) + sum(s["running"] for s in swarm_services), - failed_24h=failed_24h, + services_healthy=sum(s["state"] == "healthy" for s in swarm_services), + running_count=len(running), + failures=failures, total_count=len(all_containers) + len(swarm_services), show_stopped=show_stopped, widget_id=widget_id, ) +CPU_AVERAGE_MINUTES = 5 + + @docker_bp.get("/widget/resources") @require_role(UserRole.viewer) async def widget_resources(): - """HTMX dashboard widget: CPU + memory usage for the busiest containers.""" + """HTMX dashboard widget: the containers using the most CPU, averaged over + the last few minutes so the order holds still between polls, with their + CPU in cores and memory in bytes.""" limit = max(1, min(20, int(request.args.get("limit", 10) or 10))) widget_id = request.args.get("wid", "0") + since = datetime.now(timezone.utc) - timedelta(minutes=CPU_AVERAGE_MINUTES) async with current_app.db_sessionmaker() as db: # Dedup before the limit, else swarm tasks reported by both managers can # fill the list with duplicates of the same container. - containers = dedup_by_container_id(list((await db.execute( - select(DockerContainer) - .where(DockerContainer.status == "running") - .order_by(DockerContainer.cpu_pct.desc().nullslast()) - )).scalars()))[:limit] - hosts = await docker_svc.host_map(db, {c.host_id for c in containers}) + running = dedup_by_container_id([ + c for c in await docker_svc.list_containers(db) if c.status == "running"]) + average = {(h, n): v for h, n, v in (await db.execute( + select(DockerMetric.host_id, DockerMetric.container_name, func.avg(DockerMetric.cpu_pct)) + .where(DockerMetric.scraped_at >= since) + .group_by(DockerMetric.host_id, DockerMetric.container_name) + )).all()} + nodes = list((await db.execute(select(DockerSwarmNode))).scalars()) + hosts = await docker_svc.host_map(db, {c.host_id for c in running}) + capacity = await docker_svc.host_capacity(db, {c.host_id for c in running}) - rows_data = [ - {"c": c, "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id} - for c in containers - ] + def cpu_of(c): + v = average.get((c.host_id, c.name)) + return c.cpu_pct if v is None else float(v) + + busiest = sorted(running, key=lambda c: (-(cpu_of(c) or 0), c.name))[:limit] + node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname} + rows_data = [{ + "c": c, + "label": docker_svc.task_label(c, node_hostname), + "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id, + "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c)), + } for c in busiest] return await render_template( "docker/widget_resources.html", rows=rows_data, - multi_host=len({c.host_id for c in containers}) > 1, + minutes=CPU_AVERAGE_MINUTES, + multi_host=len({c.host_id for c in busiest}) > 1, widget_id=widget_id, ) @@ -281,22 +314,50 @@ async def widget_resources(): @docker_bp.get("/host/") @require_role(UserRole.viewer) async def host_panel(host_id: str): - """Per-host Docker fragment embedded on the core Hosts hub via HTMX. + """Per-host Docker fragment embedded on the core Hosts hub via HTMX: + problems first (with exit code and since when), then running, then + stopped. Swarm tasks Swarm has already replaced are history and left out. Renders nothing when the host reports no containers, so hosts without Docker don't carry an empty card. """ + now = datetime.now(timezone.utc) async with current_app.db_sessionmaker() as db: containers = await docker_svc.list_containers(db, host_id) + if not containers: + return "" + services = list((await db.execute(select(DockerSwarmService))).scalars()) + nodes = list((await db.execute(select(DockerSwarmNode))).scalars()) + failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24)) + capacity = await docker_svc.host_capacity(db, {host_id}) - if not containers: - return "" - running = sum(1 for c in containers if c.status == "running") + replaced = replaced_tasks(containers, services) + node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname} + hour_ago = now - timedelta(hours=1) + crash = docker_svc.crash_loop_deaths(current_app.config) + rows = [] + for c in containers: + if (c.host_id, c.name) in replaced: + continue + events = failures.get((c.host_id, c.name), []) + recent = sum(1 for e in events if e.at >= hour_ago) + problems = docker_svc.container_problems(c, recent, bool(events), crash) + rows.append({ + "c": c, "label": docker_svc.task_label(c, node_hostname), "problems": problems, + "since": docker_svc.problem_since(problems, events, hour_ago), + "usage": docker_svc.usage(c, *capacity.get(host_id, (None, None))), + "uptime": _uptime(c.started_at) if c.status == "running" else None, + }) + rows.sort(key=lambda r: ( + docker_svc.PROBLEMS.index(r["problems"][0]) if r["problems"] else len(docker_svc.PROBLEMS), + r["c"].status != "running", r["label"].lower())) + running = sum(1 for r in rows if r["c"].status == "running") return await render_template( "docker/host_panel.html", - containers=containers, - running=running, - stopped=len(containers) - running, + rows=rows, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY, + running=running, stopped=len(rows) - running, + problems=sum(1 for r in rows if r["problems"]), + replaced=len(replaced), ) @@ -319,6 +380,15 @@ async def container_detail(host_id: str, name: str): c = await db.get(DockerContainer, (host_id, name)) host = await db.get(Host, host_id) events = await docker_svc.container_events(db, host_id, name) if c is not None else [] + node_name = None + if c is not None and c.node_id: + node_name = (await db.execute( + select(DockerSwarmNode.hostname) + .where(DockerSwarmNode.node_id == c.node_id, DockerSwarmNode.hostname != "") + .limit(1) + )).scalar() + cpus, mem_total = (await docker_svc.host_capacity(db, [host_id])).get(host_id, (None, None)) \ + if c is not None else (None, None) if c is None: return await render_template( @@ -332,6 +402,10 @@ async def container_detail(host_id: str, name: str): host=host, host_id=host_id, name=name, + label=docker_svc.task_label(c, {c.node_id: node_name} if node_name else None), + node_name=node_name, + usage=docker_svc.usage(c, cpus, mem_total), + exit_failed=docker_svc.exit_is_failure(c.exit_code, c.oom_killed), ports=json.loads(c.ports_json) if c.ports_json else [], uptime=_uptime(c.started_at) if c.status == "running" else None, net_rx=human_bytes(c.net_rx_bytes), net_tx=human_bytes(c.net_tx_bytes), @@ -415,7 +489,11 @@ async def swarm(): now = datetime.now(timezone.utc) deaths = docker_svc.crash_loop_deaths(current_app.config) svc_dicts = [] - for s in sorted(svc_by_name.values(), key=lambda s: s.service_name): + for s in svc_by_name.values(): + tasks = parse_tasks(s) + summary = task_summary(tasks, now, deaths, node_names) + state = ("healthy" if s.running >= s.desired and s.desired > 0 + else "degraded" if s.running > 0 else "down") placement = [] for p in (json.loads(s.placement_json) if s.placement_json else []): nid = p.get("node_id", "") @@ -429,18 +507,21 @@ async def swarm(): "healthy": s.running >= s.desired and s.desired > 0, "degraded": 0 < s.running < s.desired, "down": s.running == 0 and s.desired > 0, - "state": ("healthy" if s.running >= s.desired and s.desired > 0 - else "degraded" if s.running > 0 else "down"), + "state": state, "placement": placement, - **task_summary(parse_tasks(s), now, deaths, node_names), + **summary, + "trouble_since": trouble_since(tasks, state, summary["crash_looping"], now), }) + # Problems first: a node that is down or not taking work, then by role. node_rows = sorted( - node_by_id.values(), key=lambda n: (n.role, (n.hostname or n.node_id).lower()) + node_by_id.values(), key=lambda n: ( + n.status == "ready" and n.availability == "active", + n.role, (n.hostname or n.node_id).lower()) ) manager_names = sorted(hosts[m].name if m in hosts else m for m in members) swarms.append({ "managers": manager_names, - "services": svc_dicts, + "services": services_problems_first(svc_dicts), "nodes": node_rows, }) diff --git a/plugins/docker/service.py b/plugins/docker/service.py index 82ed4dc..adcd022 100644 --- a/plugins/docker/service.py +++ b/plugins/docker/service.py @@ -6,7 +6,7 @@ Quart's pipeline) call the same code. """ from __future__ import annotations -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from sqlalchemy import func, select @@ -30,25 +30,159 @@ def crash_loop_deaths(config) -> int: return int(th.get("warn") or CRASH_LOOP_DEATHS) -def failed_24h(dead_names, swarm_services) -> int: - """Failures in the last 24h for the dashboard: distinct plain containers - that died or were OOM-killed, plus each swarm service's failed tasks. +# Exit codes a deliberate stop leaves: SIGTERM (143), and SIGKILL (137) when +# the process outlives the stop timeout. Without an OOM kill they are someone +# stopping the container, not the container failing. +STOP_EXITS = (137, 143) - A swarm task's container dies whenever Swarm replaces it (a rolling update - kills it with 137), so its die event is not a failure. Services with - Swarm's task records count their failed tasks instead; a service without - them (an agent older than 1.8.0) falls back to its tasks' die events. + +def exit_is_failure(exit_code, oom_killed: bool = False) -> bool: + """Whether a container's exit is a failure: an OOM kill, or a non-zero + exit other than a stop's.""" + return bool(oom_killed) or exit_code not in (None, 0, *STOP_EXITS) + + +def _event_exit(detail) -> int | None: + """The exit code in a die event's detail ("exit 1"), or None.""" + text = (detail or "").strip() + if text.startswith("exit "): + try: + return int(text[5:]) + except ValueError: + return None + return None + + +def event_is_failure(e) -> bool: + """Whether a die/oom event is a failure. An oom always is; a die is unless + its exit is a stop's (or clean).""" + if e.event == "oom": + return True + return e.event == "die" and exit_is_failure(_event_exit(e.detail)) + + +def failures_24h(events, swarm_services, host_name: dict) -> list[dict]: + """What failed in the last 24h, newest first, for the dashboard count and + the Docker page's list of it: one row per plain container that died or was + OOM-killed, and one per swarm service with failed tasks. + + `events` are die/oom events of the last 24h. A swarm task's container dies + whenever Swarm replaces it, so a service with Swarm's task records counts + its failed tasks instead of its containers' deaths; a service without them + (an agent older than 1.8.0) falls back to its tasks' deaths. + + Each row: {kind: "container"|"service", name, host_id, host, count, at, + detail}. """ by_prefix = {s["name"] + ".": s for s in swarm_services} - plain, fallback = 0, set() - for name in dead_names: - svc = next((s for p, s in by_prefix.items() if name.startswith(p)), None) - if svc is None: - plain += 1 - elif not svc["tasks_known"]: - fallback.add(name) - tasks = sum(s["failures_24h"] or 0 for s in swarm_services if s["tasks_known"]) - return plain + len(fallback) + tasks + rows: dict[tuple, dict] = {} + # Each manager reports every swarm task, so one task's death can arrive + # once per manager: a service counts each dead task container once. + task_deaths: dict[tuple, set] = {} + for e in sorted(events, key=lambda e: e.at, reverse=True): + if not event_is_failure(e): + continue + svc = next((s for p, s in by_prefix.items() if e.container_name.startswith(p)), None) + if svc is not None and svc["tasks_known"]: + continue + kind, name = ("service", svc["name"]) if svc else ("container", e.container_name) + key = (kind, name) if svc else (kind, e.host_id, name) + if svc is not None: + seen = task_deaths.setdefault(key, set()) + if e.container_name in seen: + continue + seen.add(e.container_name) + row = rows.get(key) + if row is None: + rows[key] = {"kind": kind, "name": name, "host_id": e.host_id, + "host": host_name.get(e.host_id, e.host_id), "count": 1, "at": e.at, + "detail": "OOM-killed" if e.event == "oom" else e.detail} + else: + row["count"] += 1 + from .swarm_view import task_time + for s in swarm_services: + if s["tasks_known"] and s["failures_24h"]: + err = s["last_error"] or {} + rows[("service", s["name"])] = { + "kind": "service", "name": s["name"], "host_id": None, "host": err.get("node"), + "count": s["failures_24h"], "at": task_time(err) if err else None, + "detail": err.get("error") or err.get("message") or err.get("state")} + epoch = datetime.min.replace(tzinfo=timezone.utc) + return sorted(rows.values(), key=lambda r: r["at"] or epoch, reverse=True) + + +def task_label(c, node_hostname: dict | None = None) -> str: + """How a container reads to a person. A swarm task is its service and + slot: "codex_db · 1" for a replica, "agent · node-3" for a global + service's task on a node. Anything else is its own name.""" + from .swarm_view import _slot_key + svc = getattr(c, "service_name", None) + if not svc: + return c.name + slot = _slot_key(c) + if slot.isdigit(): + return f"{svc} · {slot}" + host = (node_hostname or {}).get(slot) or (node_hostname or {}).get(getattr(c, "node_id", None)) + return f"{svc} · {host}" if host else svc + + +def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = None) -> dict: + """A container's CPU and memory in terms a reader can weigh. + + Docker's CPU% is per core (100% = one core busy), so it is shown as cores + and graded as a share of the host's CPUs when the host's core count is + known. Docker's memory limit is the host's RAM when none is set, so memory + is shown in bytes, against a limit only when one is actually set; mem_pct + is then a share of that limit, otherwise of the host's memory. `cpu_pct` + replaces the container's last sample (an average, say) when given. + """ + cpu = getattr(c, "cpu_pct", None) if cpu_pct is None else cpu_pct + limit = getattr(c, "mem_limit_bytes", None) + limited = bool(limit and mem_total and limit < mem_total * 0.95) + return { + "cores": None if cpu is None else cpu / 100.0, + "cpu_share": None if cpu is None or not cpus else cpu / cpus, + "cpus": cpus, + "mem_bytes": getattr(c, "mem_usage_bytes", None), + "mem_limit": limit if limited else None, + "mem_pct": getattr(c, "mem_pct", None), + "mem_of": "its limit" if limited else ("the host's memory" if mem_total else None), + } + + +async def host_capacity(db, host_ids) -> dict[str, tuple[int | None, int | None]]: + """{host_id: (CPU cores, memory bytes)} from the host agent's latest + readings: cores counted from its per-core CPU series, memory from + available ÷ (1 − used%). (None, None) for a host without an agent.""" + from sqlalchemy import or_ + from steward.models.metrics import PluginMetricLatest + hosts = await host_map(db, set(host_ids)) + if not hosts: + return {} + names = {h.name: hid for hid, h in hosts.items()} + rows = (await db.execute( + select(PluginMetricLatest.resource_name, PluginMetricLatest.metric_name, + PluginMetricLatest.value) + .where(PluginMetricLatest.source_module == "host_agent", + or_(PluginMetricLatest.resource_name.in_(list(names)), + *(PluginMetricLatest.resource_name.like(escape_like(n) + ":core%", escape="\\") + for n in names))) + )).all() + cores: dict[str, int] = {} + mem: dict[str, dict] = {} + for res, metric, value in rows: + if res in names: + mem.setdefault(res, {})[metric] = value + elif metric == "cpu_pct": + host = res.rsplit(":core", 1)[0] + cores[host] = cores.get(host, 0) + 1 + out = {} + for name, hid in names.items(): + m = mem.get(name, {}) + avail, used = m.get("mem_available_bytes"), m.get("mem_used_pct") + total = int(avail / (1 - used / 100)) if avail and used is not None and used < 99.5 else None + out[hid] = (cores.get(name), total) + return out def bucket(values: list[float], target: int = 40) -> list[float]: @@ -210,7 +344,8 @@ async def log_context(db, host_id: str, name: str, ts, n: int) -> tuple[list, li async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]: - """{(host_id, container_name): [die/oom events since `since`, newest first]}.""" + """{(host_id, container_name): [die/oom events since `since`, newest first]}, + leaving out a die that was a stop (event_is_failure).""" rows = (await db.execute( select(DockerEvent) .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since) @@ -218,10 +353,21 @@ async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list] )).scalars() out: dict[tuple[str, str], list] = {} for e in rows: - out.setdefault((e.host_id, e.container_name), []).append(e) + if event_is_failure(e): + out.setdefault((e.host_id, e.container_name), []).append(e) return out +async def failure_events(db, since) -> list: + """Die and OOM events since `since`, newest first; failures_24h() decides + which of them are failures.""" + return list((await db.execute( + select(DockerEvent) + .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since) + .order_by(DockerEvent.at.desc()) + )).scalars()) + + async def retained_names(db, want: str, *, exact: bool, host_id: str | None = None, limit: int = 20) -> list[tuple[str, str, datetime]]: """(host_id, container_name, last seen) of containers Steward still holds @@ -303,7 +449,7 @@ def container_problems(c, failures_1h: int, failed_24h: bool, out.append("unhealthy") if c.oom_killed: out.append("oom_killed") - if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h: + if status in ("exited", "dead") and exit_is_failure(c.exit_code) and failed_24h: out.append("exited_nonzero") return out diff --git a/plugins/docker/swarm_view.py b/plugins/docker/swarm_view.py index 90a4b41..ad2de80 100644 --- a/plugins/docker/swarm_view.py +++ b/plugins/docker/swarm_view.py @@ -350,3 +350,31 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d standalone = sorted(groups.values(), key=lambda g: g["host"].lower()) return {"services": out_services, "standalone": standalone, "superseded": superseded} + + +def service_trouble(s: dict) -> int | None: + """How bad a built service is, for sorting and for Needs attention: 0 a + crash loop, 1 no replica running, 2 short of replicas, 3 healthy but with + failed tasks in the last 24h; None when nothing is wrong.""" + if s.get("crash_looping"): + return 0 + if s.get("state") == "down": + return 1 + if s.get("state") == "degraded": + return 2 + if s.get("failures_24h"): + return 3 + return None + + +def services_problems_first(services: list[dict]) -> list[dict]: + """Services in trouble first, worst first, then the rest by name.""" + return sorted(services, key=lambda s: ( + 4 if service_trouble(s) is None else service_trouble(s), s["name"].lower())) + + +def failing_services(services: list[dict]) -> list[dict]: + """The services that belong in Needs attention: a crash loop, or short of + their desired replicas. A healthy service whose failed tasks Swarm has + already replaced is history, shown on the service, not a problem now.""" + return [s for s in services_problems_first(services) if (service_trouble(s) or 9) <= 2] diff --git a/plugins/docker/templates/docker/_usage.html b/plugins/docker/templates/docker/_usage.html new file mode 100644 index 0000000..b270c62 --- /dev/null +++ b/plugins/docker/templates/docker/_usage.html @@ -0,0 +1,26 @@ +{#- A container's CPU and memory as service.usage() describes them (#5209). + + Docker counts CPU per core (100% = one core busy), so CPU reads as cores, + toned by its share of the host's CPUs when the host's core count is known. + Memory reads in bytes, against a limit only when one is actually set; + mem_pct is then a share of that limit, otherwise of the host's memory. -#} + +{% macro cores(n) -%} +{{ "%.2f"|format(n) if n < 1 else "%.1f"|format(n) }} {{ "core" if n == 1 else "cores" }} +{%- endmacro %} + +{% macro cpu(u) -%} +{%- if u.cores is none -%}— +{%- else -%} +{{ cores(u.cores) }} +{%- endif -%} +{%- endmacro %} + +{% macro mem(u) -%} +{%- if u.mem_bytes is none -%}— +{%- else -%} +{{ u.mem_bytes|bytes }}{% if u.mem_limit %} / {{ u.mem_limit|bytes }}{% endif %} +{%- endif -%} +{%- endmacro %} diff --git a/plugins/docker/templates/docker/container_detail.html b/plugins/docker/templates/docker/container_detail.html index bb0c986..5931077 100644 --- a/plugins/docker/templates/docker/container_detail.html +++ b/plugins/docker/templates/docker/container_detail.html @@ -1,8 +1,11 @@ -{# docker/container_detail.html — full detail page for one container #} +{# docker/container_detail.html — full detail page for one container. A swarm + task is titled by its service and slot, with the raw container name under + it; CPU reads in cores and memory in bytes (see _usage.html). #} {% extends "base.html" %} {% from "_macros.html" import crumbs %} -{% block title %}{{ name }} — Docker — Steward{% endblock %} -{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %} +{% from "docker/_usage.html" import cpu, mem %} +{% block title %}{{ label or name }} — Docker — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (label or name, "")]) }}{% endblock %} {% block content %} {% if container is none %} @@ -18,11 +21,14 @@ {# ── Header ──────────────────────────────────────────────────────────────── #}
-

{{ container.name }}

+

{{ label }}

{% if container.health == 'healthy' %}healthy {% elif container.health == 'unhealthy' %}unhealthy {% elif container.health == 'starting' %}starting{% endif %}
+{% if label != container.name %} +
{{ container.name }}
+{% endif %}
{{ container.status }}{% if uptime %} · up {{ uptime }}{% endif %} {% if host %} · on {{ host.name }}{% endif %} @@ -38,16 +44,27 @@ {% endmacro %}
- {{ fact("CPU", "%.1f%%" | format(container.cpu_pct) if container.cpu_pct is not none else "—") }} - {{ fact("Memory", "%.1f%%" | format(container.mem_pct) if container.mem_pct is not none else "—") }} + {{ fact("CPU", cpu(usage) if container.status == 'running' else "—") }} + {{ fact("Memory", mem(usage) if container.status == 'running' else "—") }} + {#- Swarm never restarts a task's container: it replaces the task with a new + one, so restart_count is always 0 there and the service's task failures + (Swarm page) are the count that means something. #} + {% if not container.service_name %} {{ fact("Restarts", container.restart_count, "var(--orange)" if container.restart_count else "") }} + {% endif %} {{ fact("Last exit", (container.exit_code ~ (" (OOM)" if container.oom_killed else "")) if container.exit_code is not none else "—", - "var(--red)" if (container.exit_code is not none and container.exit_code != 0) else "") }} - {{ fact("Net in", net_rx) }} - {{ fact("Net out", net_tx) }} - {{ fact("Block read", blk_read) }} - {{ fact("Block write", blk_write) }} + "var(--red)" if exit_failed else "") }} + {{ fact("Net in since start", net_rx) }} + {{ fact("Net out since start", net_tx) }} + {{ fact("Disk read since start", blk_read) }} + {{ fact("Disk written since start", blk_write) }}
+{% if container.service_name %} +
+ Swarm replaces a failed task with a new container rather than restarting it; the + service's failed tasks are on the Swarm page. +
+{% endif %} {# ── Image / placement ───────────────────────────────────────────────────── #}
@@ -61,7 +78,9 @@ Swarm service{{ container.service_name }} {% endif %} {% if container.node_id %} - Node{{ container.node_id }} + Node + {% if node_name %}{{ node_name }} + {% else %}{{ container.node_id }}{% endif %} {% endif %} {% if ports %} Ports diff --git a/plugins/docker/templates/docker/host_panel.html b/plugins/docker/templates/docker/host_panel.html index 942aef4..882e744 100644 --- a/plugins/docker/templates/docker/host_panel.html +++ b/plugins/docker/templates/docker/host_panel.html @@ -1,22 +1,58 @@ -{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub #} +{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub: + problems first with what is wrong and since when, then running, then + stopped. Tasks Swarm has replaced are left out (counted in the header). #} +{% from "_macros.html" import status_badge, status_dot %} +{% from "docker/_usage.html" import cpu, mem %}
-
+

Docker

+ {% if problems %}{{ problems }} with problems · {% endif %} {{ running }} running{% if stopped %} · {{ stopped }} stopped{% endif %} + {% if replaced %} · {{ replaced }} replaced task{{ "s" if replaced != 1 }} hidden{% endif %} All →
-
- {% for c in containers %} -
- - {{ c.name }} - - {% if c.cpu_pct is not none %}{{ "%.1f" | format(c.cpu_pct) }}%{% endif %} - {% if c.mem_pct is not none %} · {{ "%.1f" | format(c.mem_pct) }}%{% endif %} - -
- {% endfor %} -
+ + + + + + + + + + {% for r in rows %} + {% set c = r.c %} + + + + + + {% endfor %} + +
ContainerCPUMemory
+
+ {{ status_dot('crit' if r.problems else ('up' if c.status == 'running' else 'inactive'), c.status|capitalize) }} +
+ {{ r.label }} + {% if r.problems %} + {% set worst = r.problems[0] %} +
+ {{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }} + + {%- for p in r.problems %} {{ says[p] }}{{ " and" if not loop.last }}{% endfor %} + {%- if c.exit_code %} · exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}{% endif %} + {%- if r.since %} · since {{ r.since|ago }}{% endif -%} + +
+ {% else %} +
+ {{ c.status }}{% if r.uptime %} · up {{ r.uptime }}{% endif %} + {%- if c.status != 'running' and c.exit_code %} · exit {{ c.exit_code }}{% endif %} +
+ {% endif %} +
+
+ 		{% if c.status == 'running' %}{{ cpu(r.usage) }}{% else %}—{% endif %}{% if c.status == 'running' %}{{ mem(r.usage) }}{% else %}—{% endif %}
diff --git a/plugins/docker/templates/docker/index.html b/plugins/docker/templates/docker/index.html index 62dfce9..d662bc3 100644 --- a/plugins/docker/templates/docker/index.html +++ b/plugins/docker/templates/docker/index.html @@ -13,4 +13,17 @@ hx-swap="innerHTML">

Loading…

+{# A link to a section (the dashboard's "failed (24h)" → #failures) lands + before the rows exist; scroll to it once, when they first arrive. #} + {% endblock %} diff --git a/plugins/docker/templates/docker/rows.html b/plugins/docker/templates/docker/rows.html index 8e6d9f8..b0a348e 100644 --- a/plugins/docker/templates/docker/rows.html +++ b/plugins/docker/templates/docker/rows.html @@ -2,18 +2,36 @@ first (service.needs_attention, the ranking the MCP tools use), the counts, then swarm services and each host's containers. #} {% from "_macros.html" import stat_tile, status_badge, status_dot %} +{% from "docker/_usage.html" import cpu, mem, cores %} {# ── Needs attention ──────────────────────────────────────────────────────── #} -{% if attention %} +{% if attention or failing_services %}

Needs attention

    + {# A swarm service in trouble: Swarm replaces its failed tasks, so the + service, not any one container, is what is failing. #} + {% for s in failing_services %} + {% set e = s.last_error %} +
  • + {% if s.crash_looping %}{{ status_badge('crit', 'Crash-looping') }} + {% elif s.state == 'down' %}{{ status_badge('crit', 'Down') }} + {% else %}{{ status_badge('warn', 'Degraded') }}{% endif %} + {{ s.name }} + + {{- s.running }} of {{ s.desired }} running + {%- if s.crash_looping %} · {{ s.failures_1h }} tasks failed in the last hour{% endif %} + {%- if e %} · last task {{ e.state }}: {{ e.error or e.message or e.state }}{% if e.exit_code %} (exit {{ e.exit_code }}){% endif %}{% if e.node %} on {{ e.node }}{% endif %}{% endif %} + {%- if s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif -%} + +
    • + {% endfor %} {% for r in attention %} {% set c = r.container %} {% set worst = r.problems[0] %}
  • {{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }} - {{ c.name }} + {{ r.label }} {%- for p in r.problems %}{{ says[p] }}{{ " and " if not loop.last }}{% endfor %} {%- if c.exit_code and worst in ('exited_nonzero', 'crash_looping', 'restarting') %} · exit {{ c.exit_code }}{% endif %} @@ -42,18 +60,45 @@
{% endif %} -{# ── Counts ───────────────────────────────────────────────────────────────── #} +{# ── Counts ────────────────────────────────────────────────────────────── + Swarm services are counted as services (a task's container comes and goes); + the container counts are the containers outside any swarm service. ────── #}
- {{ stat_tile("Needs attention", attention|length, state="crit" if attention else "inactive") }} + {{ stat_tile("Needs attention", attention|length + failing_services|length, state="crit" if attention or failing_services else "inactive") }} {{ stat_tile("Unhealthy", unhealthy, state="crit" if unhealthy else "inactive") }} {{ stat_tile("Restarting", restarting, state="crit" if restarting else "inactive", hint="restarting or crash-looping") }} - {{ stat_tile("Running", running, state="up" if running else "inactive") }} - {{ stat_tile("Stopped", stopped, state="inactive") }} + {{ stat_tile("Failed (24h)", failures|length, state="warn" if failures else "inactive", hint="containers and services; listed below", href="#failures" if failures else none) }} + {% if swarm_services %}{{ stat_tile("Swarm services", services_healthy ~ "/" ~ swarm_services|length, state="up" if services_healthy == swarm_services|length else "warn", hint="healthy", href="/plugins/docker/swarm") }}{% endif %} + {{ stat_tile("Containers running", running, state="up" if running else "inactive", hint="outside swarm services" if swarm_services else none) }} + {{ stat_tile("Containers stopped", stopped, state="inactive") }} {% if stale %}{{ stat_tile("Stale", stale, state="stale", hint="no recent report") }}{% endif %} - {% if swarm_services %}{{ stat_tile("Services", swarm_services|length, href="/plugins/docker/swarm") }}{% endif %} - {{ stat_tile("Hosts", host_groups|length) }} + {{ stat_tile("Docker hosts", docker_hosts) }}
+{# ── What failed in the last 24h: what the dashboard's "failed (24h)" counts. + A plain container's death, or a swarm service's failed tasks; a container + someone stopped (exit 137/143) is not a failure. ───────────────────────── #} +{% if failures %} +
+

Failed in the last 24h

+
    + {% for f in failures %} +
  • + {{ status_badge('warn', 'Service' if f.kind == 'service' else 'Container') }} + {% if f.kind == 'service' %}{{ f.name }} + {% else %}{{ f.name }}{% endif %} + + {{- f.count }} {{ "failure" if f.count == 1 else "failures" }} + {%- if f.detail %} · last: {{ f.detail }}{% endif %} + {%- if f.host %} · {{ f.host }}{% endif %} + {%- if f.at %} · {{ f.at|ago }}{% endif -%} + +
    • + {% endfor %} +
+
+{% endif %} + {# ── Swarm services: collapsed per-service, each current replica labelled with its host (tasks Swarm replaced are left out). Ghost replicas (tasks on nodes with no Steward agent) come from the managers' placement data so the picture @@ -72,7 +117,7 @@ {{ s.name }} {{ s.mode }} - {{ s.running }}/{{ s.desired }} running + {{ s.running }}/{{ s.desired }} running{% if s.state != 'healthy' and s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif %}
{% with style="padding-left:1.1rem;margin-bottom:0.35rem;" %}{% include "docker/_swarm_failures.html" %}{% endwith %} @@ -88,7 +133,7 @@
{{ r.host }} - {% if r.cpu_pct is not none %}{{ "%.0f" | format(r.cpu_pct) }}%{% endif %} + {% if r.cpu_pct is not none %}{{ cores(r.cpu_pct / 100) }}{% endif %}
{% endif %} {% endfor %} @@ -119,8 +164,8 @@ Container Image Ports - CPU % - Mem % + CPU + Memory @@ -144,7 +189,7 @@
{{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %} {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %} - · exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %} + · exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %} {% endif %} {% if c.restart_count %} · ⟳{{ c.restart_count }}{% endif %}
@@ -164,20 +209,8 @@
{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}
{% endfor %} - - {% if c.cpu_pct is not none %} - {{ "%.1f" | format(c.cpu_pct) }}% - {% else %} - — - {% endif %} - - - {% if c.mem_pct is not none %} - {{ "%.1f" | format(c.mem_pct) }}% - {% else %} - — - {% endif %} - + {{ cpu(item.usage) }} + {{ mem(item.usage) }} {% endfor %} {% endfor %} diff --git a/plugins/docker/templates/docker/swarm.html b/plugins/docker/templates/docker/swarm.html index efb3342..4da31c3 100644 --- a/plugins/docker/templates/docker/swarm.html +++ b/plugins/docker/templates/docker/swarm.html @@ -1,4 +1,6 @@ -{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement #} +{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement. + Services in trouble come first (crash loop, down, short), with since-times; + nodes that are down or not taking work come first. #} {% extends "base.html" %} {% from "_macros.html" import crumbs %} {% block title %}Swarm — Docker — Steward{% endblock %} @@ -34,6 +36,7 @@ {{ s.running }}/{{ s.desired }} + {% if s.state != 'healthy' and s.trouble_since %}
since {{ s.trouble_since|ago }}
{% endif %} {{ s.image or "—" }} diff --git a/plugins/docker/templates/docker/widget.html b/plugins/docker/templates/docker/widget.html index 44dd5d8..9d0ce9a 100644 --- a/plugins/docker/templates/docker/widget.html +++ b/plugins/docker/templates/docker/widget.html @@ -1,21 +1,32 @@ -{# docker/widget.html — dashboard widget: container status overview, by host #} +{# docker/widget.html — dashboard widget: what Docker is running and what has + failed. Swarm services first, troubled ones at the top with since-times; then + the containers outside any service, by host, problems first. The failed count + links to the Docker page's list of exactly what it counts. #} +{% from "docker/_usage.html" import cpu %} {% if total_count == 0 %}

No containers reported yet.

{% else %} -
-
+
+ {% if swarm_services %} +
+ {{ services_healthy }}/{{ swarm_services|length }} + services healthy +
+ {% endif %} +
{{ running_count }} - running -
-
- {{ failed_24h }} - failed (24h) + containers running
+ + {{ failures|length }} + failed in 24h +
-{# Swarm services — collapsed, each with its replicas' host chips (dashed = a - node with no Steward agent, count-only from placement). #} +{# Swarm services — troubled first, each with its replicas' host chips + (dashed = a node with no Steward agent, count-only from placement). #} {% if swarm_services %}
Swarm services
@@ -24,8 +35,9 @@
{{ s.name }} - {% if s.crash_looping %}crash-looping{% endif %} - {{ s.running }}/{{ s.desired }} + {% if s.crash_looping %}crash-looping{% endif %} + {% if s.state != 'healthy' and s.trouble_since %}since {{ s.trouble_since|ago }}{% endif %} + {{ s.running }}/{{ s.desired }}
{% for r in s.replicas %} @@ -41,25 +53,25 @@
{% endif %} +{% if host_groups and swarm_services %} +
Other containers
+{% endif %} {% for g in host_groups %} {% if multi_host %} -
{{ g.host_name }}
+
{{ g.host_name }}
{% endif %}
- {% for c in g.containers %} + {% for item in g.containers %} + {% set c = item.c %}
- + - {{ c.name }}{% if c.health == 'unhealthy' %} ●{% elif c.health == 'healthy' %} ●{% endif %}{% if c.restart_count %} ⟳{{ c.restart_count }}{% endif %} + {{ c.name }}{% if c.health == 'unhealthy' %} unhealthy{% endif %}{% if c.status == 'restarting' %} restarting{% endif %} {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %} - exit {{ c.exit_code }} - {% endif %} - {% if c.cpu_pct is not none %} - - {{ "%.1f" | format(c.cpu_pct) }}% - + exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %} {% endif %} + {% if c.status == 'running' %}{{ cpu(item.usage) }}{% endif %}
{% endfor %}
diff --git a/plugins/docker/templates/docker/widget_resources.html b/plugins/docker/templates/docker/widget_resources.html index c4605fa..8d792fd 100644 --- a/plugins/docker/templates/docker/widget_resources.html +++ b/plugins/docker/templates/docker/widget_resources.html @@ -1,28 +1,41 @@ -{# docker/widget_resources.html — dashboard widget: the busiest running containers - by CPU, each with labeled CPU + memory utilisation bars. #} +{# docker/widget_resources.html — dashboard widget: the containers using the + most CPU (averaged over the last few minutes, so the order holds still), + each with its CPU in cores and memory in bytes. The CPU bar is a share of + the host's CPUs; the memory bar a share of the container's limit, or of + the host's memory when no limit is set. #} +{% from "docker/_usage.html" import cpu, mem %} {% if not rows %}
No running containers.
{% else %} -{% macro bar(label, pct, warn, crit) %} -
+{% macro bar(label, pct, tone, text, title) %} +
{{ label }}
-
+ {% endif %}
- {{ "%.1f" | format(pct) }}% + {{ text }}
{% endmacro %} +
Busiest by CPU, {{ minutes }}-minute average
{% for r in rows %} - {% set c = r.c %} + {% set u = r.usage %}
-
- {{ c.name }}{% if multi_host %} · {{ r.host_name }}{% endif %} +
+ {{ r.label }}{% if multi_host %} · {{ r.host_name }}{% endif %}
- {% if c.cpu_pct is not none %}{{ bar("CPU", c.cpu_pct, 50, 80) }}{% endif %} - {% if c.mem_pct is not none %}{{ bar("MEM", c.mem_pct, 70, 90) }}{% endif %} + {% if u.cores is not none %} + {{ bar("CPU", u.cpu_share, status_class(u.cpu_share, 'cpu'), cpu(u), + ("%.1f%% of the host's %d core%s" % (u.cpu_share, u.cpus, "" if u.cpus == 1 else "s")) if u.cpus else "Host core count unknown") }} + {% endif %} + {% if u.mem_bytes is not none %} + {{ bar("MEM", u.mem_pct if u.mem_of else none, status_class(u.mem_pct, 'mem') if u.mem_of else 'unknown', mem(u), + ("%.1f%% of %s" % (u.mem_pct, u.mem_of)) if u.mem_of and u.mem_pct is not none else "") }} + {% endif %}
{% endfor %}
diff --git a/steward/core/widgets.py b/steward/core/widgets.py index 8660d6d..ae6d769 100644 --- a/steward/core/widgets.py +++ b/steward/core/widgets.py @@ -209,8 +209,8 @@ WIDGET_REGISTRY: dict[str, dict] = { }, "docker_containers": { "key": "docker_containers", - "label": "Docker — Containers", - "description": "Container status overview — running/stopped counts and per-container CPU", + "label": "Docker — Services & failures", + "description": "What Docker is running and what failed: swarm services and containers, problems first, with the last 24h's failures", "hx_url": "/plugins/docker/widget", "detail_url": "/plugins/docker/", "plugin": "docker", @@ -230,8 +230,8 @@ WIDGET_REGISTRY: dict[str, dict] = { }, "docker_resources": { "key": "docker_resources", - "label": "Docker — Resources", - "description": "CPU and memory usage bars for running containers", + "label": "Docker — Busiest containers", + "description": "The containers using the most CPU (cores, 5-minute average) and their memory", "hx_url": "/plugins/docker/widget/resources", "detail_url": "/plugins/docker/", "plugin": "docker", diff --git a/tests/plugins/docker/test_docker_page.py b/tests/plugins/docker/test_docker_page.py index 343af45..ff47963 100644 --- a/tests/plugins/docker/test_docker_page.py +++ b/tests/plugins/docker/test_docker_page.py @@ -43,22 +43,24 @@ def test_stale_is_older_than_the_agents_window(): async def test_a_crash_loop_comes_first_and_a_stale_row_says_so(app): loop = _c("api", status="restarting", exit_code=1, restart_count=7) old = _c("cache", scraped_at=NOW - timedelta(minutes=30), cpu_pct=95.0) - attention = [{"container": loop, "problems": ["restarting", "crash_looping"], "replaced": False, + attention = [{"container": loop, "label": "api", "problems": ["restarting", "crash_looping"], "replaced": False, "since": datetime.now(timezone.utc) - timedelta(minutes=8), "stale": False}] groups = [{"host": None, "host_id": "h1", "host_name": "nas", "running": 1, "stopped": 1, "grouped": False, "subgroups": [{"label": None, "containers": [ - {"container": loop, "ports": [], "uptime": None, "stale": False}, - {"container": old, "ports": [], "uptime": "2h", "stale": True}]}]}] + {"container": loop, "ports": [], "uptime": None, "stale": False, + "usage": svc.usage(loop, 1, None), "exit_failed": True}, + {"container": old, "ports": [], "uptime": "2h", "stale": True, + "usage": svc.usage(old, 1, None), "exit_failed": False}]}]}] async with app.test_request_context("/plugins/docker/rows"): page = await render_plugin_template( app, "docker", "docker/rows.html", attention=attention, says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=1, stale=1, host_groups=groups, swarm_services=[], - running=1, stopped=1, total=2) + running=1, stopped=1, docker_hosts=1, failures=[], failing_services=[]) page = re.sub(r">\s+<", "><", " ".join(page.split())) card = page[page.index("Needs attention"):page.index("")] assert 'Restarting' in card - assert 'href="/plugins/docker/container/h1/api">api' in card + assert 'href="/plugins/docker/container/h1/api" title="api">api' in card assert "is restarting and keeps dying · exit 1 · since 8m ago" in card assert "cache" not in card assert page.index("Needs attention") < page.index("Restarting1' in page assert 'Stale1' in page assert 'class="badge badge-warn" title="Last report' in page # the stale row - assert '95.0%' in page # CPU through Settings → Thresholds + # CPU in cores, toned by its share of the host's CPUs through Settings → Thresholds. + assert ('0.95 cores') in page assert "history" not in page.lower() and "" not in page and "No containers reported yet" in page @@ -89,7 +94,7 @@ async def test_a_host_whose_docker_stopped_answering_says_so(app): page = await render_plugin_template( app, "docker", "docker/rows.html", docker_down=down, attention=[], says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[], - swarm_services=[], running=0, stopped=0, total=0) + swarm_services=[], running=0, stopped=0, docker_hosts=0, failures=[], failing_services=[]) page = re.sub(r">\s+<", "><", " ".join(page.split())) card = page[page.index("Docker not answering"):page.index("")] assert 'No Dockernas' in card @@ -102,5 +107,35 @@ async def test_no_card_while_every_docker_answers(app): page = await render_plugin_template( app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[], - swarm_services=[], running=0, stopped=0, total=0) + swarm_services=[], running=0, stopped=0, docker_hosts=0, failures=[], failing_services=[]) assert "Docker not answering" not in page + + +async def test_a_failing_service_leads_and_the_failures_are_listed(app): + now = datetime.now(timezone.utc) + sick = {"name": "codex_db", "mode": "replicated", "running": 0, "desired": 1, "state": "down", + "crash_looping": True, "failures_1h": 7, "failures_24h": 9, "tasks_known": True, + "trouble_since": now - timedelta(minutes=20), "replicas": [], "previous_tasks": 7, + "last_error": {"state": "failed", "error": "task: non-zero exit (1)", "exit_code": 1, + "node": "nas", "at": None}} + failures = [{"kind": "service", "name": "codex_db", "host_id": None, "host": "nas", "count": 9, + "at": now - timedelta(minutes=2), "detail": "task: non-zero exit (1)"}, + {"kind": "container", "name": "backup", "host_id": "h1", "host": "nas", "count": 1, + "at": now - timedelta(hours=3), "detail": "exit 2"}] + async with app.test_request_context("/plugins/docker/rows"): + page = await render_plugin_template( + app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS, + severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[], + swarm_services=[sick], services_healthy=0, failing_services=[sick], failures=failures, + running=0, stopped=0, docker_hosts=1) + page = re.sub(r">\s+<", "><", " ".join(page.split())) + card = page[page.index("Needs attention"):page.index("")] + assert 'Crash-loopingcodex_db' in card + assert "0 of 1 running · 7 tasks failed in the last hour · last task failed: task: non-zero exit (1)" in card + assert "since Swarm services0/1' in page + listed = page[page.index('id="failures"'):] + listed = listed[:listed.index("")] + assert listed.index("codex_db") < listed.index("backup") # newest first + assert "9 failures · last: task: non-zero exit (1) · nas" in listed + assert 'href="/plugins/docker/container/h1/backup">backup' in listed diff --git a/tests/plugins/docker/test_mcp_tools.py b/tests/plugins/docker/test_mcp_tools.py index d4036a8..759e9ea 100644 --- a/tests/plugins/docker/test_mcp_tools.py +++ b/tests/plugins/docker/test_mcp_tools.py @@ -34,12 +34,21 @@ def test_container_problems_most_severe_first(): def test_exited_nonzero_needs_a_recent_death(): - crashed = _c(status="exited", exit_code=137) + crashed = _c(status="exited", exit_code=1) assert t.container_problems(crashed, 1, True) == ["exited_nonzero"] # A container that died weeks ago and was left stopped is not news. assert t.container_problems(crashed, 0, False) == [] +def test_a_stop_is_not_an_exit_failure(): + # 137 (SIGKILL) and 143 (SIGTERM) are how `docker stop` ends a container; + # without an OOM kill that is someone stopping it, not it failing (#5209). + for code in (137, 143): + assert t.container_problems(_c(status="exited", exit_code=code), 1, True) == [] + oom = _c(status="exited", exit_code=137, oom_killed=True) + assert t.container_problems(oom, 1, True) == ["oom_killed"] + + def test_cut_middle_keeps_both_ends_and_sizes_the_gap(): assert t.cut_middle("short") == "short" line = "A" * 10 + "B" * 100 + "C" * 10 diff --git a/tests/plugins/docker/test_service.py b/tests/plugins/docker/test_service.py index dd669c4..8d2f866 100644 --- a/tests/plugins/docker/test_service.py +++ b/tests/plugins/docker/test_service.py @@ -1,5 +1,11 @@ """Unit tests for the docker plugin's shared query helpers (no DB).""" +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace as NS + from plugins.docker import service as svc +from plugins.docker import swarm_view as sv + +NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc) def test_escape_like_makes_wildcards_literal(): @@ -27,3 +33,78 @@ def test_prune_extra_vars_mapping(): } # System prune stays conservative — no prune_all_images key. assert svc.prune_extra_vars("system") == {"prune_target": "system"} + + +def test_a_stop_is_not_a_failure(): + # docker stop ends a container with 143 (SIGTERM) or 137 (SIGKILL). + assert not svc.exit_is_failure(None) and not svc.exit_is_failure(0) + assert not svc.exit_is_failure(137) and not svc.exit_is_failure(143) + assert svc.exit_is_failure(1) and svc.exit_is_failure(2) + assert svc.exit_is_failure(137, oom_killed=True) + + +def _ev(name, event, detail="", mins=5, host="h1"): + return NS(container_name=name, host_id=host, event=event, detail=detail, at=NOW - timedelta(minutes=mins)) + + +def test_event_is_failure_reads_the_exit_from_the_die_detail(): + assert svc.event_is_failure(_ev("a", "die", "exit 1")) + assert not svc.event_is_failure(_ev("a", "die", "exit 143")) + assert not svc.event_is_failure(_ev("a", "die", "exit 0")) + assert svc.event_is_failure(_ev("a", "oom")) + assert not svc.event_is_failure(_ev("a", "start")) + + +def _service(name, *, known=True, failures=0, error=None): + return {"name": name, "tasks_known": known, "failures_24h": failures if known else None, + "last_error": error} + + +def test_failures_24h_counts_services_once_and_skips_stops(): + events = [_ev("backup", "die", "exit 2", mins=30), _ev("backup", "die", "exit 2", mins=90), + _ev("web", "die", "exit 143", mins=10), # someone stopped it + _ev("app.1.abc", "die", "exit 1", mins=3), # Swarm counts its tasks + _ev("old.1.x", "die", "exit 1", mins=4), _ev("old.2.y", "die", "exit 1", mins=40), + _ev("old.1.x", "die", "exit 1", mins=4, host="h2")] # a second manager's copy + services = [_service("app", failures=3, error={"state": "failed", "error": "boom", "node": "nas", + "at": (NOW - timedelta(minutes=1)).isoformat()}), + _service("old", known=False)] # agent before 1.8.0 + rows = svc.failures_24h(events, services, {"h1": "nas"}) + got = {(r["kind"], r["name"]): r for r in rows} + assert set(got) == {("container", "backup"), ("service", "app"), ("service", "old")} + assert got[("container", "backup")]["count"] == 2 and got[("container", "backup")]["host"] == "nas" + assert got[("service", "app")]["count"] == 3 and got[("service", "app")]["detail"] == "boom" + assert got[("service", "old")]["count"] == 2 # its tasks' deaths + assert [r["name"] for r in rows] == ["app", "old", "backup"] # newest first + + +def test_task_label_names_the_service_and_slot(): + assert svc.task_label(NS(name="nginx", service_name=None)) == "nginx" + assert svc.task_label(NS(name="codex_db.1.37ma", service_name="codex_db")) == "codex_db · 1" + agent = NS(name="agent.n0d3id.t4sk", service_name="agent", node_id="n0d3id") + assert svc.task_label(agent, {"n0d3id": "node-3"}) == "agent · node-3" + assert svc.task_label(agent) == "agent" + + +def test_usage_reads_cpu_in_cores_and_memory_against_a_real_limit(): + gib = 1024 ** 3 + c = NS(cpu_pct=150.0, mem_usage_bytes=gib, mem_limit_bytes=16 * gib, mem_pct=6.25) + u = svc.usage(c, 4, 16 * gib) + assert u["cores"] == 1.5 and u["cpu_share"] == 37.5 + # A limit of the host's whole memory is Docker saying "no limit". + assert u["mem_limit"] is None and u["mem_of"] == "the host's memory" + limited = svc.usage(NS(cpu_pct=None, mem_usage_bytes=gib, mem_limit_bytes=2 * gib, mem_pct=50.0), 4, 16 * gib) + assert limited["cores"] is None and limited["mem_limit"] == 2 * gib and limited["mem_of"] == "its limit" + # Unknown host size: cores still read, no share to grade against. + assert svc.usage(c, None, None)["cpu_share"] is None + assert svc.usage(c, 4, 16 * gib, cpu_pct=50.0)["cores"] == 0.5 # an average in place + + +def test_services_in_trouble_come_first_worst_first(): + def s(name, state, loop=False, f24=0): + return {"name": name, "state": state, "crash_looping": loop, "failures_24h": f24} + services = [s("alpha", "healthy"), s("beta", "healthy", f24=2), s("gamma", "degraded"), + s("delta", "down"), s("eps", "down", loop=True)] + assert [x["name"] for x in sv.services_problems_first(services)] == ["eps", "delta", "gamma", "beta", "alpha"] + # A recovered service's failed tasks are history: not in Needs attention. + assert [x["name"] for x in sv.failing_services(services)] == ["eps", "delta", "gamma"]