bvandeusenandClaude Opus 5.5
61b22056d5
#5209 "Docker views: readable names, CPU/memory in cores and bytes, failures listed with since-time" (milestone 477 "UI problems first"). - A swarm task reads as its service and slot ("codex_db · 1", or the node for a global service), with the raw name in the title: Docker page, host panel, busiest widget, container detail. - CPU reads in cores (Docker's 100% = one core), toned by its share of the host's CPUs from the agent's per-core series. Memory reads in bytes, against a limit only when one is really set (Docker reports host RAM otherwise). - Exit 137/143 without an OOM kill is a stop, not a failure: no exited_nonzero, no crash-loop death, no "failed (24h)", dim exit code. - "Failed (24h)" is a list (Docker page #failures, linked from the widget and the tile): plain containers by their deaths, swarm services by Swarm's failed tasks; a second manager's copy of a task death counts once. - Failing swarm services lead Needs attention; services sort problems first with since-times on the page, the widget and the Swarm page; nodes that are down or drained sort first. - Busiest widget orders by a 5-minute CPU average so the list holds still. - Host panel: a table, problems first with what is wrong and since when; replaced swarm tasks hidden and counted. - Container detail: node hostname, I/O labelled "since start", Restarts hidden for swarm tasks with a note on where task failures are. - Widget names: "Docker — Services & failures", "Docker — Busiest containers". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
181 lines
9.2 KiB
Python
181 lines
9.2 KiB
Python
"""The docker plugin's MCP tools: what counts as a problem, and how it is reported."""
|
|
import inspect
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace as NS
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
import plugins.docker as docker_plugin
|
|
from plugins.docker import mcp_tools as t
|
|
from steward.models.users import UserRole
|
|
|
|
NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
def _c(**kw):
|
|
base = dict(name="web", host_id="h1", image="nginx", status="running", health=None,
|
|
exit_code=None, oom_killed=False, restart_count=0, cpu_pct=1.0, mem_pct=2.0,
|
|
mem_usage_bytes=1, mem_limit_bytes=2, started_at=NOW - timedelta(hours=1),
|
|
compose_project=None, service_name=None, scraped_at=NOW - timedelta(seconds=30))
|
|
base.update(kw)
|
|
return NS(**base)
|
|
|
|
|
|
def test_container_problems_most_severe_first():
|
|
assert t.container_problems(_c(), 0, False) == []
|
|
assert t.container_problems(_c(status="restarting"), 4, True) == ["restarting", "crash_looping"]
|
|
# The crash-loop count comes from Settings → Thresholds.
|
|
assert t.container_problems(_c(), 2, True) == []
|
|
assert t.container_problems(_c(), 2, True, crash_loop_deaths=2) == ["crash_looping"]
|
|
assert t.container_problems(_c(health="unhealthy", oom_killed=True), 0, False) == ["unhealthy", "oom_killed"]
|
|
# An unhealthy flag on a stopped container is stale; it isn't failing a healthcheck now.
|
|
assert t.container_problems(_c(status="exited", health="unhealthy", exit_code=0), 0, False) == []
|
|
|
|
|
|
def test_exited_nonzero_needs_a_recent_death():
|
|
crashed = _c(status="exited", exit_code=1)
|
|
assert t.container_problems(crashed, 1, True) == ["exited_nonzero"]
|
|
# A container that died weeks ago and was left stopped is not news.
|
|
assert t.container_problems(crashed, 0, False) == []
|
|
|
|
|
|
def test_a_stop_is_not_an_exit_failure():
|
|
# 137 (SIGKILL) and 143 (SIGTERM) are how `docker stop` ends a container;
|
|
# without an OOM kill that is someone stopping it, not it failing (#5209).
|
|
for code in (137, 143):
|
|
assert t.container_problems(_c(status="exited", exit_code=code), 1, True) == []
|
|
oom = _c(status="exited", exit_code=137, oom_killed=True)
|
|
assert t.container_problems(oom, 1, True) == ["oom_killed"]
|
|
|
|
|
|
def test_cut_middle_keeps_both_ends_and_sizes_the_gap():
|
|
assert t.cut_middle("short") == "short"
|
|
line = "A" * 10 + "B" * 100 + "C" * 10
|
|
cut = t.cut_middle(line, limit=20)
|
|
assert cut.startswith("A" * 10) and cut.endswith("C" * 10)
|
|
assert "[… 100 characters omitted …]" in cut
|
|
|
|
|
|
def test_container_row_uptime_only_while_running():
|
|
row = t.container_row(_c(), {"h1": "nas"}, [], NOW)
|
|
assert row["host_name"] == "nas" and row["uptime_s"] == 3600 and row["reported_age_s"] == 30
|
|
assert t.container_row(_c(status="exited"), {}, [], NOW)["uptime_s"] is None
|
|
|
|
|
|
def test_container_findings_one_per_container_with_every_problem_named():
|
|
rows = [t.container_row(_c(name="db", status="restarting", exit_code=1), {"h1": "nas"},
|
|
["restarting", "crash_looping"], NOW),
|
|
t.container_row(_c(name="ok"), {}, [], NOW)]
|
|
(f,) = t.container_findings(rows)
|
|
assert f["severity"] == "critical" and f["kind"] == "container_restarting"
|
|
assert "is restarting and keeps dying" in f["summary"] and "on nas" in f["summary"]
|
|
assert "(exit 1)" in f["summary"]
|
|
assert f["next"] == 'get_container(container="db", host="h1")'
|
|
|
|
|
|
def test_service_findings_by_state():
|
|
svcs = [{"name": "a", "state": "healthy", "running": 2, "desired": 2},
|
|
{"name": "b", "state": "degraded", "running": 1, "desired": 3},
|
|
{"name": "c", "state": "down", "running": 0, "desired": 1}]
|
|
got = {f["subject"]: f["severity"] for f in t.service_findings(svcs)}
|
|
assert got == {"b": "warning", "c": "critical"}
|
|
|
|
|
|
def test_service_findings_name_the_crash_loop_and_swarms_error():
|
|
err = {"error": "task: non-zero exit (1)", "message": "started", "state": "failed",
|
|
"exit_code": 1, "node": "docker03"}
|
|
svcs = [{"name": "db", "state": "down", "running": 0, "desired": 1, "crash_looping": True,
|
|
"failures_1h": 4, "failures_24h": 5, "last_error": err},
|
|
{"name": "app", "state": "down", "running": 0, "desired": 1, "crash_looping": False,
|
|
"failures_1h": 0, "failures_24h": 0,
|
|
"last_error": {"error": "No such image: app:dev", "state": "rejected",
|
|
"exit_code": None, "node": None, "message": None}},
|
|
{"name": "web", "state": "healthy", "running": 2, "desired": 2, "crash_looping": False,
|
|
"failures_1h": 0, "failures_24h": 1, "last_error": err},
|
|
{"name": "ok", "state": "healthy", "running": 1, "desired": 1, "crash_looping": False,
|
|
"failures_1h": 0, "failures_24h": 0, "last_error": None}]
|
|
got = {f["subject"]: f for f in t.service_findings(svcs)}
|
|
assert set(got) == {"db", "app", "web"}
|
|
db = got["db"]
|
|
assert db["kind"] == "swarm_service_crash_looping" and db["severity"] == "critical"
|
|
assert "4 tasks failed in the last hour" in db["summary"]
|
|
assert "last task error: task: non-zero exit (1), exit 1 on docker03" in db["summary"]
|
|
# Down without a loop still names why: here Swarm could not even start it.
|
|
assert got["app"]["kind"] == "swarm_service_down"
|
|
assert got["app"]["summary"].endswith("last task error: No such image: app:dev")
|
|
# Recovered, but a task died in the last day: worth knowing, not critical.
|
|
assert got["web"]["kind"] == "swarm_task_failed" and got["web"]["severity"] == "warning"
|
|
assert "had 1 failed task in the last 24h" in got["web"]["summary"]
|
|
|
|
|
|
def test_a_replaced_swarm_task_is_marked_and_judged_on_its_service():
|
|
task = _c(name="app.1.old", status="exited", exit_code=137, service_name="app")
|
|
row = t.container_row(task, {}, [], NOW, replaced=True)
|
|
assert row["swarm_replaced"] is True and row["problems"] == []
|
|
assert t.container_row(_c(), {}, [], NOW)["swarm_replaced"] is None # not a swarm task
|
|
|
|
|
|
def test_plugin_exposes_the_mcp_hooks():
|
|
reg = MagicMock()
|
|
docker_plugin.register_mcp_tools(reg)
|
|
regs = {call.args[0].__name__: call.kwargs for call in reg.call_args_list}
|
|
assert set(regs) == {"list_containers", "get_container", "list_swarm_services",
|
|
"search_container_logs", "get_docker_disk_usage", "prune_docker"}
|
|
for name, kw in regs.items():
|
|
if name == "prune_docker":
|
|
assert (kw["role"], kw["read_only"]) == (UserRole.admin, False)
|
|
else:
|
|
assert (kw["role"], kw["read_only"]) == (UserRole.viewer, True), name
|
|
assert callable(docker_plugin.mcp_overview)
|
|
|
|
|
|
def test_prune_preview_sizes_what_each_target_removes():
|
|
usage = NS(images_reclaimable=3000, build_cache_size=70)
|
|
assert t.prune_preview("containers", usage, 4) == {"stopped_containers": 4}
|
|
assert t.prune_preview("images", usage, 4) == {"unused_images_bytes": 3000}
|
|
system = t.prune_preview("system", usage, 4)
|
|
assert system["stopped_containers"] == 4 and system["build_cache_bytes"] == 70
|
|
assert t.prune_preview("images", None, 0) == {"unused_images_bytes": None}
|
|
|
|
|
|
async def test_prune_docker_refuses_an_unknown_target_before_reading_anything(monkeypatch):
|
|
def _no_db():
|
|
raise AssertionError("touched the database")
|
|
monkeypatch.setattr(t, "db_session", _no_db)
|
|
with pytest.raises(ValueError, match="'volumes' is not one of containers, images, system"):
|
|
await t.prune_docker("nas", "volumes")
|
|
assert inspect.signature(t.prune_docker).parameters["check"].default is True
|
|
|
|
|
|
def _death(event, minutes_ago):
|
|
return NS(event=event, at=NOW - timedelta(minutes=minutes_ago))
|
|
|
|
|
|
def test_problem_since_dates_each_problem_from_its_own_evidence():
|
|
hour_ago = NOW - timedelta(hours=1)
|
|
deaths = [_death("die", 5), _death("die", 20), _death("die", 50), _death("die", 300)]
|
|
# A crash loop began with its first death inside the hour, not the older one.
|
|
assert t.problem_since(["crash_looping"], deaths, hour_ago) == NOW - timedelta(minutes=50)
|
|
# An error exit dates from the death that ended it.
|
|
assert t.problem_since(["exited_nonzero"], deaths, hour_ago) == NOW - timedelta(minutes=5)
|
|
oom = [_death("die", 2), _death("oom", 3)]
|
|
assert t.problem_since(["oom_killed", "exited_nonzero"], oom, hour_ago) == NOW - timedelta(minutes=3)
|
|
turned = NOW - timedelta(hours=2)
|
|
assert t.problem_since(["unhealthy"], [], hour_ago, turned) == turned
|
|
# No retained evidence: unknown, not now.
|
|
assert t.problem_since(["unhealthy"], [], hour_ago) is None
|
|
assert t.problem_since([], deaths, hour_ago) is None
|
|
|
|
|
|
def test_findings_carry_when_the_problem_began():
|
|
began = NOW - timedelta(minutes=40)
|
|
row = t.container_row(_c(name="db", status="exited", exit_code=1), {}, ["exited_nonzero"],
|
|
NOW, since=began)
|
|
assert row["problem_since"] == "2026-10-06T11:20:00Z" and row["problem_age_s"] == 2400
|
|
(f,) = t.container_findings([row])
|
|
assert f["since"] == "2026-10-06T11:20:00Z" and f["age_s"] is not None
|
|
svc = {"name": "db", "state": "down", "running": 0, "desired": 1, "trouble_since": began}
|
|
(g,) = t.service_findings([svc])
|
|
assert g["since"] == "2026-10-06T11:20:00Z"
|