bvandeusenandClaude Opus 5.5
1b82d8ad83
Nine services that lost tasks in one node's bad three minutes stayed on the overview as warnings for a day, 17 hours after they recovered. Against the UI ruling (only what is current on the dashboard), a service running all its replicas is now history: its failed tasks stay on the Docker page, not on the overview or the dashboard's failed count. Services still short of replicas that started failing within five minutes of each other read as one finding naming them, dated from the first. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
199 lines
10 KiB
Python
199 lines
10 KiB
Python
"""The docker plugin's MCP tools: what counts as a problem, and how it is reported."""
|
|
import inspect
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace as NS
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
import plugins.docker as docker_plugin
|
|
from plugins.docker import mcp_tools as t
|
|
from steward.models.users import UserRole
|
|
|
|
NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
def _c(**kw):
|
|
base = dict(name="web", host_id="h1", image="nginx", status="running", health=None,
|
|
exit_code=None, oom_killed=False, restart_count=0, cpu_pct=1.0, mem_pct=2.0,
|
|
mem_usage_bytes=1, mem_limit_bytes=2, started_at=NOW - timedelta(hours=1),
|
|
compose_project=None, service_name=None, scraped_at=NOW - timedelta(seconds=30))
|
|
base.update(kw)
|
|
return NS(**base)
|
|
|
|
|
|
def test_container_problems_most_severe_first():
|
|
assert t.container_problems(_c(), 0, False) == []
|
|
assert t.container_problems(_c(status="restarting"), 4, True) == ["restarting", "crash_looping"]
|
|
# The crash-loop count comes from Settings → Thresholds.
|
|
assert t.container_problems(_c(), 2, True) == []
|
|
assert t.container_problems(_c(), 2, True, crash_loop_deaths=2) == ["crash_looping"]
|
|
assert t.container_problems(_c(health="unhealthy", oom_killed=True), 0, False) == ["unhealthy", "oom_killed"]
|
|
# An unhealthy flag on a stopped container is stale; it isn't failing a healthcheck now.
|
|
assert t.container_problems(_c(status="exited", health="unhealthy", exit_code=0), 0, False) == []
|
|
|
|
|
|
def test_exited_nonzero_needs_a_recent_death():
|
|
crashed = _c(status="exited", exit_code=1)
|
|
assert t.container_problems(crashed, 1, True) == ["exited_nonzero"]
|
|
# A container that died weeks ago and was left stopped is not news.
|
|
assert t.container_problems(crashed, 0, False) == []
|
|
|
|
|
|
def test_a_stop_is_not_an_exit_failure():
|
|
# 137 (SIGKILL) and 143 (SIGTERM) are how `docker stop` ends a container;
|
|
# without an OOM kill that is someone stopping it, not it failing (#5209).
|
|
for code in (137, 143):
|
|
assert t.container_problems(_c(status="exited", exit_code=code), 1, True) == []
|
|
oom = _c(status="exited", exit_code=137, oom_killed=True)
|
|
assert t.container_problems(oom, 1, True) == ["oom_killed"]
|
|
|
|
|
|
def test_cut_middle_keeps_both_ends_and_sizes_the_gap():
|
|
assert t.cut_middle("short") == "short"
|
|
line = "A" * 10 + "B" * 100 + "C" * 10
|
|
cut = t.cut_middle(line, limit=20)
|
|
assert cut.startswith("A" * 10) and cut.endswith("C" * 10)
|
|
assert "[… 100 characters omitted …]" in cut
|
|
|
|
|
|
def test_container_row_uptime_only_while_running():
|
|
row = t.container_row(_c(), {"h1": "nas"}, [], NOW)
|
|
assert row["host_name"] == "nas" and row["uptime_s"] == 3600 and row["reported_age_s"] == 30
|
|
assert t.container_row(_c(status="exited"), {}, [], NOW)["uptime_s"] is None
|
|
|
|
|
|
def test_container_findings_one_per_container_with_every_problem_named():
|
|
rows = [t.container_row(_c(name="db", status="restarting", exit_code=1), {"h1": "nas"},
|
|
["restarting", "crash_looping"], NOW),
|
|
t.container_row(_c(name="ok"), {}, [], NOW)]
|
|
(f,) = t.container_findings(rows)
|
|
assert f["severity"] == "critical" and f["kind"] == "container_restarting"
|
|
assert "is restarting and keeps dying" in f["summary"] and "on nas" in f["summary"]
|
|
assert "(exit 1)" in f["summary"]
|
|
assert f["next"] == 'get_container(container="db", host="h1")'
|
|
|
|
|
|
def test_service_findings_by_state():
|
|
svcs = [{"name": "a", "state": "healthy", "running": 2, "desired": 2},
|
|
{"name": "b", "state": "degraded", "running": 1, "desired": 3},
|
|
{"name": "c", "state": "down", "running": 0, "desired": 1}]
|
|
got = {f["subject"]: f["severity"] for f in t.service_findings(svcs)}
|
|
assert got == {"b": "warning", "c": "critical"}
|
|
|
|
|
|
def test_service_findings_name_the_crash_loop_and_swarms_error():
|
|
err = {"error": "task: non-zero exit (1)", "message": "started", "state": "failed",
|
|
"exit_code": 1, "node": "docker03"}
|
|
svcs = [{"name": "db", "state": "down", "running": 0, "desired": 1, "crash_looping": True,
|
|
"failures_1h": 4, "failures_24h": 5, "last_error": err},
|
|
{"name": "app", "state": "down", "running": 0, "desired": 1, "crash_looping": False,
|
|
"failures_1h": 0, "failures_24h": 0,
|
|
"last_error": {"error": "No such image: app:dev", "state": "rejected",
|
|
"exit_code": None, "node": None, "message": None}},
|
|
{"name": "web", "state": "healthy", "running": 2, "desired": 2, "crash_looping": False,
|
|
"failures_1h": 0, "failures_24h": 1, "last_error": err},
|
|
{"name": "ok", "state": "healthy", "running": 1, "desired": 1, "crash_looping": False,
|
|
"failures_1h": 0, "failures_24h": 0, "last_error": None}]
|
|
got = {f["subject"]: f for f in t.service_findings(svcs)}
|
|
assert set(got) == {"db", "app"} # web is back to all its replicas: history (#5722)
|
|
db = got["db"]
|
|
assert db["kind"] == "swarm_service_crash_looping" and db["severity"] == "critical"
|
|
assert "4 tasks failed in the last hour" in db["summary"]
|
|
assert "last task error: task: non-zero exit (1), exit 1 on docker03" in db["summary"]
|
|
# Down without a loop still names why: here Swarm could not even start it.
|
|
assert got["app"]["kind"] == "swarm_service_down"
|
|
assert got["app"]["summary"].endswith("last task error: No such image: app:dev")
|
|
|
|
|
|
def test_a_replaced_swarm_task_is_marked_and_judged_on_its_service():
|
|
task = _c(name="app.1.old", status="exited", exit_code=137, service_name="app")
|
|
row = t.container_row(task, {}, [], NOW, replaced=True)
|
|
assert row["swarm_replaced"] is True and row["problems"] == []
|
|
assert t.container_row(_c(), {}, [], NOW)["swarm_replaced"] is None # not a swarm task
|
|
|
|
|
|
def test_plugin_exposes_the_mcp_hooks():
|
|
reg = MagicMock()
|
|
docker_plugin.register_mcp_tools(reg)
|
|
regs = {call.args[0].__name__: call.kwargs for call in reg.call_args_list}
|
|
assert set(regs) == {"list_containers", "get_container", "list_swarm_services",
|
|
"search_container_logs", "get_docker_disk_usage", "prune_docker"}
|
|
for name, kw in regs.items():
|
|
if name == "prune_docker":
|
|
assert (kw["role"], kw["read_only"]) == (UserRole.admin, False)
|
|
else:
|
|
assert (kw["role"], kw["read_only"]) == (UserRole.viewer, True), name
|
|
assert callable(docker_plugin.mcp_overview)
|
|
|
|
|
|
def test_prune_preview_sizes_what_each_target_removes():
|
|
usage = NS(images_reclaimable=3000, build_cache_size=70)
|
|
assert t.prune_preview("containers", usage, 4) == {"stopped_containers": 4}
|
|
assert t.prune_preview("images", usage, 4) == {"unused_images_bytes": 3000}
|
|
system = t.prune_preview("system", usage, 4)
|
|
assert system["stopped_containers"] == 4 and system["build_cache_bytes"] == 70
|
|
assert t.prune_preview("images", None, 0) == {"unused_images_bytes": None}
|
|
|
|
|
|
async def test_prune_docker_refuses_an_unknown_target_before_reading_anything(monkeypatch):
|
|
def _no_db():
|
|
raise AssertionError("touched the database")
|
|
monkeypatch.setattr(t, "db_session", _no_db)
|
|
with pytest.raises(ValueError, match="'volumes' is not one of containers, images, system"):
|
|
await t.prune_docker("nas", "volumes")
|
|
assert inspect.signature(t.prune_docker).parameters["check"].default is True
|
|
|
|
|
|
def _death(event, minutes_ago):
|
|
return NS(event=event, at=NOW - timedelta(minutes=minutes_ago))
|
|
|
|
|
|
def test_problem_since_dates_each_problem_from_its_own_evidence():
|
|
hour_ago = NOW - timedelta(hours=1)
|
|
deaths = [_death("die", 5), _death("die", 20), _death("die", 50), _death("die", 300)]
|
|
# A crash loop began with its first death inside the hour, not the older one.
|
|
assert t.problem_since(["crash_looping"], deaths, hour_ago) == NOW - timedelta(minutes=50)
|
|
# An error exit dates from the death that ended it.
|
|
assert t.problem_since(["exited_nonzero"], deaths, hour_ago) == NOW - timedelta(minutes=5)
|
|
oom = [_death("die", 2), _death("oom", 3)]
|
|
assert t.problem_since(["oom_killed", "exited_nonzero"], oom, hour_ago) == NOW - timedelta(minutes=3)
|
|
turned = NOW - timedelta(hours=2)
|
|
assert t.problem_since(["unhealthy"], [], hour_ago, turned) == turned
|
|
# No retained evidence: unknown, not now.
|
|
assert t.problem_since(["unhealthy"], [], hour_ago) is None
|
|
assert t.problem_since([], deaths, hour_ago) is None
|
|
|
|
|
|
def test_findings_carry_when_the_problem_began():
|
|
began = NOW - timedelta(minutes=40)
|
|
row = t.container_row(_c(name="db", status="exited", exit_code=1), {}, ["exited_nonzero"],
|
|
NOW, since=began)
|
|
assert row["problem_since"] == "2026-10-06T11:20:00Z" and row["problem_age_s"] == 2400
|
|
(f,) = t.container_findings([row])
|
|
assert f["since"] == "2026-10-06T11:20:00Z" and f["age_s"] is not None
|
|
svc = {"name": "db", "state": "down", "running": 0, "desired": 1, "trouble_since": began}
|
|
(g,) = t.service_findings([svc])
|
|
assert g["since"] == "2026-10-06T11:20:00Z"
|
|
|
|
|
|
def test_services_that_failed_together_are_one_finding_and_recovered_ones_none():
|
|
"""#5722: nine services lost tasks within three minutes on one node's bad
|
|
moment, and stayed on the overview for a day after they were back. Those
|
|
back to full replicas are history; those still short and started together
|
|
are one finding naming them, dated from the first."""
|
|
at = NOW - timedelta(minutes=30)
|
|
svcs = [{"name": "auth", "state": "degraded", "running": 1, "desired": 2, "trouble_since": at},
|
|
{"name": "wiki", "state": "down", "running": 0, "desired": 1,
|
|
"trouble_since": at + timedelta(minutes=2)},
|
|
{"name": "later", "state": "degraded", "running": 1, "desired": 2,
|
|
"trouble_since": at + timedelta(minutes=20)},
|
|
{"name": "back", "state": "healthy", "running": 1, "desired": 1, "failures_24h": 3,
|
|
"trouble_since": at}]
|
|
together, alone = t.service_findings(svcs)
|
|
assert together["kind"] == "swarm_services_short" and together["severity"] == "critical"
|
|
assert together["subject"] == "2 swarm services"
|
|
assert "auth (1 of 2 replicas running), wiki (0 of 1 replicas running)" in together["summary"]
|
|
assert together["since"] == "2026-10-06T11:30:00Z"
|
|
assert alone["subject"] == "later" and alone["kind"] == "swarm_service_degraded"
|