bvandeusenandClaude Opus 5.5
56063f27a4
The agent sent Docker's working set (usage less inactive_file), which still counts recently used file cache. A file-heavy container with no limit read 191 GB while its processes held 2.2 GB, and the host's own figure, which already leaves cache out, disagreed with it. Agent 1.23.0 sends mem_used_bytes / mem_used_pct (anon + shmem, so Postgres's shared buffers count; v1: rss + shmem) and mem_cache_bytes apart. The new names keep the two meanings apart in history and in learned normals: docker_013 renames the columns, clears the old values and adds mem_cache_bytes. The Docker page, swarm by-node sums, Busiest Containers, container detail, normals and the MCP tools all read the new figure. File cache is shown only on the container's page and in get_container. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
226 lines
12 KiB
Python
226 lines
12 KiB
Python
"""The docker plugin's MCP tools: what counts as a problem, and how it is reported."""
|
|
import inspect
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace as NS
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
import plugins.docker as docker_plugin
|
|
from plugins.docker import mcp_tools as t
|
|
from steward.models.users import UserRole
|
|
|
|
NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
def _c(**kw):
|
|
base = dict(name="web", host_id="h1", image="nginx", status="running", health=None,
|
|
exit_code=None, oom_killed=False, restart_count=0, cpu_pct=1.0, mem_used_pct=2.0,
|
|
mem_used_bytes=1, mem_limit_bytes=2, started_at=NOW - timedelta(hours=1),
|
|
compose_project=None, service_name=None, scraped_at=NOW - timedelta(seconds=30))
|
|
base.update(kw)
|
|
return NS(**base)
|
|
|
|
|
|
def test_container_problems_most_severe_first():
|
|
assert t.container_problems(_c(), 0, False) == []
|
|
assert t.container_problems(_c(status="restarting"), 4, True) == ["restarting", "crash_looping"]
|
|
# The crash-loop count comes from Settings → Thresholds.
|
|
assert t.container_problems(_c(), 2, True) == []
|
|
assert t.container_problems(_c(), 2, True, crash_loop_deaths=2) == ["crash_looping"]
|
|
assert t.container_problems(_c(health="unhealthy", oom_killed=True), 0, False) == ["unhealthy", "oom_killed"]
|
|
# An unhealthy flag on a stopped container is stale; it isn't failing a healthcheck now.
|
|
assert t.container_problems(_c(status="exited", health="unhealthy", exit_code=0), 0, False) == []
|
|
|
|
|
|
def test_exited_nonzero_needs_a_recent_death():
|
|
crashed = _c(status="exited", exit_code=1)
|
|
assert t.container_problems(crashed, 1, True) == ["exited_nonzero"]
|
|
# A container that died weeks ago and was left stopped is not news.
|
|
assert t.container_problems(crashed, 0, False) == []
|
|
|
|
|
|
def test_a_stop_is_not_an_exit_failure():
|
|
# 137 (SIGKILL) and 143 (SIGTERM) are how `docker stop` ends a container;
|
|
# without an OOM kill that is someone stopping it, not it failing (#5209).
|
|
for code in (137, 143):
|
|
assert t.container_problems(_c(status="exited", exit_code=code), 1, True) == []
|
|
oom = _c(status="exited", exit_code=137, oom_killed=True)
|
|
assert t.container_problems(oom, 1, True) == ["oom_killed"]
|
|
|
|
|
|
def test_cut_middle_keeps_both_ends_and_sizes_the_gap():
|
|
assert t.cut_middle("short") == "short"
|
|
line = "A" * 10 + "B" * 100 + "C" * 10
|
|
cut = t.cut_middle(line, limit=20)
|
|
assert cut.startswith("A" * 10) and cut.endswith("C" * 10)
|
|
assert "[… 100 characters omitted …]" in cut
|
|
|
|
|
|
def test_container_row_uptime_only_while_running():
|
|
row = t.container_row(_c(), {"h1": "nas"}, [], NOW)
|
|
assert row["host_name"] == "nas" and row["uptime_s"] == 3600 and row["reported_age_s"] == 30
|
|
assert t.container_row(_c(status="exited"), {}, [], NOW)["uptime_s"] is None
|
|
|
|
|
|
def test_container_findings_one_per_container_with_every_problem_named():
|
|
rows = [t.container_row(_c(name="db", status="restarting", exit_code=1), {"h1": "nas"},
|
|
["restarting", "crash_looping"], NOW),
|
|
t.container_row(_c(name="ok"), {}, [], NOW)]
|
|
(f,) = t.container_findings(rows)
|
|
assert f["severity"] == "critical" and f["kind"] == "container_restarting"
|
|
assert "is restarting and keeps dying" in f["summary"] and "on nas" in f["summary"]
|
|
assert "(exit 1)" in f["summary"]
|
|
assert f["next"] == 'get_container(container="db", host="h1")'
|
|
|
|
|
|
def test_service_findings_by_state():
|
|
svcs = [{"name": "a", "state": "healthy", "running": 2, "desired": 2},
|
|
{"name": "b", "state": "degraded", "running": 1, "desired": 3},
|
|
{"name": "c", "state": "down", "running": 0, "desired": 1}]
|
|
got = {f["subject"]: f["severity"] for f in t.service_findings(svcs)}
|
|
assert got == {"b": "warning", "c": "critical"}
|
|
|
|
|
|
def test_service_findings_name_the_crash_loop_and_swarms_error():
|
|
err = {"error": "task: non-zero exit (1)", "message": "started", "state": "failed",
|
|
"exit_code": 1, "node": "docker03"}
|
|
svcs = [{"name": "db", "state": "down", "running": 0, "desired": 1, "crash_looping": True,
|
|
"failures_1h": 4, "failures_24h": 5, "last_error": err},
|
|
{"name": "app", "state": "down", "running": 0, "desired": 1, "crash_looping": False,
|
|
"failures_1h": 0, "failures_24h": 0,
|
|
"last_error": {"error": "No such image: app:dev", "state": "rejected",
|
|
"exit_code": None, "node": None, "message": None}},
|
|
{"name": "web", "state": "healthy", "running": 2, "desired": 2, "crash_looping": False,
|
|
"failures_1h": 0, "failures_24h": 1, "last_error": err},
|
|
{"name": "ok", "state": "healthy", "running": 1, "desired": 1, "crash_looping": False,
|
|
"failures_1h": 0, "failures_24h": 0, "last_error": None}]
|
|
got = {f["subject"]: f for f in t.service_findings(svcs)}
|
|
assert set(got) == {"db", "app"} # web is back to all its replicas: history (#5722)
|
|
db = got["db"]
|
|
assert db["kind"] == "swarm_service_crash_looping" and db["severity"] == "critical"
|
|
assert "4 tasks failed in the last hour" in db["summary"]
|
|
assert "last task error: task: non-zero exit (1), exit 1 on docker03" in db["summary"]
|
|
# Down without a loop still names why: here Swarm could not even start it.
|
|
assert got["app"]["kind"] == "swarm_service_down"
|
|
assert got["app"]["summary"].endswith("last task error: No such image: app:dev")
|
|
|
|
|
|
def test_a_replaced_swarm_task_is_marked_and_judged_on_its_service():
|
|
task = _c(name="app.1.old", status="exited", exit_code=137, service_name="app")
|
|
row = t.container_row(task, {}, [], NOW, replaced=True)
|
|
assert row["swarm_replaced"] is True and row["problems"] == []
|
|
assert t.container_row(_c(), {}, [], NOW)["swarm_replaced"] is None # not a swarm task
|
|
|
|
|
|
def test_plugin_exposes_the_mcp_hooks():
|
|
reg = MagicMock()
|
|
docker_plugin.register_mcp_tools(reg)
|
|
regs = {call.args[0].__name__: call.kwargs for call in reg.call_args_list}
|
|
assert set(regs) == {"list_containers", "get_container", "list_swarm_services",
|
|
"search_container_logs", "get_docker_disk_usage", "prune_docker"}
|
|
for name, kw in regs.items():
|
|
if name == "prune_docker":
|
|
assert (kw["role"], kw["read_only"]) == (UserRole.admin, False)
|
|
else:
|
|
assert (kw["role"], kw["read_only"]) == (UserRole.viewer, True), name
|
|
assert callable(docker_plugin.mcp_overview)
|
|
|
|
|
|
def test_prune_preview_sizes_what_each_target_removes():
|
|
usage = NS(images_reclaimable=3000, build_cache_size=70)
|
|
assert t.prune_preview("containers", usage, 4) == {"stopped_containers": 4}
|
|
assert t.prune_preview("images", usage, 4) == {"unused_images_bytes": 3000}
|
|
system = t.prune_preview("system", usage, 4)
|
|
assert system["stopped_containers"] == 4 and system["build_cache_bytes"] == 70
|
|
assert t.prune_preview("images", None, 0) == {"unused_images_bytes": None}
|
|
|
|
|
|
async def test_prune_docker_refuses_an_unknown_target_before_reading_anything(monkeypatch):
|
|
def _no_db():
|
|
raise AssertionError("touched the database")
|
|
monkeypatch.setattr(t, "db_session", _no_db)
|
|
with pytest.raises(ValueError, match="'volumes' is not one of containers, images, system"):
|
|
await t.prune_docker("nas", "volumes")
|
|
assert inspect.signature(t.prune_docker).parameters["check"].default is True
|
|
|
|
|
|
def _death(event, minutes_ago):
|
|
return NS(event=event, at=NOW - timedelta(minutes=minutes_ago))
|
|
|
|
|
|
def test_problem_since_dates_each_problem_from_its_own_evidence():
|
|
hour_ago = NOW - timedelta(hours=1)
|
|
deaths = [_death("die", 5), _death("die", 20), _death("die", 50), _death("die", 300)]
|
|
# A crash loop began with its first death inside the hour, not the older one.
|
|
assert t.problem_since(["crash_looping"], deaths, hour_ago) == NOW - timedelta(minutes=50)
|
|
# An error exit dates from the death that ended it.
|
|
assert t.problem_since(["exited_nonzero"], deaths, hour_ago) == NOW - timedelta(minutes=5)
|
|
oom = [_death("die", 2), _death("oom", 3)]
|
|
assert t.problem_since(["oom_killed", "exited_nonzero"], oom, hour_ago) == NOW - timedelta(minutes=3)
|
|
turned = NOW - timedelta(hours=2)
|
|
assert t.problem_since(["unhealthy"], [], hour_ago, turned) == turned
|
|
# No retained evidence: unknown, not now.
|
|
assert t.problem_since(["unhealthy"], [], hour_ago) is None
|
|
assert t.problem_since([], deaths, hour_ago) is None
|
|
|
|
|
|
def test_findings_carry_when_the_problem_began():
|
|
began = NOW - timedelta(minutes=40)
|
|
row = t.container_row(_c(name="db", status="exited", exit_code=1), {}, ["exited_nonzero"],
|
|
NOW, since=began)
|
|
assert row["problem_since"] == "2026-10-06T11:20:00Z" and row["problem_age_s"] == 2400
|
|
(f,) = t.container_findings([row])
|
|
assert f["since"] == "2026-10-06T11:20:00Z" and f["age_s"] is not None
|
|
svc = {"name": "db", "state": "down", "running": 0, "desired": 1, "trouble_since": began}
|
|
(g,) = t.service_findings([svc])
|
|
assert g["since"] == "2026-10-06T11:20:00Z"
|
|
|
|
|
|
def test_services_that_failed_together_are_one_finding_and_recovered_ones_none():
|
|
"""#5722: nine services lost tasks within three minutes on one node's bad
|
|
moment, and stayed on the overview for a day after they were back. Those
|
|
back to full replicas are history; those still short and started together
|
|
are one finding naming them, dated from the first."""
|
|
at = NOW - timedelta(minutes=30)
|
|
svcs = [{"name": "auth", "state": "degraded", "running": 1, "desired": 2, "trouble_since": at},
|
|
{"name": "wiki", "state": "down", "running": 0, "desired": 1,
|
|
"trouble_since": at + timedelta(minutes=2)},
|
|
{"name": "later", "state": "degraded", "running": 1, "desired": 2,
|
|
"trouble_since": at + timedelta(minutes=20)},
|
|
{"name": "back", "state": "healthy", "running": 1, "desired": 1, "failures_24h": 3,
|
|
"trouble_since": at}]
|
|
together, alone = t.service_findings(svcs)
|
|
assert together["kind"] == "swarm_services_short" and together["severity"] == "critical"
|
|
assert together["subject"] == "2 swarm services"
|
|
assert "auth (1 of 2 replicas running), wiki (0 of 1 replicas running)" in together["summary"]
|
|
assert together["since"] == "2026-10-06T11:30:00Z"
|
|
assert alone["subject"] == "later" and alone["kind"] == "swarm_service_degraded"
|
|
|
|
|
|
def _node(node_id, hostname, status="ready", availability="active", role="worker", minutes=0):
|
|
return NS(node_id=node_id, hostname=hostname, status=status, availability=availability,
|
|
role=role, leader=False, host_id="mgr",
|
|
updated_at=datetime(2026, 10, 15, 12, 0, tzinfo=timezone.utc) - timedelta(minutes=minutes))
|
|
|
|
|
|
def test_unique_nodes_keep_the_newest_report_of_each():
|
|
old, new = _node("n1", "a", status="down", minutes=5), _node("n1", "a")
|
|
assert t.unique_nodes([old, new, _node("n2", "b")]) == [new, _node("n2", "b")]
|
|
|
|
|
|
def test_node_findings_name_a_down_node_unless_maintenance_is_rebooting_it():
|
|
nodes = [_node("n1", "Web-1", status="down", role="manager"),
|
|
_node("n2", "web-2", availability="drain"), # kept drained on purpose (#5777)
|
|
_node("n3", "web-3", status="unknown", availability="pause"),
|
|
_node("n4", "web-4", availability="drain", status="down"), # being rebooted
|
|
_node("n5", "web-5")]
|
|
out = {f["subject"]: f for f in t.node_findings(nodes, {"web-4": "rebooting"})}
|
|
assert set(out) == {"Web-1", "web-3"}
|
|
assert out["Web-1"]["severity"] == "critical" and out["Web-1"]["kind"] == "swarm_node_down"
|
|
assert out["Web-1"]["summary"] == "swarm manager Web-1 is down"
|
|
assert out["web-3"]["summary"] == "swarm worker web-3 is unknown and paused"
|
|
# Once its job is over, the same node drained and down is named.
|
|
[f] = t.node_findings([_node("n4", "web-4", availability="drain", status="down")], {})
|
|
assert f["summary"] == "swarm worker web-4 is down and drained"
|