bvandeusenandClaude Opus 5.5
56063f27a4
The agent sent Docker's working set (usage less inactive_file), which still counts recently used file cache. A file-heavy container with no limit read 191 GB while its processes held 2.2 GB, and the host's own figure, which already leaves cache out, disagreed with it. Agent 1.23.0 sends mem_used_bytes / mem_used_pct (anon + shmem, so Postgres's shared buffers count; v1: rss + shmem) and mem_cache_bytes apart. The new names keep the two meanings apart in history and in learned normals: docker_013 renames the columns, clears the old values and adds mem_cache_bytes. The Docker page, swarm by-node sums, Busiest Containers, container detail, normals and the MCP tools all read the new figure. File cache is shown only on the container's page and in get_container. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
214 lines
11 KiB
Python
214 lines
11 KiB
Python
"""Unit tests for the swarm-aware container view builder (pure, no DB/ORM)."""
|
|
import json
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace
|
|
|
|
from plugins.docker.swarm_view import (
|
|
build_swarm_services, last_task_error, task_failures, trouble_since,
|
|
)
|
|
|
|
NODE_HOSTNAME = {"nA": "docker01", "nB": "docker02", "nC": "docker03"}
|
|
HOST_NAME = {"h1": "docker01", "h2": "docker02"} # agents run on docker01/02 only
|
|
|
|
|
|
NOW = datetime(2026, 10, 6, 13, 30, tzinfo=timezone.utc)
|
|
|
|
|
|
def _c(host_id, name, *, service=None, node=None, status="running", cpu=1.0,
|
|
task=None, started=None):
|
|
return SimpleNamespace(host_id=host_id, name=name, service_name=service,
|
|
node_id=node, status=status, cpu_pct=cpu, mem_used_pct=2.0,
|
|
health=None, restart_count=0, container_id=name,
|
|
task_id=task, started_at=started)
|
|
|
|
|
|
def _svc(name, running, desired, placement, mode="replicated", tasks=None, updated=None):
|
|
return SimpleNamespace(service_name=name, running=running, desired=desired,
|
|
mode=mode, placement_json=json.dumps(placement),
|
|
tasks_json=json.dumps(tasks or []), updated_at=updated)
|
|
|
|
|
|
def _task(tid, *, state, desired="shutdown", ago_min=5, error="", exit_code=None, node="nA",
|
|
slot=1):
|
|
at = (NOW - timedelta(minutes=ago_min)).isoformat().replace("+00:00", "Z")
|
|
return {"task_id": tid, "slot": slot, "node_id": node, "desired_state": desired,
|
|
"state": state, "message": "", "error": error, "exit_code": exit_code, "at": at}
|
|
|
|
|
|
def _view(containers, services):
|
|
return build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME, now=NOW)
|
|
|
|
|
|
def test_real_replicas_plus_agentless_ghost():
|
|
containers = [
|
|
_c("h1", "web.1.aaa", service="web", node="nA"),
|
|
_c("h2", "web.2.bbb", service="web", node="nB"),
|
|
_c("h1", "db", node=None), # standalone, non-swarm
|
|
]
|
|
services = [_svc("web", running=3, desired=3,
|
|
placement=[{"node_id": "nA", "running": 1},
|
|
{"node_id": "nB", "running": 1},
|
|
{"node_id": "nC", "running": 1}])] # nC has no agent
|
|
|
|
view = build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME)
|
|
|
|
assert len(view["services"]) == 1
|
|
svc = view["services"][0]
|
|
assert svc["name"] == "web" and svc["state"] == "healthy"
|
|
reals = [r for r in svc["replicas"] if not r["ghost"]]
|
|
ghosts = [r for r in svc["replicas"] if r["ghost"]]
|
|
assert sorted(r["host"] for r in reals) == ["docker01", "docker02"]
|
|
assert len(ghosts) == 1 and ghosts[0]["host"] == "docker03" and ghosts[0]["count"] == 1
|
|
|
|
# The non-swarm container is grouped by host, untouched.
|
|
assert len(view["standalone"]) == 1
|
|
g = view["standalone"][0]
|
|
assert g["host"] == "docker01" and [c.name for c in g["containers"]] == ["db"]
|
|
|
|
|
|
def test_placement_fully_covered_has_no_ghosts():
|
|
containers = [_c("h1", "api.1.x", service="api", node="nA"),
|
|
_c("h2", "api.2.y", service="api", node="nB")]
|
|
services = [_svc("api", 2, 2, [{"node_id": "nA", "running": 1},
|
|
{"node_id": "nB", "running": 1}])]
|
|
view = build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME)
|
|
assert all(not r["ghost"] for r in view["services"][0]["replicas"])
|
|
|
|
|
|
def test_multiple_tasks_same_node_partially_agentless():
|
|
# Placement says 2 running on nA, but we only have 1 local row → 1 ghost on nA.
|
|
containers = [_c("h1", "cache.1.x", service="cache", node="nA")]
|
|
services = [_svc("cache", 2, 2, [{"node_id": "nA", "running": 2}])]
|
|
view = build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME)
|
|
ghosts = [r for r in view["services"][0]["replicas"] if r["ghost"]]
|
|
assert len(ghosts) == 1 and ghosts[0]["count"] == 1 and ghosts[0]["host"] == "docker01"
|
|
|
|
|
|
def test_service_without_a_service_row_is_synthesised():
|
|
# Container labeled with a service, but no manager reported the service row.
|
|
containers = [_c("h1", "orphan.1.x", service="orphan", node="nA")]
|
|
view = build_swarm_services(containers, [], NODE_HOSTNAME, HOST_NAME)
|
|
svc = view["services"][0]
|
|
assert svc["name"] == "orphan" and svc["running"] == 1 and svc["desired"] == 1
|
|
assert all(not r["ghost"] for r in svc["replicas"])
|
|
|
|
|
|
def test_down_service_state():
|
|
services = [_svc("idle", 0, 2, [])]
|
|
view = build_swarm_services([], services, NODE_HOSTNAME, HOST_NAME)
|
|
assert view["services"][0]["state"] == "down"
|
|
|
|
|
|
# ── task awareness (agent ≥ 1.8.0) ───────────────────────────────────────────
|
|
|
|
def test_a_task_swarm_replaced_is_not_a_replica():
|
|
# A stop-first redeploy: the old task was killed (exit 137) and Swarm set its
|
|
# desired state to shutdown. It is history, not a dead replica.
|
|
containers = [_c("h1", "app.1.old", service="app", node="nA", status="exited", task="old"),
|
|
_c("h2", "app.1.new", service="app", node="nB", task="new")]
|
|
tasks = [_task("new", state="running", desired="running", ago_min=1, node="nB"),
|
|
_task("old", state="shutdown", ago_min=2, exit_code=137)]
|
|
view = _view(containers, [_svc("app", 1, 1, [{"node_id": "nB", "running": 1}], tasks=tasks)])
|
|
(svc,) = view["services"]
|
|
assert [r["name"] for r in svc["replicas"]] == ["app.1.new"]
|
|
assert svc["previous_tasks"] == 1 and svc["state"] == "healthy"
|
|
assert view["superseded"] == {("h1", "app.1.old")}
|
|
# A deliberate shutdown is not a failure.
|
|
assert svc["failures_24h"] == 0 and svc["last_error"] is None and not svc["crash_looping"]
|
|
|
|
|
|
def test_without_task_data_the_slot_decides():
|
|
# An older agent sends no tasks: a stopped container is replaced when its slot
|
|
# has a running task, or one that started later.
|
|
t0 = NOW - timedelta(hours=2)
|
|
containers = [_c("h1", "lib.1.a", service="lib", status="exited", started=t0),
|
|
_c("h1", "lib.1.b", service="lib", status="exited", started=t0 + timedelta(minutes=5)),
|
|
_c("h1", "lib.1.c", service="lib", started=t0 + timedelta(minutes=10)),
|
|
_c("h1", "lib.2.d", service="lib", status="exited", started=t0)]
|
|
view = _view(containers, [_svc("lib", 1, 2, [])])
|
|
(svc,) = view["services"]
|
|
# Slot 2's only task is stopped with nothing newer: still the current task.
|
|
assert sorted(r["name"] for r in svc["replicas"]) == ["lib.1.c", "lib.2.d"]
|
|
assert svc["previous_tasks"] == 2
|
|
assert svc["tasks_known"] is False and svc["failures_1h"] is None
|
|
assert svc["crash_looping"] is False
|
|
|
|
|
|
def test_a_crash_loop_is_counted_across_task_retries():
|
|
# Each retry is a new task and a new container that dies once; only the
|
|
# service sees the loop.
|
|
tasks = [_task("t5", state="starting", desired="running", ago_min=0)]
|
|
tasks += [_task(f"t{i}", state="failed", ago_min=10 * i, exit_code=1,
|
|
error="task: non-zero exit (1)") for i in range(1, 5)]
|
|
tasks.append(_task("t9", state="failed", ago_min=180, exit_code=1))
|
|
view = build_swarm_services([], [_svc("db", 0, 1, [], tasks=tasks)], NODE_HOSTNAME, HOST_NAME,
|
|
now=NOW, crash_loop_deaths=3)
|
|
(svc,) = view["services"]
|
|
assert svc["failures_1h"] == 4 and svc["failures_24h"] == 5
|
|
assert svc["crash_looping"] is True and svc["state"] == "down"
|
|
err = svc["last_error"]
|
|
assert err["error"] == "task: non-zero exit (1)" and err["exit_code"] == 1
|
|
assert err["node"] == "docker01" and err["state"] == "failed"
|
|
# The threshold comes from Settings → Thresholds.
|
|
lax = build_swarm_services([], [_svc("db", 0, 1, [], tasks=tasks)], NODE_HOSTNAME, HOST_NAME,
|
|
now=NOW, crash_loop_deaths=5)
|
|
assert lax["services"][0]["crash_looping"] is False
|
|
|
|
|
|
def test_a_rejected_or_stuck_task_explains_itself():
|
|
rejected = [_task("t1", state="rejected", desired="running", error="No such image: codex:dev")]
|
|
assert last_task_error(rejected)["error"] == "No such image: codex:dev"
|
|
# A task Swarm is still trying to place carries its reason too.
|
|
pending = [_task("t2", state="pending", desired="running", error="no suitable node")]
|
|
assert last_task_error(pending)["state"] == "pending"
|
|
assert last_task_error([_task("t3", state="running", desired="running")]) is None
|
|
assert task_failures(rejected, NOW - timedelta(hours=1)) == 1
|
|
|
|
|
|
def test_the_newest_manager_report_wins():
|
|
# Two managers report the same service; the one updated last (here: the one
|
|
# with task data) is used.
|
|
old = _svc("web", 1, 1, [], updated=NOW - timedelta(minutes=5))
|
|
new = _svc("web", 1, 1, [], tasks=[_task("t1", state="failed", ago_min=3)],
|
|
updated=NOW - timedelta(seconds=30))
|
|
(svc,) = _view([], [old, new])["services"]
|
|
assert svc["tasks_known"] is True and svc["failures_1h"] == 1
|
|
|
|
|
|
|
|
def _ago(minutes):
|
|
return NOW - timedelta(minutes=minutes)
|
|
|
|
|
|
def test_trouble_since_for_each_kind_of_trouble():
|
|
loop = [_task("t1", state="failed", ago_min=2), _task("t2", state="failed", ago_min=20),
|
|
_task("t3", state="failed", ago_min=45), _task("t4", state="failed", ago_min=90)]
|
|
# A crash loop: its first failure inside the hour.
|
|
assert trouble_since(loop, "down", True, NOW) == _ago(45)
|
|
# Down: the oldest of the newest troubled tasks, back to a running one.
|
|
down = [_task("t1", state="pending", ago_min=1, error="no suitable node"),
|
|
_task("t2", state="rejected", ago_min=30, error="No such image"),
|
|
_task("t3", state="running", desired="running", ago_min=600),
|
|
_task("t4", state="failed", ago_min=900)]
|
|
assert trouble_since(down, "degraded", False, NOW) == _ago(30)
|
|
# Another slot's newer, healthy task does not hide the failing one.
|
|
two = [_task("a2", state="running", desired="running", ago_min=5, slot=1),
|
|
_task("b1", state="failed", ago_min=8, slot=2),
|
|
_task("b0", state="failed", ago_min=40, slot=2),
|
|
_task("a1", state="failed", ago_min=300, slot=1)]
|
|
assert trouble_since(two, "degraded", False, NOW) == _ago(40)
|
|
# Healthy again, but a task failed today: dated from that failure.
|
|
recovered = [_task("t1", state="running", desired="running", ago_min=10),
|
|
_task("t2", state="failed", ago_min=60 * 5),
|
|
_task("t3", state="failed", ago_min=60 * 30)]
|
|
assert trouble_since(recovered, "healthy", False, NOW) == _ago(60 * 5)
|
|
# No task records: unknown.
|
|
assert trouble_since([], "down", False, NOW) is None
|
|
|
|
|
|
def test_the_view_dates_a_down_service():
|
|
tasks = [_task("t1", state="failed", ago_min=3, exit_code=1),
|
|
_task("t2", state="failed", ago_min=70, exit_code=1)]
|
|
(s,) = _view([], [_svc("db", 0, 1, [], tasks=tasks)])["services"]
|
|
assert s["state"] == "down" and s["trouble_since"] == _ago(70)
|