bvandeusenandClaude Opus 5.5
bac2859455
Milestone 465 "Swarm task awareness". Fixes three defects from the live MCP review #4973: - #5008 "Swarm task history read as live state" - #5009 "Crash-loop detection is blind to Swarm services" - #5011 "Swarm task errors are not captured" Agent and ingest (agent 1.8.0): - A manager's agent now reports each service's newest tasks: slot, node, desired and current state, Swarm's Err/Message, exit code and time. - They are stored in docker_swarm_services.tasks_json (migration docker_010_swarm_tasks). Swarm view (pure): - A task container is "replaced" when Swarm set its task's desired state off `running`. Without the task on record, a newer or running task in the same slot decides. - Replaced containers are no longer replicas or crashes. - Each service gets, from the task records: - failures_1h and failures_24h (failed or rejected tasks); - crash_looping (Settings → Thresholds count); - last_error, in Swarm's own words. MCP: - New findings: swarm_service_crash_looping (critical) and swarm_task_failed (warning, on a healthy service with a failure in 24h). - Down and degraded findings name the last task error. - list_swarm_services carries the failure fields. - Container rows mark swarm_replaced and give a replaced task no problems. Web: - The containers page swarm cards and the Swarm topology page show crash loops, recent failures and the last task error. - The dashboard widget flags a crash loop, and its "failed (24h)" count no longer counts a rolling update's killed task. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
175 lines
8.8 KiB
Python
175 lines
8.8 KiB
Python
"""Unit tests for the swarm-aware container view builder (pure, no DB/ORM)."""
|
|
import json
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace
|
|
|
|
from plugins.docker.swarm_view import build_swarm_services, last_task_error, task_failures
|
|
|
|
NODE_HOSTNAME = {"nA": "docker01", "nB": "docker02", "nC": "docker03"}
|
|
HOST_NAME = {"h1": "docker01", "h2": "docker02"} # agents run on docker01/02 only
|
|
|
|
|
|
NOW = datetime(2026, 10, 6, 13, 30, tzinfo=timezone.utc)
|
|
|
|
|
|
def _c(host_id, name, *, service=None, node=None, status="running", cpu=1.0,
|
|
task=None, started=None):
|
|
return SimpleNamespace(host_id=host_id, name=name, service_name=service,
|
|
node_id=node, status=status, cpu_pct=cpu, mem_pct=2.0,
|
|
health=None, restart_count=0, container_id=name,
|
|
task_id=task, started_at=started)
|
|
|
|
|
|
def _svc(name, running, desired, placement, mode="replicated", tasks=None, updated=None):
|
|
return SimpleNamespace(service_name=name, running=running, desired=desired,
|
|
mode=mode, placement_json=json.dumps(placement),
|
|
tasks_json=json.dumps(tasks or []), updated_at=updated)
|
|
|
|
|
|
def _task(tid, *, state, desired="shutdown", ago_min=5, error="", exit_code=None, node="nA",
|
|
slot=1):
|
|
at = (NOW - timedelta(minutes=ago_min)).isoformat().replace("+00:00", "Z")
|
|
return {"task_id": tid, "slot": slot, "node_id": node, "desired_state": desired,
|
|
"state": state, "message": "", "error": error, "exit_code": exit_code, "at": at}
|
|
|
|
|
|
def _view(containers, services):
|
|
return build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME, now=NOW)
|
|
|
|
|
|
def test_real_replicas_plus_agentless_ghost():
|
|
containers = [
|
|
_c("h1", "web.1.aaa", service="web", node="nA"),
|
|
_c("h2", "web.2.bbb", service="web", node="nB"),
|
|
_c("h1", "db", node=None), # standalone, non-swarm
|
|
]
|
|
services = [_svc("web", running=3, desired=3,
|
|
placement=[{"node_id": "nA", "running": 1},
|
|
{"node_id": "nB", "running": 1},
|
|
{"node_id": "nC", "running": 1}])] # nC has no agent
|
|
|
|
view = build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME)
|
|
|
|
assert len(view["services"]) == 1
|
|
svc = view["services"][0]
|
|
assert svc["name"] == "web" and svc["state"] == "healthy"
|
|
reals = [r for r in svc["replicas"] if not r["ghost"]]
|
|
ghosts = [r for r in svc["replicas"] if r["ghost"]]
|
|
assert sorted(r["host"] for r in reals) == ["docker01", "docker02"]
|
|
assert len(ghosts) == 1 and ghosts[0]["host"] == "docker03" and ghosts[0]["count"] == 1
|
|
|
|
# The non-swarm container is grouped by host, untouched.
|
|
assert len(view["standalone"]) == 1
|
|
g = view["standalone"][0]
|
|
assert g["host"] == "docker01" and [c.name for c in g["containers"]] == ["db"]
|
|
|
|
|
|
def test_placement_fully_covered_has_no_ghosts():
|
|
containers = [_c("h1", "api.1.x", service="api", node="nA"),
|
|
_c("h2", "api.2.y", service="api", node="nB")]
|
|
services = [_svc("api", 2, 2, [{"node_id": "nA", "running": 1},
|
|
{"node_id": "nB", "running": 1}])]
|
|
view = build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME)
|
|
assert all(not r["ghost"] for r in view["services"][0]["replicas"])
|
|
|
|
|
|
def test_multiple_tasks_same_node_partially_agentless():
|
|
# Placement says 2 running on nA, but we only have 1 local row → 1 ghost on nA.
|
|
containers = [_c("h1", "cache.1.x", service="cache", node="nA")]
|
|
services = [_svc("cache", 2, 2, [{"node_id": "nA", "running": 2}])]
|
|
view = build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME)
|
|
ghosts = [r for r in view["services"][0]["replicas"] if r["ghost"]]
|
|
assert len(ghosts) == 1 and ghosts[0]["count"] == 1 and ghosts[0]["host"] == "docker01"
|
|
|
|
|
|
def test_service_without_a_service_row_is_synthesised():
|
|
# Container labeled with a service, but no manager reported the service row.
|
|
containers = [_c("h1", "orphan.1.x", service="orphan", node="nA")]
|
|
view = build_swarm_services(containers, [], NODE_HOSTNAME, HOST_NAME)
|
|
svc = view["services"][0]
|
|
assert svc["name"] == "orphan" and svc["running"] == 1 and svc["desired"] == 1
|
|
assert all(not r["ghost"] for r in svc["replicas"])
|
|
|
|
|
|
def test_down_service_state():
|
|
services = [_svc("idle", 0, 2, [])]
|
|
view = build_swarm_services([], services, NODE_HOSTNAME, HOST_NAME)
|
|
assert view["services"][0]["state"] == "down"
|
|
|
|
|
|
# ── task awareness (agent ≥ 1.8.0) ───────────────────────────────────────────
|
|
|
|
def test_a_task_swarm_replaced_is_not_a_replica():
|
|
# A stop-first redeploy: the old task was killed (exit 137) and Swarm set its
|
|
# desired state to shutdown. It is history, not a dead replica.
|
|
containers = [_c("h1", "app.1.old", service="app", node="nA", status="exited", task="old"),
|
|
_c("h2", "app.1.new", service="app", node="nB", task="new")]
|
|
tasks = [_task("new", state="running", desired="running", ago_min=1, node="nB"),
|
|
_task("old", state="shutdown", ago_min=2, exit_code=137)]
|
|
view = _view(containers, [_svc("app", 1, 1, [{"node_id": "nB", "running": 1}], tasks=tasks)])
|
|
(svc,) = view["services"]
|
|
assert [r["name"] for r in svc["replicas"]] == ["app.1.new"]
|
|
assert svc["previous_tasks"] == 1 and svc["state"] == "healthy"
|
|
assert view["superseded"] == {("h1", "app.1.old")}
|
|
# A deliberate shutdown is not a failure.
|
|
assert svc["failures_24h"] == 0 and svc["last_error"] is None and not svc["crash_looping"]
|
|
|
|
|
|
def test_without_task_data_the_slot_decides():
|
|
# An older agent sends no tasks: a stopped container is replaced when its slot
|
|
# has a running task, or one that started later.
|
|
t0 = NOW - timedelta(hours=2)
|
|
containers = [_c("h1", "lib.1.a", service="lib", status="exited", started=t0),
|
|
_c("h1", "lib.1.b", service="lib", status="exited", started=t0 + timedelta(minutes=5)),
|
|
_c("h1", "lib.1.c", service="lib", started=t0 + timedelta(minutes=10)),
|
|
_c("h1", "lib.2.d", service="lib", status="exited", started=t0)]
|
|
view = _view(containers, [_svc("lib", 1, 2, [])])
|
|
(svc,) = view["services"]
|
|
# Slot 2's only task is stopped with nothing newer: still the current task.
|
|
assert sorted(r["name"] for r in svc["replicas"]) == ["lib.1.c", "lib.2.d"]
|
|
assert svc["previous_tasks"] == 2
|
|
assert svc["tasks_known"] is False and svc["failures_1h"] is None
|
|
assert svc["crash_looping"] is False
|
|
|
|
|
|
def test_a_crash_loop_is_counted_across_task_retries():
|
|
# Each retry is a new task and a new container that dies once; only the
|
|
# service sees the loop.
|
|
tasks = [_task("t5", state="starting", desired="running", ago_min=0)]
|
|
tasks += [_task(f"t{i}", state="failed", ago_min=10 * i, exit_code=1,
|
|
error="task: non-zero exit (1)") for i in range(1, 5)]
|
|
tasks.append(_task("t9", state="failed", ago_min=180, exit_code=1))
|
|
view = build_swarm_services([], [_svc("db", 0, 1, [], tasks=tasks)], NODE_HOSTNAME, HOST_NAME,
|
|
now=NOW, crash_loop_deaths=3)
|
|
(svc,) = view["services"]
|
|
assert svc["failures_1h"] == 4 and svc["failures_24h"] == 5
|
|
assert svc["crash_looping"] is True and svc["state"] == "down"
|
|
err = svc["last_error"]
|
|
assert err["error"] == "task: non-zero exit (1)" and err["exit_code"] == 1
|
|
assert err["node"] == "docker01" and err["state"] == "failed"
|
|
# The threshold comes from Settings → Thresholds.
|
|
lax = build_swarm_services([], [_svc("db", 0, 1, [], tasks=tasks)], NODE_HOSTNAME, HOST_NAME,
|
|
now=NOW, crash_loop_deaths=5)
|
|
assert lax["services"][0]["crash_looping"] is False
|
|
|
|
|
|
def test_a_rejected_or_stuck_task_explains_itself():
|
|
rejected = [_task("t1", state="rejected", desired="running", error="No such image: codex:dev")]
|
|
assert last_task_error(rejected)["error"] == "No such image: codex:dev"
|
|
# A task Swarm is still trying to place carries its reason too.
|
|
pending = [_task("t2", state="pending", desired="running", error="no suitable node")]
|
|
assert last_task_error(pending)["state"] == "pending"
|
|
assert last_task_error([_task("t3", state="running", desired="running")]) is None
|
|
assert task_failures(rejected, NOW - timedelta(hours=1)) == 1
|
|
|
|
|
|
def test_the_newest_manager_report_wins():
|
|
# Two managers report the same service; the one updated last (here: the one
|
|
# with task data) is used.
|
|
old = _svc("web", 1, 1, [], updated=NOW - timedelta(minutes=5))
|
|
new = _svc("web", 1, 1, [], tasks=[_task("t1", state="failed", ago_min=3)],
|
|
updated=NOW - timedelta(seconds=30))
|
|
(svc,) = _view([], [old, new])["services"]
|
|
assert svc["tasks_known"] is True and svc["failures_1h"] == 1
|
|
|