"""Unit tests for the swarm-aware container view builder (pure, no DB/ORM).""" import json from datetime import datetime, timedelta, timezone from types import SimpleNamespace from plugins.docker.swarm_view import ( build_swarm_services, last_task_error, task_failures, trouble_since, ) NODE_HOSTNAME = {"nA": "docker01", "nB": "docker02", "nC": "docker03"} HOST_NAME = {"h1": "docker01", "h2": "docker02"} # agents run on docker01/02 only NOW = datetime(2026, 10, 6, 13, 30, tzinfo=timezone.utc) def _c(host_id, name, *, service=None, node=None, status="running", cpu=1.0, task=None, started=None): return SimpleNamespace(host_id=host_id, name=name, service_name=service, node_id=node, status=status, cpu_pct=cpu, mem_used_pct=2.0, health=None, restart_count=0, container_id=name, task_id=task, started_at=started) def _svc(name, running, desired, placement, mode="replicated", tasks=None, updated=None): return SimpleNamespace(service_name=name, running=running, desired=desired, mode=mode, placement_json=json.dumps(placement), tasks_json=json.dumps(tasks or []), updated_at=updated) def _task(tid, *, state, desired="shutdown", ago_min=5, error="", exit_code=None, node="nA", slot=1): at = (NOW - timedelta(minutes=ago_min)).isoformat().replace("+00:00", "Z") return {"task_id": tid, "slot": slot, "node_id": node, "desired_state": desired, "state": state, "message": "", "error": error, "exit_code": exit_code, "at": at} def _view(containers, services): return build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME, now=NOW) def test_real_replicas_plus_agentless_ghost(): containers = [ _c("h1", "web.1.aaa", service="web", node="nA"), _c("h2", "web.2.bbb", service="web", node="nB"), _c("h1", "db", node=None), # standalone, non-swarm ] services = [_svc("web", running=3, desired=3, placement=[{"node_id": "nA", "running": 1}, {"node_id": "nB", "running": 1}, {"node_id": "nC", "running": 1}])] # nC has no agent view = build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME) assert len(view["services"]) == 1 svc = view["services"][0] assert svc["name"] == "web" and svc["state"] == "healthy" reals = [r for r in svc["replicas"] if not r["ghost"]] ghosts = [r for r in svc["replicas"] if r["ghost"]] assert sorted(r["host"] for r in reals) == ["docker01", "docker02"] assert len(ghosts) == 1 and ghosts[0]["host"] == "docker03" and ghosts[0]["count"] == 1 # The non-swarm container is grouped by host, untouched. assert len(view["standalone"]) == 1 g = view["standalone"][0] assert g["host"] == "docker01" and [c.name for c in g["containers"]] == ["db"] def test_placement_fully_covered_has_no_ghosts(): containers = [_c("h1", "api.1.x", service="api", node="nA"), _c("h2", "api.2.y", service="api", node="nB")] services = [_svc("api", 2, 2, [{"node_id": "nA", "running": 1}, {"node_id": "nB", "running": 1}])] view = build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME) assert all(not r["ghost"] for r in view["services"][0]["replicas"]) def test_multiple_tasks_same_node_partially_agentless(): # Placement says 2 running on nA, but we only have 1 local row → 1 ghost on nA. containers = [_c("h1", "cache.1.x", service="cache", node="nA")] services = [_svc("cache", 2, 2, [{"node_id": "nA", "running": 2}])] view = build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME) ghosts = [r for r in view["services"][0]["replicas"] if r["ghost"]] assert len(ghosts) == 1 and ghosts[0]["count"] == 1 and ghosts[0]["host"] == "docker01" def test_service_without_a_service_row_is_synthesised(): # Container labeled with a service, but no manager reported the service row. containers = [_c("h1", "orphan.1.x", service="orphan", node="nA")] view = build_swarm_services(containers, [], NODE_HOSTNAME, HOST_NAME) svc = view["services"][0] assert svc["name"] == "orphan" and svc["running"] == 1 and svc["desired"] == 1 assert all(not r["ghost"] for r in svc["replicas"]) def test_down_service_state(): services = [_svc("idle", 0, 2, [])] view = build_swarm_services([], services, NODE_HOSTNAME, HOST_NAME) assert view["services"][0]["state"] == "down" # ── task awareness (agent ≥ 1.8.0) ─────────────────────────────────────────── def test_a_task_swarm_replaced_is_not_a_replica(): # A stop-first redeploy: the old task was killed (exit 137) and Swarm set its # desired state to shutdown. It is history, not a dead replica. containers = [_c("h1", "app.1.old", service="app", node="nA", status="exited", task="old"), _c("h2", "app.1.new", service="app", node="nB", task="new")] tasks = [_task("new", state="running", desired="running", ago_min=1, node="nB"), _task("old", state="shutdown", ago_min=2, exit_code=137)] view = _view(containers, [_svc("app", 1, 1, [{"node_id": "nB", "running": 1}], tasks=tasks)]) (svc,) = view["services"] assert [r["name"] for r in svc["replicas"]] == ["app.1.new"] assert svc["previous_tasks"] == 1 and svc["state"] == "healthy" assert view["superseded"] == {("h1", "app.1.old")} # A deliberate shutdown is not a failure. assert svc["failures_24h"] == 0 and svc["last_error"] is None and not svc["crash_looping"] def test_without_task_data_the_slot_decides(): # An older agent sends no tasks: a stopped container is replaced when its slot # has a running task, or one that started later. t0 = NOW - timedelta(hours=2) containers = [_c("h1", "lib.1.a", service="lib", status="exited", started=t0), _c("h1", "lib.1.b", service="lib", status="exited", started=t0 + timedelta(minutes=5)), _c("h1", "lib.1.c", service="lib", started=t0 + timedelta(minutes=10)), _c("h1", "lib.2.d", service="lib", status="exited", started=t0)] view = _view(containers, [_svc("lib", 1, 2, [])]) (svc,) = view["services"] # Slot 2's only task is stopped with nothing newer: still the current task. assert sorted(r["name"] for r in svc["replicas"]) == ["lib.1.c", "lib.2.d"] assert svc["previous_tasks"] == 2 assert svc["tasks_known"] is False and svc["failures_1h"] is None assert svc["crash_looping"] is False def test_a_crash_loop_is_counted_across_task_retries(): # Each retry is a new task and a new container that dies once; only the # service sees the loop. tasks = [_task("t5", state="starting", desired="running", ago_min=0)] tasks += [_task(f"t{i}", state="failed", ago_min=10 * i, exit_code=1, error="task: non-zero exit (1)") for i in range(1, 5)] tasks.append(_task("t9", state="failed", ago_min=180, exit_code=1)) view = build_swarm_services([], [_svc("db", 0, 1, [], tasks=tasks)], NODE_HOSTNAME, HOST_NAME, now=NOW, crash_loop_deaths=3) (svc,) = view["services"] assert svc["failures_1h"] == 4 and svc["failures_24h"] == 5 assert svc["crash_looping"] is True and svc["state"] == "down" err = svc["last_error"] assert err["error"] == "task: non-zero exit (1)" and err["exit_code"] == 1 assert err["node"] == "docker01" and err["state"] == "failed" # The threshold comes from Settings → Thresholds. lax = build_swarm_services([], [_svc("db", 0, 1, [], tasks=tasks)], NODE_HOSTNAME, HOST_NAME, now=NOW, crash_loop_deaths=5) assert lax["services"][0]["crash_looping"] is False def test_a_rejected_or_stuck_task_explains_itself(): rejected = [_task("t1", state="rejected", desired="running", error="No such image: codex:dev")] assert last_task_error(rejected)["error"] == "No such image: codex:dev" # A task Swarm is still trying to place carries its reason too. pending = [_task("t2", state="pending", desired="running", error="no suitable node")] assert last_task_error(pending)["state"] == "pending" assert last_task_error([_task("t3", state="running", desired="running")]) is None assert task_failures(rejected, NOW - timedelta(hours=1)) == 1 def test_the_newest_manager_report_wins(): # Two managers report the same service; the one updated last (here: the one # with task data) is used. old = _svc("web", 1, 1, [], updated=NOW - timedelta(minutes=5)) new = _svc("web", 1, 1, [], tasks=[_task("t1", state="failed", ago_min=3)], updated=NOW - timedelta(seconds=30)) (svc,) = _view([], [old, new])["services"] assert svc["tasks_known"] is True and svc["failures_1h"] == 1 def _ago(minutes): return NOW - timedelta(minutes=minutes) def test_trouble_since_for_each_kind_of_trouble(): loop = [_task("t1", state="failed", ago_min=2), _task("t2", state="failed", ago_min=20), _task("t3", state="failed", ago_min=45), _task("t4", state="failed", ago_min=90)] # A crash loop: its first failure inside the hour. assert trouble_since(loop, "down", True, NOW) == _ago(45) # Down: the oldest of the newest troubled tasks, back to a running one. down = [_task("t1", state="pending", ago_min=1, error="no suitable node"), _task("t2", state="rejected", ago_min=30, error="No such image"), _task("t3", state="running", desired="running", ago_min=600), _task("t4", state="failed", ago_min=900)] assert trouble_since(down, "degraded", False, NOW) == _ago(30) # Another slot's newer, healthy task does not hide the failing one. two = [_task("a2", state="running", desired="running", ago_min=5, slot=1), _task("b1", state="failed", ago_min=8, slot=2), _task("b0", state="failed", ago_min=40, slot=2), _task("a1", state="failed", ago_min=300, slot=1)] assert trouble_since(two, "degraded", False, NOW) == _ago(40) # Healthy again, but a task failed today: dated from that failure. recovered = [_task("t1", state="running", desired="running", ago_min=10), _task("t2", state="failed", ago_min=60 * 5), _task("t3", state="failed", ago_min=60 * 30)] assert trouble_since(recovered, "healthy", False, NOW) == _ago(60 * 5) # No task records: unknown. assert trouble_since([], "down", False, NOW) is None def test_the_view_dates_a_down_service(): tasks = [_task("t1", state="failed", ago_min=3, exit_code=1), _task("t2", state="failed", ago_min=70, exit_code=1)] (s,) = _view([], [_svc("db", 0, 1, [], tasks=tasks)])["services"] assert s["state"] == "down" and s["trouble_since"] == _ago(70)