bvandeusenandClaude Opus 5.5
e9341c5639
#5013 "Overview findings for containers and Swarm services carry since=null" - Container problems are dated from their own evidence: a crash or restart loop from its first death in the hour, an OOM kill or error exit from the death that ended it, a failing healthcheck from its last turn to unhealthy. list_containers and get_container carry problem_since / problem_age_s. - Swarm services carry trouble_since: a crash loop's first failure in the hour, a short service's oldest troubled task in each failing slot, a recovered service's first failure in 24h. list_swarm_services shows it. - get_overview findings from the docker plugin now have since and age_s, so they rank by how long they have stood. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
214 lines
11 KiB
Python
214 lines
11 KiB
Python
"""Unit tests for the swarm-aware container view builder (pure, no DB/ORM)."""
|
|
import json
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace
|
|
|
|
from plugins.docker.swarm_view import (
|
|
build_swarm_services, last_task_error, task_failures, trouble_since,
|
|
)
|
|
|
|
NODE_HOSTNAME = {"nA": "docker01", "nB": "docker02", "nC": "docker03"}
|
|
HOST_NAME = {"h1": "docker01", "h2": "docker02"} # agents run on docker01/02 only
|
|
|
|
|
|
NOW = datetime(2026, 10, 6, 13, 30, tzinfo=timezone.utc)
|
|
|
|
|
|
def _c(host_id, name, *, service=None, node=None, status="running", cpu=1.0,
|
|
task=None, started=None):
|
|
return SimpleNamespace(host_id=host_id, name=name, service_name=service,
|
|
node_id=node, status=status, cpu_pct=cpu, mem_pct=2.0,
|
|
health=None, restart_count=0, container_id=name,
|
|
task_id=task, started_at=started)
|
|
|
|
|
|
def _svc(name, running, desired, placement, mode="replicated", tasks=None, updated=None):
|
|
return SimpleNamespace(service_name=name, running=running, desired=desired,
|
|
mode=mode, placement_json=json.dumps(placement),
|
|
tasks_json=json.dumps(tasks or []), updated_at=updated)
|
|
|
|
|
|
def _task(tid, *, state, desired="shutdown", ago_min=5, error="", exit_code=None, node="nA",
|
|
slot=1):
|
|
at = (NOW - timedelta(minutes=ago_min)).isoformat().replace("+00:00", "Z")
|
|
return {"task_id": tid, "slot": slot, "node_id": node, "desired_state": desired,
|
|
"state": state, "message": "", "error": error, "exit_code": exit_code, "at": at}
|
|
|
|
|
|
def _view(containers, services):
|
|
return build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME, now=NOW)
|
|
|
|
|
|
def test_real_replicas_plus_agentless_ghost():
|
|
containers = [
|
|
_c("h1", "web.1.aaa", service="web", node="nA"),
|
|
_c("h2", "web.2.bbb", service="web", node="nB"),
|
|
_c("h1", "db", node=None), # standalone, non-swarm
|
|
]
|
|
services = [_svc("web", running=3, desired=3,
|
|
placement=[{"node_id": "nA", "running": 1},
|
|
{"node_id": "nB", "running": 1},
|
|
{"node_id": "nC", "running": 1}])] # nC has no agent
|
|
|
|
view = build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME)
|
|
|
|
assert len(view["services"]) == 1
|
|
svc = view["services"][0]
|
|
assert svc["name"] == "web" and svc["state"] == "healthy"
|
|
reals = [r for r in svc["replicas"] if not r["ghost"]]
|
|
ghosts = [r for r in svc["replicas"] if r["ghost"]]
|
|
assert sorted(r["host"] for r in reals) == ["docker01", "docker02"]
|
|
assert len(ghosts) == 1 and ghosts[0]["host"] == "docker03" and ghosts[0]["count"] == 1
|
|
|
|
# The non-swarm container is grouped by host, untouched.
|
|
assert len(view["standalone"]) == 1
|
|
g = view["standalone"][0]
|
|
assert g["host"] == "docker01" and [c.name for c in g["containers"]] == ["db"]
|
|
|
|
|
|
def test_placement_fully_covered_has_no_ghosts():
|
|
containers = [_c("h1", "api.1.x", service="api", node="nA"),
|
|
_c("h2", "api.2.y", service="api", node="nB")]
|
|
services = [_svc("api", 2, 2, [{"node_id": "nA", "running": 1},
|
|
{"node_id": "nB", "running": 1}])]
|
|
view = build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME)
|
|
assert all(not r["ghost"] for r in view["services"][0]["replicas"])
|
|
|
|
|
|
def test_multiple_tasks_same_node_partially_agentless():
|
|
# Placement says 2 running on nA, but we only have 1 local row → 1 ghost on nA.
|
|
containers = [_c("h1", "cache.1.x", service="cache", node="nA")]
|
|
services = [_svc("cache", 2, 2, [{"node_id": "nA", "running": 2}])]
|
|
view = build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME)
|
|
ghosts = [r for r in view["services"][0]["replicas"] if r["ghost"]]
|
|
assert len(ghosts) == 1 and ghosts[0]["count"] == 1 and ghosts[0]["host"] == "docker01"
|
|
|
|
|
|
def test_service_without_a_service_row_is_synthesised():
|
|
# Container labeled with a service, but no manager reported the service row.
|
|
containers = [_c("h1", "orphan.1.x", service="orphan", node="nA")]
|
|
view = build_swarm_services(containers, [], NODE_HOSTNAME, HOST_NAME)
|
|
svc = view["services"][0]
|
|
assert svc["name"] == "orphan" and svc["running"] == 1 and svc["desired"] == 1
|
|
assert all(not r["ghost"] for r in svc["replicas"])
|
|
|
|
|
|
def test_down_service_state():
|
|
services = [_svc("idle", 0, 2, [])]
|
|
view = build_swarm_services([], services, NODE_HOSTNAME, HOST_NAME)
|
|
assert view["services"][0]["state"] == "down"
|
|
|
|
|
|
# ── task awareness (agent ≥ 1.8.0) ───────────────────────────────────────────
|
|
|
|
def test_a_task_swarm_replaced_is_not_a_replica():
|
|
# A stop-first redeploy: the old task was killed (exit 137) and Swarm set its
|
|
# desired state to shutdown. It is history, not a dead replica.
|
|
containers = [_c("h1", "app.1.old", service="app", node="nA", status="exited", task="old"),
|
|
_c("h2", "app.1.new", service="app", node="nB", task="new")]
|
|
tasks = [_task("new", state="running", desired="running", ago_min=1, node="nB"),
|
|
_task("old", state="shutdown", ago_min=2, exit_code=137)]
|
|
view = _view(containers, [_svc("app", 1, 1, [{"node_id": "nB", "running": 1}], tasks=tasks)])
|
|
(svc,) = view["services"]
|
|
assert [r["name"] for r in svc["replicas"]] == ["app.1.new"]
|
|
assert svc["previous_tasks"] == 1 and svc["state"] == "healthy"
|
|
assert view["superseded"] == {("h1", "app.1.old")}
|
|
# A deliberate shutdown is not a failure.
|
|
assert svc["failures_24h"] == 0 and svc["last_error"] is None and not svc["crash_looping"]
|
|
|
|
|
|
def test_without_task_data_the_slot_decides():
|
|
# An older agent sends no tasks: a stopped container is replaced when its slot
|
|
# has a running task, or one that started later.
|
|
t0 = NOW - timedelta(hours=2)
|
|
containers = [_c("h1", "lib.1.a", service="lib", status="exited", started=t0),
|
|
_c("h1", "lib.1.b", service="lib", status="exited", started=t0 + timedelta(minutes=5)),
|
|
_c("h1", "lib.1.c", service="lib", started=t0 + timedelta(minutes=10)),
|
|
_c("h1", "lib.2.d", service="lib", status="exited", started=t0)]
|
|
view = _view(containers, [_svc("lib", 1, 2, [])])
|
|
(svc,) = view["services"]
|
|
# Slot 2's only task is stopped with nothing newer: still the current task.
|
|
assert sorted(r["name"] for r in svc["replicas"]) == ["lib.1.c", "lib.2.d"]
|
|
assert svc["previous_tasks"] == 2
|
|
assert svc["tasks_known"] is False and svc["failures_1h"] is None
|
|
assert svc["crash_looping"] is False
|
|
|
|
|
|
def test_a_crash_loop_is_counted_across_task_retries():
|
|
# Each retry is a new task and a new container that dies once; only the
|
|
# service sees the loop.
|
|
tasks = [_task("t5", state="starting", desired="running", ago_min=0)]
|
|
tasks += [_task(f"t{i}", state="failed", ago_min=10 * i, exit_code=1,
|
|
error="task: non-zero exit (1)") for i in range(1, 5)]
|
|
tasks.append(_task("t9", state="failed", ago_min=180, exit_code=1))
|
|
view = build_swarm_services([], [_svc("db", 0, 1, [], tasks=tasks)], NODE_HOSTNAME, HOST_NAME,
|
|
now=NOW, crash_loop_deaths=3)
|
|
(svc,) = view["services"]
|
|
assert svc["failures_1h"] == 4 and svc["failures_24h"] == 5
|
|
assert svc["crash_looping"] is True and svc["state"] == "down"
|
|
err = svc["last_error"]
|
|
assert err["error"] == "task: non-zero exit (1)" and err["exit_code"] == 1
|
|
assert err["node"] == "docker01" and err["state"] == "failed"
|
|
# The threshold comes from Settings → Thresholds.
|
|
lax = build_swarm_services([], [_svc("db", 0, 1, [], tasks=tasks)], NODE_HOSTNAME, HOST_NAME,
|
|
now=NOW, crash_loop_deaths=5)
|
|
assert lax["services"][0]["crash_looping"] is False
|
|
|
|
|
|
def test_a_rejected_or_stuck_task_explains_itself():
|
|
rejected = [_task("t1", state="rejected", desired="running", error="No such image: codex:dev")]
|
|
assert last_task_error(rejected)["error"] == "No such image: codex:dev"
|
|
# A task Swarm is still trying to place carries its reason too.
|
|
pending = [_task("t2", state="pending", desired="running", error="no suitable node")]
|
|
assert last_task_error(pending)["state"] == "pending"
|
|
assert last_task_error([_task("t3", state="running", desired="running")]) is None
|
|
assert task_failures(rejected, NOW - timedelta(hours=1)) == 1
|
|
|
|
|
|
def test_the_newest_manager_report_wins():
|
|
# Two managers report the same service; the one updated last (here: the one
|
|
# with task data) is used.
|
|
old = _svc("web", 1, 1, [], updated=NOW - timedelta(minutes=5))
|
|
new = _svc("web", 1, 1, [], tasks=[_task("t1", state="failed", ago_min=3)],
|
|
updated=NOW - timedelta(seconds=30))
|
|
(svc,) = _view([], [old, new])["services"]
|
|
assert svc["tasks_known"] is True and svc["failures_1h"] == 1
|
|
|
|
|
|
|
|
def _ago(minutes):
|
|
return NOW - timedelta(minutes=minutes)
|
|
|
|
|
|
def test_trouble_since_for_each_kind_of_trouble():
|
|
loop = [_task("t1", state="failed", ago_min=2), _task("t2", state="failed", ago_min=20),
|
|
_task("t3", state="failed", ago_min=45), _task("t4", state="failed", ago_min=90)]
|
|
# A crash loop: its first failure inside the hour.
|
|
assert trouble_since(loop, "down", True, NOW) == _ago(45)
|
|
# Down: the oldest of the newest troubled tasks, back to a running one.
|
|
down = [_task("t1", state="pending", ago_min=1, error="no suitable node"),
|
|
_task("t2", state="rejected", ago_min=30, error="No such image"),
|
|
_task("t3", state="running", desired="running", ago_min=600),
|
|
_task("t4", state="failed", ago_min=900)]
|
|
assert trouble_since(down, "degraded", False, NOW) == _ago(30)
|
|
# Another slot's newer, healthy task does not hide the failing one.
|
|
two = [_task("a2", state="running", desired="running", ago_min=5, slot=1),
|
|
_task("b1", state="failed", ago_min=8, slot=2),
|
|
_task("b0", state="failed", ago_min=40, slot=2),
|
|
_task("a1", state="failed", ago_min=300, slot=1)]
|
|
assert trouble_since(two, "degraded", False, NOW) == _ago(40)
|
|
# Healthy again, but a task failed today: dated from that failure.
|
|
recovered = [_task("t1", state="running", desired="running", ago_min=10),
|
|
_task("t2", state="failed", ago_min=60 * 5),
|
|
_task("t3", state="failed", ago_min=60 * 30)]
|
|
assert trouble_since(recovered, "healthy", False, NOW) == _ago(60 * 5)
|
|
# No task records: unknown.
|
|
assert trouble_since([], "down", False, NOW) is None
|
|
|
|
|
|
def test_the_view_dates_a_down_service():
|
|
tasks = [_task("t1", state="failed", ago_min=3, exit_code=1),
|
|
_task("t2", state="failed", ago_min=70, exit_code=1)]
|
|
(s,) = _view([], [_svc("db", 0, 1, [], tasks=tasks)])["services"]
|
|
assert s["state"] == "down" and s["trouble_since"] == _ago(70)
|