"""A maintenance run spreads swarm services back out after its last host: what it moves, what it leaves and why, and how the step reaches the manager and comes back.""" from datetime import datetime, timedelta, timezone from types import SimpleNamespace as NS from plugins.host_agent import rebalance as rb NOW = datetime(2026, 10, 25, 14, 0, tzinfo=timezone.utc) NODES = {"n1": "node-1", "n2": "node-2"} def _svc(name, node="n1", running=1, desired=None, mode="replicated", image="auto"): return {"name": name, "mode": mode, "desired": running if desired is None else desired, "running": running, "image": f"registry.example/{name}@sha256:{name}" if image == "auto" else image, "placement": [{"node_id": node, "running": running}] if running else []} def _held(*names): return {f"registry.example/{n}@sha256:{n}" for n in names} def test_services_on_the_fuller_node_move_until_the_nodes_are_even(): services = [_svc(n) for n in "abcdef"] moves, left = rb.choose(services, NODES, {"n2": _held(*"abcdef")}) assert moves == ["a", "b", "c"] and left == [] def test_nothing_moves_when_the_nodes_are_already_even(): services = [_svc("a"), _svc("b", "n2"), _svc("c")] assert rb.choose(services, NODES, {"n1": _held("b"), "n2": _held("a", "c")}) == ([], []) def test_a_service_whose_image_the_emptier_node_lacks_stays_and_says_why(): services = [_svc("a"), _svc("b"), _svc("c"), _svc("d")] moves, left = rb.choose(services, NODES, {"n2": _held("b")}) assert moves == ["b"] assert left == [{"service": "a", "reason": "its image isn't on node-2"}, {"service": "c", "reason": "its image isn't on node-2"}, {"service": "d", "reason": "its image isn't on node-2"}] def test_an_unknown_image_list_moves_nothing_there(): moves, left = rb.choose([_svc("a"), _svc("b")], NODES, {}) assert moves == [] assert left[0]["reason"] == "no image list from node-2 (its agent needs 1.26.0)" def test_kept_unpinned_short_and_global_services_stay(): services = [_svc("keep"), _svc("tag", image=None), _svc("short", running=1, desired=2), _svc("agent", mode="global"), _svc("move")] moves, left = rb.choose(services, NODES, {"n2": _held("keep", "tag", "short", "agent", "move")}, keep=["keep"]) assert moves == ["move"] reasons = {x["service"]: x["reason"] for x in left} # Even after one move (4 on n1 against 1): what else could have moved, and why it didn't. assert reasons == {"keep": "kept in place by the plan", "short": "below its replicas", "tag": "its image isn't pinned to a digest"} def test_a_service_with_all_its_replicas_on_one_node_goes_first(): services = [_svc("a"), _svc("b"), _svc("c"), _svc("workers", running=2)] moves, _ = rb.choose(services, NODES, {"n2": _held("a", "b", "c", "workers")}) assert moves[0] == "workers" and len(moves) == 2 # 5 on n1: the workers' one, then a def test_a_service_already_spread_or_on_the_emptier_node_is_not_moved(): split = {"name": "split", "mode": "replicated", "desired": 2, "running": 2, "image": "registry.example/split@sha256:split", "placement": [{"node_id": "n1", "running": 1}, {"node_id": "n2", "running": 1}]} services = [split, _svc("a"), _svc("b"), _svc("d"), _svc("c", "n2")] # 4 on n1, 2 on n2 moves, _ = rb.choose(services, NODES, {"n2": _held("split", "a", "b", "d")}) assert "split" not in moves and moves == ["a"] def test_one_node_has_nothing_to_spread_to(): assert rb.choose([_svc("a"), _svc("b")], {"n1": "node-1"}, {}) == ([], []) def test_a_pinned_image_reads_the_way_docker_reports_it(): assert rb.pinned("traefik:v3.1", "sha256:abc") == "traefik@sha256:abc" assert rb.pinned("docker.io/library/redis:7", "sha256:abc") == "redis@sha256:abc" assert rb.pinned("registry.example:5000/team/app:dev", "sha256:abc") == \ "registry.example:5000/team/app@sha256:abc" assert rb.pinned("app:dev", None) is None assert rb.held(["traefik@sha256:abc", "docker.io/library/redis@sha256:def", "junk"]) == \ {"traefik@sha256:abc", "redis@sha256:def"} def test_the_plan_names_the_services_it_keeps_once_each(): assert rb.keep_list(" web_proxy, metrics_app,\nweb_proxy ,, ") == ["web_proxy", "metrics_app"] assert rb.keep_list(None) == [] def test_only_a_new_enough_agent_rebalances(): assert rb.can_rebalance("1.26.0") and rb.can_rebalance("2.0.0") assert not rb.can_rebalance("1.25.3") and not rb.can_rebalance(None) def _run(**rebalance): return NS(id="run-1", rebalance=rebalance or None) def test_the_step_is_handed_until_the_manager_says_it_started(): run = _run(state="handed", step_id="run-1-rebalance", planned=["a", "b"], seen=False, at=NOW.isoformat(), left=[]) assert rb.handoff(run) == {"id": "run-1-rebalance", "action": "rebalance", "services": ["a", "b"]} assert rb.apply_result(run, {"id": "run-1-rebalance", "state": "running"}, NOW) assert rb.handoff(run) is None assert not rb.apply_result(run, {"id": "other", "state": "done"}, NOW) def test_the_result_says_what_moved_and_the_run_ends_on_it(): run = _run(state="handed", step_id="run-1-rebalance", planned=["a", "b"], seen=True, at=NOW.isoformat(), left=[{"service": "c", "reason": "its image isn't on node-2"}]) assert rb.apply_result(run, {"id": "run-1-rebalance", "state": "done", "ok": True, "moved": ["a", "b"], "left": [], "detail": "moved 2 of 2"}, NOW) assert run.rebalance["state"] == "done" and run.rebalance["moved"] == ["a", "b"] assert rb.outcome(run) == ("done", "spread 2 swarm services back out; 1 left where they are") assert rb.label(run) == ("Spread 2 swarm services back out; 1 left where they are", False, False) # A result already applied changes nothing. assert not rb.apply_result(run, {"id": "run-1-rebalance", "state": "done"}, NOW) def test_a_move_that_didnt_settle_fails_the_run(): run = _run(state="handed", step_id="run-1-rebalance", planned=["a"], seen=True, at=NOW.isoformat(), left=[]) rb.apply_result(run, {"id": "run-1-rebalance", "state": "failed", "ok": False, "moved": [], "left": [{"service": "a", "reason": "didn't settle: paused"}], "detail": "moved 0 of 1; didn't settle: a"}, NOW) assert rb.outcome(run) == ("failed", "spreading swarm services back out: moved 0 of 1; didn't settle: a") assert rb.label(run)[1] is True def test_a_manager_that_never_answers_times_out(): run = _run(state="handed", step_id="run-1-rebalance", planned=["a"], seen=True, at=NOW.isoformat(), left=[]) assert not rb.check_deadline(run, NOW + rb.DEADLINE) assert rb.check_deadline(run, NOW + rb.DEADLINE + timedelta(seconds=1)) assert run.rebalance["state"] == "timed_out" and rb.outcome(run)[0] == "failed" def test_a_run_with_nothing_to_move_ends_done_and_says_why(): assert rb.outcome(_run()) == ("done", None) and rb.label(_run()) is None nothing = _run(state="nothing", left=[], detail="no swarm manager has the maintenance helper") assert rb.outcome(nothing) == ( "done", "swarm services not spread back out: no swarm manager has the maintenance helper") even = _run(state="nothing", left=[]) assert rb.outcome(even) == ("done", None) assert rb.label(even) == ("Nothing to spread back out", False, False)