feat(maintenance): a run spreads swarm services back out after its last host, moving only what the emptier node can start (#6398)

Each weekly run drains Docker02 then Docker03, and the swarm never moves a
task back by itself, so every run ended with nearly every service on one
node. On 2026-10-25 the operator had that spread out by hand.

- Agent 1.26.0 reports each swarm service's pinned digest, and every
  repo@sha256 its host holds (from /system/df, every 10 min).
- Once a run's last host is back in service, Steward picks services on the
  fullest node to move to the emptiest until they are within one task. It
  takes services with several replicas on one node first. A service moves only
  when the emptier node already holds its exact image. One the plan keeps in
  place, one below its replicas, and one with no pinned digest stay put, and
  the run says why.
- A swarm manager's helper carries it out through a new action, rebalance,
  with the service names (validated, at most 48). It force-updates each in
  turn with the spec unchanged and no registry lookup, skips any a
  constraint places, and gives each 5 min to settle.
- The run ends done with what moved, or failed if a move didn't settle or
  the manager didn't answer in 60 min. A docker-scoped window covers the
  restarts, and the step goes on the timeline.
- Plans gain "Spread swarm services back out" (on by default) and "Keep in
  place", on the create form and in an "After the run" form on each plan.
  list_maintenance_plans shows both and the latest run's spread.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_mcp_maintenance.py

@@ -110,7 +110,9 @@ def test_maintenance_plans_jobs_and_starting_one_through_mcp(app):
         plan = plans[ids["plan"]]
         assert plan["group"] == f"mm-{tag}" and plan["schedule"].startswith("Sundays at 03:00")
         assert plan["next_slot_at"] and plan["enabled"] is True
+        assert plan["spread_after_run"] is True and plan["keep_in_place"] == []
         run = plan["latest_run"]
+        assert run["spread"] is None            # it failed before its last host
         assert run["state"] == "failed" and run["hosts_done"] == 1
         assert run["hosts"] == [f"mm-failed-{tag}", f"mm-busy-{tag}"]
         assert run["skipped"] == [{"host": f"mm-free-{tag}", "reason": "no agent"}]

tests/plugins/host_agent/test_agent_docker.py

@@ -286,10 +286,12 @@ def test_swarm_services_rolls_up_replicas():
     assert len(out["agent"].pop("tasks")) == 2
     assert out["web"] == {"service_name": "web", "mode": "replicated",
                           "desired": 3, "running": 2, "image": "nginx:1",
+                          "image_digest": "sha256:deadbeef",
                           "placement": [{"node_id": "n1", "running": 2}]}
     # Global mode has no fixed replica count → desired derived from desired tasks.
     assert out["agent"] == {"service_name": "agent", "mode": "global",
                             "desired": 2, "running": 2, "image": "agent:latest",
+                            "image_digest": None,
                             "placement": [{"node_id": "n1", "running": 1},
                                           {"node_id": "n2", "running": 1}]}
 
@@ -476,8 +478,9 @@ def test_collect_disk_usage_computes_reclaimable(monkeypatch):
     df = {
         "LayersSize": 1000,
         "Images": [
-            {"Id": "sha256:a", "RepoTags": ["app:1"], "Size": 300, "Containers": 1},
-            {"Id": "sha256:b", "RepoTags": None, "Size": 700, "Containers": 0},
+            {"Id": "sha256:a", "RepoTags": ["app:1"], "Size": 300, "Containers": 1,
+             "RepoDigests": ["registry.example:5000/app@sha256:aaa", "app@sha256:aaa"]},
+            {"Id": "sha256:b", "RepoTags": None, "Size": 700, "Containers": 0, "RepoDigests": None},
         ],
         "Containers": [{"Id": "c1", "SizeRw": 25}, {"Id": "c2", "SizeRw": 75}],
         "Volumes": [{"Name": "v1", "UsageData": {"Size": 40}},
@@ -495,6 +498,8 @@ def test_collect_disk_usage_computes_reclaimable(monkeypatch):
     assert out["volumes_count"] == 2 and out["volumes_size"] == 40  # negative skipped
     assert out["build_cache_size"] == 15
     assert len(out["images"]) == 2
+    # Every image the host holds, by digest: what a swarm service is checked against.
+    assert out["digests"] == ["app@sha256:aaa", "registry.example:5000/app@sha256:aaa"]
 
 
 def test_collect_disk_usage_silent_skip_when_no_socket():

tests/plugins/host_agent/test_maint_helper.py

@@ -103,6 +103,46 @@ def test_a_reboot_says_so_before_it_goes(maint, tmp_path, monkeypatch):
     assert called == [["systemctl", "reboot"]]
 
 
+def test_a_rebalance_names_services_and_nothing_else_may(maint, tmp_path):
+    req = {"id": "run-1-rebalance", "action": "rebalance", "services": ["web_app", "db.1"]}
+    assert maint.read_request(_write(tmp_path / "r.json", req))["services"] == ["web_app", "db.1"]
+    for bad in ([], ["--detach"], ["web app"], ["web;rm"], "web_app", ["x"] * (maint.SERVICES_MAX + 1)):
+        assert maint.read_request(_write(tmp_path / "r.json", {**req, "services": bad})) is None
+    # Only a rebalance carries services.
+    assert maint.read_request(_write(tmp_path / "r.json",
+                                     {"id": "job-1", "action": "update", "services": ["web_app"]})) is None
+
+
+def test_a_rebalance_force_updates_each_service_and_leaves_one_a_constraint_places(maint, monkeypatch):
+    monkeypatch.setattr(maint.shutil, "which", lambda _n: "/usr/bin/docker")
+    calls = []
+    placement = {"web": "null", "pinned": '{"Constraints":["node.hostname==a"]}', "spread": "{}"}
+
+    def run(cmd, deadline=0, env=None):
+        calls.append(cmd)
+        name = cmd[-1]
+        if cmd[2] == "inspect":
+            return (name in placement, placement.get(name, "no such service"))
+        return (name != "spread", "update paused")
+    monkeypatch.setattr(maint, "run", run)
+    out = maint.do_rebalance(["web", "pinned", "gone", "spread"])
+    assert out["moved"] == ["web"] and out["ok"] is False
+    assert out["left"] == [{"service": "pinned", "reason": "placed by a constraint"},
+                           {"service": "gone", "reason": "not found on this manager"},
+                           {"service": "spread", "reason": "didn't settle: update paused"}]
+    assert ["docker", "service", "update", "--force", "--detach=false", "--quiet", "web"] in calls
+    assert not any(c[2] == "update" and c[-1] in ("pinned", "gone") for c in calls)
+
+
+def test_a_rebalance_starts_no_move_once_out_of_time(maint, monkeypatch):
+    monkeypatch.setattr(maint.shutil, "which", lambda _n: "/usr/bin/docker")
+    monkeypatch.setattr(maint, "run", lambda cmd, deadline=0, env=None: (True, "null"))
+    ticks = iter([0, 0, maint.REBALANCE_SECONDS + 1])
+    out = maint.do_rebalance(["a", "b"], clock=lambda: next(ticks))
+    assert out["moved"] == ["a"] and out["left"] == [{"service": "b", "reason": "out of time"}]
+    assert out["ok"] is True
+
+
 # ── the agent's side ─────────────────────────────────────────────────────────
 
 def test_the_agent_leaves_a_whole_request_the_helper_accepts(maint, tmp_path):
@@ -111,6 +151,13 @@ def test_the_agent_leaves_a_whole_request_the_helper_accepts(maint, tmp_path):
     assert maint.read_request(str(tmp_path / "job-3.json"))["action"] == "drain"
 
 
+def test_the_agent_passes_a_rebalance_its_services(maint, tmp_path):
+    a.take_maintenance([{"id": "run-1-rebalance", "action": "rebalance", "services": ["web", "db"]}],
+                       set(), requests_dir=str(tmp_path))
+    req = maint.read_request(str(tmp_path / "run-1-rebalance.json"))
+    assert req["action"] == "rebalance" and req["services"] == ["web", "db"]
+
+
 def test_the_agent_refuses_to_ask_for_anything_else(tmp_path):
     with pytest.raises(ValueError):
         a.request_maintenance("job-3", "shell", requests_dir=str(tmp_path))

tests/plugins/host_agent/test_rebalance.py

@@ -0,0 +1,151 @@
+"""A maintenance run spreads swarm services back out after its last host:
+what it moves, what it leaves and why, and how the step reaches the
+manager and comes back."""
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace as NS
+
+from plugins.host_agent import rebalance as rb
+
+NOW = datetime(2026, 10, 25, 14, 0, tzinfo=timezone.utc)
+NODES = {"n1": "node-1", "n2": "node-2"}
+
+
+def _svc(name, node="n1", running=1, desired=None, mode="replicated", image="auto"):
+    return {"name": name, "mode": mode, "desired": running if desired is None else desired,
+            "running": running,
+            "image": f"registry.example/{name}@sha256:{name}" if image == "auto" else image,
+            "placement": [{"node_id": node, "running": running}] if running else []}
+
+
+def _held(*names):
+    return {f"registry.example/{n}@sha256:{n}" for n in names}
+
+
+def test_services_on_the_fuller_node_move_until_the_nodes_are_even():
+    services = [_svc(n) for n in "abcdef"]
+    moves, left = rb.choose(services, NODES, {"n2": _held(*"abcdef")})
+    assert moves == ["a", "b", "c"] and left == []
+
+
+def test_nothing_moves_when_the_nodes_are_already_even():
+    services = [_svc("a"), _svc("b", "n2"), _svc("c")]
+    assert rb.choose(services, NODES, {"n1": _held("b"), "n2": _held("a", "c")}) == ([], [])
+
+
+def test_a_service_whose_image_the_emptier_node_lacks_stays_and_says_why():
+    services = [_svc("a"), _svc("b"), _svc("c"), _svc("d")]
+    moves, left = rb.choose(services, NODES, {"n2": _held("b")})
+    assert moves == ["b"]
+    assert left == [{"service": "a", "reason": "its image isn't on node-2"},
+                    {"service": "c", "reason": "its image isn't on node-2"},
+                    {"service": "d", "reason": "its image isn't on node-2"}]
+
+
+def test_an_unknown_image_list_moves_nothing_there():
+    moves, left = rb.choose([_svc("a"), _svc("b")], NODES, {})
+    assert moves == []
+    assert left[0]["reason"] == "no image list from node-2 (its agent needs 1.26.0)"
+
+
+def test_kept_unpinned_short_and_global_services_stay():
+    services = [_svc("keep"), _svc("tag", image=None), _svc("short", running=1, desired=2),
+                _svc("agent", mode="global"), _svc("move")]
+    moves, left = rb.choose(services, NODES, {"n2": _held("keep", "tag", "short", "agent", "move")},
+                            keep=["keep"])
+    assert moves == ["move"]
+    reasons = {x["service"]: x["reason"] for x in left}
+    # Even after one move (4 on n1 against 1): what else could have moved, and why it didn't.
+    assert reasons == {"keep": "kept in place by the plan", "short": "below its replicas",
+                       "tag": "its image isn't pinned to a digest"}
+
+
+def test_a_service_with_all_its_replicas_on_one_node_goes_first():
+    services = [_svc("a"), _svc("b"), _svc("c"), _svc("workers", running=2)]
+    moves, _ = rb.choose(services, NODES, {"n2": _held("a", "b", "c", "workers")})
+    assert moves[0] == "workers" and len(moves) == 2      # 5 on n1: the workers' one, then a
+
+
+def test_a_service_already_spread_or_on_the_emptier_node_is_not_moved():
+    split = {"name": "split", "mode": "replicated", "desired": 2, "running": 2,
+             "image": "registry.example/split@sha256:split",
+             "placement": [{"node_id": "n1", "running": 1}, {"node_id": "n2", "running": 1}]}
+    services = [split, _svc("a"), _svc("b"), _svc("d"), _svc("c", "n2")]   # 4 on n1, 2 on n2
+    moves, _ = rb.choose(services, NODES, {"n2": _held("split", "a", "b", "d")})
+    assert "split" not in moves and moves == ["a"]
+
+
+def test_one_node_has_nothing_to_spread_to():
+    assert rb.choose([_svc("a"), _svc("b")], {"n1": "node-1"}, {}) == ([], [])
+
+
+def test_a_pinned_image_reads_the_way_docker_reports_it():
+    assert rb.pinned("traefik:v3.1", "sha256:abc") == "traefik@sha256:abc"
+    assert rb.pinned("docker.io/library/redis:7", "sha256:abc") == "redis@sha256:abc"
+    assert rb.pinned("registry.example:5000/team/app:dev", "sha256:abc") == \
+        "registry.example:5000/team/app@sha256:abc"
+    assert rb.pinned("app:dev", None) is None
+    assert rb.held(["traefik@sha256:abc", "docker.io/library/redis@sha256:def", "junk"]) == \
+        {"traefik@sha256:abc", "redis@sha256:def"}
+
+
+def test_the_plan_names_the_services_it_keeps_once_each():
+    assert rb.keep_list(" web_proxy, metrics_app,\nweb_proxy ,, ") == ["web_proxy", "metrics_app"]
+    assert rb.keep_list(None) == []
+
+
+def test_only_a_new_enough_agent_rebalances():
+    assert rb.can_rebalance("1.26.0") and rb.can_rebalance("2.0.0")
+    assert not rb.can_rebalance("1.25.3") and not rb.can_rebalance(None)
+
+
+def _run(**rebalance):
+    return NS(id="run-1", rebalance=rebalance or None)
+
+
+def test_the_step_is_handed_until_the_manager_says_it_started():
+    run = _run(state="handed", step_id="run-1-rebalance", planned=["a", "b"], seen=False,
+               at=NOW.isoformat(), left=[])
+    assert rb.handoff(run) == {"id": "run-1-rebalance", "action": "rebalance", "services": ["a", "b"]}
+    assert rb.apply_result(run, {"id": "run-1-rebalance", "state": "running"}, NOW)
+    assert rb.handoff(run) is None
+    assert not rb.apply_result(run, {"id": "other", "state": "done"}, NOW)
+
+
+def test_the_result_says_what_moved_and_the_run_ends_on_it():
+    run = _run(state="handed", step_id="run-1-rebalance", planned=["a", "b"], seen=True,
+               at=NOW.isoformat(), left=[{"service": "c", "reason": "its image isn't on node-2"}])
+    assert rb.apply_result(run, {"id": "run-1-rebalance", "state": "done", "ok": True,
+                                 "moved": ["a", "b"], "left": [], "detail": "moved 2 of 2"}, NOW)
+    assert run.rebalance["state"] == "done" and run.rebalance["moved"] == ["a", "b"]
+    assert rb.outcome(run) == ("done", "spread 2 swarm services back out; 1 left where they are")
+    assert rb.label(run) == ("Spread 2 swarm services back out; 1 left where they are", False, False)
+    # A result already applied changes nothing.
+    assert not rb.apply_result(run, {"id": "run-1-rebalance", "state": "done"}, NOW)
+
+
+def test_a_move_that_didnt_settle_fails_the_run():
+    run = _run(state="handed", step_id="run-1-rebalance", planned=["a"], seen=True,
+               at=NOW.isoformat(), left=[])
+    rb.apply_result(run, {"id": "run-1-rebalance", "state": "failed", "ok": False, "moved": [],
+                          "left": [{"service": "a", "reason": "didn't settle: paused"}],
+                          "detail": "moved 0 of 1; didn't settle: a"}, NOW)
+    assert rb.outcome(run) == ("failed", "spreading swarm services back out: moved 0 of 1; didn't settle: a")
+    assert rb.label(run)[1] is True
+
+
+def test_a_manager_that_never_answers_times_out():
+    run = _run(state="handed", step_id="run-1-rebalance", planned=["a"], seen=True,
+               at=NOW.isoformat(), left=[])
+    assert not rb.check_deadline(run, NOW + rb.DEADLINE)
+    assert rb.check_deadline(run, NOW + rb.DEADLINE + timedelta(seconds=1))
+    assert run.rebalance["state"] == "timed_out" and rb.outcome(run)[0] == "failed"
+
+
+def test_a_run_with_nothing_to_move_ends_done_and_says_why():
+    assert rb.outcome(_run()) == ("done", None) and rb.label(_run()) is None
+    nothing = _run(state="nothing", left=[], detail="no swarm manager has the maintenance helper")
+    assert rb.outcome(nothing) == (
+        "done", "swarm services not spread back out: no swarm manager has the maintenance helper")
+    even = _run(state="nothing", left=[])
+    assert rb.outcome(even) == ("done", None)
+    assert rb.label(even) == ("Nothing to spread back out", False, False)