feat(maintenance): a run spreads swarm services back out after its last host, moving only what the emptier node can start (#6398)

Each weekly run drains Docker02 then Docker03, and the swarm never moves a
task back by itself, so every run ended with nearly every service on one
node. On 2026-10-25 the operator had that spread out by hand.

- Agent 1.26.0 reports each swarm service's pinned digest, and every
  repo@sha256 its host holds (from /system/df, every 10 min).
- Once a run's last host is back in service, Steward picks services on the
  fullest node to move to the emptiest until they are within one task. It
  takes services with several replicas on one node first. A service moves only
  when the emptier node already holds its exact image. One the plan keeps in
  place, one below its replicas, and one with no pinned digest stay put, and
  the run says why.
- A swarm manager's helper carries it out through a new action, rebalance,
  with the service names (validated, at most 48). It force-updates each in
  turn with the spec unchanged and no registry lookup, skips any a
  constraint places, and gives each 5 min to settle.
- The run ends done with what moved, or failed if a move didn't settle or
  the manager didn't answer in 60 min. A docker-scoped window covers the
  restarts, and the step goes on the timeline.
- Plans gain "Spread swarm services back out" (on by default) and "Keep in
  place", on the create form and in an "After the run" form on each plan.
  list_maintenance_plans shows both and the latest run's spread.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/plugins/host_agent/test_maint_helper.py

@@ -103,6 +103,46 @@ def test_a_reboot_says_so_before_it_goes(maint, tmp_path, monkeypatch):
     assert called == [["systemctl", "reboot"]]
 
 
+def test_a_rebalance_names_services_and_nothing_else_may(maint, tmp_path):
+    req = {"id": "run-1-rebalance", "action": "rebalance", "services": ["web_app", "db.1"]}
+    assert maint.read_request(_write(tmp_path / "r.json", req))["services"] == ["web_app", "db.1"]
+    for bad in ([], ["--detach"], ["web app"], ["web;rm"], "web_app", ["x"] * (maint.SERVICES_MAX + 1)):
+        assert maint.read_request(_write(tmp_path / "r.json", {**req, "services": bad})) is None
+    # Only a rebalance carries services.
+    assert maint.read_request(_write(tmp_path / "r.json",
+                                     {"id": "job-1", "action": "update", "services": ["web_app"]})) is None
+
+
+def test_a_rebalance_force_updates_each_service_and_leaves_one_a_constraint_places(maint, monkeypatch):
+    monkeypatch.setattr(maint.shutil, "which", lambda _n: "/usr/bin/docker")
+    calls = []
+    placement = {"web": "null", "pinned": '{"Constraints":["node.hostname==a"]}', "spread": "{}"}
+
+    def run(cmd, deadline=0, env=None):
+        calls.append(cmd)
+        name = cmd[-1]
+        if cmd[2] == "inspect":
+            return (name in placement, placement.get(name, "no such service"))
+        return (name != "spread", "update paused")
+    monkeypatch.setattr(maint, "run", run)
+    out = maint.do_rebalance(["web", "pinned", "gone", "spread"])
+    assert out["moved"] == ["web"] and out["ok"] is False
+    assert out["left"] == [{"service": "pinned", "reason": "placed by a constraint"},
+                           {"service": "gone", "reason": "not found on this manager"},
+                           {"service": "spread", "reason": "didn't settle: update paused"}]
+    assert ["docker", "service", "update", "--force", "--detach=false", "--quiet", "web"] in calls
+    assert not any(c[2] == "update" and c[-1] in ("pinned", "gone") for c in calls)
+
+
+def test_a_rebalance_starts_no_move_once_out_of_time(maint, monkeypatch):
+    monkeypatch.setattr(maint.shutil, "which", lambda _n: "/usr/bin/docker")
+    monkeypatch.setattr(maint, "run", lambda cmd, deadline=0, env=None: (True, "null"))
+    ticks = iter([0, 0, maint.REBALANCE_SECONDS + 1])
+    out = maint.do_rebalance(["a", "b"], clock=lambda: next(ticks))
+    assert out["moved"] == ["a"] and out["left"] == [{"service": "b", "reason": "out of time"}]
+    assert out["ok"] is True
+
+
 # ── the agent's side ─────────────────────────────────────────────────────────
 
 def test_the_agent_leaves_a_whole_request_the_helper_accepts(maint, tmp_path):
@@ -111,6 +151,13 @@ def test_the_agent_leaves_a_whole_request_the_helper_accepts(maint, tmp_path):
     assert maint.read_request(str(tmp_path / "job-3.json"))["action"] == "drain"
 
 
+def test_the_agent_passes_a_rebalance_its_services(maint, tmp_path):
+    a.take_maintenance([{"id": "run-1-rebalance", "action": "rebalance", "services": ["web", "db"]}],
+                       set(), requests_dir=str(tmp_path))
+    req = maint.read_request(str(tmp_path / "run-1-rebalance.json"))
+    assert req["action"] == "rebalance" and req["services"] == ["web", "db"]
+
+
 def test_the_agent_refuses_to_ask_for_anything_else(tmp_path):
     with pytest.raises(ValueError):
         a.request_maintenance("job-3", "shell", requests_dir=str(tmp_path))