feat(maintenance): a run spreads swarm services back out after its last host, moving only what the emptier node can start (#6398)

Each weekly run drains Docker02 then Docker03, and the swarm never moves a
task back by itself, so every run ended with nearly every service on one
node. On 2026-10-25 the operator had that spread out by hand.

- Agent 1.26.0 reports each swarm service's pinned digest, and every
  repo@sha256 its host holds (from /system/df, every 10 min).
- Once a run's last host is back in service, Steward picks services on the
  fullest node to move to the emptiest until they are within one task. It
  takes services with several replicas on one node first. A service moves only
  when the emptier node already holds its exact image. One the plan keeps in
  place, one below its replicas, and one with no pinned digest stay put, and
  the run says why.
- A swarm manager's helper carries it out through a new action, rebalance,
  with the service names (validated, at most 48). It force-updates each in
  turn with the spec unchanged and no registry lookup, skips any a
  constraint places, and gives each 5 min to settle.
- The run ends done with what moved, or failed if a move didn't settle or
  the manager didn't answer in 60 min. A docker-scoped window covers the
  restarts, and the step goes on the timeline.
- Plans gain "Spread swarm services back out" (on by default) and "Keep in
  place", on the create form and in an "After the run" form on each plan.
  list_maintenance_plans shows both and the latest run's spread.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/routes.py

@@ -1442,8 +1442,10 @@ async def update_via_ansible():
 # ── Maintenance cadence (#5768) ───────────────────────────────────────────────
 
 def _run_rows(run, jobs: dict, names: dict) -> list[dict]:
-    """A run's hosts in order, each with where it stands, then those passed over."""
+    """A run's hosts in order, each with where it stands, then those passed
+    over, then the spreading of its swarm services back out."""
     import json as _json
+    from . import rebalance
     rows = []
     for i, host_id in enumerate(_json.loads(run.host_ids)):
         job = jobs.get((run.id, host_id))
@@ -1462,6 +1464,13 @@ def _run_rows(run, jobs: dict, names: dict) -> list[dict]:
     for s in _json.loads(run.skipped):
         rows.append({"host_id": s["host_id"], "name": names.get(s["host_id"], s["host_id"]),
                      "text": f"Skipped: {s['reason']}", "bad": False, "busy": False, "detail": None})
+    swarm = rebalance.label(run)
+    if swarm is not None:
+        rb = run.rebalance or {}
+        said = [f"moved: {', '.join(rb['moved'])}"] if rb.get("moved") else []
+        said += [f"{x['service']}: {x['reason']}" for x in rb.get("left") or []]
+        rows.append({"host_id": None, "name": "Swarm services", "text": swarm[0], "bad": swarm[1],
+                     "busy": swarm[2], "detail": "; ".join(said) or None})
     return rows
 
 
@@ -1532,6 +1541,7 @@ async def maintenance_plan_create():
         return _error(400, "invalid", "weekday, time (HH:MM) and window must be numbers")
     policy = form.get("reboot_policy") or "if_required"
     tz = (form.get("time_zone") or "UTC").strip()
+    spread, keep = _spread_form(form)
     if not name or not 0 <= weekday <= 6 or not 30 <= window <= 24 * 60 \
             or policy not in maintenance.REBOOT_POLICIES:
         return _error(400, "invalid", "name, a weekday, a window of 30 min to 24 h and a reboot policy are needed")
@@ -1543,6 +1553,7 @@ async def maintenance_plan_create():
         plan = HostMaintenancePlan(name=name, group_id=form["group_id"], weekday=weekday,
                                    at_time=f"{cadence.parse_at(at_time):%H:%M}", time_zone=tz,
                                    window_minutes=window, reboot_policy=policy, enabled=True,
+                                   spread=spread, keep_in_place=keep,
                                    created_by=session.get("user_id"),
                                    # the first slot is the next one, not one already past
                                    last_slot_at=datetime.now(timezone.utc))
@@ -1553,6 +1564,13 @@ async def maintenance_plan_create():
     return redirect(url_for("host_agent.maintenance_page"))
 
 
+def _spread_form(form) -> tuple[bool, str]:
+    """(spread, keep_in_place) from a plan form: the box, and the services
+    named to stay where they are, tidied to one comma-separated list."""
+    from .rebalance import keep_list
+    return form.get("spread") == "on", ", ".join(keep_list(form.get("keep_in_place")))[:2000]
+
+
 async def _plan_action(plan_id: str, act):
     from .models import HostMaintenancePlan
     async with current_app.db_sessionmaker() as db:
@@ -1578,6 +1596,22 @@ async def maintenance_plan_toggle(plan_id: str):
     return resp
 
 
+@host_agent_bp.post("/maintenance/plans/<plan_id>/spread")
+@require_role(UserRole.admin)
+async def maintenance_plan_spread(plan_id: str):
+    """Whether the plan's runs end by spreading swarm services back out, and
+    which services stay where they are."""
+    form = await request.form
+    spread, keep = _spread_form(form)
+
+    async def act(db, plan):
+        plan.spread, plan.keep_in_place = spread, keep
+    resp = await _plan_action(plan_id, act)
+    await audit_request("host_agent.maintenance_plan_spread", "maintenance_plan", plan_id,
+                        {"spread": spread, "keep_in_place": keep})
+    return resp
+
+
 @host_agent_bp.post("/maintenance/plans/<plan_id>/delete")
 @require_role(UserRole.admin)
 async def maintenance_plan_delete(plan_id: str):