feat(maintenance): a run spreads swarm services back out after its last host, moving only what the emptier node can start (#6398)

Each weekly run drains Docker02 then Docker03, and the swarm never moves a
task back by itself, so every run ended with nearly every service on one
node. On 2026-10-25 the operator had that spread out by hand.

- Agent 1.26.0 reports each swarm service's pinned digest, and every
  repo@sha256 its host holds (from /system/df, every 10 min).
- Once a run's last host is back in service, Steward picks services on the
  fullest node to move to the emptiest until they are within one task. It
  takes services with several replicas on one node first. A service moves only
  when the emptier node already holds its exact image. One the plan keeps in
  place, one below its replicas, and one with no pinned digest stay put, and
  the run says why.
- A swarm manager's helper carries it out through a new action, rebalance,
  with the service names (validated, at most 48). It force-updates each in
  turn with the spec unchanged and no registry lookup, skips any a
  constraint places, and gives each 5 min to settle.
- The run ends done with what moved, or failed if a move didn't settle or
  the manager didn't answer in 60 min. A docker-scoped window covers the
  restarts, and the step goes on the timeline.
- Plans gain "Spread swarm services back out" (on by default) and "Keep in
  place", on the create form and in an "After the run" form on each plan.
  list_maintenance_plans shows both and the latest run's spread.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/files/maint.py

@@ -8,7 +8,8 @@ REQUESTS, the one directory it may write. This helper reads each request
 without following links, refuses anything outside the list, deletes it, does
 the action, and writes the outcome to RESULTS (root's own directory, which the
 agent reads back and reports to Steward). A stolen agent token can therefore
-make its host update, reboot or drain, and nothing else.
+make its host update, reboot or drain, or on a swarm manager restart named
+services where the swarm puts them, and nothing else.
 
 One action only reads: `drives` (#5789) asks smartctl for each disk's health,
 which needs root, and writes a summary to DRIVES, never a result: it is not a
@@ -16,6 +17,12 @@ maintenance job. `check` (#6098) changes nothing but the package lists: it
 says what an update would upgrade, whether Docker is among it, and which
 containers run outside the swarm.
 
+`rebalance` spreads swarm services back out after a maintenance run left them
+on one node. Steward names the services, having checked that the emptier node
+holds each one's image; the helper force-updates them one at a time, spec
+unchanged and no image pulled, so the swarm places each on its emptiest node.
+A service a constraint or a preference places is left alone.
+
 Stdlib only; served by Steward at /plugins/host_agent/files/maint.py.
 """
 from __future__ import annotations
@@ -32,7 +39,7 @@ from datetime import datetime, timezone
 REQUESTS = "/var/lib/steward-agent/requests"
 RESULTS = "/var/lib/steward-maint/results"
 DRIVES = "/var/lib/steward-maint/drives.json"
-ACTIONS = ("check", "update", "reboot", "drain", "undrain", "drives")
+ACTIONS = ("check", "update", "reboot", "drain", "undrain", "drives", "rebalance")
 REQUEST_MAX_BYTES = 4096
 RESULTS_KEPT_SECONDS = 7 * 86400
 UPDATE_DEADLINE_SECONDS = 3000
@@ -42,6 +49,12 @@ OUTPUT_KEPT_CHARS = 2000
 LISTING_KEPT_CHARS = 1_000_000
 _ID = re.compile(r"^[A-Za-z0-9-]{1,64}$")
 _NODE = re.compile(r"^[a-z0-9]{25}$")
+# A swarm service's name as Docker allows it, never a flag.
+_SERVICE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,62}$")
+SERVICES_MAX = 48
+MOVE_DEADLINE_SECONDS = 300
+# A rebalance starts no move after this long.
+REBALANCE_SECONDS = 2400
 
 
 def now_iso() -> str:
@@ -70,12 +83,19 @@ def read_request(path: str):
 
 
 def valid(req) -> bool:
-    """A request names a known action, has a plain id, and a node id only in
-    the swarm's own form."""
+    """A request names a known action, has a plain id, a node id only in the
+    swarm's own form, and, for a rebalance alone, a list of service names."""
     if not isinstance(req, dict) or req.get("action") not in ACTIONS:
         return False
     if not isinstance(req.get("id"), str) or not _ID.match(req["id"]):
         return False
+    services = req.get("services")
+    if req["action"] == "rebalance":
+        if not (isinstance(services, list) and 0 < len(services) <= SERVICES_MAX
+                and all(isinstance(s, str) and _SERVICE.match(s) for s in services)):
+            return False
+    elif services is not None:
+        return False
     node = req.get("node")
     return node is None or (isinstance(node, str) and bool(_NODE.match(node)))
 
@@ -344,6 +364,52 @@ def do_node(availability: str, node: str | None) -> dict:
     return {"ok": ok, "node": node, "detail": out.strip()[-500:]}
 
 
+def placed_by_rule(name: str) -> bool | None:
+    """Whether a constraint or a preference decides where the service runs
+    (a force-update would leave it where it is); None when it can't be read."""
+    ok, out = run(["docker", "service", "inspect", "--format",
+                   "{{json .Spec.TaskTemplate.Placement}}", name], deadline=30)
+    if not ok:
+        return None
+    try:
+        placement = json.loads(out.strip() or "null") or {}
+    except ValueError:
+        return None
+    return bool(placement.get("Constraints") or placement.get("Preferences"))
+
+
+def do_rebalance(services: list, clock=time.monotonic) -> dict:
+    """Force-update each named service in turn, waiting for it to settle, so
+    the swarm starts it again on its emptiest node. Fails when a move didn't
+    settle; what was left and why is in `left`."""
+    if not shutil.which("docker"):
+        return {"ok": False, "detail": "no docker on this host"}
+    began = clock()
+    moved, left, failed = [], [], []
+    for name in services:
+        if clock() - began > REBALANCE_SECONDS:
+            left.append({"service": name, "reason": "out of time"})
+            continue
+        rule = placed_by_rule(name)
+        if rule is None:
+            left.append({"service": name, "reason": "not found on this manager"})
+            continue
+        if rule:
+            left.append({"service": name, "reason": "placed by a constraint"})
+            continue
+        ok, out = run(["docker", "service", "update", "--force", "--detach=false", "--quiet", name],
+                      deadline=MOVE_DEADLINE_SECONDS)
+        if ok:
+            moved.append(name)
+        else:
+            failed.append(name)
+            left.append({"service": name, "reason": f"didn't settle: {out.strip()[-120:]}"})
+    detail = f"moved {len(moved)} of {len(services)}"
+    if failed:
+        detail += "; didn't settle: " + ", ".join(failed)
+    return {"ok": not failed, "moved": moved, "left": left, "detail": detail}
+
+
 # ── drive health (#5789) ─────────────────────────────────────────────────────
 # smartctl ≥ 7.0 answers in JSON, the same for SATA, SAS and NVMe drives on any
 # distribution; an older one is reported, never parsed as text. The helper
@@ -509,6 +575,8 @@ def handle(req: dict) -> dict:
         outcome = do_update()
     elif action in ("drain", "undrain"):
         outcome = do_node("drain" if action == "drain" else "active", req.get("node"))
+    elif action == "rebalance":
+        outcome = do_rebalance(req["services"])
     else:                                            # reboot
         result = {**base, "state": "rebooting", "ok": True, "finished_at": now_iso()}
         write_result(result)