feat(maintenance): a run spreads swarm services back out after its last host, moving only what the emptier node can start (#6398)

Each weekly run drains Docker02 then Docker03, and the swarm never moves a
task back by itself, so every run ended with nearly every service on one
node. On 2026-10-25 the operator had that spread out by hand.

- Agent 1.26.0 reports each swarm service's pinned digest, and every
  repo@sha256 its host holds (from /system/df, every 10 min).
- Once a run's last host is back in service, Steward picks services on the
  fullest node to move to the emptiest until they are within one task. It
  takes services with several replicas on one node first. A service moves only
  when the emptier node already holds its exact image. One the plan keeps in
  place, one below its replicas, and one with no pinned digest stay put, and
  the run says why.
- A swarm manager's helper carries it out through a new action, rebalance,
  with the service names (validated, at most 48). It force-updates each in
  turn with the spec unchanged and no registry lookup, skips any a
  constraint places, and gives each 5 min to settle.
- The run ends done with what moved, or failed if a move didn't settle or
  the manager didn't answer in 60 min. A docker-scoped window covers the
  restarts, and the step goes on the timeline.
- Plans gain "Spread swarm services back out" (on by default) and "Keep in
  place", on the create form and in an "After the run" form on each plan.
  list_maintenance_plans shows both and the latest run's spread.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/agent.py

@@ -22,7 +22,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.25.0"
+AGENT_VERSION = "1.26.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -585,6 +585,9 @@ DOCKER_DEADLINE_SECONDS = 20.0
 # /system/df walks every image, container and volume: asked this often, not
 # every sample. Steward keeps the last answer between reports.
 DOCKER_DF_INTERVAL_SECONDS = 600.0
+# How many of a host's image digests a report carries: what Steward checks a
+# swarm service's pinned image against before moving it to this node.
+DOCKER_DIGESTS_MAX = 400
 
 # Container-log collection (m79). Logs ride in the same push as metrics, so one
 # interval's batch is capped to keep a chatty container from bloating a POST (and
@@ -1251,11 +1254,14 @@ def _swarm_services(services: list, tasks: list) -> list:
             desired = (mode_obj.get("Replicated") or {}).get("Replicas", 0) or 0
         image = (((spec.get("TaskTemplate") or {}).get("ContainerSpec") or {})
                  .get("Image") or "")
-        image = image.split("@", 1)[0]  # drop the @sha256:… digest suffix
+        image, _, digest = image.partition("@")  # the @sha256:… digest apart
         out.append({
             "service_name": name, "mode": mode,
             "desired": desired, "running": running_by_svc.get(sid, 0),
             "image": image,
+            # The digest a stack deploy pinned, which a node must hold to start
+            # the service without the registry.
+            "image_digest": digest if digest.startswith("sha256:") else None,
             # task→node placement of the running replicas (cross-node — a manager
             # sees every task, even ones whose containers live on other hosts).
             "placement": [{"node_id": n, "running": cnt}
@@ -1383,6 +1389,10 @@ def collect_disk_usage(socket_path: str):
         "volumes_size": sum(_vol_size(v) for v in volumes if _vol_size(v) > 0),
         "build_cache_size": sum(int(b.get("Size") or 0) for b in build_cache),
         "images": _disk_images(images),
+        # Every image this host holds, by repository digest ("repo@sha256:…"),
+        # so Steward can tell whether a swarm service could start here.
+        "digests": sorted({d for im in images for d in (im.get("RepoDigests") or [])
+                           if isinstance(d, str) and "@sha256:" in d})[:DOCKER_DIGESTS_MAX],
     }
 
 
@@ -1667,13 +1677,13 @@ def collect_reboot(path: str = REBOOT_REQUIRED, modules: str = KERNEL_MODULES):
 # The agent can't update or reboot its host: it runs unprivileged under
 # NoNewPrivileges. It leaves a request in MAINT_REQUESTS, the one directory its
 # unit lets it write; steward-maint.path starts the root helper (maint.py),
-# which does only check / update / reboot / drain / undrain and writes each
+# which does only check / update / reboot / drain / undrain / rebalance and writes each
 # outcome to MAINT_RESULTS. The agent reads those back and reports each new state once.
 
 MAINT_REQUESTS = "/var/lib/steward-agent/requests"
 MAINT_RESULTS = "/var/lib/steward-maint/results"
 MAINT_PATH_UNIT = "/etc/systemd/system/steward-maint.path"
-MAINT_ACTIONS = ("check", "update", "reboot", "drain", "undrain", "drives")
+MAINT_ACTIONS = ("check", "update", "reboot", "drain", "undrain", "drives", "rebalance")
 MAINT_RESULT_MAX_BYTES = 8192
 # The helper keeps results a week; only this recent a one is reported, so a
 # job a reboot interrupted still reaches Steward from the agent's next start
@@ -1683,7 +1693,9 @@ _MAINT_ID = re.compile(r"^[A-Za-z0-9-]{1,64}$")
 _MAINT_FIELDS = ("action", "state", "ok", "packages", "reboot_required", "node",
                  "detail", "started_at", "finished_at",
                  # a check's answer (#6098)
-                 "pending", "docker_update", "docker_packages", "containers")
+                 "pending", "docker_update", "docker_packages", "containers",
+                 # a rebalance's answer
+                 "moved", "left")
 
 
 def maint_helper_installed(unit: str = MAINT_PATH_UNIT,
@@ -1693,14 +1705,17 @@ def maint_helper_installed(unit: str = MAINT_PATH_UNIT,
 
 
 def request_maintenance(job_id: str, action: str, node: str | None = None,
-                        requests_dir: str = MAINT_REQUESTS) -> None:
+                        requests_dir: str = MAINT_REQUESTS, services=None) -> None:
     """Leave the helper a request, whole: written to a dot-file the path unit
-    doesn't match, then renamed to `<id>.json`."""
+    doesn't match, then renamed to `<id>.json`. A rebalance names the swarm
+    services to move; the helper checks each name itself."""
     if action not in MAINT_ACTIONS or not _MAINT_ID.match(job_id or ""):
         raise ValueError(f"not a maintenance request: {action!r} {job_id!r}")
     req = {"id": job_id, "action": action}
     if node:
         req["node"] = node
+    if action == "rebalance":
+        req["services"] = [s for s in services if isinstance(s, str)] if isinstance(services, list) else []
     tmp = os.path.join(requests_dir, f".{job_id}.tmp")
     with open(tmp, "w") as fh:
         json.dump(req, fh)
@@ -1718,7 +1733,8 @@ def take_maintenance(steps, requested: set, requests_dir: str = MAINT_REQUESTS)
             continue
         try:
             request_maintenance(str(step.get("id") or ""), str(step.get("action") or ""),
-                                step.get("node"), requests_dir=requests_dir)
+                                step.get("node"), requests_dir=requests_dir,
+                                services=step.get("services"))
         except (ValueError, OSError) as exc:
             _log("WARN", f"maintenance step {step.get('id')!r} not taken: {exc}")
             continue