feat(maintenance): a run spreads swarm services back out after its last host, moving only what the emptier node can start (#6398)

Each weekly run drains Docker02 then Docker03, and the swarm never moves a
task back by itself, so every run ended with nearly every service on one
node. On 2026-10-25 the operator had that spread out by hand.

- Agent 1.26.0 reports each swarm service's pinned digest, and every
  repo@sha256 its host holds (from /system/df, every 10 min).
- Once a run's last host is back in service, Steward picks services on the
  fullest node to move to the emptiest until they are within one task. It
  takes services with several replicas on one node first. A service moves only
  when the emptier node already holds its exact image. One the plan keeps in
  place, one below its replicas, and one with no pinned digest stay put, and
  the run says why.
- A swarm manager's helper carries it out through a new action, rebalance,
  with the service names (validated, at most 48). It force-updates each in
  turn with the spec unchanged and no registry lookup, skips any a
  constraint places, and gives each 5 min to settle.
- The run ends done with what moved, or failed if a move didn't settle or
  the manager didn't answer in 60 min. A docker-scoped window covers the
  restarts, and the step goes on the timeline.
- Plans gain "Spread swarm services back out" (on by default) and "Keep in
  place", on the create form and in an "After the run" form on each plan.
  list_maintenance_plans shows both and the latest run's spread.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/timeline.py

@@ -1,6 +1,7 @@
 # plugins/host_agent/timeline.py
 """What the host agent puts on the timeline: each maintenance job once it
-ends, and each stretch a host was away (a reboot, a crash, lost contact, the
+ends, the swarm services a maintenance run spread back out after its last
+host, and each stretch a host was away (a reboot, a crash, lost contact, the
 agent restarting). One entry per job and per downtime row, kept up to date
 as the row changes: a downtime that ends gets its length, a failed job
 whose host came back late reads as back.
@@ -8,6 +9,7 @@ whose host came back late reads as back.
 from __future__ import annotations
 
 import logging
+from datetime import datetime
 
 from sqlalchemy import select
 
@@ -23,6 +25,8 @@ TYPES = (
      "A host's agent updated it, and rebooted it when the update needed one."),
     ("maintenance.failed", "Maintenance failed", "automation", "host_os", True,
      "A maintenance job failed or timed out."),
+    ("maintenance.rebalanced", "Swarm services spread back out", "automation", "service", True,
+     "After a maintenance run, its swarm services were moved back onto the nodes it had drained."),
     ("host.rebooted", "Host rebooted", "observed", "host_os", True,
      "A host went down and came back, announced or not (a crash, a power cut)."),
     ("host.lost_contact", "Agent lost contact", "observed", "network", True,
@@ -35,8 +39,9 @@ TYPES = (
 def register() -> None:
     for t in TYPES:
         catalog.register_type(*t)
-    from .models import HostDowntime, HostMaintenanceJob
+    from .models import HostDowntime, HostMaintenanceJob, HostMaintenanceRun
     catalog.register_ref("maintenance_job", HostMaintenanceJob)
+    catalog.register_ref("maintenance_run", HostMaintenanceRun)
     catalog.register_ref("host_downtime", HostDowntime)
 
 
@@ -62,6 +67,26 @@ async def job_entry(session, job, host_name: str | None = None):
         actor=await username(session, job.created_by) or "Steward")
 
 
+async def run_entry(session, run):
+    """The entry for a run's spreading of swarm services back out, once it ended."""
+    from . import rebalance
+    from .models import HostMaintenancePlan
+    rb = run.rebalance or {}
+    if rb.get("state") not in ("done", "failed", "timed_out"):
+        return None
+    plan = await session.get(HostMaintenancePlan, run.plan_id)
+    summary = rebalance.outcome(run)[1] or "swarm services spread back out"
+    return await record_or_update(
+        session, ("maintenance_run", run.id), type="maintenance.rebalanced",
+        at=datetime.fromisoformat(rb["at"]),
+        until=datetime.fromisoformat(rb["finished_at"]) if rb.get("finished_at") else None,
+        summary=f"{plan.name if plan else 'Maintenance'}: {summary}",
+        hosts=[], subject=None,
+        detail={"state": rb["state"], "moved": rb.get("moved") or [], "left": rb.get("left") or [],
+                "note": rb.get("detail"), "url": "/plugins/host_agent/maintenance"},
+        actor=await username(session, plan.created_by if plan else None) or "Steward")
+
+
 _DOWNTIME_TYPES = {"reboot": "host.rebooted", "unclean": "host.rebooted",
                    "lost_contact": "host.lost_contact", "agent_stop": "host.agent_restarted"}