feat(host_agent): Steward hands a host a maintenance job and follows it through the reboot (#5767)

#5767 "Steward hands a host a maintenance job and follows it to the end,
through its own restarts", milestone 514.

A host_maintenance_jobs row per job: queued → updating → [draining →]
rebooting → [undraining →] done, or failed / timed_out. Steward's answer to
an ingest post carries the step that host's agent should carry out now; the
agent (1.18.0) leaves each step id for the root helper once, and the helper
refuses an id it already ran, so repeats are harmless. Results come back as
the #5766 maintenance events, matched by step id; the reboot completes when
the host's `started` event has a boot_at after the reboot was handed out.

The job reboots only when the update says one is needed (or never, if asked).
A swarm node is drained first by itself if it's a manager or by a manager
with the helper, and made active after; with no manager to drain it the job
is refused. A minute tick times out a job past its state's deadline. A
maintenance window over host_agent/<host> lasts the rest of the job.

Everything is on the row, so a Steward restart, or the reboot of the host
Steward runs on, resumes from the next post. Ingest runs it in its own
savepoint so a job going wrong never costs the samples.

The Agent panel leads with a running job and says what a finished one did;
Manage agent has Update now (with "never reboot"), or says why a job can't
start. Integration test: hand off, Steward restarts, the post-boot events
close the job.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/routes.py

@@ -17,6 +17,7 @@ from steward.models.users import UserRole
 from sqlalchemy import select, or_, and_
 from datetime import timedelta
 
+from steward.core.audit import audit_request
 from steward.core.formatting import worst_reading
 from steward.core.health import ingest as ingest_tally
 from steward.core.metric_series import points as metric_points
@@ -26,6 +27,7 @@ from steward.hosts import service as host_svc
 from steward.models.hosts import Host
 from steward.models.metrics import PluginMetric
 from .downtime import recent as recent_downtime, record_events
+from . import maintenance
 from .models import HostAgentRegistration
 from .scheduler import arrivals
 # Query helpers live in a model-free module so integration tests can import them
@@ -347,6 +349,7 @@ async def ingest():
 
     token_hash = _hash_token(raw)
     phases = _Phases()
+    maint_steps: list = []
     samples, dropped = newest_samples(samples, MAX_SAMPLES_PER_INGEST)
     if dropped:
         current_app.logger.warning(
@@ -461,6 +464,20 @@ async def ingest():
             # was a restart without a shutdown.
             if events:
                 await record_events(session, host.id, events, reg.last_seen_at)
+            # Maintenance jobs (#5767): results and boots move a job on, then
+            # whatever step is due for this host's agent goes back in the answer.
+            # Each in its own savepoint: a job going wrong never costs the samples.
+            try:
+                async with session.begin_nested():
+                    if events:
+                        await maintenance.apply_events(session, host.id, events, received_at)
+                    if reg.maint_helper:
+                        maint_steps = await maintenance.handoffs(
+                            session, host.id, host.name, received_at)
+            except Exception:
+                maint_steps = []
+                current_app.logger.exception(
+                    "host_agent ingest: maintenance jobs failed for host=%s", host.name)
 
             md = payload.get("metadata") or {}
             changed = False
@@ -518,7 +535,10 @@ async def ingest():
     phases.report(host.name, accepted, len(metric_rows),
                   sum(len(c) for _, c in docker_snapshots),
                   sum(len(lines) for _, lines in docker_log_batches))
-    return jsonify({"ok": True, "samples_accepted": accepted, "samples_dropped": dropped}), 200
+    answer = {"ok": True, "samples_accepted": accepted, "samples_dropped": dropped}
+    if maint_steps:
+        answer["maintenance"] = maint_steps
+    return jsonify(answer), 200
 
 
 # The agent gives up on a post after 10 s (agent.py); an ingest past this
@@ -964,6 +984,14 @@ async def host_panel(host_id: str):
         target = (await db.execute(select(AnsibleTarget).where(
             AnsibleTarget.host_id == host_id))).scalar_one_or_none()
         downtime = await recent_downtime(db, host_id, datetime.now(timezone.utc)) if reg else []
+        # Maintenance (#5767): the running or latest job, and whether one can start.
+        maint_job = await maintenance.latest_job(db, host_id, datetime.now(timezone.utc)) if reg else None
+        maint_blocker = None
+        if reg is not None:
+            try:
+                await maintenance.plan_job(db, host, reg)
+            except maintenance.CannotStart as exc:
+                maint_blocker = str(exc)
 
     ls = reg.last_seen_at if reg else None
     # "Deployed" is gated on the agent actually checking in (last_seen_at), NOT on
@@ -976,6 +1004,8 @@ async def host_panel(host_id: str):
     return await render_template(
         "panel.html",
         host=host, reg=reg, reporting=reporting, stale=stale, target=target, downtime=downtime,
+        maint_job=maint_job, maint_blocker=maint_blocker,
+        maint_summary=maintenance.summary(maint_job) if maint_job else None,
         ansible_available=has_capability("ansible.run_playbook"),
         managed_key_set=bool((ansible_cfg.get("ssh_public_key") or "").strip()),
     )
@@ -1060,6 +1090,30 @@ async def rotate_token(host_id: str):
     return redirect(url_for("host_agent.fleet") + f"?new_token={raw}&host_id={host_id}")
 
 
+@host_agent_bp.post("/fleet/<host_id>/maintain")
+@require_role(UserRole.admin)
+async def start_maintenance(host_id: str):
+    """Update this host now, rebooting it only if the update needs it (#5767).
+    The job starts with the agent's next post and is followed on the panel."""
+    form = await request.form
+    policy = "never" if form.get("reboot_policy") == "never" else "if_required"
+    user_id = session.get("user_id")
+    async with current_app.db_sessionmaker() as db:
+        host = await db.get(Host, host_id)
+        if host is None:
+            return _error(404, "not_found")
+        reg = (await db.execute(select(HostAgentRegistration).where(
+            HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
+        try:
+            job = await maintenance.create_job(db, host, reg, user_id, policy)
+        except maintenance.CannotStart as exc:
+            return _error(409, "cannot_start", f"{host.name} can't be updated now: {exc}")
+        await db.commit()
+    await audit_request("host_agent.maintenance", "host", host_id,
+                        {"job_id": job.id, "reboot_policy": policy})
+    return redirect(f"/hosts/{host_id}")
+
+
 @host_agent_bp.post("/fleet/<host_id>/delete")
 @require_role(UserRole.admin)
 async def delete_registration(host_id: str):