feat(host_agent): Steward hands a host a maintenance job and follows it through the reboot (#5767)

#5767 "Steward hands a host a maintenance job and follows it to the end,
through its own restarts", milestone 514.

A host_maintenance_jobs row per job: queued → updating → [draining →]
rebooting → [undraining →] done, or failed / timed_out. Steward's answer to
an ingest post carries the step that host's agent should carry out now; the
agent (1.18.0) leaves each step id for the root helper once, and the helper
refuses an id it already ran, so repeats are harmless. Results come back as
the #5766 maintenance events, matched by step id; the reboot completes when
the host's `started` event has a boot_at after the reboot was handed out.

The job reboots only when the update says one is needed (or never, if asked).
A swarm node is drained first by itself if it's a manager or by a manager
with the helper, and made active after; with no manager to drain it the job
is refused. A minute tick times out a job past its state's deadline. A
maintenance window over host_agent/<host> lasts the rest of the job.

Everything is on the row, so a Steward restart, or the reboot of the host
Steward runs on, resumes from the next post. Ingest runs it in its own
savepoint so a job going wrong never costs the samples.

The Agent panel leads with a running job and says what a finished one did;
Manage agent has Update now (with "never reboot"), or says why a job can't
start. Integration test: hand off, Steward restarts, the post-boot events
close the job.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_host_page.py

@@ -279,3 +279,32 @@ async def test_manage_agent_says_whether_the_maintenance_helper_is_installed(app
             reg=SimpleNamespace(**{**vars(reg), "maint_helper": None}), **kw)
     assert "Maintenance helper installed" in installed
     assert "No maintenance helper yet: Update agent installs it" in missing
+
+
+async def test_the_agent_card_follows_a_maintenance_job_and_says_why_it_cant_start_one(app):
+    """#5767: a running job leads what the host has waiting; Update now is
+    offered only when a job can start, else the reason is shown."""
+    from datetime import datetime, timedelta, timezone
+    from plugins.host_agent.maintenance import summary
+    from tests._templates import render_plugin_template
+    now = datetime.now(timezone.utc)
+    host = SimpleNamespace(id="h1", name="box", address="10.0.0.5")
+    reg = SimpleNamespace(agent_version="1.18.0", last_seen_at=now, maint_helper=True,
+                          updates_checked_at=now, updates_pending=4, updates_security=0,
+                          reboot_required=False)
+    job = SimpleNamespace(state="rebooting", state_at=now - timedelta(minutes=2), finished_at=None,
+                          detail=None, packages=4, back_at=None, reboot_needed=True,
+                          reboot_policy="if_required")
+    kw = dict(host=host, reg=reg, reporting=True, stale=False, target=None, ansible_available=False,
+              managed_key_set=False, downtime=[])
+    async with app.test_request_context("/plugins/host_agent/panel/h1"):
+        session["user_role"] = "admin"
+        busy = await render_plugin_template(
+            app, "host_agent", "panel.html", maint_job=job, maint_summary=summary(job),
+            maint_blocker="a maintenance job is already running on it", **kw)
+        free = await render_plugin_template(app, "host_agent", "panel.html", maint_job=None,
+                                            maint_blocker=None, **kw)
+    assert busy.index("Maintenance: Rebooting") < busy.index("4 updates waiting")
+    assert "Can't update from here: a maintenance job is already running on it." in busy
+    assert ">Update now</button>" not in busy
+    assert ">Update now</button>" in free and 'name="reboot_policy" value="never"' in free

tests/integration/test_host_maintenance_job.py

@@ -0,0 +1,118 @@
+"""Integration (#5767): Steward hands a host a maintenance job and follows it
+to the end, through a restart of its own, from the agent's posts alone.
+
+Rows are seeded and read with plain SQL: importing the plugin's models again
+after the app loaded the plugin raises "Table already defined" (see
+test_host_agent_alerts)."""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+def _app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+async def _seed(app):
+    from sqlalchemy import text
+    from steward.models.hosts import Host
+    from steward.models.users import User
+    tag = uuid.uuid4().hex[:8]
+    token = f"tok-{tag}-{uuid.uuid4().hex}"
+    job_id = str(uuid.uuid4())
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            user = User(username=f"mj-{tag}", email=f"mj-{tag}@test", password_hash="x")
+            host = Host(name=f"maint-{tag}", address="10.9.9.7")
+            s.add_all([user, host])
+            await s.flush()
+            await s.execute(text(
+                "INSERT INTO host_agent_registrations "
+                "(id, host_id, token_hash, token_created_at, last_seen_at, maint_helper, "
+                " created_at, updated_at) "
+                "VALUES (:id, :host, :hash, now(), now(), true, now(), now())"),
+                {"id": str(uuid.uuid4()), "host": host.id,
+                 "hash": hashlib.sha256(token.encode()).hexdigest()})
+            await s.execute(text(
+                "INSERT INTO host_maintenance_jobs "
+                "(id, host_id, state, state_at, reboot_policy, step_seen, created_by, created_at) "
+                "VALUES (:id, :host, 'queued', now(), 'if_required', false, :user, now())"),
+                {"id": job_id, "host": host.id, "user": user.id})
+            host_id = host.id
+    return host_id, token, job_id
+
+
+async def _post(app, token, events=(), sample=True):
+    now = datetime.now(timezone.utc).isoformat()
+    resp = await app.test_client().post(
+        "/plugins/host_agent/ingest",
+        json={"samples": [{"ts": now, "cpu_pct": 2.0}] if sample else [],
+              "events": list(events)},
+        headers={"Authorization": f"Bearer {token}"})
+    assert resp.status_code == 200, await resp.get_data(as_text=True)
+    return (await resp.get_json()).get("maintenance", [])
+
+
+async def _job(app, job_id):
+    from sqlalchemy import text
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            return (await s.execute(text(
+                "SELECT j.state, j.packages, j.reboot_needed, j.back_at, w.end_at "
+                "FROM host_maintenance_jobs j LEFT JOIN maintenance_windows w ON w.id = j.window_id "
+                "WHERE j.id = :id"), {"id": job_id})).one()
+
+
+def test_a_job_is_followed_through_a_steward_restart_and_the_host_reboot():
+    app = _app()
+
+    async def _first_half():
+        host_id, token, job_id = await _seed(app)
+        # The agent's next post starts the job: update first, under a window.
+        assert await _post(app, token) == [{"id": f"{job_id}-update", "action": "update"}]
+        state, *_, window_end = await _job(app, job_id)
+        assert state == "updating" and window_end > datetime.now(timezone.utc)
+        # The helper took it: the step is no longer handed out.
+        running = {"kind": "maintenance", "id": f"{job_id}-update", "action": "update",
+                   "state": "running"}
+        assert await _post(app, token, [running]) == []
+        return token, job_id
+
+    token, job_id = asyncio.run(_first_half())
+
+    # Steward restarts: a new process, nothing in memory but the database.
+    app = _app()
+
+    async def _second_half():
+        done = {"kind": "maintenance", "id": f"{job_id}-update", "action": "update",
+                "state": "done", "ok": True, "packages": 12, "reboot_required": True}
+        assert await _post(app, token, [done]) == [{"id": f"{job_id}-reboot", "action": "reboot"}]
+        assert (await _job(app, job_id))[:3] == ("rebooting", 12, True)
+        # The agent restarting on a host that stayed up doesn't count as the reboot.
+        old_boot = datetime.now(timezone.utc) - timedelta(days=3)
+        await _post(app, token, [{"kind": "started", "at": datetime.now(timezone.utc).isoformat(),
+                                  "boot_at": old_boot.isoformat()}])
+        assert (await _job(app, job_id))[0] == "rebooting"
+        # After the reboot: the helper's result and the agent's boot, in one post.
+        boot = datetime.now(timezone.utc)
+        rebooted = {"kind": "maintenance", "id": f"{job_id}-reboot", "action": "reboot",
+                    "state": "rebooting", "ok": True}
+        started = {"kind": "started", "at": (boot + timedelta(seconds=20)).isoformat(),
+                   "boot_at": boot.isoformat()}
+        assert await _post(app, token, [rebooted, started]) == []
+        state, packages, reboot_needed, back_at, window_end = await _job(app, job_id)
+        assert state == "done" and back_at is not None
+        assert window_end <= datetime.now(timezone.utc)
+
+    asyncio.run(_second_half())

tests/plugins/host_agent/test_maintenance.py

@@ -0,0 +1,154 @@
+# tests/plugins/host_agent/test_maintenance.py
+"""#5767: a host's maintenance job, step by step, from the agent's reports.
+
+The transitions are model-free, so a job here is a SimpleNamespace with the
+row's attributes; the database side is tests/integration/test_host_maintenance_job.py.
+"""
+import json
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+
+import pytest
+
+from plugins.host_agent import agent as a
+from plugins.host_agent import maintenance as m
+
+T0 = datetime(2026, 10, 14, 23, 0, tzinfo=timezone.utc)
+
+
+def _job(**kw):
+    base = dict(id="job-1", host_id="h1", state="queued", state_at=T0, created_at=T0,
+                reboot_policy="if_required", step_id=None, step_seen=False, node_id=None,
+                drain_host_id=None, packages=None, reboot_needed=None, back_at=None,
+                detail=None, finished_at=None)
+    base.update(kw)
+    return SimpleNamespace(**base)
+
+
+def _result(job, state, **kw):
+    return {"kind": "maintenance", "id": job.step_id, "state": state, **kw}
+
+
+def test_an_update_with_no_reboot_needed_is_done_after_the_update():
+    job = _job()
+    m.start(job, T0)
+    assert m.handoff(job) == {"id": "job-1-update", "action": "update"}
+    assert m.apply_result(job, _result(job, "running"), T0)
+    assert m.handoff(job) is None                       # taken: not handed again
+    m.apply_result(job, _result(job, "done", ok=True, packages=3, reboot_required=False), T0)
+    assert (job.state, job.packages, job.finished_at) == ("done", 3, T0)
+    assert m.summary(job) == "Done: 3 packages updated, no reboot needed"
+
+
+def test_a_swarm_node_is_drained_rebooted_and_made_active_again():
+    job = _job(node_id="n" * 25, drain_host_id="mgr")
+    m.start(job, T0)
+    m.apply_result(job, _result(job, "done", ok=True, packages=12, reboot_required=True), T0)
+    assert job.state == "draining" and m.executor(job) == "mgr"
+    assert m.handoff(job) == {"id": "job-1-drain", "action": "drain", "node": "n" * 25}
+    m.apply_result(job, _result(job, "done", ok=True), T0)
+    assert job.state == "rebooting" and m.executor(job) == "h1"
+    assert m.handoff(job) == {"id": "job-1-reboot", "action": "reboot"}
+    t1 = T0 + timedelta(minutes=4)
+    assert m.apply_boot(job, T0 + timedelta(minutes=2), t1)
+    assert job.state == "undraining" and job.back_at == t1 and m.executor(job) == "mgr"
+    m.apply_result(job, _result(job, "done", ok=True), t1)
+    assert job.state == "done"
+    assert m.summary(job) == "Done: 12 packages updated, rebooted"
+
+
+def test_a_job_that_may_never_reboot_stops_after_the_update():
+    job = _job(reboot_policy="never")
+    m.start(job, T0)
+    m.apply_result(job, _result(job, "done", ok=True, packages=1, reboot_required=True), T0)
+    assert job.state == "done"
+    assert "reboot needed, not allowed" in m.summary(job)
+
+
+def test_only_a_boot_after_the_reboot_was_handed_out_completes_it():
+    job = _job(state="rebooting", step_id="job-1-reboot", state_at=T0)
+    assert not m.apply_boot(job, T0 - timedelta(days=3), T0)    # the agent restarted, not the host
+    assert not m.apply_boot(job, None, T0)
+    assert job.state == "rebooting"
+    assert m.apply_boot(job, T0 - timedelta(seconds=30), T0)     # within the clock slack
+    assert job.state == "done"
+
+
+def test_a_failed_step_fails_the_job_and_says_which():
+    job = _job(state="undraining", step_id="job-1-undrain", node_id="n" * 25, drain_host_id="mgr")
+    m.apply_result(job, _result(job, "failed", ok=False, detail="This node is not a swarm manager."), T0)
+    assert job.state == "failed"
+    assert job.detail.startswith("undrain failed: This node is not a swarm manager.")
+    assert "still drained" in job.detail
+
+
+def test_a_late_or_repeated_result_does_nothing():
+    job = _job()
+    m.start(job, T0)
+    stale = {"kind": "maintenance", "id": "job-1-drain", "state": "done", "ok": True}
+    assert not m.apply_result(job, stale, T0)
+    assert m.apply_result(job, _result(job, "running"), T0)
+    assert not m.apply_result(job, _result(job, "running"), T0)
+    m.apply_result(job, _result(job, "done", ok=True, reboot_required=False), T0)
+    assert not m.apply_result(job, {"kind": "maintenance", "id": "job-1-update", "state": "done"}, T0)
+
+
+@pytest.mark.parametrize("state,minutes,expected", [
+    ("queued", 59, False), ("queued", 61, True),
+    ("updating", 91, True), ("rebooting", 29, False), ("rebooting", 31, True),
+])
+def test_a_job_is_given_up_past_its_state_deadline(state, minutes, expected):
+    job = _job(state=state, step_id=f"job-1-{m.STEP_ACTION.get(state, 'x')}")
+    assert m.check_deadline(job, T0 + timedelta(minutes=minutes)) is expected
+    if expected:
+        assert job.state == "timed_out" and "within" in job.detail
+
+
+def test_the_window_lasts_the_rest_of_the_job_and_ends_with_it():
+    job = _job(state="rebooting", step_id="job-1-reboot")
+    assert m.window_end(job, T0) == T0 + timedelta(minutes=30 + 10)
+    job.state = "done"
+    assert m.window_end(job, T0) == T0
+
+
+def test_a_manager_drains_itself_and_a_worker_is_drained_by_a_manager():
+    nodes = {"docker02": "a" * 25, "docker03": "b" * 25}
+    managers = {"h2": "a" * 25}
+    assert m.pick_drainer(nodes, managers, "Docker02", "h2", {"h2", "h3"}) == ("a" * 25, "h2")
+    assert m.pick_drainer(nodes, managers, "Docker03", "h3", {"h2", "h3"}) == ("b" * 25, "h2")
+    assert m.pick_drainer(nodes, managers, "Util01", "h4", {"h2"}) == (None, None)
+    with pytest.raises(m.CannotStart, match="no swarm manager"):
+        m.pick_drainer(nodes, managers, "Docker03", "h3", {"h3"})
+
+
+# ── the agent's side ─────────────────────────────────────────────────────────
+
+def test_the_agent_leaves_each_handed_step_once(tmp_path):
+    requested: set = set()
+    steps = [{"id": "job-1-update", "action": "update"}, {"id": "job-1-x", "action": "shell"}, "junk"]
+    assert a.take_maintenance(steps, requested, requests_dir=str(tmp_path)) == ["job-1-update"]
+    assert json.loads((tmp_path / "job-1-update.json").read_text()) == {
+        "id": "job-1-update", "action": "update"}
+    (tmp_path / "job-1-update.json").unlink()          # the helper took it
+    assert a.take_maintenance(steps, requested, requests_dir=str(tmp_path)) == []
+    assert not (tmp_path / "job-1-update.json").exists()
+    assert a.take_maintenance(None, requested, requests_dir=str(tmp_path)) == []
+
+
+def test_the_agent_reads_stewards_answer_when_asked(monkeypatch):
+    class Resp:
+        status = 200
+
+        def read(self, n):
+            return b'{"ok": true, "maintenance": [{"id": "job-1-reboot", "action": "reboot"}]}'
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *exc):
+            return False
+
+    monkeypatch.setattr(a.urllib.request, "urlopen", lambda req, timeout: Resp())
+    answer: dict = {}
+    assert a.post_payload("https://x", "tok", {}, answer) == (True, 200)
+    assert answer["maintenance"] == [{"id": "job-1-reboot", "action": "reboot"}]