fix(maintenance): a job given up after draining its node makes the node active again (#6387)

On 2026-10-25 the 09:00 job on VDNT-Docker02 drained the node, which ran
Traefik and Steward. The agent's result reaches Steward through Traefik,
and Traefik took 11.5 minutes to start on Docker03 (its image had to be
pulled). The job gave up at the 10-minute drain deadline and finished
timed_out where it stood, so the node stayed drained for hours.

- A job that fails or times out while its node may be drained (the drain
  was handed out, or went through and wasn't undone) goes to undraining
  first. The outcome is kept in the new column ends_as, and the job
  finishes as that once the node is active again. If the undrain goes
  wrong too, detail says both.
- The drain deadline is 30 minutes instead of 10, so a result held up while
  the proxy moves still lands.
- Migration host_agent_012_ends_as. get_maintenance_job returns ends_as.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/plugins/host_agent/test_maintenance.py

@@ -21,7 +21,7 @@ def _job(**kw):
                 reboot_policy="if_required", step_id=None, step_seen=False, node_id=None,
                 drain_host_id=None, packages=None, reboot_needed=None, back_at=None,
                 detail=None, finished_at=None, drained_at=None, docker_update=None, checked=None,
-                came_back_at=None)
+                came_back_at=None, ends_as=None)
     base.update(kw)
     return SimpleNamespace(**base)
 
@@ -239,6 +239,73 @@ def test_the_new_states_are_given_up_too(state, minutes):
     assert m.check_deadline(job, T0 + timedelta(minutes=minutes)) and job.state == "timed_out"
 
 
+def _drained_job(**kw):
+    return _job(node_id="n" * 25, drain_host_id="mgr", docker_update=True, **kw)
+
+
+def test_a_drain_whose_result_is_late_still_gets_its_time():
+    # Draining the node that runs the proxy moves it, and the drain's result
+    # waits for it to run again elsewhere.
+    job = _drained_job(state="draining", step_id="job-1-drain")
+    assert not m.check_deadline(job, T0 + timedelta(minutes=29))
+    m.apply_result(job, _result(job, "done", ok=True), T0 + timedelta(minutes=12))
+    assert job.state == "settling"
+
+
+def test_a_drain_given_up_makes_the_node_active_again_before_the_job_ends():
+    job = _drained_job(state="draining", step_id="job-1-drain")
+    assert m.check_deadline(job, T0 + timedelta(minutes=31))
+    assert job.state == "undraining" and job.finished_at is None
+    assert m.executor(job) == "mgr"
+    assert m.handoff(job) == {"id": "job-1-undrain", "action": "undrain", "node": "n" * 25}
+    late = {"kind": "maintenance", "id": "job-1-drain", "state": "done", "ok": True}
+    assert not m.apply_result(job, late, T0 + timedelta(minutes=32))   # the drain's late result
+    t1 = T0 + timedelta(minutes=33)
+    m.apply_result(job, _result(job, "done", ok=True), t1)
+    assert (job.state, job.finished_at) == ("timed_out", t1)
+    assert job.detail == "the node wasn't drained within 30 min"
+    assert m.still_failed(job)
+
+
+@pytest.mark.parametrize("state, step, minutes", [
+    ("settling", None, 16), ("updating", "job-1-update", 91), ("rebooting", "job-1-reboot", 31),
+])
+def test_a_job_given_up_after_its_drain_makes_the_node_active_again(state, step, minutes):
+    job = _drained_job(state=state, step_id=step, drained_at=T0)
+    assert m.check_deadline(job, T0 + timedelta(minutes=minutes))
+    assert job.state == "undraining" and job.ends_as == "timed_out"
+
+
+def test_a_failed_step_after_the_drain_makes_the_node_active_again_and_fails():
+    job = _drained_job(state="updating", step_id="job-1-update", drained_at=T0)
+    m.apply_result(job, _result(job, "failed", ok=False, detail="dpkg lock"), T0)
+    assert job.state == "undraining"
+    m.apply_result(job, _result(job, "done", ok=True), T0)
+    assert job.state == "failed" and job.detail == "update failed: dpkg lock"
+
+
+def test_an_undrain_that_goes_wrong_after_giving_up_says_both():
+    job = _drained_job(state="draining", step_id="job-1-drain")
+    m.check_deadline(job, T0 + timedelta(minutes=31))
+    assert m.check_deadline(job, T0 + timedelta(minutes=42))
+    assert job.state == "timed_out"
+    assert job.detail == ("the node wasn't drained within 30 min; "
+                          "then the node wasn't made active again within 10 min")
+    job = _drained_job(state="updating", step_id="job-1-update", drained_at=T0)
+    m.apply_result(job, _result(job, "failed", ok=False, detail="dpkg lock"), T0)
+    m.apply_result(job, _result(job, "failed", ok=False, detail="not a manager"), T0)
+    assert job.state == "failed"
+    assert job.detail == ("update failed: dpkg lock; then undrain failed: not a manager "
+                          "(the node is still drained)")
+
+
+def test_a_job_that_never_drained_ends_where_it_is_given_up():
+    job = _job(state="updating", step_id="job-1-update", node_id="n" * 25, drain_host_id="mgr")
+    assert m.check_deadline(job, T0 + timedelta(minutes=91)) and job.state == "timed_out"
+    kept = _job(state="draining", step_id="job-1-drain", node_id="n" * 25)    # kept drained: no drainer
+    assert m.check_deadline(kept, T0 + timedelta(minutes=31)) and kept.state == "timed_out"
+
+
 # ── the agent's side ─────────────────────────────────────────────────────────
 
 def test_the_agent_leaves_each_handed_step_once(tmp_path):

tests/plugins/host_agent/test_mcp_tools.py

@@ -35,7 +35,7 @@ def test_a_job_row_says_what_it_did_and_when_it_gives_up():
              node_id="n1", drain_host_id="h2", detail=None, created_by=None, run_id="r1",
              created_at=NOW - timedelta(minutes=20), finished_at=None,
              drained_at=NOW - timedelta(minutes=11), docker_update=False, came_back_at=None,
-             checked={"pending": 12, "docker_packages": [], "containers": []})
+             ends_as=None, checked={"pending": 12, "docker_packages": [], "containers": []})
     row = _job_row(job, {"h1": "web-1", "h2": "mgr-1"}, NOW)
     assert (row["host"], row["drained_by"], row["label"], row["active"]) == (
         "web-1", "mgr-1", "Rebooting", True)