fix(ansible): a run that waited says so in its stored output, not only live (#5158)

The "[queued — waiting for run X on the same host]" and run-slot lines were
broadcast to live viewers and never stored, so a run that sat queued had no
record of why once the live view was gone (CI 8439: the overlap test found
the order right and the explanation missing). _claim_hosts now returns the
line it waited on and start_run seeds the run's output with every wait.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/ansible/executor.py

@@ -96,16 +96,17 @@ def _host_gate(app: "Quart") -> dict:
     return gate
 
 
-async def _claim_hosts(app: "Quart", run_id: str, hosts: set[str]) -> bool:
+async def _claim_hosts(app: "Quart", run_id: str, hosts: set[str]) -> str | None:
     """Hold every host in hosts for run_id, waiting while another run holds
-    any of them. Returns whether it had to wait. Raises _Aborted when the run
+    any of them. Returns the line saying what it waited for, or None when it
-    is cancelled while waiting, HostBusy after HOST_WAIT_SECONDS."""
+    didn't wait. Raises _Aborted when the run is cancelled while waiting,
+    HostBusy after HOST_WAIT_SECONDS."""
     from steward.models.ansible import AnsibleRunStatus
 
     gate = _host_gate(app)
     held: dict[str, str] = gate["held"]
     give_up = time.monotonic() + HOST_WAIT_SECONDS
-    waited = False
+    waited: str | None = None
     while True:
         busy = sorted({held[h] for h in hosts if h in held and held[h] != run_id})
         if not busy:
@@ -119,11 +120,11 @@ async def _claim_hosts(app: "Quart", run_id: str, hosts: set[str]) -> bool:
             names = ", ".join(sorted(h for h in hosts if h in held))
             raise HostBusy(f"gave up after {HOST_WAIT_SECONDS:.0f} s waiting for "
                            f"{names}, held by run {', '.join(busy)}")
-        if not waited:
+        if waited is None:
-            waited = True
+            waited = (f"[queued — waiting for run {', '.join(busy)} "
+                      "to finish on the same host]")
             await _set_status(app, run_id, AnsibleRunStatus.queued)
-            _broadcast(run_id, f"[queued — waiting for run {', '.join(busy)} "
+            _broadcast(run_id, waited)
-                               "to finish on the same host]")
         changed = gate["changed"]
         try:
             # Woken by a release; the short timeout also notices a cancel.
@@ -557,12 +558,19 @@ async def start_run(
     slot_held = False
     try:
         # The hosts first, so a run waiting on a host doesn't sit on a run slot.
-        queued = await _claim_hosts(app, run_id, hosts)
+        # What it waited for goes in the stored output too, so a run that sat
+        # queued says why after the live view is gone.
+        waits: list[str] = []
+        host_wait = await _claim_hosts(app, run_id, hosts)
+        if host_wait:
+            waits.append(host_wait)
         if sem.locked():
-            if not queued:
+            if not waits:
                 await _set_status(app, run_id, AnsibleRunStatus.queued)
-            queued = True
+            waits.append("[queued — waiting for an Ansible run slot]")
-            _broadcast(run_id, "[queued — waiting for an Ansible run slot]")
+            _broadcast(run_id, waits[-1])
+        queued = bool(waits)
+        db_output = "\n".join(waits)
         await sem.acquire()
         slot_held = True
         if run_id in _cancelled:  # cancelled while queued — abort before launching

tests/core/test_ansible_overlap.py

@@ -47,18 +47,18 @@ def test_a_second_run_on_the_same_host_waits_for_the_first(gate):
     app = gate.app
 
     async def go():
-        assert await executor._claim_hosts(app, "a", {"web"}) is False
+        assert await executor._claim_hosts(app, "a", {"web"}) is None
         b = asyncio.create_task(executor._claim_hosts(app, "b", {"web", "db"}))
         await asyncio.sleep(0.05)
         assert not b.done()
         held = app._ansible_host_gate["held"]
         assert "db" not in held                 # all or nothing: b holds no half
-        assert await executor._claim_hosts(app, "c", {"db"}) is False
+        assert await executor._claim_hosts(app, "c", {"db"}) is None
         executor._release_hosts(app, "c", {"db"})
         await asyncio.sleep(0.05)
         assert not b.done()                     # web is still a's
         executor._release_hosts(app, "a", {"web"})
-        assert await asyncio.wait_for(b, 5) is True
+        assert "waiting for run a" in await asyncio.wait_for(b, 5)
         assert held == {"web": "b", "db": "b"}
 
     asyncio.run(go())