diff --git a/steward/ansible/executor.py b/steward/ansible/executor.py index 3c6415e..c2fdefa 100644 --- a/steward/ansible/executor.py +++ b/steward/ansible/executor.py @@ -96,16 +96,17 @@ def _host_gate(app: "Quart") -> dict: return gate -async def _claim_hosts(app: "Quart", run_id: str, hosts: set[str]) -> bool: +async def _claim_hosts(app: "Quart", run_id: str, hosts: set[str]) -> str | None: """Hold every host in hosts for run_id, waiting while another run holds - any of them. Returns whether it had to wait. Raises _Aborted when the run - is cancelled while waiting, HostBusy after HOST_WAIT_SECONDS.""" + any of them. Returns the line saying what it waited for, or None when it + didn't wait. Raises _Aborted when the run is cancelled while waiting, + HostBusy after HOST_WAIT_SECONDS.""" from steward.models.ansible import AnsibleRunStatus gate = _host_gate(app) held: dict[str, str] = gate["held"] give_up = time.monotonic() + HOST_WAIT_SECONDS - waited = False + waited: str | None = None while True: busy = sorted({held[h] for h in hosts if h in held and held[h] != run_id}) if not busy: @@ -119,11 +120,11 @@ async def _claim_hosts(app: "Quart", run_id: str, hosts: set[str]) -> bool: names = ", ".join(sorted(h for h in hosts if h in held)) raise HostBusy(f"gave up after {HOST_WAIT_SECONDS:.0f} s waiting for " f"{names}, held by run {', '.join(busy)}") - if not waited: - waited = True + if waited is None: + waited = (f"[queued — waiting for run {', '.join(busy)} " + "to finish on the same host]") await _set_status(app, run_id, AnsibleRunStatus.queued) - _broadcast(run_id, f"[queued — waiting for run {', '.join(busy)} " - "to finish on the same host]") + _broadcast(run_id, waited) changed = gate["changed"] try: # Woken by a release; the short timeout also notices a cancel. @@ -557,12 +558,19 @@ async def start_run( slot_held = False try: # The hosts first, so a run waiting on a host doesn't sit on a run slot. - queued = await _claim_hosts(app, run_id, hosts) + # What it waited for goes in the stored output too, so a run that sat + # queued says why after the live view is gone. + waits: list[str] = [] + host_wait = await _claim_hosts(app, run_id, hosts) + if host_wait: + waits.append(host_wait) if sem.locked(): - if not queued: + if not waits: await _set_status(app, run_id, AnsibleRunStatus.queued) - queued = True - _broadcast(run_id, "[queued — waiting for an Ansible run slot]") + waits.append("[queued — waiting for an Ansible run slot]") + _broadcast(run_id, waits[-1]) + queued = bool(waits) + db_output = "\n".join(waits) await sem.acquire() slot_held = True if run_id in _cancelled: # cancelled while queued — abort before launching diff --git a/tests/core/test_ansible_overlap.py b/tests/core/test_ansible_overlap.py index 555726d..5d0914b 100644 --- a/tests/core/test_ansible_overlap.py +++ b/tests/core/test_ansible_overlap.py @@ -47,18 +47,18 @@ def test_a_second_run_on_the_same_host_waits_for_the_first(gate): app = gate.app async def go(): - assert await executor._claim_hosts(app, "a", {"web"}) is False + assert await executor._claim_hosts(app, "a", {"web"}) is None b = asyncio.create_task(executor._claim_hosts(app, "b", {"web", "db"})) await asyncio.sleep(0.05) assert not b.done() held = app._ansible_host_gate["held"] assert "db" not in held # all or nothing: b holds no half - assert await executor._claim_hosts(app, "c", {"db"}) is False + assert await executor._claim_hosts(app, "c", {"db"}) is None executor._release_hosts(app, "c", {"db"}) await asyncio.sleep(0.05) assert not b.done() # web is still a's executor._release_hosts(app, "a", {"web"}) - assert await asyncio.wait_for(b, 5) is True + assert "waiting for run a" in await asyncio.wait_for(b, 5) assert held == {"web": "b", "db": "b"} asyncio.run(go())