bvandeusenandClaude Opus 5.5
3c24c633df
- One run per host: start_run claims every host it touches (addresses read from the inventory Steward wrote; a source's own inventory file is claimed as that file) before it takes a run slot. The claim is all-or-nothing. A run waiting on a host shows queued and names the run it waits for, honours a cancel, and fails after HOST_WAIT_SECONDS (1 h) with which host and run. - A due schedule whose previous run is still queued or running skips the firing: last_run_at moves on, last_run_id still names the run going, and last_error says "Skipped: the previous run was still going". The Schedules page shows that beside the run (muted), and a launch error in red. - An alert rule that fires while the action run from an earlier firing is still going starts no new run; the event says so and names the run. An in-process set closes the gap between the check and the run being linked. - docs/core/ansible.md: queued/cancelled statuses and the overlap rules. Tests: tests/core/test_ansible_overlap.py (claim keys, waiting, all-or-nothing, cancel, deadline) and tests/integration/test_ansible_overlap.py (two real runs on one host take turns; schedule skip then fire; alert action skip then start). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
94 lines
3.4 KiB
Python
94 lines
3.4 KiB
Python
"""One run per host (#5158): what a run claims, and how a second run waits."""
|
|
import asyncio
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from steward.ansible import executor
|
|
from steward.ansible.inventory_gen import generate_inventory, inventory_to_yaml
|
|
|
|
|
|
def _target(name, address, groups=()):
|
|
return SimpleNamespace(name=name, address=address, ansible_vars={},
|
|
groups=[SimpleNamespace(name=g, ansible_vars={}) for g in groups])
|
|
|
|
|
|
def test_a_run_claims_the_addresses_in_the_inventory_steward_wrote():
|
|
inv = inventory_to_yaml(generate_inventory(
|
|
[_target("web", "10.0.0.5", ["linux"]), _target("db", "DB.lan", ["linux"])]))
|
|
# A host listed again under its group (no vars there) is still its address.
|
|
assert executor.run_hosts(inv, None) == {"10.0.0.5", "db.lan"}
|
|
|
|
|
|
def test_a_host_without_an_address_is_claimed_by_name():
|
|
assert executor.run_hosts("all:\n hosts:\n box1:\n token: x\n", None) == {"box1"}
|
|
|
|
|
|
def test_a_run_on_a_source_inventory_claims_the_file():
|
|
assert executor.run_hosts(None, "inventories/prod.ini") == {"inventory:inventories/prod.ini"}
|
|
assert executor.run_hosts(None, "") == set()
|
|
|
|
|
|
@pytest.fixture
|
|
def gate(monkeypatch):
|
|
statuses: list[tuple[str, str]] = []
|
|
|
|
async def fake_set_status(_app, run_id, status):
|
|
statuses.append((run_id, status.value))
|
|
|
|
monkeypatch.setattr(executor, "_set_status", fake_set_status)
|
|
yield SimpleNamespace(app=SimpleNamespace(), statuses=statuses)
|
|
for rid in ("a", "b", "c"):
|
|
executor._run_lines.pop(rid, None)
|
|
executor._cancelled.discard(rid)
|
|
|
|
|
|
def test_a_second_run_on_the_same_host_waits_for_the_first(gate):
|
|
app = gate.app
|
|
|
|
async def go():
|
|
assert await executor._claim_hosts(app, "a", {"web"}) is False
|
|
b = asyncio.create_task(executor._claim_hosts(app, "b", {"web", "db"}))
|
|
await asyncio.sleep(0.05)
|
|
assert not b.done()
|
|
held = app._ansible_host_gate["held"]
|
|
assert "db" not in held # all or nothing: b holds no half
|
|
assert await executor._claim_hosts(app, "c", {"db"}) is False
|
|
executor._release_hosts(app, "c", {"db"})
|
|
await asyncio.sleep(0.05)
|
|
assert not b.done() # web is still a's
|
|
executor._release_hosts(app, "a", {"web"})
|
|
assert await asyncio.wait_for(b, 5) is True
|
|
assert held == {"web": "b", "db": "b"}
|
|
|
|
asyncio.run(go())
|
|
assert ("b", "queued") in gate.statuses
|
|
assert any("waiting for run a" in line for line in executor._run_lines["b"])
|
|
|
|
|
|
def test_a_run_cancelled_while_waiting_for_a_host_never_claims_it(gate):
|
|
app = gate.app
|
|
|
|
async def go():
|
|
await executor._claim_hosts(app, "a", {"web"})
|
|
b = asyncio.create_task(executor._claim_hosts(app, "b", {"web"}))
|
|
await asyncio.sleep(0.05)
|
|
executor._cancelled.add("b")
|
|
with pytest.raises(executor._Aborted):
|
|
await asyncio.wait_for(b, 5)
|
|
assert app._ansible_host_gate["held"] == {"web": "a"}
|
|
|
|
asyncio.run(go())
|
|
|
|
|
|
def test_a_run_gives_up_waiting_for_a_host_and_says_which(gate, monkeypatch):
|
|
monkeypatch.setattr(executor, "HOST_WAIT_SECONDS", 0.2)
|
|
app = gate.app
|
|
|
|
async def go():
|
|
await executor._claim_hosts(app, "a", {"web"})
|
|
with pytest.raises(executor.HostBusy, match="waiting for web, held by run a"):
|
|
await asyncio.wait_for(executor._claim_hosts(app, "b", {"web"}), 5)
|
|
|
|
asyncio.run(go())
|