"""Integration: how an Ansible run ends (#5156). ansible-playbook is replaced by a small Python script, run as a real process through the executor, so reading, the process group and the final status are the real paths. Needs Postgres for the run row. """ from __future__ import annotations import asyncio import os import sys import time import uuid import pytest pytestmark = pytest.mark.integration RECAP = ("PLAY RECAP ****\\n" "localhost : ok=1 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0") # Starts a child in the same process group and records its pid, so a test can # check nothing the run started outlives it. SPAWN_CHILD = ( "import subprocess, sys, time\n" "c = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n" "open(sys.argv[1], 'w').write(str(c.pid))\n" ) @pytest.fixture def app(): if not os.environ.get("STEWARD_DATABASE_URL"): pytest.skip("needs Postgres") from steward.app import create_app return create_app(testing=False) def _use_script(monkeypatch, script: str, *args: str) -> None: from steward.ansible import executor monkeypatch.setattr(executor, "build_ansible_command", lambda *_a, **_k: [sys.executable, "-c", script, *args]) async def _new_run(app) -> str: from steward.models.ansible import AnsibleRun, AnsibleRunStatus run_id = str(uuid.uuid4()) async with app.db_sessionmaker() as s: async with s.begin(): s.add(AnsibleRun(id=run_id, playbook_path="t.yml", inventory_path="inv.ini", source_name="t", triggered_by=None, status=AnsibleRunStatus.running)) return run_id async def _row(app, run_id): from sqlalchemy import select from steward.models.ansible import AnsibleRun async with app.db_sessionmaker() as s: return (await s.execute(select(AnsibleRun).where(AnsibleRun.id == run_id))).scalar_one() def _gone(pid: int, within: float = 5.0) -> bool: """The process has exited (a zombie left for init to reap counts).""" deadline = time.monotonic() + within while time.monotonic() < deadline: try: with open(f"/proc/{pid}/status") as f: if "State:\tZ" in f.read(): return True except FileNotFoundError: return True time.sleep(0.1) return False def test_a_line_longer_than_the_reader_limit_is_cut_not_fatal(app, monkeypatch, tmp_path): from steward.ansible import executor from steward.models.ansible import AnsibleRunStatus _use_script(monkeypatch, f"print('x' * 200000); print('after'); print('{RECAP}')") async def _go(): run_id = await _new_run(app) await executor.start_run(app, run_id, "t.yml", "inv.ini", str(tmp_path)) return await _row(app, run_id) run = asyncio.run(_go()) assert run.status == AnsibleRunStatus.success, run.output assert f"[line cut at {executor._MAX_LINE_BYTES} bytes]" in run.output assert "after" in run.output def test_an_error_mid_run_ends_the_whole_process_group(app, monkeypatch, tmp_path): from steward.ansible import executor from steward.models.ansible import AnsibleRunStatus pidfile = tmp_path / "child.pid" _use_script(monkeypatch, SPAWN_CHILD + "print('boom', flush=True)\ntime.sleep(60)\n", str(pidfile)) real_scrub = executor.scrub def scrub(line, secrets): if line == "boom": raise RuntimeError("reader failed") return real_scrub(line, secrets) monkeypatch.setattr(executor, "scrub", scrub) async def _go(): run_id = await _new_run(app) await asyncio.wait_for( executor.start_run(app, run_id, "t.yml", "inv.ini", str(tmp_path)), 30) return await _row(app, run_id) run = asyncio.run(_go()) assert run.status == AnsibleRunStatus.failed assert "reader failed" in run.output assert _gone(int(pidfile.read_text())) def test_a_run_whose_task_is_cancelled_is_not_left_running(app, monkeypatch, tmp_path): from steward.ansible import executor from steward.models.ansible import AnsibleRunStatus pidfile = tmp_path / "child.pid" _use_script(monkeypatch, SPAWN_CHILD + "print('started', flush=True)\ntime.sleep(60)\n", str(pidfile)) async def _go(): run_id = await _new_run(app) task = asyncio.create_task( executor.start_run(app, run_id, "t.yml", "inv.ini", str(tmp_path))) deadline = time.monotonic() + 15 while "started" not in executor._run_lines.get(run_id, []): assert time.monotonic() < deadline, "the run never started" await asyncio.sleep(0.1) task.cancel() # Steward shutting down mid-run with pytest.raises(asyncio.CancelledError): await task return await _row(app, run_id) run = asyncio.run(_go()) assert run.status == AnsibleRunStatus.cancelled assert run.finished_at is not None assert _gone(int(pidfile.read_text())) def test_a_cancel_landing_as_the_run_finishes_keeps_its_result(app, monkeypatch, tmp_path): from steward.ansible import executor from steward.models.ansible import AnsibleRunStatus _use_script(monkeypatch, f"print('{RECAP}')") real_scrub = executor.scrub run_ids: list[str] = [] def scrub(line, secrets): if line.startswith("localhost :"): # the last line: the cancel arrives now executor._cancelled.add(run_ids[0]) return real_scrub(line, secrets) monkeypatch.setattr(executor, "scrub", scrub) async def _go(): run_ids.append(await _new_run(app)) await executor.start_run(app, run_ids[0], "t.yml", "inv.ini", str(tmp_path)) return await _row(app, run_ids[0]) run = asyncio.run(_go()) assert run.status == AnsibleRunStatus.success