FabledSteward/tests/integration/test_ansible_run_end.py

"""Integration: how an Ansible run ends (#5156).

ansible-playbook is replaced by a small Python script, run as a real process
through the executor, so reading, the process group and the final status are
the real paths. Needs Postgres for the run row.
"""
from __future__ import annotations

import asyncio
import os
import sys
import time
import uuid

import pytest

pytestmark = pytest.mark.integration

RECAP = ("PLAY RECAP ****\\n"
         "localhost : ok=1 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0")

# Starts a child in the same process group and records its pid, so a test can
# check nothing the run started outlives it.
SPAWN_CHILD = (
    "import subprocess, sys, time\n"
    "c = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n"
    "open(sys.argv[1], 'w').write(str(c.pid))\n"
)


@pytest.fixture
def app():
    if not os.environ.get("STEWARD_DATABASE_URL"):
        pytest.skip("needs Postgres")
    from steward.app import create_app
    return create_app(testing=False)


def _use_script(monkeypatch, script: str, *args: str) -> None:
    from steward.ansible import executor
    monkeypatch.setattr(executor, "build_ansible_command",
                        lambda *_a, **_k: [sys.executable, "-c", script, *args])


async def _new_run(app) -> str:
    from steward.models.ansible import AnsibleRun, AnsibleRunStatus
    run_id = str(uuid.uuid4())
    async with app.db_sessionmaker() as s:
        async with s.begin():
            s.add(AnsibleRun(id=run_id, playbook_path="t.yml", inventory_path="inv.ini",
                             source_name="t", triggered_by=None,
                             status=AnsibleRunStatus.running))
    return run_id


async def _row(app, run_id):
    from sqlalchemy import select
    from steward.models.ansible import AnsibleRun
    async with app.db_sessionmaker() as s:
        return (await s.execute(select(AnsibleRun).where(AnsibleRun.id == run_id))).scalar_one()


def _gone(pid: int, within: float = 5.0) -> bool:
    """The process has exited (a zombie left for init to reap counts)."""
    deadline = time.monotonic() + within
    while time.monotonic() < deadline:
        try:
            with open(f"/proc/{pid}/status") as f:
                if "State:\tZ" in f.read():
                    return True
        except FileNotFoundError:
            return True
        time.sleep(0.1)
    return False


def test_a_line_longer_than_the_reader_limit_is_cut_not_fatal(app, monkeypatch, tmp_path):
    from steward.ansible import executor
    from steward.models.ansible import AnsibleRunStatus
    _use_script(monkeypatch, f"print('x' * 200000); print('after'); print('{RECAP}')")

    async def _go():
        run_id = await _new_run(app)
        await executor.start_run(app, run_id, "t.yml", "inv.ini", str(tmp_path))
        return await _row(app, run_id)

    run = asyncio.run(_go())
    assert run.status == AnsibleRunStatus.success, run.output
    assert f"[line cut at {executor._MAX_LINE_BYTES} bytes]" in run.output
    assert "after" in run.output


def test_an_error_mid_run_ends_the_whole_process_group(app, monkeypatch, tmp_path):
    from steward.ansible import executor
    from steward.models.ansible import AnsibleRunStatus
    pidfile = tmp_path / "child.pid"
    _use_script(monkeypatch, SPAWN_CHILD + "print('boom', flush=True)\ntime.sleep(60)\n",
                str(pidfile))
    real_scrub = executor.scrub

    def scrub(line, secrets):
        if line == "boom":
            raise RuntimeError("reader failed")
        return real_scrub(line, secrets)

    monkeypatch.setattr(executor, "scrub", scrub)

    async def _go():
        run_id = await _new_run(app)
        await asyncio.wait_for(
            executor.start_run(app, run_id, "t.yml", "inv.ini", str(tmp_path)), 30)
        return await _row(app, run_id)

    run = asyncio.run(_go())
    assert run.status == AnsibleRunStatus.failed
    assert "reader failed" in run.output
    assert _gone(int(pidfile.read_text()))


def test_a_run_whose_task_is_cancelled_is_not_left_running(app, monkeypatch, tmp_path):
    from steward.ansible import executor
    from steward.models.ansible import AnsibleRunStatus
    pidfile = tmp_path / "child.pid"
    _use_script(monkeypatch, SPAWN_CHILD + "print('started', flush=True)\ntime.sleep(60)\n",
                str(pidfile))

    async def _go():
        run_id = await _new_run(app)
        task = asyncio.create_task(
            executor.start_run(app, run_id, "t.yml", "inv.ini", str(tmp_path)))
        deadline = time.monotonic() + 15
        while "started" not in executor._run_lines.get(run_id, []):
            assert time.monotonic() < deadline, "the run never started"
            await asyncio.sleep(0.1)
        task.cancel()                       # Steward shutting down mid-run
        with pytest.raises(asyncio.CancelledError):
            await task
        return await _row(app, run_id)

    run = asyncio.run(_go())
    assert run.status == AnsibleRunStatus.cancelled
    assert run.finished_at is not None
    assert _gone(int(pidfile.read_text()))


def test_a_cancel_landing_as_the_run_finishes_keeps_its_result(app, monkeypatch, tmp_path):
    from steward.ansible import executor
    from steward.models.ansible import AnsibleRunStatus
    _use_script(monkeypatch, f"print('{RECAP}')")
    real_scrub = executor.scrub
    run_ids: list[str] = []

    def scrub(line, secrets):
        if line.startswith("localhost :"):   # the last line: the cancel arrives now
            executor._cancelled.add(run_ids[0])
        return real_scrub(line, secrets)

    monkeypatch.setattr(executor, "scrub", scrub)

    async def _go():
        run_ids.append(await _new_run(app))
        await executor.start_run(app, run_ids[0], "t.yml", "inv.ini", str(tmp_path))
        return await _row(app, run_ids[0])

    run = asyncio.run(_go())
    assert run.status == AnsibleRunStatus.success