fix(ansible): runs end cleanly: output read in chunks, the process group always ended, a cancelled task recorded (#5156)

- Output is read in 64 KiB chunks and split into lines here; a line past
  64 KiB is cut and marked instead of raising out of readline's limit.
- ansible-playbook runs in its own session; whatever ends the read (finish,
  an error, the task being cancelled) SIGTERMs then SIGKILLs its whole group,
  so no ssh or worker it forked outlives the run.
- A run whose task is cancelled (shutdown) records 'cancelled' with
  finished_at before re-raising, instead of staying 'running'.
- A cancel that lands after the process exited 0 no longer turns a
  successful run into 'cancelled'.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_ansible_executor.py

@@ -28,6 +28,34 @@ def test_parse_recap_empty_without_recap():
     assert parse_recap(["TASK [x] ***", "ok: [h]"]) == {}
 
 
+class _Stream:
+    def __init__(self, chunks):
+        self.chunks = list(chunks)
+
+    async def read(self, _n):
+        return self.chunks.pop(0) if self.chunks else b""
+
+
+def _read_lines(chunks, monkeypatch, limit=8):
+    import asyncio
+    from steward.ansible import executor
+    monkeypatch.setattr(executor, "_MAX_LINE_BYTES", limit)
+
+    async def _go():
+        return [line async for line in executor._lines(_Stream(chunks))]
+    return asyncio.run(_go())
+
+
+def test_lines_split_across_chunks(monkeypatch):
+    assert _read_lines([b"ab", b"c\nde", b"f\n", b"tail"], monkeypatch) == [b"abc", b"def", b"tail"]
+
+
+def test_a_line_past_the_limit_is_cut_and_the_rest_dropped(monkeypatch):
+    # #5156: a long line can't stall or break the reader; the next line is intact.
+    got = _read_lines([b"0123456789", b"abcdef", b"gh\nnext\n"], monkeypatch)
+    assert got == [b"0123456789"[:8] + b" [line cut at 8 bytes]", b"next"]
+
+
 def test_cancel_run_flags_untracked_run():
     rid = "nonexistent-run-id"
     assert cancel_run(rid) is True

tests/integration/test_ansible_run_end.py

@@ -0,0 +1,166 @@
+"""Integration: how an Ansible run ends (#5156).
+
+ansible-playbook is replaced by a small Python script, run as a real process
+through the executor, so reading, the process group and the final status are
+the real paths. Needs Postgres for the run row.
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+import time
+import uuid
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+RECAP = ("PLAY RECAP ****\\n"
+         "localhost : ok=1 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0")
+
+# Starts a child in the same process group and records its pid, so a test can
+# check nothing the run started outlives it.
+SPAWN_CHILD = (
+    "import subprocess, sys, time\n"
+    "c = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n"
+    "open(sys.argv[1], 'w').write(str(c.pid))\n"
+)
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def _use_script(monkeypatch, script: str, *args: str) -> None:
+    from steward.ansible import executor
+    monkeypatch.setattr(executor, "build_ansible_command",
+                        lambda *_a, **_k: [sys.executable, "-c", script, *args])
+
+
+async def _new_run(app) -> str:
+    from steward.models.ansible import AnsibleRun, AnsibleRunStatus
+    run_id = str(uuid.uuid4())
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            s.add(AnsibleRun(id=run_id, playbook_path="t.yml", inventory_path="inv.ini",
+                             source_name="t", triggered_by=None,
+                             status=AnsibleRunStatus.running))
+    return run_id
+
+
+async def _row(app, run_id):
+    from sqlalchemy import select
+    from steward.models.ansible import AnsibleRun
+    async with app.db_sessionmaker() as s:
+        return (await s.execute(select(AnsibleRun).where(AnsibleRun.id == run_id))).scalar_one()
+
+
+def _gone(pid: int, within: float = 5.0) -> bool:
+    """The process has exited (a zombie left for init to reap counts)."""
+    deadline = time.monotonic() + within
+    while time.monotonic() < deadline:
+        try:
+            with open(f"/proc/{pid}/status") as f:
+                if "State:\tZ" in f.read():
+                    return True
+        except FileNotFoundError:
+            return True
+        time.sleep(0.1)
+    return False
+
+
+def test_a_line_longer_than_the_reader_limit_is_cut_not_fatal(app, monkeypatch, tmp_path):
+    from steward.ansible import executor
+    from steward.models.ansible import AnsibleRunStatus
+    _use_script(monkeypatch, f"print('x' * 200000); print('after'); print('{RECAP}')")
+
+    async def _go():
+        run_id = await _new_run(app)
+        await executor.start_run(app, run_id, "t.yml", "inv.ini", str(tmp_path))
+        return await _row(app, run_id)
+
+    run = asyncio.run(_go())
+    assert run.status == AnsibleRunStatus.success, run.output
+    assert f"[line cut at {executor._MAX_LINE_BYTES} bytes]" in run.output
+    assert "after" in run.output
+
+
+def test_an_error_mid_run_ends_the_whole_process_group(app, monkeypatch, tmp_path):
+    from steward.ansible import executor
+    from steward.models.ansible import AnsibleRunStatus
+    pidfile = tmp_path / "child.pid"
+    _use_script(monkeypatch, SPAWN_CHILD + "print('boom', flush=True)\ntime.sleep(60)\n",
+                str(pidfile))
+    real_scrub = executor.scrub
+
+    def scrub(line, secrets):
+        if line == "boom":
+            raise RuntimeError("reader failed")
+        return real_scrub(line, secrets)
+
+    monkeypatch.setattr(executor, "scrub", scrub)
+
+    async def _go():
+        run_id = await _new_run(app)
+        await asyncio.wait_for(
+            executor.start_run(app, run_id, "t.yml", "inv.ini", str(tmp_path)), 30)
+        return await _row(app, run_id)
+
+    run = asyncio.run(_go())
+    assert run.status == AnsibleRunStatus.failed
+    assert "reader failed" in run.output
+    assert _gone(int(pidfile.read_text()))
+
+
+def test_a_run_whose_task_is_cancelled_is_not_left_running(app, monkeypatch, tmp_path):
+    from steward.ansible import executor
+    from steward.models.ansible import AnsibleRunStatus
+    pidfile = tmp_path / "child.pid"
+    _use_script(monkeypatch, SPAWN_CHILD + "print('started', flush=True)\ntime.sleep(60)\n",
+                str(pidfile))
+
+    async def _go():
+        run_id = await _new_run(app)
+        task = asyncio.create_task(
+            executor.start_run(app, run_id, "t.yml", "inv.ini", str(tmp_path)))
+        deadline = time.monotonic() + 15
+        while "started" not in executor._run_lines.get(run_id, []):
+            assert time.monotonic() < deadline, "the run never started"
+            await asyncio.sleep(0.1)
+        task.cancel()                       # Steward shutting down mid-run
+        with pytest.raises(asyncio.CancelledError):
+            await task
+        return await _row(app, run_id)
+
+    run = asyncio.run(_go())
+    assert run.status == AnsibleRunStatus.cancelled
+    assert run.finished_at is not None
+    assert _gone(int(pidfile.read_text()))
+
+
+def test_a_cancel_landing_as_the_run_finishes_keeps_its_result(app, monkeypatch, tmp_path):
+    from steward.ansible import executor
+    from steward.models.ansible import AnsibleRunStatus
+    _use_script(monkeypatch, f"print('{RECAP}')")
+    real_scrub = executor.scrub
+    run_ids: list[str] = []
+
+    def scrub(line, secrets):
+        if line.startswith("localhost :"):   # the last line: the cancel arrives now
+            executor._cancelled.add(run_ids[0])
+        return real_scrub(line, secrets)
+
+    monkeypatch.setattr(executor, "scrub", scrub)
+
+    async def _go():
+        run_ids.append(await _new_run(app))
+        await executor.start_run(app, run_ids[0], "t.yml", "inv.ini", str(tmp_path))
+        return await _row(app, run_ids[0])
+
+    run = asyncio.run(_go())
+    assert run.status == AnsibleRunStatus.success