fix(ansible): runs end cleanly: output read in chunks, the process group always ended, a cancelled task recorded (#5156)

- Output is read in 64 KiB chunks and split into lines here; a line past
  64 KiB is cut and marked instead of raising out of readline's limit.
- ansible-playbook runs in its own session; whatever ends the read (finish,
  an error, the task being cancelled) SIGTERMs then SIGKILLs its whole group,
  so no ssh or worker it forked outlives the run.
- A run whose task is cancelled (shutdown) records 'cancelled' with
  finished_at before re-raising, instead of staying 'running'.
- A cancel that lands after the process exited 0 no longer turns a
  successful run into 'cancelled'.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/ansible/executor.py

@@ -6,6 +6,7 @@ import logging
 import os
 import re
 import shutil
+import signal
 import tempfile
 import time
 from dataclasses import dataclass
@@ -26,6 +27,8 @@ _FLUSH_SECS = 5.0
 _MEM_LINES_CAP = 5000   # bounded per-run in-memory replay buffer (SSE late-joiners)
 _MEM_TTL_SECS = 300     # drop in-memory state this long after a run completes
 _FAILURE_CAP = 50       # max failed-task lines captured into structured results
+_READ_CHUNK = 65536     # output is read in chunks, never a whole line at once (#5156)
+_MAX_LINE_BYTES = 65536  # a longer line is cut here and marked, not allowed to stall
 
 # Full run logs persist here (survive restart + the 1 MB DB cap). Override via env.
 ARTIFACT_DIR = os.environ.get("STEWARD_ANSIBLE_LOG_DIR", "/data/ansible/runs")
@@ -261,20 +264,63 @@ async def _set_status(app: "Quart", run_id: str, status) -> None:
             )
 
 
-async def _terminate_then_kill(proc: asyncio.subprocess.Process, grace: float = 10.0) -> None:
+def _signal_group(proc: asyncio.subprocess.Process, sig: int) -> None:
+    """Send sig to the run's whole process group (ansible-playbook and the
+    ssh and worker processes it forks), or to the process alone when it has
+    no group of its own."""
     try:
-        proc.terminate()
+        os.killpg(proc.pid, sig)
     except ProcessLookupError:
-        return
-    try:
-        await asyncio.wait_for(proc.wait(), timeout=grace)
-    except asyncio.TimeoutError:
+        pass
+    except OSError:
         try:
-            proc.kill()
+            proc.send_signal(sig)
         except ProcessLookupError:
             pass
 
 
+async def _terminate_then_kill(proc: asyncio.subprocess.Process, grace: float = 10.0) -> None:
+    """SIGTERM the run's process group, then SIGKILL it if it outlives grace.
+    The group is killed even once the leader has exited, so no worker it
+    forked is left behind (#5156)."""
+    if proc.returncode is None:
+        _signal_group(proc, signal.SIGTERM)
+        try:
+            await asyncio.wait_for(proc.wait(), timeout=grace)
+        except asyncio.TimeoutError:
+            pass
+    _signal_group(proc, signal.SIGKILL)
+
+
+async def _lines(stream: asyncio.StreamReader):
+    """The stream's lines, read in chunks so no line length can stall or break
+    the reader: a line past _MAX_LINE_BYTES is cut there and marked, and the
+    rest of it is dropped up to its newline (#5156)."""
+    pending = b""
+    skipping = False
+    while True:
+        chunk = await stream.read(_READ_CHUNK)
+        if not chunk:
+            break
+        pending += chunk
+        while True:
+            nl = pending.find(b"\n")
+            if nl < 0:
+                break
+            line, pending = pending[:nl], pending[nl + 1:]
+            if skipping:
+                skipping = False
+                continue
+            yield line
+        if len(pending) > _MAX_LINE_BYTES and not skipping:
+            yield pending[:_MAX_LINE_BYTES] + b" [line cut at %d bytes]" % _MAX_LINE_BYTES
+            pending, skipping = b"", True
+        elif skipping:
+            pending = b""
+    if pending and not skipping:
+        yield pending
+
+
 def cancel_run(run_id: str) -> bool:
     """Request cancellation of a run. Terminates the process if it's running;
     a queued run is flagged so start_run aborts before launch. Returns False
@@ -404,6 +450,8 @@ async def start_run(
 
     final_status = "failed"
     logf = None
+    proc = None
+    interrupted = False
     try:
         if run_id in _cancelled:  # cancelled while queued — abort before launching
             raise _Aborted
@@ -463,17 +511,19 @@ async def start_run(
                 if ssh_user:
                     user_args += ["--user", ssh_user]
 
+            # Its own session, so the whole group (ssh, forks) can be signalled.
             proc = await asyncio.create_subprocess_exec(
                 *cmd, *cred_args, *boot_args, *user_args, *ev_args,
                 stdout=asyncio.subprocess.PIPE,
                 stderr=asyncio.subprocess.STDOUT,
                 cwd=cwd,
                 env=ansible_env(creds, os.environ),
+                start_new_session=True,
             )
             _run_procs[run_id] = proc
             assert proc.stdout is not None
 
-            async for raw_line in proc.stdout:
+            async for raw_line in _lines(proc.stdout):
                 line = scrub(raw_line.decode(errors="replace").rstrip("\n\r"), secrets)
                 _broadcast(run_id, line)
                 if logf is not None:
@@ -496,7 +546,8 @@ async def start_run(
 
             await proc.wait()
             recap = parse_recap(_run_lines.get(run_id, []))
-            if run_id in _cancelled:
+            # A cancel that lands after the run finished doesn't undo its result.
+            if run_id in _cancelled and proc.returncode != 0:
                 final_status = "cancelled"
             elif proc.returncode != 0:
                 final_status = "failed"
@@ -511,10 +562,21 @@ async def start_run(
             else:
                 final_status = "success"
         finally:
+            # Whatever ended the read (the run finishing, an error, the task
+            # being cancelled), nothing the run started outlives it.
+            if proc is not None:
+                await asyncio.shield(_terminate_then_kill(proc, grace=5.0))
             shutil.rmtree(tmpdir, ignore_errors=True)
 
     except _Aborted:
         final_status = "cancelled"
+    except asyncio.CancelledError:
+        # The task itself was cancelled (shutdown): record that the run did
+        # not finish rather than leave it marked running (#5156).
+        final_status, interrupted = "cancelled", True
+        msg = "[run error] Steward stopped the run before it finished."
+        _broadcast(run_id, msg)
+        db_output = (db_output + "\n" + msg) if db_output else msg
     except Exception as exc:
         logger.exception("Ansible run %s failed with exception", run_id)
         final_status = "cancelled" if run_id in _cancelled else "failed"
@@ -533,5 +595,11 @@ async def start_run(
         sem.release()
 
     results = {"hosts": parse_recap(_run_lines.get(run_id, [])), "failures": failures}
-    await _flush(final_status, results=results)
+    try:
+        await asyncio.shield(_flush(final_status, results=results))
+    except Exception:
+        logger.exception("Ansible run %s: could not record its final status %s",
+                         run_id, final_status)
     _broadcast(run_id, _Done(status=final_status))
+    if interrupted:
+        raise asyncio.CancelledError