fix(ansible): git sources work in the image and survive a redeploy; pulls time out; alert vars go in a JSON file (#5157)

- git ships in the image; checkouts live under /data/ansible/sources
  (STEWARD_ANSIBLE_SOURCE_DIR), on the persistent volume.
- One task, ansible_git_pulls, reads the sources every 30 s and pulls each
  as its interval passes: a source added in Settings is pulled without a
  restart, a removed one is forgotten.
- git_pull: a 120 s timeout ending the process group, GIT_TERMINAL_PROMPT=0,
  pull --ff-only, a failed clone removed; it raises GitPullError with git's
  message, so the Sync button stops saying Synced after a failure (and the
  message is now escaped).
- An alert action's vars and the firing's steward_alert_* vars reach
  ansible-playbook as one JSON extra-vars file, not -e key=value, which
  Ansible split on whitespace (a rule named 'High CPU on web').

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/ansible/sources.py

@@ -3,8 +3,10 @@ import asyncio
 import logging
 import os
 import re
+import shutil
 import stat
 import tempfile
+import time
 from pathlib import Path
 
 logger = logging.getLogger(__name__)
@@ -76,6 +78,9 @@ BUILTIN_SOURCE_NAME = "steward-builtin"
 # persistent /data volume; created lazily on first save.
 LOCAL_SOURCE_NAME = "steward-local"
 USER_PLAYBOOK_DIR = os.environ.get("STEWARD_PLAYBOOK_DIR", "/data/ansible/playbooks")
+# Git sources are checked out here, on the persistent /data volume, so a
+# redeploy keeps them instead of re-cloning into the container's layer (#5157).
+SOURCE_CACHE_DIR = os.environ.get("STEWARD_ANSIBLE_SOURCE_DIR", "/data/ansible/sources")
 
 
 def _builtin_source() -> dict:
@@ -124,7 +129,7 @@ def get_sources(ansible_cfg: dict) -> list[dict]:
     Config structure in config.yaml::
 
         ansible:
-          cache_dir: /var/cache/steward/ansible
+          cache_dir: /data/ansible/sources
           sources:
             - name: my-playbooks
               type: local
@@ -136,7 +141,7 @@ def get_sources(ansible_cfg: dict) -> list[dict]:
               pull_interval_seconds: 3600
     """
     sources = ansible_cfg.get("sources", [])
-    cache_dir = ansible_cfg.get("cache_dir", "/var/cache/steward/ansible")
+    cache_dir = ansible_cfg.get("cache_dir") or SOURCE_CACHE_DIR
     # Always expose the bundled (read-only) + local (writable) first-party sources.
     result = [_builtin_source(), _local_source()]
     for src in sources:
@@ -343,8 +348,43 @@ ASKPASS_SCRIPT = (
 )
 
 
-async def git_pull(source: dict) -> None:
-    """Clone the git repo if absent; pull if already present.
+# A clone or pull past this is cut, so a hung remote can't hold a source's
+# pull (or the Sync button) open forever (#5157).
+GIT_TIMEOUT_SECONDS = 120.0
+
+
+class GitPullError(RuntimeError):
+    """A clone or pull that failed or ran past its timeout; says which and why."""
+
+
+async def _git(argv: list[str], env: dict, timeout: float, what: str) -> None:
+    from steward.ansible.executor import _terminate_then_kill
+
+    proc = await asyncio.create_subprocess_exec(
+        *argv,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+        env=env,
+        start_new_session=True,   # its own group, so a timeout ends ssh/helpers too
+    )
+    try:
+        _, stderr = await asyncio.wait_for(proc.communicate(), timeout)
+    except asyncio.TimeoutError:
+        raise GitPullError(f"{what} did not finish within {timeout:g} s") from None
+    finally:
+        if proc.returncode is None:
+            await asyncio.shield(_terminate_then_kill(proc, grace=2.0))
+    if proc.returncode != 0:
+        raise GitPullError(f"{what} failed: {stderr.decode(errors='replace').strip()[-500:]}")
+
+
+async def git_pull(source: dict, timeout: float = GIT_TIMEOUT_SECONDS) -> None:
+    """Clone the git repo if absent; fast-forward it if already present.
+
+    Raises GitPullError when git fails or runs past `timeout`; a clone that
+    did not finish is removed, so the next attempt clones afresh rather than
+    pulling into a broken checkout. git never prompts (GIT_TERMINAL_PROMPT=0):
+    a missing credential fails at once instead of waiting on input.
 
     When source['http_token'] is set, git asks a temporary GIT_ASKPASS script,
     which prints the token from the child's environment, so it never touches
@@ -353,7 +393,7 @@ async def git_pull(source: dict) -> None:
     path = Path(source["path"])
     token = source.get("http_token", "")
 
-    env = None
+    env = {**os.environ, "GIT_TERMINAL_PROMPT": "0"}
     askpass_path = None
     if token:
         fd, askpass_path = tempfile.mkstemp(prefix="steward-askpass-", suffix=".sh")
@@ -362,41 +402,50 @@ async def git_pull(source: dict) -> None:
         finally:
             os.close(fd)
         os.chmod(askpass_path, stat.S_IRWXU)
-        env = {**os.environ, "GIT_ASKPASS": askpass_path, ASKPASS_TOKEN_ENV: token}
+        env.update({"GIT_ASKPASS": askpass_path, ASKPASS_TOKEN_ENV: token})
 
     try:
         if not (path / ".git").exists():
             path.mkdir(parents=True, exist_ok=True)
-            proc = await asyncio.create_subprocess_exec(
-                "git", "clone",
-                "--branch", source["branch"],
-                "--single-branch",
-                source["url"], str(path),
-                stdout=asyncio.subprocess.PIPE,
-                stderr=asyncio.subprocess.PIPE,
-                env=env,
-            )
-            _, stderr = await proc.communicate()
-            if proc.returncode != 0:
-                logger.error(
-                    "git clone failed for %r: %s",
-                    source["name"],
-                    stderr.decode(errors="replace"),
-                )
+            try:
+                await _git(["git", "clone", "--branch", source["branch"], "--single-branch",
+                            source["url"], str(path)],
+                           env, timeout, f"git clone of {source['name']!r}")
+            except BaseException:
+                shutil.rmtree(path, ignore_errors=True)
+                raise
         else:
-            proc = await asyncio.create_subprocess_exec(
-                "git", "-C", str(path), "pull",
-                stdout=asyncio.subprocess.PIPE,
-                stderr=asyncio.subprocess.PIPE,
-                env=env,
-            )
-            _, stderr = await proc.communicate()
-            if proc.returncode != 0:
-                logger.error(
-                    "git pull failed for %r: %s",
-                    source["name"],
-                    stderr.decode(errors="replace"),
-                )
+            await _git(["git", "-C", str(path), "pull", "--ff-only"],
+                       env, timeout, f"git pull of {source['name']!r}")
     finally:
         if askpass_path and os.path.exists(askpass_path):
             os.unlink(askpass_path)
+
+
+# When each git source was last pulled (monotonic seconds), by name.
+_last_pull: dict[str, float] = {}
+
+
+async def pull_due_sources(ansible_cfg: dict, now: float | None = None) -> list[str]:
+    """Pull every git source whose pull_interval_seconds has passed.
+
+    Run by one scheduled task on a short tick, reading the sources as they are
+    now, so a source added in Settings is pulled within a tick and an edited
+    interval applies without a restart (#5157). A failed pull is logged and
+    waits out its interval like a good one. Returns the names pulled.
+    """
+    now = time.monotonic() if now is None else now
+    sources = get_sources(ansible_cfg)
+    due = [src for src in sources
+           if src["type"] == "git"
+           and now - _last_pull.get(src["name"], float("-inf")) >= src["pull_interval_seconds"]]
+    names = {src["name"] for src in sources}
+    for gone in [n for n in _last_pull if n not in names]:
+        del _last_pull[gone]
+    for src in due:
+        _last_pull[src["name"]] = now
+        try:
+            await git_pull(src)
+        except Exception:
+            logger.exception("Ansible source %r: pull failed", src["name"])
+    return [src["name"] for src in due]