fix(ansible): git sources work in the image and survive a redeploy; pulls time out; alert vars go in a JSON file (#5157)

- git ships in the image; checkouts live under /data/ansible/sources
  (STEWARD_ANSIBLE_SOURCE_DIR), on the persistent volume.
- One task, ansible_git_pulls, reads the sources every 30 s and pulls each
  as its interval passes: a source added in Settings is pulled without a
  restart, a removed one is forgotten.
- git_pull: a 120 s timeout ending the process group, GIT_TERMINAL_PROMPT=0,
  pull --ff-only, a failed clone removed; it raises GitPullError with git's
  message, so the Sync button stops saying Synced after a failure (and the
  message is now escaped).
- An alert action's vars and the firing's steward_alert_* vars reach
  ansible-playbook as one JSON extra-vars file, not -e key=value, which
  Ansible split on whitespace (a rule named 'High CPU on web').

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/ansible/sources.py

@@ -3,8 +3,10 @@ import asyncio
 import logging
 import os
 import re
+import shutil
 import stat
 import tempfile
+import time
 from pathlib import Path
 
 logger = logging.getLogger(__name__)
@@ -76,6 +78,9 @@ BUILTIN_SOURCE_NAME = "steward-builtin"
 # persistent /data volume; created lazily on first save.
 LOCAL_SOURCE_NAME = "steward-local"
 USER_PLAYBOOK_DIR = os.environ.get("STEWARD_PLAYBOOK_DIR", "/data/ansible/playbooks")
+# Git sources are checked out here, on the persistent /data volume, so a
+# redeploy keeps them instead of re-cloning into the container's layer (#5157).
+SOURCE_CACHE_DIR = os.environ.get("STEWARD_ANSIBLE_SOURCE_DIR", "/data/ansible/sources")
 
 
 def _builtin_source() -> dict:
@@ -124,7 +129,7 @@ def get_sources(ansible_cfg: dict) -> list[dict]:
     Config structure in config.yaml::
 
         ansible:
-          cache_dir: /var/cache/steward/ansible
+          cache_dir: /data/ansible/sources
           sources:
             - name: my-playbooks
               type: local
@@ -136,7 +141,7 @@ def get_sources(ansible_cfg: dict) -> list[dict]:
               pull_interval_seconds: 3600
     """
     sources = ansible_cfg.get("sources", [])
-    cache_dir = ansible_cfg.get("cache_dir", "/var/cache/steward/ansible")
+    cache_dir = ansible_cfg.get("cache_dir") or SOURCE_CACHE_DIR
     # Always expose the bundled (read-only) + local (writable) first-party sources.
     result = [_builtin_source(), _local_source()]
     for src in sources:
@@ -343,8 +348,43 @@ ASKPASS_SCRIPT = (
 )
 
 
-async def git_pull(source: dict) -> None:
-    """Clone the git repo if absent; pull if already present.
+# A clone or pull past this is cut, so a hung remote can't hold a source's
+# pull (or the Sync button) open forever (#5157).
+GIT_TIMEOUT_SECONDS = 120.0
+
+
+class GitPullError(RuntimeError):
+    """A clone or pull that failed or ran past its timeout; says which and why."""
+
+
+async def _git(argv: list[str], env: dict, timeout: float, what: str) -> None:
+    from steward.ansible.executor import _terminate_then_kill
+
+    proc = await asyncio.create_subprocess_exec(
+        *argv,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+        env=env,
+        start_new_session=True,   # its own group, so a timeout ends ssh/helpers too
+    )
+    try:
+        _, stderr = await asyncio.wait_for(proc.communicate(), timeout)
+    except asyncio.TimeoutError:
+        raise GitPullError(f"{what} did not finish within {timeout:g} s") from None
+    finally:
+        if proc.returncode is None:
+            await asyncio.shield(_terminate_then_kill(proc, grace=2.0))
+    if proc.returncode != 0:
+        raise GitPullError(f"{what} failed: {stderr.decode(errors='replace').strip()[-500:]}")
+
+
+async def git_pull(source: dict, timeout: float = GIT_TIMEOUT_SECONDS) -> None:
+    """Clone the git repo if absent; fast-forward it if already present.
+
+    Raises GitPullError when git fails or runs past `timeout`; a clone that
+    did not finish is removed, so the next attempt clones afresh rather than
+    pulling into a broken checkout. git never prompts (GIT_TERMINAL_PROMPT=0):
+    a missing credential fails at once instead of waiting on input.
 
     When source['http_token'] is set, git asks a temporary GIT_ASKPASS script,
     which prints the token from the child's environment, so it never touches
@@ -353,7 +393,7 @@ async def git_pull(source: dict) -> None:
     path = Path(source["path"])
     token = source.get("http_token", "")
 
-    env = None
+    env = {**os.environ, "GIT_TERMINAL_PROMPT": "0"}
     askpass_path = None
     if token:
         fd, askpass_path = tempfile.mkstemp(prefix="steward-askpass-", suffix=".sh")
@@ -362,41 +402,50 @@ async def git_pull(source: dict) -> None:
         finally:
             os.close(fd)
         os.chmod(askpass_path, stat.S_IRWXU)
-        env = {**os.environ, "GIT_ASKPASS": askpass_path, ASKPASS_TOKEN_ENV: token}
+        env.update({"GIT_ASKPASS": askpass_path, ASKPASS_TOKEN_ENV: token})
 
     try:
         if not (path / ".git").exists():
             path.mkdir(parents=True, exist_ok=True)
-            proc = await asyncio.create_subprocess_exec(
-                "git", "clone",
-                "--branch", source["branch"],
-                "--single-branch",
-                source["url"], str(path),
-                stdout=asyncio.subprocess.PIPE,
-                stderr=asyncio.subprocess.PIPE,
-                env=env,
-            )
-            _, stderr = await proc.communicate()
-            if proc.returncode != 0:
-                logger.error(
-                    "git clone failed for %r: %s",
-                    source["name"],
-                    stderr.decode(errors="replace"),
-                )
+            try:
+                await _git(["git", "clone", "--branch", source["branch"], "--single-branch",
+                            source["url"], str(path)],
+                           env, timeout, f"git clone of {source['name']!r}")
+            except BaseException:
+                shutil.rmtree(path, ignore_errors=True)
+                raise
         else:
-            proc = await asyncio.create_subprocess_exec(
-                "git", "-C", str(path), "pull",
-                stdout=asyncio.subprocess.PIPE,
-                stderr=asyncio.subprocess.PIPE,
-                env=env,
-            )
-            _, stderr = await proc.communicate()
-            if proc.returncode != 0:
-                logger.error(
-                    "git pull failed for %r: %s",
-                    source["name"],
-                    stderr.decode(errors="replace"),
-                )
+            await _git(["git", "-C", str(path), "pull", "--ff-only"],
+                       env, timeout, f"git pull of {source['name']!r}")
     finally:
         if askpass_path and os.path.exists(askpass_path):
             os.unlink(askpass_path)
+
+
+# When each git source was last pulled (monotonic seconds), by name.
+_last_pull: dict[str, float] = {}
+
+
+async def pull_due_sources(ansible_cfg: dict, now: float | None = None) -> list[str]:
+    """Pull every git source whose pull_interval_seconds has passed.
+
+    Run by one scheduled task on a short tick, reading the sources as they are
+    now, so a source added in Settings is pulled within a tick and an edited
+    interval applies without a restart (#5157). A failed pull is logged and
+    waits out its interval like a good one. Returns the names pulled.
+    """
+    now = time.monotonic() if now is None else now
+    sources = get_sources(ansible_cfg)
+    due = [src for src in sources
+           if src["type"] == "git"
+           and now - _last_pull.get(src["name"], float("-inf")) >= src["pull_interval_seconds"]]
+    names = {src["name"] for src in sources}
+    for gone in [n for n in _last_pull if n not in names]:
+        del _last_pull[gone]
+    for src in due:
+        _last_pull[src["name"]] = now
+        try:
+            await git_pull(src)
+        except Exception:
+            logger.exception("Ansible source %r: pull failed", src["name"])
+    return [src["name"] for src in due]

steward/app.py

@@ -434,26 +434,21 @@ def _register_core_tasks(app: Quart) -> None:
         ),
     ])
 
-    # Register git-pull tasks for each ansible git source
-    ansible_cfg = app.config.get("ANSIBLE", {})
-    from .ansible.sources import get_sources
-    for source in get_sources(ansible_cfg):
-        if source["type"] != "git":
-            continue
-        _source = source
+    # Pull Ansible git sources as each falls due. One task reads the sources
+    # as they are on every tick, so one added in Settings is pulled within
+    # half a minute, no restart (#5157).
+    async def run_git_pulls():
+        from .ansible.sources import pull_due_sources
+        await pull_due_sources(app.config.get("ANSIBLE", {}))
 
-        async def run_git_pull(src=_source):
-            from .ansible.sources import git_pull
-            await git_pull(src)
-
-        app._task_registry.append(
-            ScheduledTask(
-                name=f"ansible_git_pull_{_source['name']}",
-                coro_factory=run_git_pull,
-                interval_seconds=_source["pull_interval_seconds"],
-                run_on_startup=True,
-            )
+    app._task_registry.append(
+        ScheduledTask(
+            name="ansible_git_pulls",
+            coro_factory=run_git_pulls,
+            interval_seconds=30,
+            run_on_startup=True,
         )
+    )
 
     # Fire due Ansible schedules (recurring playbook runs). Checks every minute;
     # each schedule's own interval gates whether it actually runs.

steward/core/alerts.py

@@ -34,21 +34,35 @@ def init_alerts(app: "Quart") -> None:
     _app = app
 
 
-def alert_extra_vars(rule: AlertRule, value: float) -> list[str]:
-    """Ansible extra-vars describing the alert that fired, as `key=value` strings.
+def alert_extra_vars(rule: AlertRule, value: float) -> dict:
+    """Ansible extra-vars describing the alert that fired.
 
     Injected into every alert-triggered playbook run so the playbook can target
-    the exact resource/metric that fired without templating. Pure function.
+    the exact resource/metric that fired without templating. They reach
+    ansible-playbook in its JSON extra-vars file, not as `-e key=value`, which
+    Ansible splits on whitespace: a rule named "High CPU on web" stays one
+    value (#5157). Pure function.
     """
-    return [
-        f"steward_alert_rule={rule.name}",
-        f"steward_alert_source={rule.source_module}",
-        f"steward_alert_resource={rule.resource_name}",
-        f"steward_alert_metric={rule.metric_name}",
-        f"steward_alert_value={value}",
-        f"steward_alert_threshold={rule.threshold}",
-        f"steward_alert_operator={rule.operator.value}",
-    ]
+    return {
+        "steward_alert_rule": rule.name,
+        "steward_alert_source": rule.source_module,
+        "steward_alert_resource": rule.resource_name,
+        "steward_alert_metric": rule.metric_name,
+        "steward_alert_value": value,
+        "steward_alert_threshold": rule.threshold,
+        "steward_alert_operator": rule.operator.value,
+    }
+
+
+def action_vars(lines) -> dict:
+    """An alert action's stored `key=value` lines as a map, split at the first
+    `=` so a value keeps its spaces, quotes and further `=`."""
+    out: dict = {}
+    for line in lines or ():
+        key, sep, val = str(line).partition("=")
+        if sep and key.strip():
+            out[key.strip()] = val
+    return out
 
 
 async def record_metric(
@@ -489,8 +503,10 @@ async def _run_ansible_action(
     playbook = action.get("playbook") or ""
     inventory = action.get("inventory") or ""
     source_name = action.get("source") or ""
+    # One JSON extra-vars file; the firing's own vars win over the action's.
     params: dict = {
-        "extra_vars": list(action.get("extra_vars") or []) + alert_extra_vars(rule, value),
+        "extra_vars_map": {**action_vars(action.get("extra_vars")),
+                           **alert_extra_vars(rule, value)},
     }
     if action.get("limit"):
         params["limit"] = action["limit"]

steward/settings/routes.py

@@ -1,5 +1,6 @@
 # steward/settings/routes.py
 from __future__ import annotations
+import html
 import logging
 from pathlib import Path
 from quart import Blueprint, current_app, render_template, request, redirect, url_for, session
@@ -505,7 +506,8 @@ async def ansible_sync_source(idx: int):
         return '<span style="color:var(--green);font-size:0.8rem;">Synced</span>'
     except Exception as exc:
         logger.exception("git pull failed for source %r", source["name"])
-        return f'<span style="color:var(--red);font-size:0.8rem;">Sync failed: {exc}</span>'
+        return ('<span style="color:var(--red);font-size:0.8rem;">Sync failed: '
+                f'{html.escape(str(exc))}</span>')
 
 
 # ── Auth (OIDC / LDAP) ────────────────────────────────────────────────────────