from __future__ import annotations import asyncio import logging import os import re import shutil import stat import tempfile import time from pathlib import Path logger = logging.getLogger(__name__) INVENTORY_NAMES = {"hosts", "inventory", "inventory.yml", "inventory.ini"} # Variable names that should be entered masked + kept out of the DB. Matched # case-insensitively as a substring of the variable name. _SECRET_VAR_RE = re.compile( r"(password|passwd|secret|token|api[_-]?key|private[_-]?key|credential)", re.I ) # A playbook self-describes via a `# description:` magic comment (Ansible rejects # unknown play keys, so a comment is the portable place). First match wins. _DESCRIPTION_RE = re.compile(r"^\s*#\s*description:\s*(.+?)\s*$", re.I | re.M) # Namespaced metadata: `# steward:: ` (category, confirm, …). _STEWARD_META_RE = re.compile(r"^\s*#\s*steward:(\w+):\s*(.+?)\s*$", re.I | re.M) _TRUTHY = {"1", "true", "yes", "on"} def _first_play_name(content: str) -> str: import yaml try: plays = yaml.safe_load(content) except yaml.YAMLError: return "" if isinstance(plays, list): for play in plays: if isinstance(play, dict) and play.get("name"): return str(play["name"]).strip() return "" def discover_playbook_meta(content: str) -> dict: """Parse Steward's playbook metadata from magic comments. Returns {description, category, confirm}. Sources: - description: ``# description:`` (primary) or ``# steward:description:``; falls back to the first play's ``name:``. - category: ``# steward:category: `` (free-text grouping label). - confirm: ``# steward:confirm: true`` → require an explicit confirmation in the run form before launching (for destructive playbooks). First match wins for each key. """ steward: dict[str, str] = {} for km in _STEWARD_META_RE.finditer(content): steward.setdefault(km.group(1).lower(), km.group(2).strip()) m = _DESCRIPTION_RE.search(content) if m: description = m.group(1).strip() elif steward.get("description"): description = steward["description"] else: description = _first_play_name(content) return { "description": description, "category": steward.get("category", ""), "confirm": str(steward.get("confirm", "")).strip().lower() in _TRUTHY, } # Name of the always-present, read-only source of first-party playbooks shipped # inside the app (maintenance tasks, host-agent install). Not operator-editable. BUILTIN_SOURCE_NAME = "steward-builtin" # Always-present, WRITABLE local source where the in-app editor saves playbooks, # so a homelab user with no git workflow can author automation. Lives on the # persistent /data volume; created lazily on first save. LOCAL_SOURCE_NAME = "steward-local" USER_PLAYBOOK_DIR = os.environ.get("STEWARD_PLAYBOOK_DIR", "/data/ansible/playbooks") # Git sources are checked out here, on the persistent /data volume, so a # redeploy keeps them instead of re-cloning into the container's layer (#5157). SOURCE_CACHE_DIR = os.environ.get("STEWARD_ANSIBLE_SOURCE_DIR", "/data/ansible/sources") def _builtin_source() -> dict: return { "name": BUILTIN_SOURCE_NAME, "type": "local", "path": str((Path(__file__).parent / "bundled").resolve()), "url": None, "branch": "main", "pull_interval_seconds": 0, "http_token": "", } def _local_source() -> dict: return { "name": LOCAL_SOURCE_NAME, "type": "local", "path": USER_PLAYBOOK_DIR, "url": None, "branch": "main", "pull_interval_seconds": 0, "http_token": "", } def is_editable_source(source: dict) -> bool: """Editable in the in-app editor: local dir sources except the read-only bundled one. Git sources are GitOps (clobbered on pull) so not editable.""" return source.get("type") == "local" and source.get("name") != BUILTIN_SOURCE_NAME def _open_token(src: dict) -> str: """The source's git token in the clear (it is stored sealed).""" from steward.core.crypto import decrypt_secret from steward.core.settings import source_token_key return decrypt_secret(src.get("http_token") or "", context=source_token_key(src.get("name", "?"))) def get_sources(ansible_cfg: dict) -> list[dict]: """Return resolved source list from ansible config section. Each source dict has: name, type, path (resolved local path), url (git only), branch (git only), pull_interval_seconds (git only). Config structure in config.yaml:: ansible: cache_dir: /data/ansible/sources sources: - name: my-playbooks type: local path: /opt/playbooks - name: infra-repo type: git url: https://github.com/user/infra.git branch: main pull_interval_seconds: 3600 """ sources = ansible_cfg.get("sources", []) cache_dir = ansible_cfg.get("cache_dir") or SOURCE_CACHE_DIR # Always expose the bundled (read-only) + local (writable) first-party sources. result = [_builtin_source(), _local_source()] for src in sources: src_type = src.get("type", "local") if src_type == "git": if not src.get("url"): raise ValueError(f"Ansible git source {src['name']!r} is missing required 'url' field") path = str(Path(cache_dir) / src["name"]) else: if not src.get("path"): raise ValueError(f"Ansible local source {src['name']!r} is missing required 'path' field") path = src.get("path", "") result.append({ "name": src["name"], "type": src_type, "path": path, "url": src.get("url"), "branch": src.get("branch", "main"), "pull_interval_seconds": int(src.get("pull_interval_seconds", 3600)), "http_token": _open_token(src) if src_type == "git" else "", }) return result def discover_playbooks(source_path: str) -> list[str]: """Recursively find .yml and .yaml files in source_path. Returns relative paths.""" root = Path(source_path) if not root.exists(): return [] playbooks = set() for ext in ("*.yml", "*.yaml"): for p in root.rglob(ext): playbooks.add(str(p.relative_to(root))) return sorted(playbooks) def discover_inventories(source_path: str) -> list[str]: """Non-recursive: return inventory filenames present in root of source_path.""" root = Path(source_path) if not root.exists(): return [] return sorted(name for name in INVENTORY_NAMES if (root / name).exists()) def host_inventory_content(host) -> str: """An ephemeral single-host inventory line for a Steward Host. Targets the host's address when set (`ansible_host=`), else just the name (resolved via DNS). Playbooks run against this should use `hosts: all` (or the host's name). Pure — `host` only needs `.name` and `.address`. """ if getattr(host, "address", ""): return f"{host.name} ansible_host={host.address}\n" return f"{host.name}\n" def read_playbook(source_path: str, relative_path: str) -> str | None: """Return contents of a playbook file, or None if not found / path escape.""" root = Path(source_path).resolve() target = (root / relative_path).resolve() # Guard against path traversal try: target.relative_to(root) except ValueError: return None if not target.exists() or not target.is_file(): return None return target.read_text(errors="replace") def _resolve_writable(source_path: str, relative_path: str) -> tuple[Path | None, str | None]: """Resolve relative_path under source_path, guarding traversal + extension. Returns (target_path, None) on success or (None, error). The root need not exist yet (steward-local is created on first save).""" rel = relative_path.strip().lstrip("/") if not rel: return None, "Filename is required" if not rel.endswith((".yml", ".yaml")): return None, "Playbook filename must end in .yml or .yaml" root = Path(source_path).resolve() target = (root / rel).resolve() try: target.relative_to(root) except ValueError: return None, "Invalid path" return target, None def write_playbook(source_path: str, relative_path: str, content: str) -> tuple[bool, str | None]: """Write a playbook into a source, creating parent dirs. Traversal-guarded.""" target, err = _resolve_writable(source_path, relative_path) if err: return False, err try: target.parent.mkdir(parents=True, exist_ok=True) target.write_text(content, encoding="utf-8") except OSError as exc: return False, f"Could not write file: {exc}" return True, None def delete_playbook(source_path: str, relative_path: str) -> tuple[bool, str | None]: """Delete a playbook from a source. Traversal-guarded; no-op if absent.""" target, err = _resolve_writable(source_path, relative_path) if err: return False, err try: if target.exists(): target.unlink() except OSError as exc: return False, f"Could not delete file: {exc}" return True, None def discover_playbook_description(content: str) -> str: """A human-readable description of what a playbook does (see discover_playbook_meta). Returns "" if none can be determined.""" return discover_playbook_meta(content)["description"] def discover_playbook_variables(content: str) -> list[dict]: """Parse a playbook and list the variables an operator can set at run time. Surfaces two sources, in this precedence (first wins on name collision): - ``vars_prompt:`` — Ansible's explicit "ask me" mechanism. ``required`` when it has no default; ``secret`` when ``private`` (Ansible's default is private=yes) or the name looks sensitive. - ``vars:`` scalar entries — shown with their default as a placeholder (NOT prefilled, so an untouched field falls through to inventory/play defaults rather than overriding them). Only top-level plays are inspected — role defaults and included var files are not traversed (kept simple + predictable). Returns a list of dicts: {name, default, secret, required, prompt}. """ import yaml try: plays = yaml.safe_load(content) except yaml.YAMLError: return [] if not isinstance(plays, list): return [] out: list[dict] = [] seen: set[str] = set() def add(name, default, secret, required, prompt=None): if not isinstance(name, str) or name in seen: return seen.add(name) out.append({ "name": name, "default": "" if default is None else default, "secret": secret, "required": required, "prompt": prompt, }) for play in plays: if not isinstance(play, dict): continue for vp in play.get("vars_prompt") or []: if not isinstance(vp, dict) or "name" not in vp: continue name = vp["name"] default = vp.get("default") private = bool(vp.get("private", True)) # Ansible defaults private=yes add(name, default, private or bool(_SECRET_VAR_RE.search(str(name))), default is None, vp.get("prompt")) play_vars = play.get("vars") if isinstance(play_vars, dict): for name, default in play_vars.items(): # Only scalar defaults map cleanly to a single input field. if isinstance(default, (str, int, float, bool)) or default is None: add(name, default, bool(_SECRET_VAR_RE.search(str(name))), False) return out def validate_playbook_yaml(content: str) -> tuple[bool, str | None]: """Cheap save-time validation: parses as YAML and is a non-empty play list.""" import yaml try: data = yaml.safe_load(content) except yaml.YAMLError as exc: return False, f"YAML error: {exc}" if data is None: return False, "Playbook is empty" if not isinstance(data, list): return False, "A playbook must be a list of plays (top-level YAML list)" return True, None # The askpass script holds no secret: git runs it with the token in its # environment, and printf prints it as data, so no character in a token can # become shell syntax. ASKPASS_TOKEN_ENV = "STEWARD_GIT_TOKEN" ASKPASS_SCRIPT = ( "#!/bin/sh\n" 'case "$1" in\n' " Username*) printf '%s\\n' oauth2 ;;\n" f" Password*) printf '%s\\n' \"${ASKPASS_TOKEN_ENV}\" ;;\n" "esac\n" ) # A clone or pull past this is cut, so a hung remote can't hold a source's # pull (or the Sync button) open forever (#5157). GIT_TIMEOUT_SECONDS = 120.0 class GitPullError(RuntimeError): """A clone or pull that failed or ran past its timeout; says which and why.""" async def _git(argv: list[str], env: dict, timeout: float, what: str) -> None: from steward.ansible.executor import _terminate_then_kill proc = await asyncio.create_subprocess_exec( *argv, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=env, start_new_session=True, # its own group, so a timeout ends ssh/helpers too ) try: _, stderr = await asyncio.wait_for(proc.communicate(), timeout) except asyncio.TimeoutError: raise GitPullError(f"{what} did not finish within {timeout:g} s") from None finally: if proc.returncode is None: await asyncio.shield(_terminate_then_kill(proc, grace=2.0)) if proc.returncode != 0: raise GitPullError(f"{what} failed: {stderr.decode(errors='replace').strip()[-500:]}") async def git_pull(source: dict, timeout: float = GIT_TIMEOUT_SECONDS) -> None: """Clone the git repo if absent; fast-forward it if already present. Raises GitPullError when git fails or runs past `timeout`; a clone that did not finish is removed, so the next attempt clones afresh rather than pulling into a broken checkout. git never prompts (GIT_TERMINAL_PROMPT=0): a missing credential fails at once instead of waiting on input. When source['http_token'] is set, git asks a temporary GIT_ASKPASS script, which prints the token from the child's environment, so it never touches .git/config, process args or a file. """ path = Path(source["path"]) token = source.get("http_token", "") env = {**os.environ, "GIT_TERMINAL_PROMPT": "0"} askpass_path = None if token: fd, askpass_path = tempfile.mkstemp(prefix="steward-askpass-", suffix=".sh") try: os.write(fd, ASKPASS_SCRIPT.encode()) finally: os.close(fd) os.chmod(askpass_path, stat.S_IRWXU) env.update({"GIT_ASKPASS": askpass_path, ASKPASS_TOKEN_ENV: token}) try: if not (path / ".git").exists(): path.mkdir(parents=True, exist_ok=True) try: await _git(["git", "clone", "--branch", source["branch"], "--single-branch", source["url"], str(path)], env, timeout, f"git clone of {source['name']!r}") except BaseException: shutil.rmtree(path, ignore_errors=True) raise else: await _git(["git", "-C", str(path), "pull", "--ff-only"], env, timeout, f"git pull of {source['name']!r}") finally: if askpass_path and os.path.exists(askpass_path): os.unlink(askpass_path) # When each git source was last pulled (monotonic seconds), by name. _last_pull: dict[str, float] = {} async def pull_due_sources(ansible_cfg: dict, now: float | None = None) -> list[str]: """Pull every git source whose pull_interval_seconds has passed. Run by one scheduled task on a short tick, reading the sources as they are now, so a source added in Settings is pulled within a tick and an edited interval applies without a restart (#5157). A failed pull is logged and waits out its interval like a good one. Returns the names pulled. """ now = time.monotonic() if now is None else now sources = get_sources(ansible_cfg) due = [src for src in sources if src["type"] == "git" and now - _last_pull.get(src["name"], float("-inf")) >= src["pull_interval_seconds"]] names = {src["name"] for src in sources} for gone in [n for n in _last_pull if n not in names]: del _last_pull[gone] for src in due: _last_pull[src["name"]] = now try: await git_pull(src) except Exception: logger.exception("Ansible source %r: pull failed", src["name"]) return [src["name"] for src in due]