feat(alerts): run an Ansible playbook on alert firing (task 250)

Rebuilds the deleted NUT/UPS automation as a general alert action: any metric
can drive a playbook run on transition-to-firing.

- models/alerts.py + migration 0014: AlertRule.ansible_action (JSON, admin-only,
  reuses the #546 param shape); AlertEvent.ansible_run_id links a firing event to
  the run it triggered
- core/alerts.py: pure alert_extra_vars() injects steward_alert_* context; on
  ('firing', event) with an action set, schedule _run_ansible_action (deferred
  after commit, same pattern as notifications) — fires once per transition,
  consecutive_failures_required is the debounce; system-triggered AnsibleRun
- alerts/routes.py: admin-only parse/validate of the action (source must be
  configured, playbook must exist); operators keep editing rules, action preserved
- rules_form.html: admin-only 'On firing -> run a playbook' section
- tests: unit for alert_extra_vars; integration drives record_metric to firing
  and asserts a system AnsibleRun ran the playbook with the injected var and the
  event linked to it

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

steward/alerts/routes.py

@@ -4,6 +4,7 @@ from quart import Blueprint, render_template, request, redirect, url_for, curren
 from steward.core.audit import log_audit
 from sqlalchemy import select
 from steward.auth.middleware import require_role
+from steward.ansible import sources as src_module
 from steward.models.alerts import AlertRule, AlertState, AlertStateEnum, AlertOperator, MaintenanceWindow
 from steward.models.hosts import Host
 from steward.models.metrics import PluginMetric
@@ -56,6 +57,61 @@ async def _get_resources(db, source_module: str) -> list[str]:
     return sorted(resources)
 
 
+def _is_admin() -> bool:
+    return session.get("user_role") == UserRole.admin.value
+
+
+def _ansible_source_names() -> list[str]:
+    try:
+        return [s["name"] for s in src_module.get_sources(current_app.config.get("ANSIBLE", {}))]
+    except Exception:
+        return []
+
+
+def _parse_ansible_action(form) -> tuple[dict | None, str | None]:
+    """Parse + validate the admin-only Ansible action from a rule form.
+
+    Returns (action_or_None, error_or_None). action is None when the action
+    checkbox is unchecked (i.e. the action is being cleared).
+    """
+    if "ansible_enabled" not in form:
+        return None, None
+    source_name = (form.get("ansible_source", "") or "").strip()
+    playbook = (form.get("ansible_playbook", "") or "").strip()
+    inventory = (form.get("ansible_inventory", "") or "").strip()
+    if not (source_name and playbook and inventory):
+        return None, "Ansible action requires source, playbook, and inventory"
+    try:
+        srcs = src_module.get_sources(current_app.config.get("ANSIBLE", {}))
+    except Exception:
+        return None, "Ansible sources are misconfigured"
+    source = next((s for s in srcs if s["name"] == source_name), None)
+    if source is None:
+        return None, f"Ansible source '{source_name}' not found"
+    if playbook not in src_module.discover_playbooks(source["path"]):
+        return None, f"Playbook '{playbook}' not found in source '{source_name}'"
+    extra_vars: list[str] = []
+    for line in (form.get("ansible_extra_vars", "") or "").splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        if "=" not in line:
+            return None, f"Invalid extra var (expected key=value): {line!r}"
+        extra_vars.append(line)
+    action: dict = {"source": source_name, "playbook": playbook, "inventory": inventory}
+    if extra_vars:
+        action["extra_vars"] = extra_vars
+    limit = (form.get("ansible_limit", "") or "").strip()
+    tags = (form.get("ansible_tags", "") or "").strip()
+    if limit:
+        action["limit"] = limit
+    if tags:
+        action["tags"] = tags
+    if "ansible_check" in form:
+        action["check"] = True
+    return action, None
+
+
 @alerts_bp.get("/")
 @require_role(UserRole.viewer)
 async def list_alerts():
@@ -120,6 +176,7 @@ async def new_rule():
         initial_source=first_source,
         resources=resources,
         metrics=METRIC_CATALOG.get(first_source, []),
+        ansible_sources=_ansible_source_names(),
     )
 
 
@@ -140,6 +197,7 @@ async def edit_rule(rule_id: str):
         initial_source=rule.source_module,
         resources=resources,
         metrics=METRIC_CATALOG.get(rule.source_module, []),
+        ansible_sources=_ansible_source_names(),
     )
 
 
@@ -149,6 +207,12 @@ async def create_rule():
     form = await request.form
     user_id = session.get("user_id")
     now = datetime.now(timezone.utc)
+    # Ansible action is admin-only; non-admins simply can't set one.
+    ansible_action = None
+    if _is_admin():
+        ansible_action, err = _parse_ansible_action(form)
+        if err:
+            return err, 400
     rule = AlertRule(
         name=form["name"].strip(),
         source_module=form["source_module"].strip(),
@@ -158,6 +222,7 @@ async def create_rule():
         threshold=float(form["threshold"]),
         consecutive_failures_required=int(form.get("consecutive_failures_required") or 1),
         enabled=True,
+        ansible_action=ansible_action,
         created_by=user_id,
     )
     state = AlertState(
@@ -182,6 +247,13 @@ async def create_rule():
 @require_role(UserRole.operator)
 async def update_rule(rule_id: str):
     form = await request.form
+    # Validate the admin-only action before opening the transaction. Non-admins
+    # leave the existing action untouched (they can still edit everything else).
+    apply_action = _is_admin()
+    if apply_action:
+        ansible_action, err = _parse_ansible_action(form)
+        if err:
+            return err, 400
     async with current_app.db_sessionmaker() as db:
         async with db.begin():
             result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
@@ -195,6 +267,8 @@ async def update_rule(rule_id: str):
             rule.operator = AlertOperator(form["operator"])
             rule.threshold = float(form["threshold"])
             rule.consecutive_failures_required = int(form.get("consecutive_failures_required") or 1)
+            if apply_action:
+                rule.ansible_action = ansible_action
     await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                     "alert_rule.updated", entity_type="alert_rule", entity_id=rule.name)
     return redirect(url_for("alerts.list_alerts"))

steward/core/alerts.py

@@ -32,6 +32,23 @@ def init_alerts(app: "Quart") -> None:
     _app = app
 
 
+def alert_extra_vars(rule: AlertRule, value: float) -> list[str]:
+    """Ansible extra-vars describing the alert that fired, as `key=value` strings.
+
+    Injected into every alert-triggered playbook run so the playbook can target
+    the exact resource/metric that fired without templating. Pure function.
+    """
+    return [
+        f"steward_alert_rule={rule.name}",
+        f"steward_alert_source={rule.source_module}",
+        f"steward_alert_resource={rule.resource_name}",
+        f"steward_alert_metric={rule.metric_name}",
+        f"steward_alert_value={value}",
+        f"steward_alert_threshold={rule.threshold}",
+        f"steward_alert_operator={rule.operator.value}",
+    ]
+
+
 async def record_metric(
     session: AsyncSession,
     source_module: str,
@@ -110,6 +127,11 @@ async def record_metric(
                         _app, notif_type, rule, value, event_id
                     )
                 )
+                # Admin-configured Ansible action runs only on transition-to-firing.
+                if notif_type == "firing" and rule.ansible_action:
+                    asyncio.create_task(
+                        _run_ansible_action(_app, rule, value, event_id)
+                    )
 
 
 def _is_breached(value: float, operator: AlertOperator, threshold: float) -> bool:
@@ -259,3 +281,67 @@ async def _dispatch_notification(
                 if webhook_r:
                     event.webhook_sent = webhook_r["sent"]
                     event.webhook_error = webhook_r.get("error")
+
+
+async def _run_ansible_action(
+    app: "Quart",
+    rule: AlertRule,
+    value: float,
+    event_id: str,
+) -> None:
+    """Run the rule's configured Ansible playbook (system-triggered).
+
+    Runs after the evaluation transaction commits (scheduled via create_task).
+    Creates a system-triggered AnsibleRun, links it to the firing AlertEvent,
+    then executes it with the static params plus injected alert context vars.
+    """
+    from steward.ansible import executor, sources as src_module
+    from steward.models.ansible import AnsibleRun, AnsibleRunStatus
+
+    action = rule.ansible_action or {}
+    playbook = action.get("playbook")
+    inventory = action.get("inventory")
+    if not playbook or not inventory:
+        logger.error("Ansible action for rule %r missing playbook/inventory", rule.name)
+        return
+
+    try:
+        srcs = src_module.get_sources(app.config.get("ANSIBLE", {}))
+    except Exception:
+        logger.exception("Ansible action for rule %r: could not load sources", rule.name)
+        return
+    source = next((s for s in srcs if s["name"] == action.get("source")), None)
+    if source is None:
+        logger.error("Ansible action for rule %r: source %r not configured",
+                     rule.name, action.get("source"))
+        return
+
+    # Static params + auto-injected alert context (all argv, never shell).
+    params: dict = {
+        "extra_vars": list(action.get("extra_vars") or []) + alert_extra_vars(rule, value),
+    }
+    if action.get("limit"):
+        params["limit"] = action["limit"]
+    if action.get("tags"):
+        params["tags"] = action["tags"]
+    if action.get("check"):
+        params["check"] = True
+
+    run_id = str(uuid.uuid4())
+    async with app.db_sessionmaker() as db:
+        async with db.begin():
+            db.add(AnsibleRun(
+                id=run_id,
+                playbook_path=playbook,
+                inventory_path=inventory,
+                source_name=source["name"],
+                triggered_by=None,
+                status=AnsibleRunStatus.running,
+                params=params,
+            ))
+            event = (await db.execute(
+                select(AlertEvent).where(AlertEvent.id == event_id))).scalar_one_or_none()
+            if event:
+                event.ansible_run_id = run_id
+
+    await executor.start_run(app, run_id, playbook, inventory, source["path"], params)

steward/migrations/versions/0014_alert_ansible_action.py

@@ -0,0 +1,31 @@
+"""Alert Ansible action: alert_rules.ansible_action + alert_events.ansible_run_id
+
+Revision ID: 0014_alert_ansible_action
+Revises: 0013_ansible_run_params
+Create Date: 2026-06-02
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "0014_alert_ansible_action"
+down_revision: Union[str, None] = "0013_ansible_run_params"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("alert_rules", sa.Column("ansible_action", sa.JSON(), nullable=True))
+    op.add_column(
+        "alert_events",
+        sa.Column(
+            "ansible_run_id", sa.String(36),
+            sa.ForeignKey("ansible_runs.id", ondelete="SET NULL"),
+            nullable=True,
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("alert_events", "ansible_run_id")
+    op.drop_column("alert_rules", "ansible_action")

steward/models/alerts.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 import enum
 import uuid
 from datetime import datetime, timezone
-from sqlalchemy import Boolean, DateTime, Enum, Float, ForeignKey, Integer, String, Text
+from sqlalchemy import JSON, Boolean, DateTime, Enum, Float, ForeignKey, Integer, String, Text
 from sqlalchemy.orm import Mapped, mapped_column
 from .base import Base
 
@@ -36,6 +36,9 @@ class AlertRule(Base):
     threshold: Mapped[float] = mapped_column(Float, nullable=False)
     consecutive_failures_required: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
     enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+    # Optional admin-only action: run an Ansible playbook on transition-to-firing.
+    # Shape: {source, playbook, inventory, extra_vars, limit, tags, check}. NULL = none.
+    ansible_action: Mapped[dict | None] = mapped_column(JSON, nullable=True)
     created_by: Mapped[str] = mapped_column(String(36), ForeignKey("users.id", ondelete="RESTRICT"), nullable=False)
     created_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
@@ -97,3 +100,7 @@ class AlertEvent(Base):
     email_error: Mapped[str | None] = mapped_column(Text, nullable=True)
     webhook_sent: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
     webhook_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    # Set when this firing event triggered an Ansible action run (audit link).
+    ansible_run_id: Mapped[str | None] = mapped_column(
+        String(36), ForeignKey("ansible_runs.id", ondelete="SET NULL"), nullable=True
+    )

steward/templates/alerts/rules_form.html

@@ -128,6 +128,57 @@
         </div>
       </details>
 
+      {# ── Ansible action (admin only) ─────────────────────────────────────── #}
+      {% if session.get("user_role") == "admin" %}
+      {% set act = rule.ansible_action if rule and rule.ansible_action else None %}
+      <div style="margin:1.25rem 0;padding:1rem;border:1px solid var(--border-mid);border-radius:6px;background:var(--bg-elevated);">
+        <label style="display:flex;align-items:center;gap:0.5rem;font-weight:600;">
+          <input type="checkbox" name="ansible_enabled" {% if act %}checked{% endif %}>
+          On firing → run an Ansible playbook
+        </label>
+        <p style="font-size:0.78rem;color:var(--text-muted);margin:0.4rem 0 0.75rem;">
+          Admin only. Runs once when the alert transitions into FIRING.
+          <code>steward_alert_*</code> context vars (resource, metric, value…) are injected automatically.
+        </p>
+        <div class="form-group">
+          <label>Source</label>
+          <select name="ansible_source">
+            <option value="">— select source —</option>
+            {% for s in ansible_sources %}
+            <option value="{{ s }}" {% if act and act.source == s %}selected{% endif %}>{{ s }}</option>
+            {% endfor %}
+          </select>
+        </div>
+        <div class="form-group">
+          <label>Playbook <span style="color:var(--text-muted);font-weight:normal;">(path within source)</span></label>
+          <input type="text" name="ansible_playbook" placeholder="playbooks/shutdown.yml"
+                 value="{{ act.playbook if act else '' }}">
+        </div>
+        <div class="form-group">
+          <label>Inventory <span style="color:var(--text-muted);font-weight:normal;">(path within source)</span></label>
+          <input type="text" name="ansible_inventory" placeholder="inventory/hosts"
+                 value="{{ act.inventory if act else '' }}">
+        </div>
+        <div class="form-group">
+          <label>Extra vars <span style="color:var(--text-muted);font-weight:normal;">(optional, one key=value per line)</span></label>
+          <textarea name="ansible_extra_vars" rows="2"
+                    style="width:100%;font-family:ui-monospace,monospace;font-size:0.85rem;">{{ act.extra_vars | join("\n") if act and act.extra_vars else "" }}</textarea>
+        </div>
+        <div class="form-group">
+          <label>Limit to host(s) <span style="color:var(--text-muted);font-weight:normal;">(optional)</span></label>
+          <input type="text" name="ansible_limit" value="{{ act.limit if act and act.limit else '' }}">
+        </div>
+        <div class="form-group">
+          <label>Tags <span style="color:var(--text-muted);font-weight:normal;">(optional)</span></label>
+          <input type="text" name="ansible_tags" value="{{ act.tags if act and act.tags else '' }}">
+        </div>
+        <label style="display:flex;align-items:center;gap:0.5rem;font-weight:normal;">
+          <input type="checkbox" name="ansible_check" {% if act and act.check %}checked{% endif %}>
+          Dry-run (--check --diff)
+        </label>
+      </div>
+      {% endif %}
+
       <div style="display:flex;gap:1rem;margin-top:1.5rem;">
         <button type="submit" class="btn">{% if rule %}Save Changes{% else %}Create Rule{% endif %}</button>
         <a href="/alerts/" class="btn btn-ghost">Cancel</a>

tests/integration/test_alert_ansible_action.py

@@ -0,0 +1,99 @@
+"""Integration: an alert transition-to-firing triggers its Ansible action (task 250).
+
+Drives the full chain against a live Postgres + real Ansible: a breaching metric
+fires the rule, which schedules a system-triggered AnsibleRun that runs a real
+connection:local playbook receiving the injected steward_alert_* context, and the
+firing AlertEvent links to the run.
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import textwrap
+import uuid
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+_NEEDS_DB = pytest.mark.skipif(
+    not os.environ.get("STEWARD_DATABASE_URL"),
+    reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)",
+)
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+@_NEEDS_DB
+def test_firing_triggers_ansible_action(app, tmp_path):
+    from sqlalchemy import select
+    from steward.core.alerts import record_metric
+    from steward.models.alerts import (
+        AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum,
+    )
+    from steward.models.ansible import AnsibleRun, AnsibleRunStatus
+    from steward.models.users import User, UserRole
+
+    (tmp_path / "fire.yml").write_text(textwrap.dedent("""\
+        - hosts: localhost
+          connection: local
+          gather_facts: false
+          tasks:
+            - debug:
+                msg: "FIRED-{{ steward_alert_metric }}"
+    """))
+    (tmp_path / "inv.ini").write_text("localhost ansible_connection=local\n")
+
+    # An ansible source the action resolves against.
+    app.config["ANSIBLE"] = {
+        "sources": [{"name": "t", "type": "local", "path": str(tmp_path)}],
+    }
+
+    uid = str(uuid.uuid4())
+    rule_id = str(uuid.uuid4())
+    action = {"source": "t", "playbook": "fire.yml", "inventory": "inv.ini"}
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add(User(
+                    id=uid, username=f"u{uid[:8]}", email=f"{uid[:8]}@e.test",
+                    password_hash="x", role=UserRole.admin, is_active=True,
+                ))
+                s.add(AlertRule(
+                    id=rule_id, name="cpu hot", source_module="host_agent",
+                    resource_name="srv1", metric_name="cpu_pct",
+                    operator=AlertOperator.gt, threshold=90.0,
+                    consecutive_failures_required=1, enabled=True,
+                    ansible_action=action, created_by=uid,
+                ))
+                s.add(AlertState(rule_id=rule_id, state=AlertStateEnum.inactive))
+
+        # Breaching value → transition to firing → schedules the action task.
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await record_metric(s, "host_agent", "srv1", "cpu_pct", 95.0)
+
+        # Let the fire-and-forget dispatch + action tasks complete.
+        pending = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
+        await asyncio.gather(*pending, return_exceptions=True)
+
+        async with app.db_sessionmaker() as s:
+            run = (await s.execute(
+                select(AnsibleRun).where(AnsibleRun.source_name == "t"))).scalars().first()
+            event = (await s.execute(
+                select(AlertEvent).where(AlertEvent.rule_id == rule_id))).scalar_one()
+            return run, event
+
+    run, event = asyncio.run(_go())
+    assert run is not None, "no AnsibleRun was created by the firing rule"
+    assert run.triggered_by is None  # system-triggered
+    assert run.status == AnsibleRunStatus.success, run.output
+    assert "FIRED-cpu_pct" in (run.output or "")  # injected context var reached the play
+    assert event.ansible_run_id == run.id  # firing event linked to the run

tests/test_alert_ansible.py

@@ -0,0 +1,30 @@
+"""Unit tests for alert -> Ansible action context vars (task 250)."""
+from steward.core.alerts import alert_extra_vars
+from steward.models.alerts import AlertOperator, AlertRule
+
+
+def test_alert_extra_vars_describes_the_firing():
+    rule = AlertRule(
+        name="UPS low battery", source_module="snmp", resource_name="ups",
+        metric_name="battery_status", operator=AlertOperator.lt, threshold=3.0,
+        created_by="u",
+    )
+    ev = alert_extra_vars(rule, 2.0)
+    assert "steward_alert_rule=UPS low battery" in ev
+    assert "steward_alert_source=snmp" in ev
+    assert "steward_alert_resource=ups" in ev
+    assert "steward_alert_metric=battery_status" in ev
+    assert "steward_alert_value=2.0" in ev
+    assert "steward_alert_threshold=3.0" in ev
+    assert "steward_alert_operator=<" in ev
+
+
+def test_alert_extra_vars_are_key_value_strings():
+    rule = AlertRule(
+        name="r", source_module="host_agent", resource_name="srv1",
+        metric_name="cpu_pct", operator=AlertOperator.gt, threshold=90.0,
+        created_by="u",
+    )
+    ev = alert_extra_vars(rule, 95.5)
+    assert all("=" in item for item in ev)
+    assert all(item.startswith("steward_alert_") for item in ev)