From 4771d17f6d2a3e90d88d7698dd0387007c113a1e Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 2 Jun 2026 14:22:29 -0400 Subject: [PATCH] feat(alerts): run an Ansible playbook on alert firing (task 250) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebuilds the deleted NUT/UPS automation as a general alert action: any metric can drive a playbook run on transition-to-firing. - models/alerts.py + migration 0014: AlertRule.ansible_action (JSON, admin-only, reuses the #546 param shape); AlertEvent.ansible_run_id links a firing event to the run it triggered - core/alerts.py: pure alert_extra_vars() injects steward_alert_* context; on ('firing', event) with an action set, schedule _run_ansible_action (deferred after commit, same pattern as notifications) — fires once per transition, consecutive_failures_required is the debounce; system-triggered AnsibleRun - alerts/routes.py: admin-only parse/validate of the action (source must be configured, playbook must exist); operators keep editing rules, action preserved - rules_form.html: admin-only 'On firing -> run a playbook' section - tests: unit for alert_extra_vars; integration drives record_metric to firing and asserts a system AnsibleRun ran the playbook with the injected var and the event linked to it Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/alerts/routes.py | 74 ++++++++++++++ steward/core/alerts.py | 86 ++++++++++++++++ .../versions/0014_alert_ansible_action.py | 31 ++++++ steward/models/alerts.py | 9 +- steward/templates/alerts/rules_form.html | 51 ++++++++++ .../integration/test_alert_ansible_action.py | 99 +++++++++++++++++++ tests/test_alert_ansible.py | 30 ++++++ 7 files changed, 379 insertions(+), 1 deletion(-) create mode 100644 steward/migrations/versions/0014_alert_ansible_action.py create mode 100644 tests/integration/test_alert_ansible_action.py create mode 100644 tests/test_alert_ansible.py diff --git a/steward/alerts/routes.py b/steward/alerts/routes.py index 9c71119..fd26249 100644 --- a/steward/alerts/routes.py +++ b/steward/alerts/routes.py @@ -4,6 +4,7 @@ from quart import Blueprint, render_template, request, redirect, url_for, curren from steward.core.audit import log_audit from sqlalchemy import select from steward.auth.middleware import require_role +from steward.ansible import sources as src_module from steward.models.alerts import AlertRule, AlertState, AlertStateEnum, AlertOperator, MaintenanceWindow from steward.models.hosts import Host from steward.models.metrics import PluginMetric @@ -56,6 +57,61 @@ async def _get_resources(db, source_module: str) -> list[str]: return sorted(resources) +def _is_admin() -> bool: + return session.get("user_role") == UserRole.admin.value + + +def _ansible_source_names() -> list[str]: + try: + return [s["name"] for s in src_module.get_sources(current_app.config.get("ANSIBLE", {}))] + except Exception: + return [] + + +def _parse_ansible_action(form) -> tuple[dict | None, str | None]: + """Parse + validate the admin-only Ansible action from a rule form. + + Returns (action_or_None, error_or_None). action is None when the action + checkbox is unchecked (i.e. the action is being cleared). + """ + if "ansible_enabled" not in form: + return None, None + source_name = (form.get("ansible_source", "") or "").strip() + playbook = (form.get("ansible_playbook", "") or "").strip() + inventory = (form.get("ansible_inventory", "") or "").strip() + if not (source_name and playbook and inventory): + return None, "Ansible action requires source, playbook, and inventory" + try: + srcs = src_module.get_sources(current_app.config.get("ANSIBLE", {})) + except Exception: + return None, "Ansible sources are misconfigured" + source = next((s for s in srcs if s["name"] == source_name), None) + if source is None: + return None, f"Ansible source '{source_name}' not found" + if playbook not in src_module.discover_playbooks(source["path"]): + return None, f"Playbook '{playbook}' not found in source '{source_name}'" + extra_vars: list[str] = [] + for line in (form.get("ansible_extra_vars", "") or "").splitlines(): + line = line.strip() + if not line: + continue + if "=" not in line: + return None, f"Invalid extra var (expected key=value): {line!r}" + extra_vars.append(line) + action: dict = {"source": source_name, "playbook": playbook, "inventory": inventory} + if extra_vars: + action["extra_vars"] = extra_vars + limit = (form.get("ansible_limit", "") or "").strip() + tags = (form.get("ansible_tags", "") or "").strip() + if limit: + action["limit"] = limit + if tags: + action["tags"] = tags + if "ansible_check" in form: + action["check"] = True + return action, None + + @alerts_bp.get("/") @require_role(UserRole.viewer) async def list_alerts(): @@ -120,6 +176,7 @@ async def new_rule(): initial_source=first_source, resources=resources, metrics=METRIC_CATALOG.get(first_source, []), + ansible_sources=_ansible_source_names(), ) @@ -140,6 +197,7 @@ async def edit_rule(rule_id: str): initial_source=rule.source_module, resources=resources, metrics=METRIC_CATALOG.get(rule.source_module, []), + ansible_sources=_ansible_source_names(), ) @@ -149,6 +207,12 @@ async def create_rule(): form = await request.form user_id = session.get("user_id") now = datetime.now(timezone.utc) + # Ansible action is admin-only; non-admins simply can't set one. + ansible_action = None + if _is_admin(): + ansible_action, err = _parse_ansible_action(form) + if err: + return err, 400 rule = AlertRule( name=form["name"].strip(), source_module=form["source_module"].strip(), @@ -158,6 +222,7 @@ async def create_rule(): threshold=float(form["threshold"]), consecutive_failures_required=int(form.get("consecutive_failures_required") or 1), enabled=True, + ansible_action=ansible_action, created_by=user_id, ) state = AlertState( @@ -182,6 +247,13 @@ async def create_rule(): @require_role(UserRole.operator) async def update_rule(rule_id: str): form = await request.form + # Validate the admin-only action before opening the transaction. Non-admins + # leave the existing action untouched (they can still edit everything else). + apply_action = _is_admin() + if apply_action: + ansible_action, err = _parse_ansible_action(form) + if err: + return err, 400 async with current_app.db_sessionmaker() as db: async with db.begin(): result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id)) @@ -195,6 +267,8 @@ async def update_rule(rule_id: str): rule.operator = AlertOperator(form["operator"]) rule.threshold = float(form["threshold"]) rule.consecutive_failures_required = int(form.get("consecutive_failures_required") or 1) + if apply_action: + rule.ansible_action = ansible_action await log_audit(current_app, session.get("user_id"), session.get("username", ""), "alert_rule.updated", entity_type="alert_rule", entity_id=rule.name) return redirect(url_for("alerts.list_alerts")) diff --git a/steward/core/alerts.py b/steward/core/alerts.py index eebd55c..40089cb 100644 --- a/steward/core/alerts.py +++ b/steward/core/alerts.py @@ -32,6 +32,23 @@ def init_alerts(app: "Quart") -> None: _app = app +def alert_extra_vars(rule: AlertRule, value: float) -> list[str]: + """Ansible extra-vars describing the alert that fired, as `key=value` strings. + + Injected into every alert-triggered playbook run so the playbook can target + the exact resource/metric that fired without templating. Pure function. + """ + return [ + f"steward_alert_rule={rule.name}", + f"steward_alert_source={rule.source_module}", + f"steward_alert_resource={rule.resource_name}", + f"steward_alert_metric={rule.metric_name}", + f"steward_alert_value={value}", + f"steward_alert_threshold={rule.threshold}", + f"steward_alert_operator={rule.operator.value}", + ] + + async def record_metric( session: AsyncSession, source_module: str, @@ -110,6 +127,11 @@ async def record_metric( _app, notif_type, rule, value, event_id ) ) + # Admin-configured Ansible action runs only on transition-to-firing. + if notif_type == "firing" and rule.ansible_action: + asyncio.create_task( + _run_ansible_action(_app, rule, value, event_id) + ) def _is_breached(value: float, operator: AlertOperator, threshold: float) -> bool: @@ -259,3 +281,67 @@ async def _dispatch_notification( if webhook_r: event.webhook_sent = webhook_r["sent"] event.webhook_error = webhook_r.get("error") + + +async def _run_ansible_action( + app: "Quart", + rule: AlertRule, + value: float, + event_id: str, +) -> None: + """Run the rule's configured Ansible playbook (system-triggered). + + Runs after the evaluation transaction commits (scheduled via create_task). + Creates a system-triggered AnsibleRun, links it to the firing AlertEvent, + then executes it with the static params plus injected alert context vars. + """ + from steward.ansible import executor, sources as src_module + from steward.models.ansible import AnsibleRun, AnsibleRunStatus + + action = rule.ansible_action or {} + playbook = action.get("playbook") + inventory = action.get("inventory") + if not playbook or not inventory: + logger.error("Ansible action for rule %r missing playbook/inventory", rule.name) + return + + try: + srcs = src_module.get_sources(app.config.get("ANSIBLE", {})) + except Exception: + logger.exception("Ansible action for rule %r: could not load sources", rule.name) + return + source = next((s for s in srcs if s["name"] == action.get("source")), None) + if source is None: + logger.error("Ansible action for rule %r: source %r not configured", + rule.name, action.get("source")) + return + + # Static params + auto-injected alert context (all argv, never shell). + params: dict = { + "extra_vars": list(action.get("extra_vars") or []) + alert_extra_vars(rule, value), + } + if action.get("limit"): + params["limit"] = action["limit"] + if action.get("tags"): + params["tags"] = action["tags"] + if action.get("check"): + params["check"] = True + + run_id = str(uuid.uuid4()) + async with app.db_sessionmaker() as db: + async with db.begin(): + db.add(AnsibleRun( + id=run_id, + playbook_path=playbook, + inventory_path=inventory, + source_name=source["name"], + triggered_by=None, + status=AnsibleRunStatus.running, + params=params, + )) + event = (await db.execute( + select(AlertEvent).where(AlertEvent.id == event_id))).scalar_one_or_none() + if event: + event.ansible_run_id = run_id + + await executor.start_run(app, run_id, playbook, inventory, source["path"], params) diff --git a/steward/migrations/versions/0014_alert_ansible_action.py b/steward/migrations/versions/0014_alert_ansible_action.py new file mode 100644 index 0000000..6048301 --- /dev/null +++ b/steward/migrations/versions/0014_alert_ansible_action.py @@ -0,0 +1,31 @@ +"""Alert Ansible action: alert_rules.ansible_action + alert_events.ansible_run_id + +Revision ID: 0014_alert_ansible_action +Revises: 0013_ansible_run_params +Create Date: 2026-06-02 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "0014_alert_ansible_action" +down_revision: Union[str, None] = "0013_ansible_run_params" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("alert_rules", sa.Column("ansible_action", sa.JSON(), nullable=True)) + op.add_column( + "alert_events", + sa.Column( + "ansible_run_id", sa.String(36), + sa.ForeignKey("ansible_runs.id", ondelete="SET NULL"), + nullable=True, + ), + ) + + +def downgrade() -> None: + op.drop_column("alert_events", "ansible_run_id") + op.drop_column("alert_rules", "ansible_action") diff --git a/steward/models/alerts.py b/steward/models/alerts.py index 88e85e6..c69b3cf 100644 --- a/steward/models/alerts.py +++ b/steward/models/alerts.py @@ -2,7 +2,7 @@ from __future__ import annotations import enum import uuid from datetime import datetime, timezone -from sqlalchemy import Boolean, DateTime, Enum, Float, ForeignKey, Integer, String, Text +from sqlalchemy import JSON, Boolean, DateTime, Enum, Float, ForeignKey, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -36,6 +36,9 @@ class AlertRule(Base): threshold: Mapped[float] = mapped_column(Float, nullable=False) consecutive_failures_required: Mapped[int] = mapped_column(Integer, nullable=False, default=1) enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + # Optional admin-only action: run an Ansible playbook on transition-to-firing. + # Shape: {source, playbook, inventory, extra_vars, limit, tags, check}. NULL = none. + ansible_action: Mapped[dict | None] = mapped_column(JSON, nullable=True) created_by: Mapped[str] = mapped_column(String(36), ForeignKey("users.id", ondelete="RESTRICT"), nullable=False) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) @@ -97,3 +100,7 @@ class AlertEvent(Base): email_error: Mapped[str | None] = mapped_column(Text, nullable=True) webhook_sent: Mapped[bool | None] = mapped_column(Boolean, nullable=True) webhook_error: Mapped[str | None] = mapped_column(Text, nullable=True) + # Set when this firing event triggered an Ansible action run (audit link). + ansible_run_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("ansible_runs.id", ondelete="SET NULL"), nullable=True + ) diff --git a/steward/templates/alerts/rules_form.html b/steward/templates/alerts/rules_form.html index 99ded85..6805827 100644 --- a/steward/templates/alerts/rules_form.html +++ b/steward/templates/alerts/rules_form.html @@ -128,6 +128,57 @@ + {# ── Ansible action (admin only) ─────────────────────────────────────── #} + {% if session.get("user_role") == "admin" %} + {% set act = rule.ansible_action if rule and rule.ansible_action else None %} +
+ +

+ Admin only. Runs once when the alert transitions into FIRING. + steward_alert_* context vars (resource, metric, value…) are injected automatically. +

+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ +
+ {% endif %} +
Cancel diff --git a/tests/integration/test_alert_ansible_action.py b/tests/integration/test_alert_ansible_action.py new file mode 100644 index 0000000..e1d2ca2 --- /dev/null +++ b/tests/integration/test_alert_ansible_action.py @@ -0,0 +1,99 @@ +"""Integration: an alert transition-to-firing triggers its Ansible action (task 250). + +Drives the full chain against a live Postgres + real Ansible: a breaching metric +fires the rule, which schedules a system-triggered AnsibleRun that runs a real +connection:local playbook receiving the injected steward_alert_* context, and the +firing AlertEvent links to the run. +""" +from __future__ import annotations + +import asyncio +import os +import textwrap +import uuid + +import pytest + +pytestmark = pytest.mark.integration + +_NEEDS_DB = pytest.mark.skipif( + not os.environ.get("STEWARD_DATABASE_URL"), + reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)", +) + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +@_NEEDS_DB +def test_firing_triggers_ansible_action(app, tmp_path): + from sqlalchemy import select + from steward.core.alerts import record_metric + from steward.models.alerts import ( + AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, + ) + from steward.models.ansible import AnsibleRun, AnsibleRunStatus + from steward.models.users import User, UserRole + + (tmp_path / "fire.yml").write_text(textwrap.dedent("""\ + - hosts: localhost + connection: local + gather_facts: false + tasks: + - debug: + msg: "FIRED-{{ steward_alert_metric }}" + """)) + (tmp_path / "inv.ini").write_text("localhost ansible_connection=local\n") + + # An ansible source the action resolves against. + app.config["ANSIBLE"] = { + "sources": [{"name": "t", "type": "local", "path": str(tmp_path)}], + } + + uid = str(uuid.uuid4()) + rule_id = str(uuid.uuid4()) + action = {"source": "t", "playbook": "fire.yml", "inventory": "inv.ini"} + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add(User( + id=uid, username=f"u{uid[:8]}", email=f"{uid[:8]}@e.test", + password_hash="x", role=UserRole.admin, is_active=True, + )) + s.add(AlertRule( + id=rule_id, name="cpu hot", source_module="host_agent", + resource_name="srv1", metric_name="cpu_pct", + operator=AlertOperator.gt, threshold=90.0, + consecutive_failures_required=1, enabled=True, + ansible_action=action, created_by=uid, + )) + s.add(AlertState(rule_id=rule_id, state=AlertStateEnum.inactive)) + + # Breaching value → transition to firing → schedules the action task. + async with app.db_sessionmaker() as s: + async with s.begin(): + await record_metric(s, "host_agent", "srv1", "cpu_pct", 95.0) + + # Let the fire-and-forget dispatch + action tasks complete. + pending = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()] + await asyncio.gather(*pending, return_exceptions=True) + + async with app.db_sessionmaker() as s: + run = (await s.execute( + select(AnsibleRun).where(AnsibleRun.source_name == "t"))).scalars().first() + event = (await s.execute( + select(AlertEvent).where(AlertEvent.rule_id == rule_id))).scalar_one() + return run, event + + run, event = asyncio.run(_go()) + assert run is not None, "no AnsibleRun was created by the firing rule" + assert run.triggered_by is None # system-triggered + assert run.status == AnsibleRunStatus.success, run.output + assert "FIRED-cpu_pct" in (run.output or "") # injected context var reached the play + assert event.ansible_run_id == run.id # firing event linked to the run diff --git a/tests/test_alert_ansible.py b/tests/test_alert_ansible.py new file mode 100644 index 0000000..d7beb9d --- /dev/null +++ b/tests/test_alert_ansible.py @@ -0,0 +1,30 @@ +"""Unit tests for alert -> Ansible action context vars (task 250).""" +from steward.core.alerts import alert_extra_vars +from steward.models.alerts import AlertOperator, AlertRule + + +def test_alert_extra_vars_describes_the_firing(): + rule = AlertRule( + name="UPS low battery", source_module="snmp", resource_name="ups", + metric_name="battery_status", operator=AlertOperator.lt, threshold=3.0, + created_by="u", + ) + ev = alert_extra_vars(rule, 2.0) + assert "steward_alert_rule=UPS low battery" in ev + assert "steward_alert_source=snmp" in ev + assert "steward_alert_resource=ups" in ev + assert "steward_alert_metric=battery_status" in ev + assert "steward_alert_value=2.0" in ev + assert "steward_alert_threshold=3.0" in ev + assert "steward_alert_operator=<" in ev + + +def test_alert_extra_vars_are_key_value_strings(): + rule = AlertRule( + name="r", source_module="host_agent", resource_name="srv1", + metric_name="cpu_pct", operator=AlertOperator.gt, threshold=90.0, + created_by="u", + ) + ev = alert_extra_vars(rule, 95.5) + assert all("=" in item for item in ev) + assert all(item.startswith("steward_alert_") for item in ev)