bvandeusenandClaude Opus 5.5
e780932673
#5212 "Plain labels: no raw ids, scopes, metric keys or unexplained jargon", batch 2: - A run records where it came from: AnsibleRun.origin ("schedule", "alert" or "mcp") and origin_id, set by the scheduler, Run now, alert actions and the MCP tools. Migration 0031 backfills past runs from alert events, schedules' last runs, and MCP audit entries. - The runs list has a Started by column: "schedule nightly", "alert rule nas down", "alice via MCP" or "alice", linked where there is a page. The raw scope tooltip is gone. - Run detail: no run id in the subtitle or body, Started by in words, and the host summary lists failed and unreachable hosts first. The live output only follows new lines while the reader is at the bottom, and when the run finishes the page reloads once so the badge and summary are final. - list_ansible_runs and get_ansible_run report origin and origin_id. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
118 lines
4.8 KiB
Python
118 lines
4.8 KiB
Python
"""Integration: an alert transition-to-firing triggers its Ansible action (task 250).
|
|
|
|
Drives the full chain against a live Postgres + real Ansible: a breaching metric
|
|
fires the rule, which schedules a system-triggered AnsibleRun that runs a real
|
|
connection:local playbook receiving the injected steward_alert_* context, and the
|
|
firing AlertEvent links to the run.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import textwrap
|
|
import uuid
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
_NEEDS_DB = pytest.mark.skipif(
|
|
not os.environ.get("STEWARD_DATABASE_URL"),
|
|
reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)",
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_firing_triggers_ansible_action(app, tmp_path):
|
|
from sqlalchemy import select
|
|
from steward.core.alerts import record_metric
|
|
from steward.models.alerts import (
|
|
AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum,
|
|
)
|
|
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
|
from steward.models.users import User, UserRole
|
|
|
|
(tmp_path / "fire.yml").write_text(textwrap.dedent("""\
|
|
- hosts: localhost
|
|
connection: local
|
|
gather_facts: false
|
|
tasks:
|
|
- debug:
|
|
msg: "FIRED-{{ steward_alert_metric }}"
|
|
"""))
|
|
# inventory.ini: an inventory name the source lists, as trigger_run's guard requires.
|
|
(tmp_path / "inventory.ini").write_text("localhost ansible_connection=local\n")
|
|
|
|
# An ansible source the action resolves against.
|
|
app.config["ANSIBLE"] = {
|
|
"sources": [{"name": "t", "type": "local", "path": str(tmp_path)}],
|
|
}
|
|
|
|
uid = str(uuid.uuid4())
|
|
rule_id = str(uuid.uuid4())
|
|
action = {"source": "t", "playbook": "fire.yml", "inventory": "inventory.ini"}
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
# User must be committed before the rule's created_by FK can reference it.
|
|
async with s.begin():
|
|
s.add(User(
|
|
id=uid, username=f"u{uid[:8]}", email=f"{uid[:8]}@e.test",
|
|
password_hash="x", role=UserRole.admin, is_active=True,
|
|
))
|
|
async with s.begin():
|
|
s.add(AlertRule(
|
|
id=rule_id, name="cpu hot", source_module="host_agent",
|
|
resource_name="srv1", metric_name="cpu_pct",
|
|
operator=AlertOperator.gt, threshold=90.0,
|
|
consecutive_failures_required=1, enabled=True,
|
|
ansible_action=action, created_by=uid,
|
|
))
|
|
s.add(AlertState(rule_id=rule_id, state=AlertStateEnum.inactive))
|
|
|
|
# Breaching value → transition to firing → schedules the action task.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await record_metric(s, "host_agent", "srv1", "cpu_pct", 95.0)
|
|
|
|
# Let the notification and the action finish, then the run the action
|
|
# started. Not "every task": a finished run leaves a cleanup task that
|
|
# sleeps for minutes (executor._schedule_cleanup).
|
|
from steward.core import alerts
|
|
loop = asyncio.get_running_loop()
|
|
deadline = loop.time() + 120
|
|
while alerts._background:
|
|
assert loop.time() < deadline, "the alert's tasks did not finish"
|
|
await asyncio.wait(set(alerts._background), timeout=deadline - loop.time())
|
|
while True:
|
|
async with app.db_sessionmaker() as s:
|
|
status = (await s.execute(
|
|
select(AnsibleRun.status).where(AnsibleRun.source_name == "t"))).scalar()
|
|
if status not in (None, AnsibleRunStatus.queued, AnsibleRunStatus.running):
|
|
break
|
|
assert loop.time() < deadline, f"the run is still {status}"
|
|
await asyncio.sleep(0.5)
|
|
|
|
async with app.db_sessionmaker() as s:
|
|
run = (await s.execute(
|
|
select(AnsibleRun).where(AnsibleRun.source_name == "t"))).scalars().first()
|
|
event = (await s.execute(
|
|
select(AlertEvent).where(AlertEvent.rule_id == rule_id))).scalar_one()
|
|
return run, event
|
|
|
|
run, event = asyncio.run(_go())
|
|
assert run is not None, "no AnsibleRun was created by the firing rule"
|
|
assert run.triggered_by is None # system-triggered
|
|
assert run.origin == "alert" and run.origin_id == event.rule_id # the runs list names the rule (#5212)
|
|
assert run.status == AnsibleRunStatus.success, run.output
|
|
assert "FIRED-cpu_pct" in (run.output or "") # injected context var reached the play
|
|
assert event.ansible_run_id == run.id # firing event linked to the run
|