bvandeusenandClaude Opus 5.5
0ec7ce4472
CI 8343: the wait-for-all-tasks loop also waited on the executor's 5-minute run-output cleanup task (_schedule_cleanup) and hit its 120s deadline. Wait on alerts._background, then poll the run to a final status. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
117 lines
4.7 KiB
Python
117 lines
4.7 KiB
Python
"""Integration: an alert transition-to-firing triggers its Ansible action (task 250).
|
|
|
|
Drives the full chain against a live Postgres + real Ansible: a breaching metric
|
|
fires the rule, which schedules a system-triggered AnsibleRun that runs a real
|
|
connection:local playbook receiving the injected steward_alert_* context, and the
|
|
firing AlertEvent links to the run.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import textwrap
|
|
import uuid
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
_NEEDS_DB = pytest.mark.skipif(
|
|
not os.environ.get("STEWARD_DATABASE_URL"),
|
|
reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)",
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_firing_triggers_ansible_action(app, tmp_path):
|
|
from sqlalchemy import select
|
|
from steward.core.alerts import record_metric
|
|
from steward.models.alerts import (
|
|
AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum,
|
|
)
|
|
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
|
from steward.models.users import User, UserRole
|
|
|
|
(tmp_path / "fire.yml").write_text(textwrap.dedent("""\
|
|
- hosts: localhost
|
|
connection: local
|
|
gather_facts: false
|
|
tasks:
|
|
- debug:
|
|
msg: "FIRED-{{ steward_alert_metric }}"
|
|
"""))
|
|
# inventory.ini: an inventory name the source lists, as trigger_run's guard requires.
|
|
(tmp_path / "inventory.ini").write_text("localhost ansible_connection=local\n")
|
|
|
|
# An ansible source the action resolves against.
|
|
app.config["ANSIBLE"] = {
|
|
"sources": [{"name": "t", "type": "local", "path": str(tmp_path)}],
|
|
}
|
|
|
|
uid = str(uuid.uuid4())
|
|
rule_id = str(uuid.uuid4())
|
|
action = {"source": "t", "playbook": "fire.yml", "inventory": "inventory.ini"}
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
# User must be committed before the rule's created_by FK can reference it.
|
|
async with s.begin():
|
|
s.add(User(
|
|
id=uid, username=f"u{uid[:8]}", email=f"{uid[:8]}@e.test",
|
|
password_hash="x", role=UserRole.admin, is_active=True,
|
|
))
|
|
async with s.begin():
|
|
s.add(AlertRule(
|
|
id=rule_id, name="cpu hot", source_module="host_agent",
|
|
resource_name="srv1", metric_name="cpu_pct",
|
|
operator=AlertOperator.gt, threshold=90.0,
|
|
consecutive_failures_required=1, enabled=True,
|
|
ansible_action=action, created_by=uid,
|
|
))
|
|
s.add(AlertState(rule_id=rule_id, state=AlertStateEnum.inactive))
|
|
|
|
# Breaching value → transition to firing → schedules the action task.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await record_metric(s, "host_agent", "srv1", "cpu_pct", 95.0)
|
|
|
|
# Let the notification and the action finish, then the run the action
|
|
# started. Not "every task": a finished run leaves a cleanup task that
|
|
# sleeps for minutes (executor._schedule_cleanup).
|
|
from steward.core import alerts
|
|
loop = asyncio.get_running_loop()
|
|
deadline = loop.time() + 120
|
|
while alerts._background:
|
|
assert loop.time() < deadline, "the alert's tasks did not finish"
|
|
await asyncio.wait(set(alerts._background), timeout=deadline - loop.time())
|
|
while True:
|
|
async with app.db_sessionmaker() as s:
|
|
status = (await s.execute(
|
|
select(AnsibleRun.status).where(AnsibleRun.source_name == "t"))).scalar()
|
|
if status not in (None, AnsibleRunStatus.queued, AnsibleRunStatus.running):
|
|
break
|
|
assert loop.time() < deadline, f"the run is still {status}"
|
|
await asyncio.sleep(0.5)
|
|
|
|
async with app.db_sessionmaker() as s:
|
|
run = (await s.execute(
|
|
select(AnsibleRun).where(AnsibleRun.source_name == "t"))).scalars().first()
|
|
event = (await s.execute(
|
|
select(AlertEvent).where(AlertEvent.rule_id == rule_id))).scalar_one()
|
|
return run, event
|
|
|
|
run, event = asyncio.run(_go())
|
|
assert run is not None, "no AnsibleRun was created by the firing rule"
|
|
assert run.triggered_by is None # system-triggered
|
|
assert run.status == AnsibleRunStatus.success, run.output
|
|
assert "FIRED-cpu_pct" in (run.output or "") # injected context var reached the play
|
|
assert event.ansible_run_id == run.id # firing event linked to the run
|