bvandeusenandClaude Opus 5.5
49607a2441
Notifications and alert actions now start only once the evaluating
transaction commits, and never for one that rolls back. record_metrics
queues them on the session (_after_commit), and after_commit/after_rollback
listeners on the sync session spawn or drop them. Spawned tasks are held in a
module set so they cannot be garbage-collected before they run.
A maintenance window no longer skips evaluation. Alerts in its scope still
change state, but their notifications and actions are held, and each event
says so ("notification muted: maintenance window <name>"; also on no-data
and resume events). notify_after_maintenance, run every minute with the
silence check as the new core task alert_reconcile, sends one notification
for an alert still firing after its window ends and runs its action. The
event's note records the release.
Alert actions go through runner.trigger_run like every other run path, so
the playbook and the inventory must be ones the source lists (#5034 guards).
The firing event links to the run, or its note says why the action did not
start. The maintenance page and form wording now say what a window does.
Tests:
- integration: a rolled-back evaluation notifies nobody, and dispatch waits
for the commit; maintenance mutes while the state moves and notifies once
when the window ends; an action whose playbook is outside its source is
refused, with the note set;
- the alert-action test uses a listed inventory name and waits, with a
deadline, for the run the action starts;
- unit: _muted_by.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
107 lines
4.2 KiB
Python
107 lines
4.2 KiB
Python
"""Integration: an alert transition-to-firing triggers its Ansible action (task 250).
|
|
|
|
Drives the full chain against a live Postgres + real Ansible: a breaching metric
|
|
fires the rule, which schedules a system-triggered AnsibleRun that runs a real
|
|
connection:local playbook receiving the injected steward_alert_* context, and the
|
|
firing AlertEvent links to the run.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import textwrap
|
|
import uuid
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
_NEEDS_DB = pytest.mark.skipif(
|
|
not os.environ.get("STEWARD_DATABASE_URL"),
|
|
reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)",
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_firing_triggers_ansible_action(app, tmp_path):
|
|
from sqlalchemy import select
|
|
from steward.core.alerts import record_metric
|
|
from steward.models.alerts import (
|
|
AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum,
|
|
)
|
|
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
|
from steward.models.users import User, UserRole
|
|
|
|
(tmp_path / "fire.yml").write_text(textwrap.dedent("""\
|
|
- hosts: localhost
|
|
connection: local
|
|
gather_facts: false
|
|
tasks:
|
|
- debug:
|
|
msg: "FIRED-{{ steward_alert_metric }}"
|
|
"""))
|
|
# inventory.ini: an inventory name the source lists, as trigger_run's guard requires.
|
|
(tmp_path / "inventory.ini").write_text("localhost ansible_connection=local\n")
|
|
|
|
# An ansible source the action resolves against.
|
|
app.config["ANSIBLE"] = {
|
|
"sources": [{"name": "t", "type": "local", "path": str(tmp_path)}],
|
|
}
|
|
|
|
uid = str(uuid.uuid4())
|
|
rule_id = str(uuid.uuid4())
|
|
action = {"source": "t", "playbook": "fire.yml", "inventory": "inventory.ini"}
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
# User must be committed before the rule's created_by FK can reference it.
|
|
async with s.begin():
|
|
s.add(User(
|
|
id=uid, username=f"u{uid[:8]}", email=f"{uid[:8]}@e.test",
|
|
password_hash="x", role=UserRole.admin, is_active=True,
|
|
))
|
|
async with s.begin():
|
|
s.add(AlertRule(
|
|
id=rule_id, name="cpu hot", source_module="host_agent",
|
|
resource_name="srv1", metric_name="cpu_pct",
|
|
operator=AlertOperator.gt, threshold=90.0,
|
|
consecutive_failures_required=1, enabled=True,
|
|
ansible_action=action, created_by=uid,
|
|
))
|
|
s.add(AlertState(rule_id=rule_id, state=AlertStateEnum.inactive))
|
|
|
|
# Breaching value → transition to firing → schedules the action task.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await record_metric(s, "host_agent", "srv1", "cpu_pct", 95.0)
|
|
|
|
# Let the dispatch, the action and the run it starts finish. The run's
|
|
# task is created by the action's task, so wait until none are left.
|
|
loop = asyncio.get_running_loop()
|
|
deadline = loop.time() + 120
|
|
while pending := [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]:
|
|
assert loop.time() < deadline, f"tasks still running: {pending}"
|
|
await asyncio.wait(pending, timeout=deadline - loop.time())
|
|
|
|
async with app.db_sessionmaker() as s:
|
|
run = (await s.execute(
|
|
select(AnsibleRun).where(AnsibleRun.source_name == "t"))).scalars().first()
|
|
event = (await s.execute(
|
|
select(AlertEvent).where(AlertEvent.rule_id == rule_id))).scalar_one()
|
|
return run, event
|
|
|
|
run, event = asyncio.run(_go())
|
|
assert run is not None, "no AnsibleRun was created by the firing rule"
|
|
assert run.triggered_by is None # system-triggered
|
|
assert run.status == AnsibleRunStatus.success, run.output
|
|
assert "FIRED-cpu_pct" in (run.output or "") # injected context var reached the play
|
|
assert event.ansible_run_id == run.id # firing event linked to the run
|