FabledSteward/tests/integration/test_alert_states.py

"""Integration: alert state against real Postgres. An acknowledgement holds
while the breach lasts, an alert clears after its OK run, and changing what a
rule watches (or disabling it) resets its state with a note on the event."""
from __future__ import annotations

import asyncio
import os
import uuid

import pytest

pytestmark = pytest.mark.integration


@pytest.fixture
def app():
    if not os.environ.get("STEWARD_DATABASE_URL"):
        pytest.skip("needs Postgres")
    from steward.app import create_app
    return create_app(testing=False)


def test_acknowledge_holds_clear_after_and_reset(app):
    from sqlalchemy import select
    from steward.alerts import service as alert_svc
    from steward.core.alerts import record_metric
    from steward.models.alerts import AlertEvent, AlertRule, AlertState, AlertStateEnum
    from steward.models.metrics import PluginMetric
    from steward.models.users import User, UserRole

    async def _go():
        tag = uuid.uuid4().hex[:8]
        res = f"box-{tag}"
        async with app.db_sessionmaker() as s:
            async with s.begin():
                user = User(username=f"al-{tag}", email=f"al-{tag}@test", password_hash="x",
                            role=UserRole.admin)
                s.add(user)
                s.add(PluginMetric(source_module="host_agent", resource_name=res,
                                   metric_name="cpu_pct", value=1.0))
                await s.flush()
                fields = await alert_svc.rule_fields(
                    s, name=f"hot {tag}", source_module="host_agent", resource_name=res,
                    metric_name="cpu_pct", operator=">", threshold=90,
                    consecutive_failures_required=1, consecutive_ok_required=2)
                rule = await alert_svc.add_rule(s, fields, created_by=user.id)
            rule_id = rule.id

        async def sample(v):
            async with app.db_sessionmaker() as s:
                async with s.begin():
                    await record_metric(s, "host_agent", res, "cpu_pct", v)

        async def state():
            async with app.db_sessionmaker() as s:
                return (await s.execute(select(AlertState).where(
                    AlertState.rule_id == rule_id))).scalar_one()

        async def events():
            async with app.db_sessionmaker() as s:
                rows = (await s.execute(select(AlertEvent).where(AlertEvent.rule_id == rule_id)
                                        .order_by(AlertEvent.transitioned_at))).scalars().all()
            return [(e.from_state, e.to_state, e.note) for e in rows]

        await sample(95)
        assert (await state()).state == AlertStateEnum.firing
        async with app.db_sessionmaker() as s:
            async with s.begin():
                await alert_svc.acknowledge(s, rule_id, None)
        for _ in range(3):
            await sample(97)  # still breached: the acknowledgement holds
        assert (await state()).state == AlertStateEnum.acknowledged
        await sample(10)      # one OK check of the two it needs
        assert (await state()).state == AlertStateEnum.acknowledged
        await sample(10)
        st = await state()
        assert st.state == AlertStateEnum.inactive and st.acknowledged_at is None
        assert await events() == [("inactive", "firing", None), ("acknowledged", "resolved", None)]

        # Changing the threshold of a firing rule resets it, saying why.
        await sample(95)
        async with app.db_sessionmaker() as s:
            async with s.begin():
                r = await s.get(AlertRule, rule_id)
                fields = await alert_svc.rule_fields(
                    s, name=r.name, source_module=r.source_module, resource_name=r.resource_name,
                    metric_name=r.metric_name, operator=">", threshold=99,
                    consecutive_failures_required=1, consecutive_ok_required=2)
                await alert_svc.apply_rule_change(s, r, fields)
        assert (await state()).state == AlertStateEnum.inactive
        assert (await events())[-1] == ("firing", "resolved", "rule changed: threshold")

        # Renaming alone does not reset; disabling does.
        await sample(100)
        async with app.db_sessionmaker() as s:
            async with s.begin():
                r = await s.get(AlertRule, rule_id)
                await alert_svc.apply_rule_change(s, r, {"name": f"renamed {tag}"})
        assert (await state()).state == AlertStateEnum.firing
        async with app.db_sessionmaker() as s:
            async with s.begin():
                r = await s.get(AlertRule, rule_id)
                await alert_svc.apply_rule_change(s, r, {}, enabled=False)
        assert (await state()).state == AlertStateEnum.inactive
        assert (await events())[-1] == ("firing", "resolved", "rule disabled")

        # Re-enabling starts the watch over: silence is measured from now.
        async with app.db_sessionmaker() as s:
            async with s.begin():
                r = await s.get(AlertRule, rule_id)
                await alert_svc.apply_rule_change(s, r, {}, enabled=True)
        st = await state()
        assert (st.state, st.last_sample_at, st.no_data_since) == (AlertStateEnum.inactive, None, None)
        assert (await events())[-1] == ("firing", "resolved", "rule disabled")

    asyncio.run(_go())


def test_record_metrics_marks_each_step(app):
    """A caller timing an ingest can tell the row write from the rule reads
    from the evaluation (#5149)."""
    from steward.alerts import service as alert_svc
    from steward.core.alerts import record_metrics
    from steward.models.metrics import PluginMetric
    from steward.models.users import User, UserRole

    class _Marks:
        def __init__(self):
            self.names = []

        def mark(self, name):
            self.names.append(name)

    def rows(res):
        return [PluginMetric(source_module="host_agent", resource_name=res,
                             metric_name="cpu_pct", value=5.0)]

    async def _go():
        tag = uuid.uuid4().hex[:8]
        watched, unwatched = f"box-{tag}", f"other-{tag}"
        async with app.db_sessionmaker() as s:
            async with s.begin():
                user = User(username=f"mk-{tag}", email=f"mk-{tag}@test", password_hash="x",
                            role=UserRole.admin)
                s.add(user)
                s.add_all(rows(watched))
                await s.flush()
                fields = await alert_svc.rule_fields(
                    s, name=f"hot {tag}", source_module="host_agent", resource_name=watched,
                    metric_name="cpu_pct", operator=">", threshold=90)
                await alert_svc.add_rule(s, fields, created_by=user.id)

        seen = {}
        for res in (watched, unwatched):
            marks = _Marks()
            async with app.db_sessionmaker() as s:
                async with s.begin():
                    await record_metrics(s, rows(res), marks)
            seen[res] = marks.names
        return seen[watched], seen[unwatched]

    watched, unwatched = asyncio.run(_go())
    assert watched == ["flush", "rules", "evaluate"]
    assert unwatched == ["flush", "rules"]   # no rule watches it: nothing to evaluate