FabledSteward/tests/core/test_alert_no_data.py

"""Silence is alertable (#5062): a series' usual gap is learned from its
samples, an outage does not become its cadence, and a series counts as silent
after three usual gaps (at least five minutes; an hour while its cadence is
unknown)."""
import asyncio
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace as NS

import pytest

from steward.core.alerts import (
    NO_DATA_FLOOR_S, NO_DATA_UNKNOWN_S, _evaluate_rule, _track_samples, next_interval,
    no_data_after, silent_since,
)
from steward.models.alerts import AlertOperator, AlertStateEnum

NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc)


@pytest.mark.parametrize("interval,gap,want", [
    (None, 60, 60),             # the first gap is the cadence
    (0.0, 30, 30),
    (60, 60, 60),               # steady
    (60, 80, 65),               # a late sample barely moves it
    (60, 3600, 90),             # an outage counts as at most three gaps
    (60, -5, 45),               # never a negative gap
])
def test_next_interval(interval, gap, want):
    assert next_interval(interval, gap) == pytest.approx(want)


def test_a_new_cadence_is_learned_in_steps():
    interval, steps = 60.0, 0
    while no_data_after(interval) < 600:  # samples now arrive every 10 minutes
        interval, steps = next_interval(interval, 600), steps + 1
    assert steps == 3


@pytest.mark.parametrize("interval,want", [
    (None, NO_DATA_UNKNOWN_S), (0.0, NO_DATA_UNKNOWN_S),
    (30, NO_DATA_FLOOR_S), (100, 300), (600, 1800),
])
def test_no_data_after(interval, want):
    assert no_data_after(interval) == want


@pytest.mark.parametrize("last_ago,interval,watch_ago,silent_after", [
    (60, 60, 9999, None),            # reporting
    (300, 60, 9999, None),           # exactly at the limit is not yet silent
    (301, 60, 9999, 300),            # silent since last + 300s
    (2000, 600, 9999, 1800),         # slow series: three of its gaps
    (None, None, 3000, None),        # never sampled, watched under an hour
    (None, None, 4000, 3600),        # never sampled, watched over an hour
])
def test_silent_since(last_ago, interval, watch_ago, silent_after):
    last = NOW - timedelta(seconds=last_ago) if last_ago is not None else None
    watch = NOW - timedelta(seconds=watch_ago)
    got = silent_since(last, interval, watch, NOW)
    if silent_after is None:
        assert got is None
    else:
        assert got == (last or watch) + timedelta(seconds=silent_after)


def test_steward_s_own_outage_is_not_the_series_silence():
    """#5441: a series last sampled 18 minutes ago, while Steward has only been
    back for 2 minutes, is not silent yet; once Steward has watched it for
    three of its gaps with nothing arriving, it is, from then."""
    last = NOW - timedelta(minutes=18)
    watch = NOW - timedelta(days=1)
    assert silent_since(last, 60, watch, NOW, awake_since=NOW - timedelta(minutes=2)) is None
    awake = NOW - timedelta(minutes=6)
    assert silent_since(last, 60, watch, NOW, awake_since=awake) == awake + timedelta(seconds=300)
    # A sample since Steward came back is what counts, when there is one.
    assert silent_since(NOW - timedelta(minutes=1), 60, watch, NOW, awake_since=awake) is None


class _Session:
    def __init__(self):
        self.added = []

    def add(self, row):
        self.added.append(row)

    async def flush(self):
        pass


def _state(**kw):
    base = dict(state=AlertStateEnum.firing, last_sample_at=None, sample_interval_s=None,
                no_data_since=None, consecutive_failure_count=0, consecutive_ok_count=0)
    return NS(**{**base, **kw})


def _samples(*ago_values):
    return [NS(recorded_at=NOW - timedelta(seconds=a), value=v) for a, v in ago_values]


def test_tracking_learns_the_gap_and_skips_late_samples():
    st = _state()
    assert _track_samples(st, _samples((180, 1), (120, 1), (60, 1)), NOW) is None
    assert (st.last_sample_at, st.sample_interval_s) == (NOW - timedelta(seconds=60), 60)
    _track_samples(st, _samples((90, 1)), NOW)  # older than the last
    assert (st.last_sample_at, st.sample_interval_s) == (NOW - timedelta(seconds=60), 60)


def test_a_sample_after_silence_clears_it_and_says_so():
    st = _state(last_sample_at=NOW - timedelta(minutes=20), sample_interval_s=60,
                no_data_since=NOW - timedelta(minutes=15))
    note = _track_samples(st, _samples((0, 42.0)), NOW)
    assert st.no_data_since is None and st.state == AlertStateEnum.firing  # still firing
    assert st.sample_interval_s == 90       # the outage counted as three gaps, at most
    assert note == "data resumed after 15m silent"


_RULE = NS(id="r", operator=AlertOperator.gt, threshold=90.0,
           consecutive_failures_required=1, consecutive_ok_required=1)


@pytest.mark.parametrize("before,value,want_event,want_notice", [
    # Back, and breaching: one event, silent to firing (#5442).
    (AlertStateEnum.inactive, 95.0, ("no_data", "firing"), "firing"),
    # Back, and fine: one event, silent back to how it was.
    (AlertStateEnum.inactive, 10.0, ("no_data", "inactive"), "data_resumed"),
    # A firing alert that went silent comes back fine: it resolves, from silent.
    (AlertStateEnum.firing, 10.0, ("no_data", "resolved"), "resolved"),
    # ... or still breaching: it stays firing, and only the resume is news.
    (AlertStateEnum.firing, 95.0, ("no_data", "firing"), "data_resumed"),
])
def test_a_resumed_series_gets_one_event_for_what_its_first_sample_did(
        before, value, want_event, want_notice):
    session, st = _Session(), _state(state=before)
    said = "data resumed after 15m silent"
    out = asyncio.run(_evaluate_rule(session, _RULE, st, value, NOW, resumed=said))
    [event] = session.added
    assert (event.from_state, event.to_state) == want_event
    assert event.note == said and event.transitioned_at == NOW
    assert [(n, e) for n, e, _ in out] == [(want_notice, event.id)]
    # Without a resume, a sample that changes nothing records nothing.
    session = _Session()
    assert asyncio.run(_evaluate_rule(session, _RULE, _state(state=st.state), value, NOW)) == []
    assert session.added == []


def test_muted_by_names_the_window_that_covers_the_rule():
    from steward.core.alerts import _muted_by
    rule = NS(source_module="host_agent", resource_name="nas")
    windows = [NS(name="other", scope_source="docker", scope_resource=None),
               NS(name="nas work", scope_source="host_agent", scope_resource="nas")]
    assert _muted_by(windows, rule) == "notification muted: maintenance window nas work"
    assert _muted_by(windows[:1], rule) is None
    assert _muted_by([NS(name="all", scope_source=None, scope_resource=None)], rule) == (
        "notification muted: maintenance window all")


def test_a_flushed_backlog_counts_as_heard_when_it_arrives():
    """#5720: an agent's backlog, flushed a couple of samples a post, is hours
    old. Judged by its newest sample, the series read silent again between
    posts and resumed twice. It counts as heard when Steward heard it, and the
    rest of the backlog reads as late."""
    st = _state(last_sample_at=NOW - timedelta(hours=3), sample_interval_s=60,
                no_data_since=NOW - timedelta(hours=2, minutes=55))
    note = _track_samples(st, _samples((2 * 3600 + 60, 5.0), (2 * 3600, 5.0)), NOW)
    assert note == "data resumed after 2h 55m silent"
    assert st.last_sample_at == NOW
    assert silent_since(st.last_sample_at, st.sample_interval_s, NOW, NOW) is None
    # The next post of the same backlog is late: no gap, no second resume.
    assert _track_samples(st, _samples((2 * 3600 - 60, 5.0)), NOW) is None
    assert st.last_sample_at == NOW