bvandeusenandClaude Opus 5.5
480b465b8e
When a silent series' sample also moved the rule (fired, resolved), the
resume and the transition were two events: the resume from the
pre-evaluation state at ingest time, the transition at the sample's
time. Ties sort by a random id, and the resume's "→ inactive" landed
after "→ firing", so the history read "silent, firing, back to normal"
at once and the episode closed while the alert was firing.
_track_samples now only clears the mark and returns its note; the first
sample's evaluation records one event from no_data to wherever that
sample took the rule ("firing" or "resolved" notification), or back to
the state it was in ("data_resumed", still only when notify_no_data).
Also fixes the #5441 integration test: its rule needs a recorded
resource to target (CI 8878).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
154 lines
6.6 KiB
Python
154 lines
6.6 KiB
Python
"""Silence is alertable (#5062): a series' usual gap is learned from its
|
|
samples, an outage does not become its cadence, and a series counts as silent
|
|
after three usual gaps (at least five minutes; an hour while its cadence is
|
|
unknown)."""
|
|
import asyncio
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace as NS
|
|
|
|
import pytest
|
|
|
|
from steward.core.alerts import (
|
|
NO_DATA_FLOOR_S, NO_DATA_UNKNOWN_S, _evaluate_rule, _track_samples, next_interval,
|
|
no_data_after, silent_since,
|
|
)
|
|
from steward.models.alerts import AlertOperator, AlertStateEnum
|
|
|
|
NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
@pytest.mark.parametrize("interval,gap,want", [
|
|
(None, 60, 60), # the first gap is the cadence
|
|
(0.0, 30, 30),
|
|
(60, 60, 60), # steady
|
|
(60, 80, 65), # a late sample barely moves it
|
|
(60, 3600, 90), # an outage counts as at most three gaps
|
|
(60, -5, 45), # never a negative gap
|
|
])
|
|
def test_next_interval(interval, gap, want):
|
|
assert next_interval(interval, gap) == pytest.approx(want)
|
|
|
|
|
|
def test_a_new_cadence_is_learned_in_steps():
|
|
interval, steps = 60.0, 0
|
|
while no_data_after(interval) < 600: # samples now arrive every 10 minutes
|
|
interval, steps = next_interval(interval, 600), steps + 1
|
|
assert steps == 3
|
|
|
|
|
|
@pytest.mark.parametrize("interval,want", [
|
|
(None, NO_DATA_UNKNOWN_S), (0.0, NO_DATA_UNKNOWN_S),
|
|
(30, NO_DATA_FLOOR_S), (100, 300), (600, 1800),
|
|
])
|
|
def test_no_data_after(interval, want):
|
|
assert no_data_after(interval) == want
|
|
|
|
|
|
@pytest.mark.parametrize("last_ago,interval,watch_ago,silent_after", [
|
|
(60, 60, 9999, None), # reporting
|
|
(300, 60, 9999, None), # exactly at the limit is not yet silent
|
|
(301, 60, 9999, 300), # silent since last + 300s
|
|
(2000, 600, 9999, 1800), # slow series: three of its gaps
|
|
(None, None, 3000, None), # never sampled, watched under an hour
|
|
(None, None, 4000, 3600), # never sampled, watched over an hour
|
|
])
|
|
def test_silent_since(last_ago, interval, watch_ago, silent_after):
|
|
last = NOW - timedelta(seconds=last_ago) if last_ago is not None else None
|
|
watch = NOW - timedelta(seconds=watch_ago)
|
|
got = silent_since(last, interval, watch, NOW)
|
|
if silent_after is None:
|
|
assert got is None
|
|
else:
|
|
assert got == (last or watch) + timedelta(seconds=silent_after)
|
|
|
|
|
|
def test_steward_s_own_outage_is_not_the_series_silence():
|
|
"""#5441: a series last sampled 18 minutes ago, while Steward has only been
|
|
back for 2 minutes, is not silent yet; once Steward has watched it for
|
|
three of its gaps with nothing arriving, it is, from then."""
|
|
last = NOW - timedelta(minutes=18)
|
|
watch = NOW - timedelta(days=1)
|
|
assert silent_since(last, 60, watch, NOW, awake_since=NOW - timedelta(minutes=2)) is None
|
|
awake = NOW - timedelta(minutes=6)
|
|
assert silent_since(last, 60, watch, NOW, awake_since=awake) == awake + timedelta(seconds=300)
|
|
# A sample since Steward came back is what counts, when there is one.
|
|
assert silent_since(NOW - timedelta(minutes=1), 60, watch, NOW, awake_since=awake) is None
|
|
|
|
|
|
class _Session:
|
|
def __init__(self):
|
|
self.added = []
|
|
|
|
def add(self, row):
|
|
self.added.append(row)
|
|
|
|
async def flush(self):
|
|
pass
|
|
|
|
|
|
def _state(**kw):
|
|
base = dict(state=AlertStateEnum.firing, last_sample_at=None, sample_interval_s=None,
|
|
no_data_since=None, consecutive_failure_count=0, consecutive_ok_count=0)
|
|
return NS(**{**base, **kw})
|
|
|
|
|
|
def _samples(*ago_values):
|
|
return [NS(recorded_at=NOW - timedelta(seconds=a), value=v) for a, v in ago_values]
|
|
|
|
|
|
def test_tracking_learns_the_gap_and_skips_late_samples():
|
|
st = _state()
|
|
assert _track_samples(st, _samples((180, 1), (120, 1), (60, 1)), NOW) is None
|
|
assert (st.last_sample_at, st.sample_interval_s) == (NOW - timedelta(seconds=60), 60)
|
|
_track_samples(st, _samples((90, 1)), NOW) # older than the last
|
|
assert (st.last_sample_at, st.sample_interval_s) == (NOW - timedelta(seconds=60), 60)
|
|
|
|
|
|
def test_a_sample_after_silence_clears_it_and_says_so():
|
|
st = _state(last_sample_at=NOW - timedelta(minutes=20), sample_interval_s=60,
|
|
no_data_since=NOW - timedelta(minutes=15))
|
|
note = _track_samples(st, _samples((0, 42.0)), NOW)
|
|
assert st.no_data_since is None and st.state == AlertStateEnum.firing # still firing
|
|
assert st.sample_interval_s == 90 # the outage counted as three gaps, at most
|
|
assert note == "data resumed after 15m silent"
|
|
|
|
|
|
_RULE = NS(id="r", operator=AlertOperator.gt, threshold=90.0,
|
|
consecutive_failures_required=1, consecutive_ok_required=1)
|
|
|
|
|
|
@pytest.mark.parametrize("before,value,want_event,want_notice", [
|
|
# Back, and breaching: one event, silent to firing (#5442).
|
|
(AlertStateEnum.inactive, 95.0, ("no_data", "firing"), "firing"),
|
|
# Back, and fine: one event, silent back to how it was.
|
|
(AlertStateEnum.inactive, 10.0, ("no_data", "inactive"), "data_resumed"),
|
|
# A firing alert that went silent comes back fine: it resolves, from silent.
|
|
(AlertStateEnum.firing, 10.0, ("no_data", "resolved"), "resolved"),
|
|
# ... or still breaching: it stays firing, and only the resume is news.
|
|
(AlertStateEnum.firing, 95.0, ("no_data", "firing"), "data_resumed"),
|
|
])
|
|
def test_a_resumed_series_gets_one_event_for_what_its_first_sample_did(
|
|
before, value, want_event, want_notice):
|
|
session, st = _Session(), _state(state=before)
|
|
said = "data resumed after 15m silent"
|
|
out = asyncio.run(_evaluate_rule(session, _RULE, st, value, NOW, resumed=said))
|
|
[event] = session.added
|
|
assert (event.from_state, event.to_state) == want_event
|
|
assert event.note == said and event.transitioned_at == NOW
|
|
assert [(n, e) for n, e, _ in out] == [(want_notice, event.id)]
|
|
# Without a resume, a sample that changes nothing records nothing.
|
|
session = _Session()
|
|
assert asyncio.run(_evaluate_rule(session, _RULE, _state(state=st.state), value, NOW)) == []
|
|
assert session.added == []
|
|
|
|
|
|
def test_muted_by_names_the_window_that_covers_the_rule():
|
|
from steward.core.alerts import _muted_by
|
|
rule = NS(source_module="host_agent", resource_name="nas")
|
|
windows = [NS(name="other", scope_source="docker", scope_resource=None),
|
|
NS(name="nas work", scope_source="host_agent", scope_resource="nas")]
|
|
assert _muted_by(windows, rule) == "notification muted: maintenance window nas work"
|
|
assert _muted_by(windows[:1], rule) is None
|
|
assert _muted_by([NS(name="all", scope_source=None, scope_resource=None)], rule) == (
|
|
"notification muted: maintenance window all")
|