bvandeusenandClaude Opus 5.5
a9bf4b7f16
After Steward or its database was down, every agent's age and every series' last sample were old, so all agent-offline rules fired and all rules went no-data at once, clearing a minute later. With per-account alert email that would have mailed everyone a dozen notices per restart. A periodic job now ticks a Watch (steward/core/watch.py) once it has reached the database; the Watch says since when runs have been unbroken (process start, or the end of a gap over 120 s). seconds_since_seen and the no-data sweep count silence from no earlier than that, so a source must stay quiet while Steward is watching before it counts as silent. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
123 lines
5.3 KiB
Python
123 lines
5.3 KiB
Python
"""Silence is alertable (#5062): a series' usual gap is learned from its
|
|
samples, an outage does not become its cadence, and a series counts as silent
|
|
after three usual gaps (at least five minutes; an hour while its cadence is
|
|
unknown)."""
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace as NS
|
|
|
|
import pytest
|
|
|
|
from steward.core.alerts import (
|
|
NO_DATA_FLOOR_S, NO_DATA_UNKNOWN_S, _track_samples, next_interval, no_data_after, silent_since,
|
|
)
|
|
from steward.models.alerts import AlertStateEnum
|
|
|
|
NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
@pytest.mark.parametrize("interval,gap,want", [
|
|
(None, 60, 60), # the first gap is the cadence
|
|
(0.0, 30, 30),
|
|
(60, 60, 60), # steady
|
|
(60, 80, 65), # a late sample barely moves it
|
|
(60, 3600, 90), # an outage counts as at most three gaps
|
|
(60, -5, 45), # never a negative gap
|
|
])
|
|
def test_next_interval(interval, gap, want):
|
|
assert next_interval(interval, gap) == pytest.approx(want)
|
|
|
|
|
|
def test_a_new_cadence_is_learned_in_steps():
|
|
interval, steps = 60.0, 0
|
|
while no_data_after(interval) < 600: # samples now arrive every 10 minutes
|
|
interval, steps = next_interval(interval, 600), steps + 1
|
|
assert steps == 3
|
|
|
|
|
|
@pytest.mark.parametrize("interval,want", [
|
|
(None, NO_DATA_UNKNOWN_S), (0.0, NO_DATA_UNKNOWN_S),
|
|
(30, NO_DATA_FLOOR_S), (100, 300), (600, 1800),
|
|
])
|
|
def test_no_data_after(interval, want):
|
|
assert no_data_after(interval) == want
|
|
|
|
|
|
@pytest.mark.parametrize("last_ago,interval,watch_ago,silent_after", [
|
|
(60, 60, 9999, None), # reporting
|
|
(300, 60, 9999, None), # exactly at the limit is not yet silent
|
|
(301, 60, 9999, 300), # silent since last + 300s
|
|
(2000, 600, 9999, 1800), # slow series: three of its gaps
|
|
(None, None, 3000, None), # never sampled, watched under an hour
|
|
(None, None, 4000, 3600), # never sampled, watched over an hour
|
|
])
|
|
def test_silent_since(last_ago, interval, watch_ago, silent_after):
|
|
last = NOW - timedelta(seconds=last_ago) if last_ago is not None else None
|
|
watch = NOW - timedelta(seconds=watch_ago)
|
|
got = silent_since(last, interval, watch, NOW)
|
|
if silent_after is None:
|
|
assert got is None
|
|
else:
|
|
assert got == (last or watch) + timedelta(seconds=silent_after)
|
|
|
|
|
|
def test_steward_s_own_outage_is_not_the_series_silence():
|
|
"""#5441: a series last sampled 18 minutes ago, while Steward has only been
|
|
back for 2 minutes, is not silent yet; once Steward has watched it for
|
|
three of its gaps with nothing arriving, it is, from then."""
|
|
last = NOW - timedelta(minutes=18)
|
|
watch = NOW - timedelta(days=1)
|
|
assert silent_since(last, 60, watch, NOW, awake_since=NOW - timedelta(minutes=2)) is None
|
|
awake = NOW - timedelta(minutes=6)
|
|
assert silent_since(last, 60, watch, NOW, awake_since=awake) == awake + timedelta(seconds=300)
|
|
# A sample since Steward came back is what counts, when there is one.
|
|
assert silent_since(NOW - timedelta(minutes=1), 60, watch, NOW, awake_since=awake) is None
|
|
|
|
|
|
class _Session:
|
|
def __init__(self):
|
|
self.added = []
|
|
|
|
def add(self, row):
|
|
self.added.append(row)
|
|
|
|
|
|
def _state(**kw):
|
|
base = dict(state=AlertStateEnum.firing, last_sample_at=None, sample_interval_s=None,
|
|
no_data_since=None)
|
|
return NS(**{**base, **kw})
|
|
|
|
|
|
def _samples(*ago_values):
|
|
return [NS(recorded_at=NOW - timedelta(seconds=a), value=v) for a, v in ago_values]
|
|
|
|
|
|
def test_tracking_learns_the_gap_and_skips_late_samples():
|
|
st, session = _state(), _Session()
|
|
assert _track_samples(session, NS(id="r"), st, _samples((180, 1), (120, 1), (60, 1)), NOW) is None
|
|
assert (st.last_sample_at, st.sample_interval_s) == (NOW - timedelta(seconds=60), 60)
|
|
_track_samples(session, NS(id="r"), st, _samples((90, 1)), NOW) # older than the last
|
|
assert (st.last_sample_at, st.sample_interval_s) == (NOW - timedelta(seconds=60), 60)
|
|
assert session.added == []
|
|
|
|
|
|
def test_a_sample_after_silence_clears_it_and_records_the_resume():
|
|
st, session = _state(last_sample_at=NOW - timedelta(minutes=20), sample_interval_s=60,
|
|
no_data_since=NOW - timedelta(minutes=15)), _Session()
|
|
event_id, value, note = _track_samples(session, NS(id="r"), st, _samples((0, 42.0)), NOW)
|
|
assert st.no_data_since is None and st.state == AlertStateEnum.firing # still firing
|
|
assert st.sample_interval_s == 90 # the outage counted as three gaps, at most
|
|
assert value == 42.0 and note == "data resumed after 15m silent"
|
|
[event] = session.added
|
|
assert (event.id, event.from_state, event.to_state) == (event_id, "no_data", "firing")
|
|
|
|
|
|
def test_muted_by_names_the_window_that_covers_the_rule():
|
|
from steward.core.alerts import _muted_by
|
|
rule = NS(source_module="host_agent", resource_name="nas")
|
|
windows = [NS(name="other", scope_source="docker", scope_resource=None),
|
|
NS(name="nas work", scope_source="host_agent", scope_resource="nas")]
|
|
assert _muted_by(windows, rule) == "notification muted: maintenance window nas work"
|
|
assert _muted_by(windows[:1], rule) is None
|
|
assert _muted_by([NS(name="all", scope_source=None, scope_resource=None)], rule) == (
|
|
"notification muted: maintenance window all")
|