bvandeusenandClaude Opus 5.5
49607a2441
Notifications and alert actions now start only once the evaluating
transaction commits, and never for one that rolls back. record_metrics
queues them on the session (_after_commit), and after_commit/after_rollback
listeners on the sync session spawn or drop them. Spawned tasks are held in a
module set so they cannot be garbage-collected before they run.
A maintenance window no longer skips evaluation. Alerts in its scope still
change state, but their notifications and actions are held, and each event
says so ("notification muted: maintenance window <name>"; also on no-data
and resume events). notify_after_maintenance, run every minute with the
silence check as the new core task alert_reconcile, sends one notification
for an alert still firing after its window ends and runs its action. The
event's note records the release.
Alert actions go through runner.trigger_run like every other run path, so
the playbook and the inventory must be ones the source lists (#5034 guards).
The firing event links to the run, or its note says why the action did not
start. The maintenance page and form wording now say what a window does.
Tests:
- integration: a rolled-back evaluation notifies nobody, and dispatch waits
for the commit; maintenance mutes while the state moves and notifies once
when the window ends; an action whose playbook is outside its source is
refused, with the note set;
- the alert-action test uses a listed inventory name and waits, with a
deadline, for the run the action starts;
- unit: _muted_by.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
110 lines
4.5 KiB
Python
110 lines
4.5 KiB
Python
"""Silence is alertable (#5062): a series' usual gap is learned from its
|
|
samples, an outage does not become its cadence, and a series counts as silent
|
|
after three usual gaps (at least five minutes; an hour while its cadence is
|
|
unknown)."""
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace as NS
|
|
|
|
import pytest
|
|
|
|
from steward.core.alerts import (
|
|
NO_DATA_FLOOR_S, NO_DATA_UNKNOWN_S, _track_samples, next_interval, no_data_after, silent_since,
|
|
)
|
|
from steward.models.alerts import AlertStateEnum
|
|
|
|
NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
@pytest.mark.parametrize("interval,gap,want", [
|
|
(None, 60, 60), # the first gap is the cadence
|
|
(0.0, 30, 30),
|
|
(60, 60, 60), # steady
|
|
(60, 80, 65), # a late sample barely moves it
|
|
(60, 3600, 90), # an outage counts as at most three gaps
|
|
(60, -5, 45), # never a negative gap
|
|
])
|
|
def test_next_interval(interval, gap, want):
|
|
assert next_interval(interval, gap) == pytest.approx(want)
|
|
|
|
|
|
def test_a_new_cadence_is_learned_in_steps():
|
|
interval, steps = 60.0, 0
|
|
while no_data_after(interval) < 600: # samples now arrive every 10 minutes
|
|
interval, steps = next_interval(interval, 600), steps + 1
|
|
assert steps == 3
|
|
|
|
|
|
@pytest.mark.parametrize("interval,want", [
|
|
(None, NO_DATA_UNKNOWN_S), (0.0, NO_DATA_UNKNOWN_S),
|
|
(30, NO_DATA_FLOOR_S), (100, 300), (600, 1800),
|
|
])
|
|
def test_no_data_after(interval, want):
|
|
assert no_data_after(interval) == want
|
|
|
|
|
|
@pytest.mark.parametrize("last_ago,interval,watch_ago,silent_after", [
|
|
(60, 60, 9999, None), # reporting
|
|
(300, 60, 9999, None), # exactly at the limit is not yet silent
|
|
(301, 60, 9999, 300), # silent since last + 300s
|
|
(2000, 600, 9999, 1800), # slow series: three of its gaps
|
|
(None, None, 3000, None), # never sampled, watched under an hour
|
|
(None, None, 4000, 3600), # never sampled, watched over an hour
|
|
])
|
|
def test_silent_since(last_ago, interval, watch_ago, silent_after):
|
|
last = NOW - timedelta(seconds=last_ago) if last_ago is not None else None
|
|
watch = NOW - timedelta(seconds=watch_ago)
|
|
got = silent_since(last, interval, watch, NOW)
|
|
if silent_after is None:
|
|
assert got is None
|
|
else:
|
|
assert got == (last or watch) + timedelta(seconds=silent_after)
|
|
|
|
|
|
class _Session:
|
|
def __init__(self):
|
|
self.added = []
|
|
|
|
def add(self, row):
|
|
self.added.append(row)
|
|
|
|
|
|
def _state(**kw):
|
|
base = dict(state=AlertStateEnum.firing, last_sample_at=None, sample_interval_s=None,
|
|
no_data_since=None)
|
|
return NS(**{**base, **kw})
|
|
|
|
|
|
def _samples(*ago_values):
|
|
return [NS(recorded_at=NOW - timedelta(seconds=a), value=v) for a, v in ago_values]
|
|
|
|
|
|
def test_tracking_learns_the_gap_and_skips_late_samples():
|
|
st, session = _state(), _Session()
|
|
assert _track_samples(session, NS(id="r"), st, _samples((180, 1), (120, 1), (60, 1)), NOW) is None
|
|
assert (st.last_sample_at, st.sample_interval_s) == (NOW - timedelta(seconds=60), 60)
|
|
_track_samples(session, NS(id="r"), st, _samples((90, 1)), NOW) # older than the last
|
|
assert (st.last_sample_at, st.sample_interval_s) == (NOW - timedelta(seconds=60), 60)
|
|
assert session.added == []
|
|
|
|
|
|
def test_a_sample_after_silence_clears_it_and_records_the_resume():
|
|
st, session = _state(last_sample_at=NOW - timedelta(minutes=20), sample_interval_s=60,
|
|
no_data_since=NOW - timedelta(minutes=15)), _Session()
|
|
event_id, value, note = _track_samples(session, NS(id="r"), st, _samples((0, 42.0)), NOW)
|
|
assert st.no_data_since is None and st.state == AlertStateEnum.firing # still firing
|
|
assert st.sample_interval_s == 90 # the outage counted as three gaps, at most
|
|
assert value == 42.0 and note == "data resumed after 15m silent"
|
|
[event] = session.added
|
|
assert (event.id, event.from_state, event.to_state) == (event_id, "no_data", "firing")
|
|
|
|
|
|
def test_muted_by_names_the_window_that_covers_the_rule():
|
|
from steward.core.alerts import _muted_by
|
|
rule = NS(source_module="host_agent", resource_name="nas")
|
|
windows = [NS(name="other", scope_source="docker", scope_resource=None),
|
|
NS(name="nas work", scope_source="host_agent", scope_resource="nas")]
|
|
assert _muted_by(windows, rule) == "notification muted: maintenance window nas work"
|
|
assert _muted_by(windows[:1], rule) is None
|
|
assert _muted_by([NS(name="all", scope_source=None, scope_resource=None)], rule) == (
|
|
"notification muted: maintenance window all")
|