bvandeusenandClaude Opus 5.5
c368e93e2c
Agent offline is an ordinary rule now. The host_agent scheduler records host_agent/<host>/seconds_since_seen every minute (from last_seen_at, or registration time for an agent that never reported), replacing the task that only logged stale agents. The catalog offers the metric, and picking it on a new rule fills "> 180". No data: each rule's state tracks its last sample and its usual gap, smoothed so that an outage counts as at most three gaps. A core task, alert_silence_check, runs every 60s and marks a rule no_data_since once its series has had no sample for three usual gaps (at least 5 minutes; an hour while the cadence is unknown). A never-sampled rule counts from when it started watching. The mark sits beside the state, so a firing alert that goes silent stays firing. Each silence records one event and sends one notification (rule option notify_no_data, default on; muted in maintenance windows). The next sample clears it with a "data resumed" event and notification. Creating, changing, disabling or re-enabling a rule restarts its watch. Migration 0029 adds the columns and seeds each existing rule's last sample and gap from its last day of data, so an upgrade does not flag every rule. Shown on the Alerts page (a NO DATA badge with its start time), in MCP list_alerts, list_alert_rules and get_host (state "no_data", no_data_since), in get_overview findings (alert_no_data, or a firing alert marked silent) and in the weekly report. The rule form has the notify option, and its metric reference now matches what traefik records and lists host_agent. Emails say "no data" and carry the event's note. The metric-catalog guard also reads positional MetricBatch.add arguments. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
99 lines
4.0 KiB
Python
99 lines
4.0 KiB
Python
"""Silence is alertable (#5062): a series' usual gap is learned from its
|
|
samples, an outage does not become its cadence, and a series counts as silent
|
|
after three usual gaps (at least five minutes; an hour while its cadence is
|
|
unknown)."""
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace as NS
|
|
|
|
import pytest
|
|
|
|
from steward.core.alerts import (
|
|
NO_DATA_FLOOR_S, NO_DATA_UNKNOWN_S, _track_samples, next_interval, no_data_after, silent_since,
|
|
)
|
|
from steward.models.alerts import AlertStateEnum
|
|
|
|
NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
@pytest.mark.parametrize("interval,gap,want", [
|
|
(None, 60, 60), # the first gap is the cadence
|
|
(0.0, 30, 30),
|
|
(60, 60, 60), # steady
|
|
(60, 80, 65), # a late sample barely moves it
|
|
(60, 3600, 90), # an outage counts as at most three gaps
|
|
(60, -5, 45), # never a negative gap
|
|
])
|
|
def test_next_interval(interval, gap, want):
|
|
assert next_interval(interval, gap) == pytest.approx(want)
|
|
|
|
|
|
def test_a_new_cadence_is_learned_in_steps():
|
|
interval, steps = 60.0, 0
|
|
while no_data_after(interval) < 600: # samples now arrive every 10 minutes
|
|
interval, steps = next_interval(interval, 600), steps + 1
|
|
assert steps == 3
|
|
|
|
|
|
@pytest.mark.parametrize("interval,want", [
|
|
(None, NO_DATA_UNKNOWN_S), (0.0, NO_DATA_UNKNOWN_S),
|
|
(30, NO_DATA_FLOOR_S), (100, 300), (600, 1800),
|
|
])
|
|
def test_no_data_after(interval, want):
|
|
assert no_data_after(interval) == want
|
|
|
|
|
|
@pytest.mark.parametrize("last_ago,interval,watch_ago,silent_after", [
|
|
(60, 60, 9999, None), # reporting
|
|
(300, 60, 9999, None), # exactly at the limit is not yet silent
|
|
(301, 60, 9999, 300), # silent since last + 300s
|
|
(2000, 600, 9999, 1800), # slow series: three of its gaps
|
|
(None, None, 3000, None), # never sampled, watched under an hour
|
|
(None, None, 4000, 3600), # never sampled, watched over an hour
|
|
])
|
|
def test_silent_since(last_ago, interval, watch_ago, silent_after):
|
|
last = NOW - timedelta(seconds=last_ago) if last_ago is not None else None
|
|
watch = NOW - timedelta(seconds=watch_ago)
|
|
got = silent_since(last, interval, watch, NOW)
|
|
if silent_after is None:
|
|
assert got is None
|
|
else:
|
|
assert got == (last or watch) + timedelta(seconds=silent_after)
|
|
|
|
|
|
class _Session:
|
|
def __init__(self):
|
|
self.added = []
|
|
|
|
def add(self, row):
|
|
self.added.append(row)
|
|
|
|
|
|
def _state(**kw):
|
|
base = dict(state=AlertStateEnum.firing, last_sample_at=None, sample_interval_s=None,
|
|
no_data_since=None)
|
|
return NS(**{**base, **kw})
|
|
|
|
|
|
def _samples(*ago_values):
|
|
return [NS(recorded_at=NOW - timedelta(seconds=a), value=v) for a, v in ago_values]
|
|
|
|
|
|
def test_tracking_learns_the_gap_and_skips_late_samples():
|
|
st, session = _state(), _Session()
|
|
assert _track_samples(session, NS(id="r"), st, _samples((180, 1), (120, 1), (60, 1)), NOW) is None
|
|
assert (st.last_sample_at, st.sample_interval_s) == (NOW - timedelta(seconds=60), 60)
|
|
_track_samples(session, NS(id="r"), st, _samples((90, 1)), NOW) # older than the last
|
|
assert (st.last_sample_at, st.sample_interval_s) == (NOW - timedelta(seconds=60), 60)
|
|
assert session.added == []
|
|
|
|
|
|
def test_a_sample_after_silence_clears_it_and_records_the_resume():
|
|
st, session = _state(last_sample_at=NOW - timedelta(minutes=20), sample_interval_s=60,
|
|
no_data_since=NOW - timedelta(minutes=15)), _Session()
|
|
event_id, value, note = _track_samples(session, NS(id="r"), st, _samples((0, 42.0)), NOW)
|
|
assert st.no_data_since is None and st.state == AlertStateEnum.firing # still firing
|
|
assert st.sample_interval_s == 90 # the outage counted as three gaps, at most
|
|
assert value == 42.0 and note == "data resumed after 15m silent"
|
|
[event] = session.added
|
|
assert (event.id, event.from_state, event.to_state) == (event_id, "no_data", "firing")
|