bvandeusenandClaude Opus 5.5
e51d969c43
A rule compared a reading with a fixed number, so Synology Disk 1's bad sectors going from 3,160 to 3,392 could say nothing: a count that only grows is soon past any line. "rose by" compares each reading with the one before it, kept on the alert's state and only for such a rule. A fall (a replaced disk) doesn't fire. On the rule form, through MCP, and worded in emails, the alert sentence, history and the timeline. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
119 lines
5.9 KiB
Python
119 lines
5.9 KiB
Python
"""Alert history read as episodes (#5316): each stretch out of quiet, how
|
|
long it lasted, its worst reading, who was told; an episode that straddles a
|
|
page of the list says so instead of guessing."""
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace as NS
|
|
|
|
from steward.alerts.episodes import episodes, fires_when, summary
|
|
from steward.models.alerts import AlertOperator
|
|
|
|
NOW = datetime(2026, 10, 8, 12, 0, tzinfo=timezone.utc)
|
|
SINCE = NOW - timedelta(days=7)
|
|
|
|
|
|
def _rule(rid="r1", metric="seconds_since_seen", op=AlertOperator.gt, threshold=180.0):
|
|
return NS(id=rid, name=f"rule {rid}", metric_name=metric, operator=op, threshold=threshold,
|
|
resource_name="VDNT-Docker02", source_module="host_agent")
|
|
|
|
|
|
def _t(rule, mins_ago, frm, to, raw=None, **kw):
|
|
return {"rule": rule, "at": NOW - timedelta(minutes=mins_ago), "from_state": frm, "to_state": to,
|
|
"raw": raw, "value": f"{raw}", "notes": kw.get("notes", []), "deliveries": kw.get("deliveries", []),
|
|
"muted_by": kw.get("muted_by"), "released": False, "ansible_run_id": kw.get("run")}
|
|
|
|
|
|
def _newest_first(*rows):
|
|
return sorted(rows, key=lambda r: r["at"], reverse=True)
|
|
|
|
|
|
def test_a_firing_and_its_resolve_are_one_episode():
|
|
r = _rule()
|
|
eps = episodes(_newest_first(_t(r, 402, "inactive", "firing", 201.0), _t(r, 389, "firing", "resolved", 19.0)),
|
|
newest_page=True, now=NOW, since=SINCE)
|
|
assert len(eps) == 1
|
|
ep = eps[0]
|
|
assert ep["worst"] == "firing" and ep["began_here"] and not ep["ongoing"]
|
|
assert ep["duration"] == 13 * 60 and ep["reading"] == "3m 21s" # the highest: it fires above
|
|
assert ep["fires_when"] == "fires above 3m"
|
|
|
|
|
|
def test_silence_then_firing_is_one_episode_and_the_worst_is_firing():
|
|
"""The live rule's shape: inactive → no data, inactive → firing (an
|
|
alert at rest records no_data as a label, not a state), no data → firing,
|
|
firing → resolved."""
|
|
r = _rule()
|
|
eps = episodes(_newest_first(
|
|
_t(r, 1397, "inactive", "no_data", notes=["no sample for 7m 54s (usually every 1m)"]),
|
|
_t(r, 1396, "inactive", "firing", 473.0),
|
|
_t(r, 1395, "no_data", "firing", 533.0, notes=["data resumed after 3m 54s silent"]),
|
|
_t(r, 1394, "firing", "resolved", 42.0)), newest_page=True, now=NOW, since=SINCE)
|
|
assert len(eps) == 1 and len(eps[0]["steps"]) == 4
|
|
assert eps[0]["worst"] == "firing" and eps[0]["duration"] == 3 * 60
|
|
assert eps[0]["notes"] == ["no sample for 7m 54s (usually every 1m)", "data resumed after 3m 54s silent"]
|
|
|
|
|
|
def test_episodes_are_newest_first_and_kept_apart_per_rule():
|
|
a, b = _rule("a"), _rule("b", "cpu_pct", AlertOperator.lt, 5.0)
|
|
eps = episodes(_newest_first(
|
|
_t(a, 60, "inactive", "firing", 200.0), _t(b, 50, "inactive", "firing", 2.0),
|
|
_t(a, 40, "firing", "resolved", 10.0), _t(b, 30, "firing", "resolved", 9.0),
|
|
_t(b, 20, "resolved", "firing", 1.0)), newest_page=True, now=NOW, since=SINCE)
|
|
assert [(e["rule"].id, e["start"]) for e in eps] == [
|
|
("b", NOW - timedelta(minutes=20)), ("b", NOW - timedelta(minutes=50)), ("a", NOW - timedelta(minutes=60))]
|
|
assert eps[0]["ongoing"] and eps[0]["duration"] == 20 * 60
|
|
assert eps[1]["reading"] == "2.0%" and eps[1]["fires_when"] == "fires below 5.0%" # the lowest: it fires below
|
|
|
|
|
|
def test_an_episode_split_by_paging_says_so():
|
|
r = _rule()
|
|
# Its end is on a newer page: this older page cannot know when it cleared.
|
|
older = episodes([_t(r, 60, "inactive", "firing", 200.0)], newest_page=False, now=NOW, since=SINCE)
|
|
assert older[0]["continues"] and not older[0]["ongoing"] and older[0]["duration"] is None
|
|
# Its start is on an older page: it began before what this page holds.
|
|
newer = episodes([_t(r, 10, "firing", "resolved", 5.0)], newest_page=True, now=NOW, since=SINCE)
|
|
assert not newer[0]["began_here"] and newer[0]["end"] == NOW - timedelta(minutes=10)
|
|
|
|
|
|
def _d(label, state="sent", error=None):
|
|
return {"label": label, "state": state, "at": None, "attempts": None, "error": error}
|
|
|
|
|
|
def test_who_was_told_is_gathered_once_per_episode():
|
|
r = _rule()
|
|
eps = episodes(_newest_first(
|
|
_t(r, 30, "inactive", "firing", 200.0, deliveries=[_d("email", "failed", "timed out"), _d("webhook")],
|
|
run="run-1"),
|
|
_t(r, 20, "firing", "resolved", 5.0, deliveries=[_d("webhook")], muted_by="patch")),
|
|
newest_page=True, now=NOW, since=SINCE)
|
|
assert eps[0]["deliveries"] == [_d("email", "failed", "timed out"), _d("webhook")]
|
|
assert eps[0]["muted_by"] == "patch" and eps[0]["runs"] == ["run-1"]
|
|
|
|
|
|
def test_the_verdict_counts_what_fired_and_for_how_long():
|
|
r = _rule()
|
|
eps = episodes(_newest_first(
|
|
_t(r, 100, "inactive", "firing", 200.0), _t(r, 90, "firing", "resolved", 5.0),
|
|
_t(r, 50, "inactive", "no_data"), _t(r, 45, "inactive", "resolved"),
|
|
_t(r, 10, "inactive", "firing", 300.0)), newest_page=True, now=NOW, since=SINCE)
|
|
assert summary(eps) == {"fired": 2, "seconds": 10 * 60 + 10 * 60, "ongoing": True}
|
|
|
|
|
|
def test_an_up_or_down_rule_says_the_word():
|
|
assert fires_when(_rule(metric="is_up", op=AlertOperator.lt, threshold=0.5)) == "fires when down"
|
|
|
|
|
|
def test_a_rule_on_a_rise_reads_as_one():
|
|
rule = _rule(metric="disk_bad_sectors", op=AlertOperator.rose, threshold=1.0)
|
|
assert fires_when(rule).startswith("fires on a rise of ")
|
|
assert AlertOperator.rose.said == "rose by" and AlertOperator.gt.said == ">"
|
|
|
|
|
|
def test_a_rise_is_against_the_reading_before_it():
|
|
from steward.core.alerts import _is_breached
|
|
rose = AlertOperator.rose
|
|
assert _is_breached(3392.0, rose, 1.0, previous=3160.0)
|
|
assert not _is_breached(3392.0, rose, 1.0, previous=3392.0) # flat
|
|
assert not _is_breached(0.0, rose, 1.0, previous=3392.0) # a replaced disk starts over
|
|
assert not _is_breached(3392.0, rose, 1.0) # nothing to have risen from
|
|
assert not _is_breached(3161.0, rose, 5.0, previous=3160.0) # less than asked
|