bvandeusenandClaude Opus 5.5
defad2e98b
#5509 "Delivery you can see: each alert's email state in history, and outgoing email on Settings -> System" (milestone 501). - Alerts -> History: an alert's email reads from the outbox (#5506): "gathering" with when it sends, "sending", "retrying" with the attempt, next try and the server's last answer, "sent", or "failed" naming each reader it didn't reach and why (outbox.delivery / email_states). Events from before the outbox keep what they recorded. The Notified column shows once an alert has a queued email. - Settings -> System gets "Outgoing email": every email of the last 7 days not yet sent, failures first, then the longest retrying, with recipient, kind, state and last error, and a count of what was sent. Failures and anything still retrying after 5 minutes lead the page's verdict (outbox.outgoing). - get_alert_history (MCP) adds `email`: the same delivery state. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
103 lines
5.1 KiB
Python
103 lines
5.1 KiB
Python
"""Alert history read as episodes (#5316): each stretch out of quiet, how
|
|
long it lasted, its worst reading, who was told; an episode that straddles a
|
|
page of the list says so instead of guessing."""
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace as NS
|
|
|
|
from steward.alerts.episodes import episodes, fires_when, summary
|
|
from steward.models.alerts import AlertOperator
|
|
|
|
NOW = datetime(2026, 10, 8, 12, 0, tzinfo=timezone.utc)
|
|
SINCE = NOW - timedelta(days=7)
|
|
|
|
|
|
def _rule(rid="r1", metric="seconds_since_seen", op=AlertOperator.gt, threshold=180.0):
|
|
return NS(id=rid, name=f"rule {rid}", metric_name=metric, operator=op, threshold=threshold,
|
|
resource_name="VDNT-Docker02", source_module="host_agent")
|
|
|
|
|
|
def _t(rule, mins_ago, frm, to, raw=None, **kw):
|
|
return {"rule": rule, "at": NOW - timedelta(minutes=mins_ago), "from_state": frm, "to_state": to,
|
|
"raw": raw, "value": f"{raw}", "notes": kw.get("notes", []), "deliveries": kw.get("deliveries", []),
|
|
"muted_by": kw.get("muted_by"), "released": False, "ansible_run_id": kw.get("run")}
|
|
|
|
|
|
def _newest_first(*rows):
|
|
return sorted(rows, key=lambda r: r["at"], reverse=True)
|
|
|
|
|
|
def test_a_firing_and_its_resolve_are_one_episode():
|
|
r = _rule()
|
|
eps = episodes(_newest_first(_t(r, 402, "inactive", "firing", 201.0), _t(r, 389, "firing", "resolved", 19.0)),
|
|
newest_page=True, now=NOW, since=SINCE)
|
|
assert len(eps) == 1
|
|
ep = eps[0]
|
|
assert ep["worst"] == "firing" and ep["began_here"] and not ep["ongoing"]
|
|
assert ep["duration"] == 13 * 60 and ep["reading"] == "3m 21s" # the highest: it fires above
|
|
assert ep["fires_when"] == "fires above 3m"
|
|
|
|
|
|
def test_silence_then_firing_is_one_episode_and_the_worst_is_firing():
|
|
"""The live rule's shape: inactive → no data, inactive → firing (an
|
|
alert at rest records no_data as a label, not a state), no data → firing,
|
|
firing → resolved."""
|
|
r = _rule()
|
|
eps = episodes(_newest_first(
|
|
_t(r, 1397, "inactive", "no_data", notes=["no sample for 7m 54s (usually every 1m)"]),
|
|
_t(r, 1396, "inactive", "firing", 473.0),
|
|
_t(r, 1395, "no_data", "firing", 533.0, notes=["data resumed after 3m 54s silent"]),
|
|
_t(r, 1394, "firing", "resolved", 42.0)), newest_page=True, now=NOW, since=SINCE)
|
|
assert len(eps) == 1 and len(eps[0]["steps"]) == 4
|
|
assert eps[0]["worst"] == "firing" and eps[0]["duration"] == 3 * 60
|
|
assert eps[0]["notes"] == ["no sample for 7m 54s (usually every 1m)", "data resumed after 3m 54s silent"]
|
|
|
|
|
|
def test_episodes_are_newest_first_and_kept_apart_per_rule():
|
|
a, b = _rule("a"), _rule("b", "cpu_pct", AlertOperator.lt, 5.0)
|
|
eps = episodes(_newest_first(
|
|
_t(a, 60, "inactive", "firing", 200.0), _t(b, 50, "inactive", "firing", 2.0),
|
|
_t(a, 40, "firing", "resolved", 10.0), _t(b, 30, "firing", "resolved", 9.0),
|
|
_t(b, 20, "resolved", "firing", 1.0)), newest_page=True, now=NOW, since=SINCE)
|
|
assert [(e["rule"].id, e["start"]) for e in eps] == [
|
|
("b", NOW - timedelta(minutes=20)), ("b", NOW - timedelta(minutes=50)), ("a", NOW - timedelta(minutes=60))]
|
|
assert eps[0]["ongoing"] and eps[0]["duration"] == 20 * 60
|
|
assert eps[1]["reading"] == "2.0%" and eps[1]["fires_when"] == "fires below 5.0%" # the lowest: it fires below
|
|
|
|
|
|
def test_an_episode_split_by_paging_says_so():
|
|
r = _rule()
|
|
# Its end is on a newer page: this older page cannot know when it cleared.
|
|
older = episodes([_t(r, 60, "inactive", "firing", 200.0)], newest_page=False, now=NOW, since=SINCE)
|
|
assert older[0]["continues"] and not older[0]["ongoing"] and older[0]["duration"] is None
|
|
# Its start is on an older page: it began before what this page holds.
|
|
newer = episodes([_t(r, 10, "firing", "resolved", 5.0)], newest_page=True, now=NOW, since=SINCE)
|
|
assert not newer[0]["began_here"] and newer[0]["end"] == NOW - timedelta(minutes=10)
|
|
|
|
|
|
def _d(label, state="sent", error=None):
|
|
return {"label": label, "state": state, "at": None, "attempts": None, "error": error}
|
|
|
|
|
|
def test_who_was_told_is_gathered_once_per_episode():
|
|
r = _rule()
|
|
eps = episodes(_newest_first(
|
|
_t(r, 30, "inactive", "firing", 200.0, deliveries=[_d("email", "failed", "timed out"), _d("webhook")],
|
|
run="run-1"),
|
|
_t(r, 20, "firing", "resolved", 5.0, deliveries=[_d("webhook")], muted_by="patch")),
|
|
newest_page=True, now=NOW, since=SINCE)
|
|
assert eps[0]["deliveries"] == [_d("email", "failed", "timed out"), _d("webhook")]
|
|
assert eps[0]["muted_by"] == "patch" and eps[0]["runs"] == ["run-1"]
|
|
|
|
|
|
def test_the_verdict_counts_what_fired_and_for_how_long():
|
|
r = _rule()
|
|
eps = episodes(_newest_first(
|
|
_t(r, 100, "inactive", "firing", 200.0), _t(r, 90, "firing", "resolved", 5.0),
|
|
_t(r, 50, "inactive", "no_data"), _t(r, 45, "inactive", "resolved"),
|
|
_t(r, 10, "inactive", "firing", 300.0)), newest_page=True, now=NOW, since=SINCE)
|
|
assert summary(eps) == {"fired": 2, "seconds": 10 * 60 + 10 * 60, "ongoing": True}
|
|
|
|
|
|
def test_an_up_or_down_rule_says_the_word():
|
|
assert fires_when(_rule(metric="is_up", op=AlertOperator.lt, threshold=0.5)) == "fires when down"
|