bvandeusenandClaude Opus 5.5
defad2e98b
#5509 "Delivery you can see: each alert's email state in history, and outgoing email on Settings -> System" (milestone 501). - Alerts -> History: an alert's email reads from the outbox (#5506): "gathering" with when it sends, "sending", "retrying" with the attempt, next try and the server's last answer, "sent", or "failed" naming each reader it didn't reach and why (outbox.delivery / email_states). Events from before the outbox keep what they recorded. The Notified column shows once an alert has a queued email. - Settings -> System gets "Outgoing email": every email of the last 7 days not yet sent, failures first, then the longest retrying, with recipient, kind, state and last error, and a count of what was sent. Failures and anything still retrying after 5 minutes lead the page's verdict (outbox.outgoing). - get_alert_history (MCP) adds `email`: the same delivery state. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
107 lines
4.7 KiB
Python
107 lines
4.7 KiB
Python
"""An alert's history read as episodes (#5316): each time a rule left quiet,
|
|
what it went through, how long it lasted, its worst reading and who was told.
|
|
|
|
The history page lists transitions a page at a time (core.paging), newest
|
|
first. `episodes` folds one page of them, so an episode can straddle two
|
|
pages: its end on a newer page (`continues`) or its start on an older one
|
|
(`began_here` false). Model-free, so it is unit-tested without a database.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime
|
|
|
|
# The states an alert rests in; leaving one opens an episode, reaching one ends it.
|
|
QUIET = frozenset({"inactive", "resolved"})
|
|
# The worst thing an episode went through, most severe first.
|
|
_SEVERITY = ("firing", "acknowledged", "no_data", "pending")
|
|
_WORDS = {">": "above", ">=": "at or above", "<": "below", "<=": "at or below",
|
|
"==": "at", "!=": "other than"}
|
|
|
|
|
|
def fires_when(rule) -> str:
|
|
"""A rule's condition as a reading's aside: "fires above 3m", "fires
|
|
when down"."""
|
|
from steward.alerts.service import rule_condition
|
|
from steward.core.formatting import metric_value
|
|
said = rule_condition(rule)
|
|
if said.startswith("is "):
|
|
return f"fires when {said[3:]}"
|
|
op = rule.operator.value
|
|
return f"fires {_WORDS.get(op, op)} {metric_value(rule.metric_name, rule.threshold)}"
|
|
|
|
|
|
def _worst_value(rule, values: list[float]) -> float | None:
|
|
"""The reading furthest into breach: the highest for a rule that fires
|
|
above, the lowest for one that fires below, else the last."""
|
|
if not values:
|
|
return None
|
|
op = rule.operator.value
|
|
if op in (">", ">="):
|
|
return max(values)
|
|
if op in ("<", "<="):
|
|
return min(values)
|
|
return values[-1]
|
|
|
|
|
|
def episodes(rows: list[dict], *, newest_page: bool, now: datetime, since: datetime) -> list[dict]:
|
|
"""Fold transitions (alert history rows, newest first, each with its
|
|
`rule`, `at`, `from_state`, `to_state`, `raw` value, `deliveries`,
|
|
`muted_by`, `released`, `ansible_run_id` and `notes`) into episodes,
|
|
newest first.
|
|
|
|
Each episode: rule, steps (oldest first), start, end (None while open),
|
|
began_here (its first step left quiet on this page), ongoing (open on the
|
|
newest page: it is still going), continues (open on an older page: it
|
|
ends on a newer one), worst (the most severe state it reached), duration
|
|
(seconds, counted from `since` when it began before the range), worst
|
|
reading, deliveries, muted_by, released and runs."""
|
|
from steward.core.formatting import metric_value
|
|
open_by_rule: dict[str, dict] = {}
|
|
out: list[dict] = []
|
|
for r in reversed(rows):
|
|
rid = r["rule"].id
|
|
ep = open_by_rule.get(rid)
|
|
if ep is None:
|
|
ep = {"rule": r["rule"], "steps": [], "start": r["at"], "end": None,
|
|
"began_here": r["from_state"] in QUIET}
|
|
out.append(ep)
|
|
ep["steps"].append(r)
|
|
if r["to_state"] in QUIET:
|
|
ep["end"] = r["at"]
|
|
open_by_rule.pop(rid, None)
|
|
else:
|
|
open_by_rule[rid] = ep
|
|
|
|
for ep in out:
|
|
steps, rule = ep["steps"], ep["rule"]
|
|
reached = {s["to_state"] for s in steps} | {s["from_state"] for s in steps}
|
|
ep["worst"] = next((s for s in _SEVERITY if s in reached), steps[-1]["to_state"])
|
|
ep["ongoing"] = ep["end"] is None and newest_page
|
|
ep["continues"] = ep["end"] is None and not newest_page
|
|
began = ep["start"] if ep["began_here"] else since
|
|
ended = ep["end"] or (now if ep["ongoing"] else None)
|
|
ep["duration"] = (ended - began).total_seconds() if ended else None
|
|
worst = _worst_value(rule, [s["raw"] for s in steps if s["raw"] is not None])
|
|
ep["reading"] = None if worst is None else metric_value(rule.metric_name, worst)
|
|
ep["fires_when"] = fires_when(rule)
|
|
seen, deliveries = set(), []
|
|
for s in steps:
|
|
for d in s["deliveries"]:
|
|
if (d["label"], d["state"]) not in seen:
|
|
seen.add((d["label"], d["state"]))
|
|
deliveries.append(d)
|
|
ep["deliveries"] = deliveries
|
|
ep["muted_by"] = next((s["muted_by"] for s in steps if s["muted_by"]), None)
|
|
ep["released"] = any(s["released"] for s in steps)
|
|
ep["runs"] = [s["ansible_run_id"] for s in steps if s["ansible_run_id"]]
|
|
ep["notes"] = [n for s in steps for n in s["notes"]]
|
|
return list(reversed(out))
|
|
|
|
|
|
def summary(eps: list[dict]) -> dict:
|
|
"""How often a rule fired in a range and for how long in all: {fired,
|
|
seconds, ongoing}."""
|
|
fired = [e for e in eps if e["worst"] in ("firing", "acknowledged")]
|
|
return {"fired": len(fired), "seconds": sum(e["duration"] or 0 for e in fired),
|
|
"ongoing": any(e["ongoing"] for e in fired)}
|