bvandeusenandClaude Opus 5.5
531a193bd1
History (#5316): - Transitions fold into episodes (steward/alerts/episodes.py): each stretch out of quiet, the worst state it reached, how long it lasted, its worst reading against the rule ('3m 21s (fires above 3m)'), who was told across its changes, and its Ansible runs. The changes themselves fold under it. - An episode cut by paging says so ('cleared later (above)', 'earlier'). - For one rule: a verdict ('Fired 4 times in the last 7d, 1h 2m in all') and no Rule column. Notified and Action show only when the range has any. Tests: - The confirm guard looks for a confirm( call, not the word in the comment that says it is gone (lesson #4365). - The Hosts viewer test looks for the add-host dialog by id, since every page now carries the confirm dialog. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
107 lines
4.7 KiB
Python
107 lines
4.7 KiB
Python
"""An alert's history read as episodes (#5316): each time a rule left quiet,
|
|
what it went through, how long it lasted, its worst reading and who was told.
|
|
|
|
The history page lists transitions a page at a time (core.paging), newest
|
|
first. `episodes` folds one page of them, so an episode can straddle two
|
|
pages: its end on a newer page (`continues`) or its start on an older one
|
|
(`began_here` false). Model-free, so it is unit-tested without a database.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime
|
|
|
|
# The states an alert rests in; leaving one opens an episode, reaching one ends it.
|
|
QUIET = frozenset({"inactive", "resolved"})
|
|
# The worst thing an episode went through, most severe first.
|
|
_SEVERITY = ("firing", "acknowledged", "no_data", "pending")
|
|
_WORDS = {">": "above", ">=": "at or above", "<": "below", "<=": "at or below",
|
|
"==": "at", "!=": "other than"}
|
|
|
|
|
|
def fires_when(rule) -> str:
|
|
"""A rule's condition as a reading's aside: "fires above 3m", "fires
|
|
when down"."""
|
|
from steward.alerts.service import rule_condition
|
|
from steward.core.formatting import metric_value
|
|
said = rule_condition(rule)
|
|
if said.startswith("is "):
|
|
return f"fires when {said[3:]}"
|
|
op = rule.operator.value
|
|
return f"fires {_WORDS.get(op, op)} {metric_value(rule.metric_name, rule.threshold)}"
|
|
|
|
|
|
def _worst_value(rule, values: list[float]) -> float | None:
|
|
"""The reading furthest into breach: the highest for a rule that fires
|
|
above, the lowest for one that fires below, else the last."""
|
|
if not values:
|
|
return None
|
|
op = rule.operator.value
|
|
if op in (">", ">="):
|
|
return max(values)
|
|
if op in ("<", "<="):
|
|
return min(values)
|
|
return values[-1]
|
|
|
|
|
|
def episodes(rows: list[dict], *, newest_page: bool, now: datetime, since: datetime) -> list[dict]:
|
|
"""Fold transitions (alert history rows, newest first, each with its
|
|
`rule`, `at`, `from_state`, `to_state`, `raw` value, `deliveries`,
|
|
`muted_by`, `released`, `ansible_run_id` and `notes`) into episodes,
|
|
newest first.
|
|
|
|
Each episode: rule, steps (oldest first), start, end (None while open),
|
|
began_here (its first step left quiet on this page), ongoing (open on the
|
|
newest page: it is still going), continues (open on an older page: it
|
|
ends on a newer one), worst (the most severe state it reached), duration
|
|
(seconds, counted from `since` when it began before the range), worst
|
|
reading, deliveries, muted_by, released and runs."""
|
|
from steward.core.formatting import metric_value
|
|
open_by_rule: dict[str, dict] = {}
|
|
out: list[dict] = []
|
|
for r in reversed(rows):
|
|
rid = r["rule"].id
|
|
ep = open_by_rule.get(rid)
|
|
if ep is None:
|
|
ep = {"rule": r["rule"], "steps": [], "start": r["at"], "end": None,
|
|
"began_here": r["from_state"] in QUIET}
|
|
out.append(ep)
|
|
ep["steps"].append(r)
|
|
if r["to_state"] in QUIET:
|
|
ep["end"] = r["at"]
|
|
open_by_rule.pop(rid, None)
|
|
else:
|
|
open_by_rule[rid] = ep
|
|
|
|
for ep in out:
|
|
steps, rule = ep["steps"], ep["rule"]
|
|
reached = {s["to_state"] for s in steps} | {s["from_state"] for s in steps}
|
|
ep["worst"] = next((s for s in _SEVERITY if s in reached), steps[-1]["to_state"])
|
|
ep["ongoing"] = ep["end"] is None and newest_page
|
|
ep["continues"] = ep["end"] is None and not newest_page
|
|
began = ep["start"] if ep["began_here"] else since
|
|
ended = ep["end"] or (now if ep["ongoing"] else None)
|
|
ep["duration"] = (ended - began).total_seconds() if ended else None
|
|
worst = _worst_value(rule, [s["raw"] for s in steps if s["raw"] is not None])
|
|
ep["reading"] = None if worst is None else metric_value(rule.metric_name, worst)
|
|
ep["fires_when"] = fires_when(rule)
|
|
seen, deliveries = set(), []
|
|
for s in steps:
|
|
for d in s["deliveries"]:
|
|
if (d[0], d[1]) not in seen:
|
|
seen.add((d[0], d[1]))
|
|
deliveries.append(d)
|
|
ep["deliveries"] = deliveries
|
|
ep["muted_by"] = next((s["muted_by"] for s in steps if s["muted_by"]), None)
|
|
ep["released"] = any(s["released"] for s in steps)
|
|
ep["runs"] = [s["ansible_run_id"] for s in steps if s["ansible_run_id"]]
|
|
ep["notes"] = [n for s in steps for n in s["notes"]]
|
|
return list(reversed(out))
|
|
|
|
|
|
def summary(eps: list[dict]) -> dict:
|
|
"""How often a rule fired in a range and for how long in all: {fired,
|
|
seconds, ongoing}."""
|
|
fired = [e for e in eps if e["worst"] in ("firing", "acknowledged")]
|
|
return {"fired": len(fired), "seconds": sum(e["duration"] or 0 for e in fired),
|
|
"ongoing": any(e["ongoing"] for e in fired)}
|