diff --git a/steward/alerts/episodes.py b/steward/alerts/episodes.py new file mode 100644 index 0000000..502544e --- /dev/null +++ b/steward/alerts/episodes.py @@ -0,0 +1,106 @@ +"""An alert's history read as episodes (#5316): each time a rule left quiet, +what it went through, how long it lasted, its worst reading and who was told. + +The history page lists transitions a page at a time (core.paging), newest +first. `episodes` folds one page of them, so an episode can straddle two +pages: its end on a newer page (`continues`) or its start on an older one +(`began_here` false). Model-free, so it is unit-tested without a database. +""" +from __future__ import annotations + +from datetime import datetime + +# The states an alert rests in; leaving one opens an episode, reaching one ends it. +QUIET = frozenset({"inactive", "resolved"}) +# The worst thing an episode went through, most severe first. +_SEVERITY = ("firing", "acknowledged", "no_data", "pending") +_WORDS = {">": "above", ">=": "at or above", "<": "below", "<=": "at or below", + "==": "at", "!=": "other than"} + + +def fires_when(rule) -> str: + """A rule's condition as a reading's aside: "fires above 3m", "fires + when down".""" + from steward.alerts.service import rule_condition + from steward.core.formatting import metric_value + said = rule_condition(rule) + if said.startswith("is "): + return f"fires when {said[3:]}" + op = rule.operator.value + return f"fires {_WORDS.get(op, op)} {metric_value(rule.metric_name, rule.threshold)}" + + +def _worst_value(rule, values: list[float]) -> float | None: + """The reading furthest into breach: the highest for a rule that fires + above, the lowest for one that fires below, else the last.""" + if not values: + return None + op = rule.operator.value + if op in (">", ">="): + return max(values) + if op in ("<", "<="): + return min(values) + return values[-1] + + +def episodes(rows: list[dict], *, newest_page: bool, now: datetime, since: datetime) -> list[dict]: + """Fold transitions (alert history rows, newest first, each with its + `rule`, `at`, `from_state`, `to_state`, `raw` value, `deliveries`, + `muted_by`, `released`, `ansible_run_id` and `notes`) into episodes, + newest first. + + Each episode: rule, steps (oldest first), start, end (None while open), + began_here (its first step left quiet on this page), ongoing (open on the + newest page: it is still going), continues (open on an older page: it + ends on a newer one), worst (the most severe state it reached), duration + (seconds, counted from `since` when it began before the range), worst + reading, deliveries, muted_by, released and runs.""" + from steward.core.formatting import metric_value + open_by_rule: dict[str, dict] = {} + out: list[dict] = [] + for r in reversed(rows): + rid = r["rule"].id + ep = open_by_rule.get(rid) + if ep is None: + ep = {"rule": r["rule"], "steps": [], "start": r["at"], "end": None, + "began_here": r["from_state"] in QUIET} + out.append(ep) + ep["steps"].append(r) + if r["to_state"] in QUIET: + ep["end"] = r["at"] + open_by_rule.pop(rid, None) + else: + open_by_rule[rid] = ep + + for ep in out: + steps, rule = ep["steps"], ep["rule"] + reached = {s["to_state"] for s in steps} | {s["from_state"] for s in steps} + ep["worst"] = next((s for s in _SEVERITY if s in reached), steps[-1]["to_state"]) + ep["ongoing"] = ep["end"] is None and newest_page + ep["continues"] = ep["end"] is None and not newest_page + began = ep["start"] if ep["began_here"] else since + ended = ep["end"] or (now if ep["ongoing"] else None) + ep["duration"] = (ended - began).total_seconds() if ended else None + worst = _worst_value(rule, [s["raw"] for s in steps if s["raw"] is not None]) + ep["reading"] = None if worst is None else metric_value(rule.metric_name, worst) + ep["fires_when"] = fires_when(rule) + seen, deliveries = set(), [] + for s in steps: + for d in s["deliveries"]: + if (d[0], d[1]) not in seen: + seen.add((d[0], d[1])) + deliveries.append(d) + ep["deliveries"] = deliveries + ep["muted_by"] = next((s["muted_by"] for s in steps if s["muted_by"]), None) + ep["released"] = any(s["released"] for s in steps) + ep["runs"] = [s["ansible_run_id"] for s in steps if s["ansible_run_id"]] + ep["notes"] = [n for s in steps for n in s["notes"]] + return list(reversed(out)) + + +def summary(eps: list[dict]) -> dict: + """How often a rule fired in a range and for how long in all: {fired, + seconds, ongoing}.""" + fired = [e for e in eps if e["worst"] in ("firing", "acknowledged")] + return {"fired": len(fired), "seconds": sum(e["duration"] or 0 for e in fired), + "ongoing": any(e["ongoing"] for e in fired)} diff --git a/steward/alerts/routes.py b/steward/alerts/routes.py index 9cfc175..fc20e60 100644 --- a/steward/alerts/routes.py +++ b/steward/alerts/routes.py @@ -161,7 +161,7 @@ def _history_row(event, rule) -> dict: return { "at": event.transitioned_at, "rule": rule, "from_state": event.from_state, "to_state": event.to_state, - "value": value, "condition": alert_svc.rule_condition(rule), + "value": value, "raw": event.metric_value, "condition": alert_svc.rule_condition(rule), "notes": notes, "muted_by": muted_by, "released": released, "deliveries": [(label, sent, error) for label, sent, error in ( ("email", event.email_sent, event.email_error), @@ -173,18 +173,31 @@ def _history_row(event, rule) -> dict: @alerts_bp.get("/history") @require_role(UserRole.viewer) async def alert_history(): - """Every alert transition in a range, newest first; one rule's with ?rule=. - A page at a time as the reader scrolls: ?before= is the next - page's rows alone (core.paging).""" + """Every alert episode in a range, newest first; one rule's with ?rule=, + which also says how often it fired and for how long in all (#5316). A + page of transitions at a time as the reader scrolls: ?before= is + the next page's episodes alone (core.paging). The Notified and Action + columns show only when something in the range was notified or acted on.""" + from steward.alerts.episodes import episodes, summary + now = datetime.now(timezone.utc) since, range_key = parse_range(request.args.get("range") or "7d") before = request.args.get("before") async with current_app.db_sessionmaker() as db: rules = await alert_svc.list_rules(db) chosen = next((r for r in rules if r.id == request.args.get("rule")), None) - rows = await alert_svc.history(db, since, chosen.id if chosen else None, PAGE_SIZE, before) + rule_id = chosen.id if chosen else None + rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before) + show_notified, show_action = await alert_svc.history_columns(db, since, rule_id) + verdict = None + if chosen and not before: + every = await alert_svc.history(db, since, rule_id, HISTORY_SUMMARY_ROWS) + verdict = summary(episodes([_history_row(e, r) for e, r in every], newest_page=True, + now=now, since=since)) rows, after = next_page(rows, lambda row: (row[0].transitioned_at, row[0].id)) - ctx = {"events": [_history_row(e, r) for e, r in rows], "after": after, - "rules": rules, "chosen": chosen, "current_range": range_key} + ctx = {"episodes": episodes([_history_row(e, r) for e, r in rows], newest_page=not before, + now=now, since=since), + "after": after, "rules": rules, "chosen": chosen, "current_range": range_key, + "show_notified": show_notified, "show_action": show_action, "verdict": verdict} if before: return await render_template("alerts/_history_page.html", **ctx) if request.headers.get("HX-Request"): @@ -192,6 +205,12 @@ async def alert_history(): return await render_template("alerts/history.html", **ctx) +# How many of one rule's transitions the history verdict reads ("fired N +# times, X in all"); a rule that flapped more often than this in the range +# is summed over its newest ones. +HISTORY_SUMMARY_ROWS = 2000 + + @alerts_bp.get("/api/rule_fields") @require_role(UserRole.viewer) async def rule_fields_api(): diff --git a/steward/alerts/service.py b/steward/alerts/service.py index 6389f7e..cca5da1 100644 --- a/steward/alerts/service.py +++ b/steward/alerts/service.py @@ -299,6 +299,21 @@ async def history(db, since: datetime, rule_id: str | None = None, return [tuple(row) for row in (await db.execute(stmt)).all()] +async def history_columns(db, since: datetime, rule_id: str | None = None) -> tuple[bool, bool]: + """(whether any transition in the range notified someone or was muted, + whether any started an Ansible run): the history page shows its Notified + and Action columns only then (#5316).""" + from steward.core.alerts import MUTED + base = select(AlertEvent.id).where(AlertEvent.transitioned_at >= since) + if rule_id: + base = base.where(AlertEvent.rule_id == rule_id) + notified = await db.scalar(select(base.where(or_( + AlertEvent.email_sent.is_not(None), AlertEvent.webhook_sent.is_not(None), + AlertEvent.note.like(f"%{MUTED}%"))).exists())) + acted = await db.scalar(select(base.where(AlertEvent.ansible_run_id.is_not(None)).exists())) + return bool(notified), bool(acted) + + async def list_maintenance_windows(db) -> list[MaintenanceWindow]: """Every window, latest start first.""" return list((await db.execute( diff --git a/steward/static/steward.css b/steward/static/steward.css index 352db6b..45a9077 100644 --- a/steward/static/steward.css +++ b/steward/static/steward.css @@ -210,6 +210,12 @@ strong, b, th { font-weight: 500; } /* The Docker page's one-line answer, above its tiles (#5298). */ .page-verdict { margin: 0 0 0.75rem; font-size: 0.95rem; } .table tr.row-off > td { opacity: 0.55; } +.table td.nowrap { white-space: nowrap; } +.cell-note { font-size: 0.78rem; color: var(--text-muted); margin-top: 0.2rem; } +/* An alert episode's own transitions, folded under it (#5316). */ +.episode-steps { margin-top: 0.3rem; font-size: 0.8rem; } +.episode-steps summary { cursor: pointer; color: var(--text-muted); } +.episode-steps ol { margin: 0.35rem 0 0; padding-left: 1.1rem; display: grid; gap: 0.25rem; } /* A problem section's heading carries its own count, so no tile repeats it (#5214). */ .section-count { margin-bottom: 0.6rem; } .section-count > span:first-child { margin-left: 0.25rem; } diff --git a/steward/templates/alerts/_history_page.html b/steward/templates/alerts/_history_page.html index 5923c27..fa9f8aa 100644 --- a/steward/templates/alerts/_history_page.html +++ b/steward/templates/alerts/_history_page.html @@ -1,51 +1,74 @@ -{# alerts/_history_page.html — one page of alert transitions, then the sentinel - that loads the next page as the reader nears it (core.paging, pref 172). #} +{# alerts/_history_page.html — one page of alert episodes (#5316), then the + sentinel that loads the next page as the reader nears it (core.paging, + pref 172). An episode is one stretch out of quiet: what it reached, for how + long, its worst reading, who was told; its transitions fold under it. #} {% from "_macros.html" import status_badge, scroll_more, scroll_end %} - {% for ev in events %} +{% set cols = 3 + (0 if chosen else 1) + (1 if show_notified else 0) + (1 if show_action else 0) %} + {% for ep in episodes %} - {{ ev.at|ago }} + + {%- if ep.began_here %}{{ ep.start|ago }}{% else %}earlier{% endif -%} + + {% if not chosen %} + + {{ ep.rule.name }} +
{{ source_label(ep.rule.source_module) }} · {{ ep.rule.resource_name }}
+ + {% endif %} - {{ ev.rule.name }} -
{{ source_label(ev.rule.source_module) }} · {{ ev.rule.resource_name }}
- - - {{ status_badge(ev.from_state) }} - → - {{ status_badge(ev.to_state) }} - {% for n in ev.notes %} -
{{ n }}
- {% endfor %} - - - {{ ev.value }} -
fires when {{ ev.condition }}
- - - {% if ev.muted_by %} - muted · {{ ev.muted_by }} - {% if ev.released %}
notified when the window ended
{% endif %} + {% if not ep.began_here and ep.steps|length == 1 %} + {%- set s = ep.steps[0] %}{{ status_badge(s.from_state) }} → {{ status_badge(s.to_state) }} + {% else %} + {{ status_badge(ep.worst) }} + {%- if ep.ongoing %} {% if ep.began_here %}for {{ ep.duration|duration }}, {% endif %}still going + {%- elif ep.continues %} cleared later (above) + {%- elif ep.began_here %} for {{ ep.duration|duration }} · cleared {{ ep.end|ago }} + {%- else %} cleared {{ ep.end|ago }}{% endif %} {% endif %} - {% for label, sent, error in ev.deliveries %} -
- {% if sent %} - {{ label }} sent - {% else %} - {{ label }} failed - {% if error %}
{{ error }}
{% endif %} + {% for n in ep.notes %}
{{ n }}
{% endfor %} + {% if ep.steps|length > 1 %} +
+ {{ ep.steps|length }} changes of state +
    + {% for s in ep.steps %} +
  1. {{ status_badge(s.from_state) }} → {{ status_badge(s.to_state) }} + {{ s.value }} · {{ s.at|ago }}
    2. + {% endfor %} +
+
+ {% endif %} + + + {%- if ep.reading %}{{ ep.reading }} ({{ ep.fires_when }}) + {%- else %}no reading ({{ ep.fires_when }}){% endif -%} + + {% if show_notified %} + + {% if ep.muted_by %} + muted · {{ ep.muted_by }} + {% if ep.released %}
notified when the window ended
{% endif %} + {% endif %} + {% for label, sent, error in ep.deliveries %} +
+ {% if sent %}{{ label }} sent + {% else %}{{ label }} failed + {% if error %}
{{ error }}
{% endif %} {% endif %}
{% endfor %} - {% if not ev.muted_by and not ev.deliveries %}—{% endif %} + {% if not ep.muted_by and not ep.deliveries %}—{% endif %} - - {% if ev.ansible_run_id %} - Ansible run - {% else %}—{% endif %} + {% endif %} + {% if show_action %} + + {% for run_id in ep.runs %}
Ansible run
+ {% else %}—{% endfor %} + {% endif %} {% endfor %} {% if after %} -{{ scroll_more("/alerts/history?before=" ~ (after|urlencode) ~ "&range=" ~ (current_range|urlencode) ~ (("&rule=" ~ (chosen.id|urlencode)) if chosen else ""), cols=6) }} +{{ scroll_more("/alerts/history?before=" ~ (after|urlencode) ~ "&range=" ~ (current_range|urlencode) ~ (("&rule=" ~ (chosen.id|urlencode)) if chosen else ""), cols=cols) }} {% else %} -{{ scroll_end(cols=6) }} +{{ scroll_end(cols=cols) }} {% endif %} diff --git a/steward/templates/alerts/_history_rows.html b/steward/templates/alerts/_history_rows.html index 548a436..7cf966d 100644 --- a/steward/templates/alerts/_history_rows.html +++ b/steward/templates/alerts/_history_rows.html @@ -1,17 +1,29 @@ -{# alerts/_history_rows.html — the history table; swapped in on a range or rule - change. Its rows come a page at a time (alerts/_history_page.html). #} -{% from "_macros.html" import status_badge, empty_state %} -{% if events %} -
- +{# alerts/_history_rows.html — the history: for one rule, how often it fired + and for how long in all; then its episodes (#5316), swapped in on a range + or rule change. The rule column shows only across rules, Notified and + Action only when the range has something to put in them. Episodes come a + page of transitions at a time (alerts/_history_page.html). #} +{% from "_macros.html" import empty_state %} +{% if chosen and verdict %} +

+ {%- if verdict.fired %}Fired {{ verdict.fired }} time{{ "s" if verdict.fired != 1 }} in the last {{ current_range }} + {%- if verdict.seconds %}, {{ verdict.seconds|duration }} in all{% endif %} + {%- if verdict.ongoing %} · firing now{% endif %} + {%- else %}Never fired in the last {{ current_range }}{% endif %} + · fires when {{ rule_condition(chosen) }} on {{ chosen.resource_name }} +

+{% endif %} +{% if episodes %} +
+
- - - - - + {% if not chosen %}{% endif %} + + + {% if show_notified %}{% endif %} + {% if show_action %}{% endif %} diff --git a/steward/templates/alerts/history.html b/steward/templates/alerts/history.html index c79825f..ebe7373 100644 --- a/steward/templates/alerts/history.html +++ b/steward/templates/alerts/history.html @@ -3,7 +3,7 @@ {% block title %}Alert History — Steward{% endblock %} {% block breadcrumb %}{{ crumbs([("Alerts", "/alerts/"), ("History", "")]) }}{% endblock %} {% block content %} -{% call page_header("Alert history", "Every change of state, newest first: the value against the rule's condition, whether the email and webhook went out, and the Ansible run an action started.") %} +{% call page_header("Alert history", "Each time a rule left quiet, newest first: what it reached and for how long, its worst reading, and who was told.") %} " in page and "" in page and "" not in page resp = await client.get(f"/alerts/history?range=24h&rule={hot_id}", headers={"HX-Request": "true"}) rows = await resp.get_data(as_text=True) assert "Rule" not in rows # one rule: no column for it + assert "Fired 1 time in the last 24h" in rows # its verdict # The next page as the reader scrolls: only older rows, without the table # around them, and the list says where it ends (#5211). @@ -84,7 +89,8 @@ def test_a_viewer_reads_the_alert_history(app): "range": "24h", "rule": hot_id, "before": cursor(now - timedelta(minutes=3), "")}) older = await resp.get_data(as_text=True) assert resp.status_code == 200 and "]*btn-danger[^>]*value="go"', base) assert "dataset.confirm" in js and "getElementById('confirm-dialog')" in js and "requestSubmit" in js assert "textContent = title" in js and "innerHTML" not in js # text, never markup - assert "window.confirm" not in js and "window.confirm" not in base + assert not re.search(r"\bconfirm\s*\(", js + base) # a call, not the comment saying it is gone assert any("data-confirm=" in p.read_text() for p in _templates())
WhenRuleChangeReadingNotifiedActionRuleWhat happenedWorst readingNotifiedAction
RuleNotifiedAction