From 531a193bd13bf7b4bfde7478fb6ce3a268eedc8b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 7 Oct 2026 23:02:01 -0400 Subject: [PATCH] feat(alerts): history reads as episodes; tests follow the confirm dialog (#5316, #5314) History (#5316): - Transitions fold into episodes (steward/alerts/episodes.py): each stretch out of quiet, the worst state it reached, how long it lasted, its worst reading against the rule ('3m 21s (fires above 3m)'), who was told across its changes, and its Ansible runs. The changes themselves fold under it. - An episode cut by paging says so ('cleared later (above)', 'earlier'). - For one rule: a verdict ('Fired 4 times in the last 7d, 1h 2m in all') and no Rule column. Notified and Action show only when the range has any. Tests: - The confirm guard looks for a confirm( call, not the word in the comment that says it is gone (lesson #4365). - The Hosts viewer test looks for the add-host dialog by id, since every page now carries the confirm dialog. Co-Authored-By: Claude Opus 5.5 --- steward/alerts/episodes.py | 106 +++++++++++++++++++ steward/alerts/routes.py | 33 ++++-- steward/alerts/service.py | 15 +++ steward/static/steward.css | 6 ++ steward/templates/alerts/_history_page.html | 97 ++++++++++------- steward/templates/alerts/_history_rows.html | 34 ++++-- steward/templates/alerts/history.html | 2 +- tests/core/test_alert_episodes.py | 98 +++++++++++++++++ tests/core/test_host_page.py | 2 +- tests/integration/test_alert_history_page.py | 12 ++- tests/test_inline_handlers.py | 2 +- 11 files changed, 346 insertions(+), 61 deletions(-) create mode 100644 steward/alerts/episodes.py create mode 100644 tests/core/test_alert_episodes.py diff --git a/steward/alerts/episodes.py b/steward/alerts/episodes.py new file mode 100644 index 0000000..502544e --- /dev/null +++ b/steward/alerts/episodes.py @@ -0,0 +1,106 @@ +"""An alert's history read as episodes (#5316): each time a rule left quiet, +what it went through, how long it lasted, its worst reading and who was told. + +The history page lists transitions a page at a time (core.paging), newest +first. `episodes` folds one page of them, so an episode can straddle two +pages: its end on a newer page (`continues`) or its start on an older one +(`began_here` false). Model-free, so it is unit-tested without a database. +""" +from __future__ import annotations + +from datetime import datetime + +# The states an alert rests in; leaving one opens an episode, reaching one ends it. +QUIET = frozenset({"inactive", "resolved"}) +# The worst thing an episode went through, most severe first. +_SEVERITY = ("firing", "acknowledged", "no_data", "pending") +_WORDS = {">": "above", ">=": "at or above", "<": "below", "<=": "at or below", + "==": "at", "!=": "other than"} + + +def fires_when(rule) -> str: + """A rule's condition as a reading's aside: "fires above 3m", "fires + when down".""" + from steward.alerts.service import rule_condition + from steward.core.formatting import metric_value + said = rule_condition(rule) + if said.startswith("is "): + return f"fires when {said[3:]}" + op = rule.operator.value + return f"fires {_WORDS.get(op, op)} {metric_value(rule.metric_name, rule.threshold)}" + + +def _worst_value(rule, values: list[float]) -> float | None: + """The reading furthest into breach: the highest for a rule that fires + above, the lowest for one that fires below, else the last.""" + if not values: + return None + op = rule.operator.value + if op in (">", ">="): + return max(values) + if op in ("<", "<="): + return min(values) + return values[-1] + + +def episodes(rows: list[dict], *, newest_page: bool, now: datetime, since: datetime) -> list[dict]: + """Fold transitions (alert history rows, newest first, each with its + `rule`, `at`, `from_state`, `to_state`, `raw` value, `deliveries`, + `muted_by`, `released`, `ansible_run_id` and `notes`) into episodes, + newest first. + + Each episode: rule, steps (oldest first), start, end (None while open), + began_here (its first step left quiet on this page), ongoing (open on the + newest page: it is still going), continues (open on an older page: it + ends on a newer one), worst (the most severe state it reached), duration + (seconds, counted from `since` when it began before the range), worst + reading, deliveries, muted_by, released and runs.""" + from steward.core.formatting import metric_value + open_by_rule: dict[str, dict] = {} + out: list[dict] = [] + for r in reversed(rows): + rid = r["rule"].id + ep = open_by_rule.get(rid) + if ep is None: + ep = {"rule": r["rule"], "steps": [], "start": r["at"], "end": None, + "began_here": r["from_state"] in QUIET} + out.append(ep) + ep["steps"].append(r) + if r["to_state"] in QUIET: + ep["end"] = r["at"] + open_by_rule.pop(rid, None) + else: + open_by_rule[rid] = ep + + for ep in out: + steps, rule = ep["steps"], ep["rule"] + reached = {s["to_state"] for s in steps} | {s["from_state"] for s in steps} + ep["worst"] = next((s for s in _SEVERITY if s in reached), steps[-1]["to_state"]) + ep["ongoing"] = ep["end"] is None and newest_page + ep["continues"] = ep["end"] is None and not newest_page + began = ep["start"] if ep["began_here"] else since + ended = ep["end"] or (now if ep["ongoing"] else None) + ep["duration"] = (ended - began).total_seconds() if ended else None + worst = _worst_value(rule, [s["raw"] for s in steps if s["raw"] is not None]) + ep["reading"] = None if worst is None else metric_value(rule.metric_name, worst) + ep["fires_when"] = fires_when(rule) + seen, deliveries = set(), [] + for s in steps: + for d in s["deliveries"]: + if (d[0], d[1]) not in seen: + seen.add((d[0], d[1])) + deliveries.append(d) + ep["deliveries"] = deliveries + ep["muted_by"] = next((s["muted_by"] for s in steps if s["muted_by"]), None) + ep["released"] = any(s["released"] for s in steps) + ep["runs"] = [s["ansible_run_id"] for s in steps if s["ansible_run_id"]] + ep["notes"] = [n for s in steps for n in s["notes"]] + return list(reversed(out)) + + +def summary(eps: list[dict]) -> dict: + """How often a rule fired in a range and for how long in all: {fired, + seconds, ongoing}.""" + fired = [e for e in eps if e["worst"] in ("firing", "acknowledged")] + return {"fired": len(fired), "seconds": sum(e["duration"] or 0 for e in fired), + "ongoing": any(e["ongoing"] for e in fired)} diff --git a/steward/alerts/routes.py b/steward/alerts/routes.py index 9cfc175..fc20e60 100644 --- a/steward/alerts/routes.py +++ b/steward/alerts/routes.py @@ -161,7 +161,7 @@ def _history_row(event, rule) -> dict: return { "at": event.transitioned_at, "rule": rule, "from_state": event.from_state, "to_state": event.to_state, - "value": value, "condition": alert_svc.rule_condition(rule), + "value": value, "raw": event.metric_value, "condition": alert_svc.rule_condition(rule), "notes": notes, "muted_by": muted_by, "released": released, "deliveries": [(label, sent, error) for label, sent, error in ( ("email", event.email_sent, event.email_error), @@ -173,18 +173,31 @@ def _history_row(event, rule) -> dict: @alerts_bp.get("/history") @require_role(UserRole.viewer) async def alert_history(): - """Every alert transition in a range, newest first; one rule's with ?rule=. - A page at a time as the reader scrolls: ?before= is the next - page's rows alone (core.paging).""" + """Every alert episode in a range, newest first; one rule's with ?rule=, + which also says how often it fired and for how long in all (#5316). A + page of transitions at a time as the reader scrolls: ?before= is + the next page's episodes alone (core.paging). The Notified and Action + columns show only when something in the range was notified or acted on.""" + from steward.alerts.episodes import episodes, summary + now = datetime.now(timezone.utc) since, range_key = parse_range(request.args.get("range") or "7d") before = request.args.get("before") async with current_app.db_sessionmaker() as db: rules = await alert_svc.list_rules(db) chosen = next((r for r in rules if r.id == request.args.get("rule")), None) - rows = await alert_svc.history(db, since, chosen.id if chosen else None, PAGE_SIZE, before) + rule_id = chosen.id if chosen else None + rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before) + show_notified, show_action = await alert_svc.history_columns(db, since, rule_id) + verdict = None + if chosen and not before: + every = await alert_svc.history(db, since, rule_id, HISTORY_SUMMARY_ROWS) + verdict = summary(episodes([_history_row(e, r) for e, r in every], newest_page=True, + now=now, since=since)) rows, after = next_page(rows, lambda row: (row[0].transitioned_at, row[0].id)) - ctx = {"events": [_history_row(e, r) for e, r in rows], "after": after, - "rules": rules, "chosen": chosen, "current_range": range_key} + ctx = {"episodes": episodes([_history_row(e, r) for e, r in rows], newest_page=not before, + now=now, since=since), + "after": after, "rules": rules, "chosen": chosen, "current_range": range_key, + "show_notified": show_notified, "show_action": show_action, "verdict": verdict} if before: return await render_template("alerts/_history_page.html", **ctx) if request.headers.get("HX-Request"): @@ -192,6 +205,12 @@ async def alert_history(): return await render_template("alerts/history.html", **ctx) +# How many of one rule's transitions the history verdict reads ("fired N +# times, X in all"); a rule that flapped more often than this in the range +# is summed over its newest ones. +HISTORY_SUMMARY_ROWS = 2000 + + @alerts_bp.get("/api/rule_fields") @require_role(UserRole.viewer) async def rule_fields_api(): diff --git a/steward/alerts/service.py b/steward/alerts/service.py index 6389f7e..cca5da1 100644 --- a/steward/alerts/service.py +++ b/steward/alerts/service.py @@ -299,6 +299,21 @@ async def history(db, since: datetime, rule_id: str | None = None, return [tuple(row) for row in (await db.execute(stmt)).all()] +async def history_columns(db, since: datetime, rule_id: str | None = None) -> tuple[bool, bool]: + """(whether any transition in the range notified someone or was muted, + whether any started an Ansible run): the history page shows its Notified + and Action columns only then (#5316).""" + from steward.core.alerts import MUTED + base = select(AlertEvent.id).where(AlertEvent.transitioned_at >= since) + if rule_id: + base = base.where(AlertEvent.rule_id == rule_id) + notified = await db.scalar(select(base.where(or_( + AlertEvent.email_sent.is_not(None), AlertEvent.webhook_sent.is_not(None), + AlertEvent.note.like(f"%{MUTED}%"))).exists())) + acted = await db.scalar(select(base.where(AlertEvent.ansible_run_id.is_not(None)).exists())) + return bool(notified), bool(acted) + + async def list_maintenance_windows(db) -> list[MaintenanceWindow]: """Every window, latest start first.""" return list((await db.execute( diff --git a/steward/static/steward.css b/steward/static/steward.css index 352db6b..45a9077 100644 --- a/steward/static/steward.css +++ b/steward/static/steward.css @@ -210,6 +210,12 @@ strong, b, th { font-weight: 500; } /* The Docker page's one-line answer, above its tiles (#5298). */ .page-verdict { margin: 0 0 0.75rem; font-size: 0.95rem; } .table tr.row-off > td { opacity: 0.55; } +.table td.nowrap { white-space: nowrap; } +.cell-note { font-size: 0.78rem; color: var(--text-muted); margin-top: 0.2rem; } +/* An alert episode's own transitions, folded under it (#5316). */ +.episode-steps { margin-top: 0.3rem; font-size: 0.8rem; } +.episode-steps summary { cursor: pointer; color: var(--text-muted); } +.episode-steps ol { margin: 0.35rem 0 0; padding-left: 1.1rem; display: grid; gap: 0.25rem; } /* A problem section's heading carries its own count, so no tile repeats it (#5214). */ .section-count { margin-bottom: 0.6rem; } .section-count > span:first-child { margin-left: 0.25rem; } diff --git a/steward/templates/alerts/_history_page.html b/steward/templates/alerts/_history_page.html index 5923c27..fa9f8aa 100644 --- a/steward/templates/alerts/_history_page.html +++ b/steward/templates/alerts/_history_page.html @@ -1,51 +1,74 @@ -{# alerts/_history_page.html — one page of alert transitions, then the sentinel - that loads the next page as the reader nears it (core.paging, pref 172). #} +{# alerts/_history_page.html — one page of alert episodes (#5316), then the + sentinel that loads the next page as the reader nears it (core.paging, + pref 172). An episode is one stretch out of quiet: what it reached, for how + long, its worst reading, who was told; its transitions fold under it. #} {% from "_macros.html" import status_badge, scroll_more, scroll_end %} - {% for ev in events %} +{% set cols = 3 + (0 if chosen else 1) + (1 if show_notified else 0) + (1 if show_action else 0) %} + {% for ep in episodes %} - {{ ev.at|ago }} + + {%- if ep.began_here %}{{ ep.start|ago }}{% else %}earlier{% endif -%} + + {% if not chosen %} + + {{ ep.rule.name }} +
{{ source_label(ep.rule.source_module) }} · {{ ep.rule.resource_name }}
+ + {% endif %} - {{ ev.rule.name }} -
{{ source_label(ev.rule.source_module) }} · {{ ev.rule.resource_name }}
- - - {{ status_badge(ev.from_state) }} - → - {{ status_badge(ev.to_state) }} - {% for n in ev.notes %} -
{{ n }}
- {% endfor %} - - - {{ ev.value }} -
fires when {{ ev.condition }}
- - - {% if ev.muted_by %} - muted · {{ ev.muted_by }} - {% if ev.released %}
notified when the window ended
{% endif %} + {% if not ep.began_here and ep.steps|length == 1 %} + {%- set s = ep.steps[0] %}{{ status_badge(s.from_state) }} → {{ status_badge(s.to_state) }} + {% else %} + {{ status_badge(ep.worst) }} + {%- if ep.ongoing %} {% if ep.began_here %}for {{ ep.duration|duration }}, {% endif %}still going + {%- elif ep.continues %} cleared later (above) + {%- elif ep.began_here %} for {{ ep.duration|duration }} · cleared {{ ep.end|ago }} + {%- else %} cleared {{ ep.end|ago }}{% endif %} {% endif %} - {% for label, sent, error in ev.deliveries %} -
- {% if sent %} - {{ label }} sent - {% else %} - {{ label }} failed - {% if error %}
{{ error }}
{% endif %} + {% for n in ep.notes %}
{{ n }}
{% endfor %} + {% if ep.steps|length > 1 %} +
+ {{ ep.steps|length }} changes of state +
    + {% for s in ep.steps %} +
  1. {{ status_badge(s.from_state) }} → {{ status_badge(s.to_state) }} + {{ s.value }} · {{ s.at|ago }}
    2. + {% endfor %} +
+
+ {% endif %} + + + {%- if ep.reading %}{{ ep.reading }} ({{ ep.fires_when }}) + {%- else %}no reading ({{ ep.fires_when }}){% endif -%} + + {% if show_notified %} + + {% if ep.muted_by %} + muted · {{ ep.muted_by }} + {% if ep.released %}
notified when the window ended
{% endif %} + {% endif %} + {% for label, sent, error in ep.deliveries %} +
+ {% if sent %}{{ label }} sent + {% else %}{{ label }} failed + {% if error %}
{{ error }}
{% endif %} {% endif %}
{% endfor %} - {% if not ev.muted_by and not ev.deliveries %}—{% endif %} + {% if not ep.muted_by and not ep.deliveries %}—{% endif %} - - {% if ev.ansible_run_id %} - Ansible run - {% else %}—{% endif %} + {% endif %} + {% if show_action %} + + {% for run_id in ep.runs %} + {% else %}—{% endfor %} + {% endif %} {% endfor %} {% if after %} -{{ scroll_more("/alerts/history?before=" ~ (after|urlencode) ~ "&range=" ~ (current_range|urlencode) ~ (("&rule=" ~ (chosen.id|urlencode)) if chosen else ""), cols=6) }} +{{ scroll_more("/alerts/history?before=" ~ (after|urlencode) ~ "&range=" ~ (current_range|urlencode) ~ (("&rule=" ~ (chosen.id|urlencode)) if chosen else ""), cols=cols) }} {% else %} -{{ scroll_end(cols=6) }} +{{ scroll_end(cols=cols) }} {% endif %} diff --git a/steward/templates/alerts/_history_rows.html b/steward/templates/alerts/_history_rows.html index 548a436..7cf966d 100644 --- a/steward/templates/alerts/_history_rows.html +++ b/steward/templates/alerts/_history_rows.html @@ -1,17 +1,29 @@ -{# alerts/_history_rows.html — the history table; swapped in on a range or rule - change. Its rows come a page at a time (alerts/_history_page.html). #} -{% from "_macros.html" import status_badge, empty_state %} -{% if events %} -
- +{# alerts/_history_rows.html — the history: for one rule, how often it fired + and for how long in all; then its episodes (#5316), swapped in on a range + or rule change. The rule column shows only across rules, Notified and + Action only when the range has something to put in them. Episodes come a + page of transitions at a time (alerts/_history_page.html). #} +{% from "_macros.html" import empty_state %} +{% if chosen and verdict %} +

+ {%- if verdict.fired %}Fired {{ verdict.fired }} time{{ "s" if verdict.fired != 1 }} in the last {{ current_range }} + {%- if verdict.seconds %}, {{ verdict.seconds|duration }} in all{% endif %} + {%- if verdict.ongoing %} · firing now{% endif %} + {%- else %}Never fired in the last {{ current_range }}{% endif %} + · fires when {{ rule_condition(chosen) }} on {{ chosen.resource_name }} +

+{% endif %} +{% if episodes %} +
+
- - - - - + {% if not chosen %}{% endif %} + + + {% if show_notified %}{% endif %} + {% if show_action %}{% endif %} diff --git a/steward/templates/alerts/history.html b/steward/templates/alerts/history.html index c79825f..ebe7373 100644 --- a/steward/templates/alerts/history.html +++ b/steward/templates/alerts/history.html @@ -3,7 +3,7 @@ {% block title %}Alert History — Steward{% endblock %} {% block breadcrumb %}{{ crumbs([("Alerts", "/alerts/"), ("History", "")]) }}{% endblock %} {% block content %} -{% call page_header("Alert history", "Every change of state, newest first: the value against the rule's condition, whether the email and webhook went out, and the Ansible run an action started.") %} +{% call page_header("Alert history", "Each time a rule left quiet, newest first: what it reached and for how long, its worst reading, and who was told.") %} " in page and "" in page and "" not in page resp = await client.get(f"/alerts/history?range=24h&rule={hot_id}", headers={"HX-Request": "true"}) rows = await resp.get_data(as_text=True) assert "Rule" not in rows # one rule: no column for it + assert "Fired 1 time in the last 24h" in rows # its verdict # The next page as the reader scrolls: only older rows, without the table # around them, and the list says where it ends (#5211). @@ -84,7 +89,8 @@ def test_a_viewer_reads_the_alert_history(app): "range": "24h", "rule": hot_id, "before": cursor(now - timedelta(minutes=3), "")}) older = await resp.get_data(as_text=True) assert resp.status_code == 200 and "]*btn-danger[^>]*value="go"', base) assert "dataset.confirm" in js and "getElementById('confirm-dialog')" in js and "requestSubmit" in js assert "textContent = title" in js and "innerHTML" not in js # text, never markup - assert "window.confirm" not in js and "window.confirm" not in base + assert not re.search(r"\bconfirm\s*\(", js + base) # a call, not the comment saying it is gone assert any("data-confirm=" in p.read_text() for p in _templates())
WhenRuleChangeReadingNotifiedActionRuleWhat happenedWorst readingNotifiedAction
RuleNotifiedAction