feat(alerts): alert history page — transitions, deliveries, mutes and actions (#5066)

#5066 "Alert history page". /alerts/history lists every alert transition
in a range (1h–30d, the shared range picker), newest first, all rules or
one (?rule=). Each row: from → to, the value against the rule's condition
("no data" for a silent one), email and webhook sent or failed with the
error, "muted · <window>" (and "notified when the window ended") taken out
of the event note, other notes, and a link to the Ansible run an action
started. Range and rule changes swap only the rows (HTMX). Viewer role.

The Alerts page links each rule's history and the whole history. The query
moves to alert_svc.history, which MCP get_alert_history now uses too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/routes.py

@@ -5,6 +5,7 @@ from steward.core.audit import log_audit
 from sqlalchemy import select
 from steward.auth.middleware import require_role
 from steward.alerts import service as alert_svc
+from steward.core.time_range import parse_range
 from steward.ansible import sources as src_module
 from steward.models.alerts import AlertRule, AlertOperator, MaintenanceWindow
 from steward.models.users import UserRole
@@ -101,6 +102,50 @@ async def list_alerts():
     return await render_template("alerts/list.html", active=active, rules=rules, operators=_OPERATORS)
 
 
+HISTORY_LIMIT = 200
+
+
+def _history_row(event, rule) -> dict:
+    """One transition as the history page shows it."""
+    from steward.core.alerts import MUTED, RELEASED
+    notes, muted_by, released = [], None, False
+    for part in (event.note or "").split("; "):
+        if part.startswith(MUTED):
+            muted_by = part[len(MUTED):].strip() or "a maintenance window"
+        elif part == RELEASED:
+            released = True
+        elif part:
+            notes.append(part)
+    value = "no data" if event.metric_value is None else f"{event.metric_value:g}"
+    return {
+        "at": event.transitioned_at, "rule": rule,
+        "from_state": event.from_state, "to_state": event.to_state,
+        "value": value, "condition": f"{rule.operator.value} {rule.threshold:g}",
+        "notes": notes, "muted_by": muted_by, "released": released,
+        "deliveries": [(label, sent, error) for label, sent, error in (
+            ("email", event.email_sent, event.email_error),
+            ("webhook", event.webhook_sent, event.webhook_error)) if sent is not None],
+        "ansible_run_id": event.ansible_run_id,
+    }
+
+
+@alerts_bp.get("/history")
+@require_role(UserRole.viewer)
+async def alert_history():
+    """Every alert transition in a range, newest first; one rule's with ?rule=."""
+    since, range_key = parse_range(request.args.get("range") or "7d")
+    async with current_app.db_sessionmaker() as db:
+        rules = await alert_svc.list_rules(db)
+        chosen = next((r for r in rules if r.id == request.args.get("rule")), None)
+        rows = await alert_svc.history(db, since, chosen.id if chosen else None, HISTORY_LIMIT)
+    ctx = {"events": [_history_row(e, r) for e, r in rows[:HISTORY_LIMIT]],
+           "truncated": len(rows) > HISTORY_LIMIT, "limit": HISTORY_LIMIT,
+           "rules": rules, "chosen": chosen, "current_range": range_key}
+    if request.headers.get("HX-Request"):
+        return await render_template("alerts/_history_rows.html", **ctx)
+    return await render_template("alerts/history.html", **ctx)
+
+
 @alerts_bp.get("/api/rule_fields")
 @require_role(UserRole.viewer)
 async def rule_fields_api():

steward/alerts/service.py

@@ -12,7 +12,7 @@ from datetime import datetime, timezone
 from sqlalchemy import or_, select
 
 from steward.models.alerts import (
-    AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
+    AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
 )
 from steward.models.metrics import PluginMetric
 from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
@@ -89,6 +89,21 @@ async def list_rules(db) -> list[AlertRule]:
     return list((await db.execute(select(AlertRule).order_by(AlertRule.name))).scalars())
 
 
+async def history(db, since: datetime, rule_id: str | None = None,
+                  limit: int = 200) -> list[tuple[AlertEvent, AlertRule]]:
+    """Alert transitions since `since`, newest first, with their rule; one
+    rule's when `rule_id` is given. Returns up to limit+1 rows so a caller can
+    tell the list was cut."""
+    stmt = (select(AlertEvent, AlertRule)
+            .join(AlertRule, AlertEvent.rule_id == AlertRule.id)
+            .where(AlertEvent.transitioned_at >= since)
+            .order_by(AlertEvent.transitioned_at.desc())
+            .limit(limit + 1))
+    if rule_id:
+        stmt = stmt.where(AlertEvent.rule_id == rule_id)
+    return [tuple(row) for row in (await db.execute(stmt)).all()]
+
+
 async def list_maintenance_windows(db) -> list[MaintenanceWindow]:
     """Every window, latest start first."""
     return list((await db.execute(

steward/mcp/tools/alerts.py

@@ -7,7 +7,7 @@ from sqlalchemy import select
 
 from steward.alerts import service as alert_svc
 from steward.mcp._context import current_caller
-from steward.models.alerts import AlertEvent, AlertOperator, AlertRule, AlertState, MaintenanceWindow
+from steward.models.alerts import AlertOperator, AlertRule, AlertState, MaintenanceWindow
 from steward.models.users import User, UserRole
 
 from ._audit import audit_mcp
@@ -95,16 +95,8 @@ async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50)
     limit = clamp_limit(limit, 50, 500)
     async with db_session() as db:
         rules = await alert_svc.list_rules(db)
-        stmt = (select(AlertEvent, AlertRule)
-                .join(AlertRule, AlertEvent.rule_id == AlertRule.id)
-                .where(AlertEvent.transitioned_at >= since)
-                .order_by(AlertEvent.transitioned_at.desc())
-                .limit(limit + 1))
-        chosen = None
-        if rule:
-            chosen = pick(rule, rules, kind="alert rule", list_tool="list_alert_rules")
-            stmt = stmt.where(AlertEvent.rule_id == chosen.id)
-        rows = (await db.execute(stmt)).all()
+        chosen = pick(rule, rules, kind="alert rule", list_tool="list_alert_rules") if rule else None
+        rows = await alert_svc.history(db, since, chosen.id if chosen else None, limit)
     events = [
         {
             "rule_id": r.id,

steward/templates/alerts/_history_rows.html

@@ -0,0 +1,75 @@
+{# alerts/_history_rows.html — the history table; swapped in on a range or rule change #}
+{% set state_badge = {"firing": "badge-red", "pending": "badge-yellow", "acknowledged": "badge-gold",
+                      "resolved": "badge-green", "inactive": "badge-dim"} %}
+{% if events %}
+<div class="card-flush" style="overflow-x:auto;">
+  <table class="table">
+    <thead>
+      <tr>
+        <th>When (UTC)</th>
+        <th>Rule</th>
+        <th>Change</th>
+        <th>Value</th>
+        <th>Notified</th>
+        <th>Action</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for ev in events %}
+      <tr>
+        <td style="font-family:ui-monospace,monospace;font-size:0.82rem;white-space:nowrap;">
+          {{ ev.at.strftime("%Y-%m-%d %H:%M:%S") }}
+        </td>
+        <td>
+          <a href="/alerts/history?rule={{ ev.rule.id }}&range={{ current_range }}">{{ ev.rule.name }}</a>
+          <div style="font-size:0.78rem;color:var(--text-dim);">{{ ev.rule.source_module }}/{{ ev.rule.resource_name }}</div>
+        </td>
+        <td style="white-space:nowrap;">
+          <span class="badge {{ state_badge.get(ev.from_state, 'badge-dim') }}">{{ ev.from_state }}</span>
+          <span style="color:var(--text-dim);">→</span>
+          <span class="badge {{ state_badge.get(ev.to_state, 'badge-dim') }}">{{ ev.to_state }}</span>
+          {% for n in ev.notes %}
+          <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.25rem;white-space:normal;">{{ n }}</div>
+          {% endfor %}
+        </td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">
+          {{ ev.rule.metric_name }} = {{ ev.value }}
+          <div style="font-size:0.75rem;color:var(--text-dim);">rule: {{ ev.condition }}</div>
+        </td>
+        <td style="font-size:0.82rem;">
+          {% if ev.muted_by %}
+          <span class="badge badge-yellow" title="A maintenance window muted this notification">muted · {{ ev.muted_by }}</span>
+          {% if ev.released %}<div style="color:var(--text-muted);margin-top:0.25rem;">notified when the window ended</div>{% endif %}
+          {% endif %}
+          {% for label, sent, error in ev.deliveries %}
+          <div style="margin-top:0.15rem;">
+            {% if sent %}
+            <span class="badge badge-green">{{ label }} sent</span>
+            {% else %}
+            <span class="badge badge-red">{{ label }} failed</span>
+            {% if error %}<div style="color:var(--text-muted);font-size:0.78rem;margin-top:0.15rem;max-width:28rem;word-break:break-word;">{{ error }}</div>{% endif %}
+            {% endif %}
+          </div>
+          {% endfor %}
+          {% if not ev.muted_by and not ev.deliveries %}<span style="color:var(--text-dim);" title="No notification was sent for this change">—</span>{% endif %}
+        </td>
+        <td style="font-size:0.82rem;">
+          {% if ev.ansible_run_id %}
+          <a href="/ansible/runs/{{ ev.ansible_run_id }}">Ansible run</a>
+          {% else %}<span style="color:var(--text-dim);">—</span>{% endif %}
+        </td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+</div>
+{% if truncated %}
+<p style="color:var(--text-muted);font-size:0.82rem;margin-top:0.75rem;">
+  Showing the newest {{ limit }}. Pick a shorter range or one rule to see the rest.
+</p>
+{% endif %}
+{% else %}
+<p style="color:var(--text-muted);">
+  No alert changed state in the last {{ current_range }}{% if chosen %} for {{ chosen.name }}{% endif %}.
+</p>
+{% endif %}

steward/templates/alerts/history.html

@@ -0,0 +1,32 @@
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs %}
+{% block title %}Alert History — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Alerts", "/alerts/"), ("History", "")]) }}{% endblock %}
+{% block content %}
+<div style="display:flex;align-items:center;justify-content:space-between;flex-wrap:wrap;gap:0.75rem;margin-bottom:1.25rem;">
+  <h1 class="page-title" style="margin-bottom:0;">Alert History</h1>
+  <div style="display:flex;align-items:center;gap:0.75rem;flex-wrap:wrap;">
+    <select id="history-rule" name="rule" aria-label="Rule"
+            hx-get="/alerts/history" hx-target="#history-rows" hx-include="#time-range"
+            hx-trigger="change" hx-push-url="false"
+            onchange="var u=new URL(window.location); this.value ? u.searchParams.set('rule', this.value) : u.searchParams.delete('rule'); history.replaceState({}, '', u);">
+      <option value="">All rules</option>
+      {% for r in rules %}
+      <option value="{{ r.id }}" {% if chosen and chosen.id == r.id %}selected{% endif %}>{{ r.name }}</option>
+      {% endfor %}
+    </select>
+    {% include "_time_range.html" %}
+  </div>
+</div>
+
+<p style="color:var(--text-muted);font-size:0.88rem;margin-bottom:1.25rem;">
+  Every change of state, newest first: what the value was against the rule's condition, whether
+  the email and webhook went out, and the Ansible run an action started.
+</p>
+
+<div id="history-rows"
+     hx-get="/alerts/history" hx-trigger="rangeChange from:body"
+     hx-include="#time-range, #history-rule" hx-swap="innerHTML">
+  {% include "alerts/_history_rows.html" %}
+</div>
+{% endblock %}

steward/templates/alerts/list.html

@@ -57,6 +57,7 @@
 <div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1rem;">
     <h2 class="section-title">Rules</h2>
     <div style="display:flex;gap:0.5rem;">
+        <a class="btn btn-ghost" href="/alerts/history">History</a>
         <a class="btn btn-ghost" href="/alerts/maintenance">Maintenance</a>
         <a class="btn" href="/alerts/rules/new">New Rule</a>
     </div>
@@ -89,6 +90,7 @@
                 </td>
                 <td style="color:var(--text-muted);text-align:center;">{{ rule.consecutive_failures_required }} / {{ rule.consecutive_ok_required }}</td>
                 <td class="td-actions">
+                    <a href="/alerts/history?rule={{ rule.id }}" class="btn btn-sm btn-ghost">History</a>
                     <a href="/alerts/rules/{{ rule.id }}/edit" class="btn btn-sm btn-ghost">Edit</a>
                     <form method="post" action="/alerts/rules/{{ rule.id }}/toggle" style="display:inline;">{{ csrf_field() }}
                         <button type="submit" class="btn btn-sm btn-ghost">