feat(alerts): alert history page — transitions, deliveries, mutes and actions (#5066)

#5066 "Alert history page". /alerts/history lists every alert transition
in a range (1h–30d, the shared range picker), newest first, all rules or
one (?rule=). Each row: from → to, the value against the rule's condition
("no data" for a silent one), email and webhook sent or failed with the
error, "muted · <window>" (and "notified when the window ended") taken out
of the event note, other notes, and a link to the Ansible run an action
started. Range and rule changes swap only the rows (HTMX). Viewer role.

The Alerts page links each rule's history and the whole history. The query
moves to alert_svc.history, which MCP get_alert_history now uses too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/routes.py

@@ -5,6 +5,7 @@ from steward.core.audit import log_audit
 from sqlalchemy import select
 from steward.auth.middleware import require_role
 from steward.alerts import service as alert_svc
+from steward.core.time_range import parse_range
 from steward.ansible import sources as src_module
 from steward.models.alerts import AlertRule, AlertOperator, MaintenanceWindow
 from steward.models.users import UserRole
@@ -101,6 +102,50 @@ async def list_alerts():
     return await render_template("alerts/list.html", active=active, rules=rules, operators=_OPERATORS)
 
 
+HISTORY_LIMIT = 200
+
+
+def _history_row(event, rule) -> dict:
+    """One transition as the history page shows it."""
+    from steward.core.alerts import MUTED, RELEASED
+    notes, muted_by, released = [], None, False
+    for part in (event.note or "").split("; "):
+        if part.startswith(MUTED):
+            muted_by = part[len(MUTED):].strip() or "a maintenance window"
+        elif part == RELEASED:
+            released = True
+        elif part:
+            notes.append(part)
+    value = "no data" if event.metric_value is None else f"{event.metric_value:g}"
+    return {
+        "at": event.transitioned_at, "rule": rule,
+        "from_state": event.from_state, "to_state": event.to_state,
+        "value": value, "condition": f"{rule.operator.value} {rule.threshold:g}",
+        "notes": notes, "muted_by": muted_by, "released": released,
+        "deliveries": [(label, sent, error) for label, sent, error in (
+            ("email", event.email_sent, event.email_error),
+            ("webhook", event.webhook_sent, event.webhook_error)) if sent is not None],
+        "ansible_run_id": event.ansible_run_id,
+    }
+
+
+@alerts_bp.get("/history")
+@require_role(UserRole.viewer)
+async def alert_history():
+    """Every alert transition in a range, newest first; one rule's with ?rule=."""
+    since, range_key = parse_range(request.args.get("range") or "7d")
+    async with current_app.db_sessionmaker() as db:
+        rules = await alert_svc.list_rules(db)
+        chosen = next((r for r in rules if r.id == request.args.get("rule")), None)
+        rows = await alert_svc.history(db, since, chosen.id if chosen else None, HISTORY_LIMIT)
+    ctx = {"events": [_history_row(e, r) for e, r in rows[:HISTORY_LIMIT]],
+           "truncated": len(rows) > HISTORY_LIMIT, "limit": HISTORY_LIMIT,
+           "rules": rules, "chosen": chosen, "current_range": range_key}
+    if request.headers.get("HX-Request"):
+        return await render_template("alerts/_history_rows.html", **ctx)
+    return await render_template("alerts/history.html", **ctx)
+
+
 @alerts_bp.get("/api/rule_fields")
 @require_role(UserRole.viewer)
 async def rule_fields_api():

steward/alerts/service.py

@@ -12,7 +12,7 @@ from datetime import datetime, timezone
 from sqlalchemy import or_, select
 
 from steward.models.alerts import (
-    AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
+    AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
 )
 from steward.models.metrics import PluginMetric
 from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
@@ -89,6 +89,21 @@ async def list_rules(db) -> list[AlertRule]:
     return list((await db.execute(select(AlertRule).order_by(AlertRule.name))).scalars())
 
 
+async def history(db, since: datetime, rule_id: str | None = None,
+                  limit: int = 200) -> list[tuple[AlertEvent, AlertRule]]:
+    """Alert transitions since `since`, newest first, with their rule; one
+    rule's when `rule_id` is given. Returns up to limit+1 rows so a caller can
+    tell the list was cut."""
+    stmt = (select(AlertEvent, AlertRule)
+            .join(AlertRule, AlertEvent.rule_id == AlertRule.id)
+            .where(AlertEvent.transitioned_at >= since)
+            .order_by(AlertEvent.transitioned_at.desc())
+            .limit(limit + 1))
+    if rule_id:
+        stmt = stmt.where(AlertEvent.rule_id == rule_id)
+    return [tuple(row) for row in (await db.execute(stmt)).all()]
+
+
 async def list_maintenance_windows(db) -> list[MaintenanceWindow]:
     """Every window, latest start first."""
     return list((await db.execute(

steward/mcp/tools/alerts.py

@@ -7,7 +7,7 @@ from sqlalchemy import select
 
 from steward.alerts import service as alert_svc
 from steward.mcp._context import current_caller
-from steward.models.alerts import AlertEvent, AlertOperator, AlertRule, AlertState, MaintenanceWindow
+from steward.models.alerts import AlertOperator, AlertRule, AlertState, MaintenanceWindow
 from steward.models.users import User, UserRole
 
 from ._audit import audit_mcp
@@ -95,16 +95,8 @@ async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50)
     limit = clamp_limit(limit, 50, 500)
     async with db_session() as db:
         rules = await alert_svc.list_rules(db)
-        stmt = (select(AlertEvent, AlertRule)
-                .join(AlertRule, AlertEvent.rule_id == AlertRule.id)
-                .where(AlertEvent.transitioned_at >= since)
-                .order_by(AlertEvent.transitioned_at.desc())
-                .limit(limit + 1))
-        chosen = None
-        if rule:
-            chosen = pick(rule, rules, kind="alert rule", list_tool="list_alert_rules")
-            stmt = stmt.where(AlertEvent.rule_id == chosen.id)
-        rows = (await db.execute(stmt)).all()
+        chosen = pick(rule, rules, kind="alert rule", list_tool="list_alert_rules") if rule else None
+        rows = await alert_svc.history(db, since, chosen.id if chosen else None, limit)
     events = [
         {
             "rule_id": r.id,

steward/templates/alerts/_history_rows.html

@@ -0,0 +1,75 @@
+{# alerts/_history_rows.html — the history table; swapped in on a range or rule change #}
+{% set state_badge = {"firing": "badge-red", "pending": "badge-yellow", "acknowledged": "badge-gold",
+                      "resolved": "badge-green", "inactive": "badge-dim"} %}
+{% if events %}
+<div class="card-flush" style="overflow-x:auto;">
+  <table class="table">
+    <thead>
+      <tr>
+        <th>When (UTC)</th>
+        <th>Rule</th>
+        <th>Change</th>
+        <th>Value</th>
+        <th>Notified</th>
+        <th>Action</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for ev in events %}
+      <tr>
+        <td style="font-family:ui-monospace,monospace;font-size:0.82rem;white-space:nowrap;">
+          {{ ev.at.strftime("%Y-%m-%d %H:%M:%S") }}
+        </td>
+        <td>
+          <a href="/alerts/history?rule={{ ev.rule.id }}&range={{ current_range }}">{{ ev.rule.name }}</a>
+          <div style="font-size:0.78rem;color:var(--text-dim);">{{ ev.rule.source_module }}/{{ ev.rule.resource_name }}</div>
+        </td>
+        <td style="white-space:nowrap;">
+          <span class="badge {{ state_badge.get(ev.from_state, 'badge-dim') }}">{{ ev.from_state }}</span>
+          <span style="color:var(--text-dim);">→</span>
+          <span class="badge {{ state_badge.get(ev.to_state, 'badge-dim') }}">{{ ev.to_state }}</span>
+          {% for n in ev.notes %}
+          <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.25rem;white-space:normal;">{{ n }}</div>
+          {% endfor %}
+        </td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">
+          {{ ev.rule.metric_name }} = {{ ev.value }}
+          <div style="font-size:0.75rem;color:var(--text-dim);">rule: {{ ev.condition }}</div>
+        </td>
+        <td style="font-size:0.82rem;">
+          {% if ev.muted_by %}
+          <span class="badge badge-yellow" title="A maintenance window muted this notification">muted · {{ ev.muted_by }}</span>
+          {% if ev.released %}<div style="color:var(--text-muted);margin-top:0.25rem;">notified when the window ended</div>{% endif %}
+          {% endif %}
+          {% for label, sent, error in ev.deliveries %}
+          <div style="margin-top:0.15rem;">
+            {% if sent %}
+            <span class="badge badge-green">{{ label }} sent</span>
+            {% else %}
+            <span class="badge badge-red">{{ label }} failed</span>
+            {% if error %}<div style="color:var(--text-muted);font-size:0.78rem;margin-top:0.15rem;max-width:28rem;word-break:break-word;">{{ error }}</div>{% endif %}
+            {% endif %}
+          </div>
+          {% endfor %}
+          {% if not ev.muted_by and not ev.deliveries %}<span style="color:var(--text-dim);" title="No notification was sent for this change">—</span>{% endif %}
+        </td>
+        <td style="font-size:0.82rem;">
+          {% if ev.ansible_run_id %}
+          <a href="/ansible/runs/{{ ev.ansible_run_id }}">Ansible run</a>
+          {% else %}<span style="color:var(--text-dim);">—</span>{% endif %}
+        </td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+</div>
+{% if truncated %}
+<p style="color:var(--text-muted);font-size:0.82rem;margin-top:0.75rem;">
+  Showing the newest {{ limit }}. Pick a shorter range or one rule to see the rest.
+</p>
+{% endif %}
+{% else %}
+<p style="color:var(--text-muted);">
+  No alert changed state in the last {{ current_range }}{% if chosen %} for {{ chosen.name }}{% endif %}.
+</p>
+{% endif %}

steward/templates/alerts/history.html

@@ -0,0 +1,32 @@
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs %}
+{% block title %}Alert History — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Alerts", "/alerts/"), ("History", "")]) }}{% endblock %}
+{% block content %}
+<div style="display:flex;align-items:center;justify-content:space-between;flex-wrap:wrap;gap:0.75rem;margin-bottom:1.25rem;">
+  <h1 class="page-title" style="margin-bottom:0;">Alert History</h1>
+  <div style="display:flex;align-items:center;gap:0.75rem;flex-wrap:wrap;">
+    <select id="history-rule" name="rule" aria-label="Rule"
+            hx-get="/alerts/history" hx-target="#history-rows" hx-include="#time-range"
+            hx-trigger="change" hx-push-url="false"
+            onchange="var u=new URL(window.location); this.value ? u.searchParams.set('rule', this.value) : u.searchParams.delete('rule'); history.replaceState({}, '', u);">
+      <option value="">All rules</option>
+      {% for r in rules %}
+      <option value="{{ r.id }}" {% if chosen and chosen.id == r.id %}selected{% endif %}>{{ r.name }}</option>
+      {% endfor %}
+    </select>
+    {% include "_time_range.html" %}
+  </div>
+</div>
+
+<p style="color:var(--text-muted);font-size:0.88rem;margin-bottom:1.25rem;">
+  Every change of state, newest first: what the value was against the rule's condition, whether
+  the email and webhook went out, and the Ansible run an action started.
+</p>
+
+<div id="history-rows"
+     hx-get="/alerts/history" hx-trigger="rangeChange from:body"
+     hx-include="#time-range, #history-rule" hx-swap="innerHTML">
+  {% include "alerts/_history_rows.html" %}
+</div>
+{% endblock %}

steward/templates/alerts/list.html

@@ -57,6 +57,7 @@
 <div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1rem;">
     <h2 class="section-title">Rules</h2>
     <div style="display:flex;gap:0.5rem;">
+        <a class="btn btn-ghost" href="/alerts/history">History</a>
         <a class="btn btn-ghost" href="/alerts/maintenance">Maintenance</a>
         <a class="btn" href="/alerts/rules/new">New Rule</a>
     </div>
@@ -89,6 +90,7 @@
                 </td>
                 <td style="color:var(--text-muted);text-align:center;">{{ rule.consecutive_failures_required }} / {{ rule.consecutive_ok_required }}</td>
                 <td class="td-actions">
+                    <a href="/alerts/history?rule={{ rule.id }}" class="btn btn-sm btn-ghost">History</a>
                     <a href="/alerts/rules/{{ rule.id }}/edit" class="btn btn-sm btn-ghost">Edit</a>
                     <form method="post" action="/alerts/rules/{{ rule.id }}/toggle" style="display:inline;">{{ csrf_field() }}
                         <button type="submit" class="btn btn-sm btn-ghost">

tests/core/test_alert_history_row.py

@@ -0,0 +1,46 @@
+"""How the alert history page reads one AlertEvent (#5066): the maintenance
+mute and its release come out of the note as their own fields, a silent
+transition reads "no data", and only attempted deliveries are listed."""
+from datetime import datetime, timezone
+from types import SimpleNamespace
+
+import pytest
+
+from steward.alerts.routes import _history_row
+from steward.core.alerts import MUTED, RELEASED
+from steward.models.alerts import AlertOperator
+
+RULE = SimpleNamespace(operator=AlertOperator(">"), threshold=90.0)
+
+
+def _event(**kw):
+    base = dict(transitioned_at=datetime(2026, 10, 6, tzinfo=timezone.utc), from_state="inactive",
+                to_state="firing", metric_value=97.5, note=None, email_sent=None, email_error=None,
+                webhook_sent=None, webhook_error=None, ansible_run_id=None)
+    return SimpleNamespace(**{**base, **kw})
+
+
+@pytest.mark.parametrize("note, notes, muted_by, released", [
+    (None, [], None, False),
+    ("rule enabled", ["rule enabled"], None, False),
+    (f"{MUTED} patch night", [], "patch night", False),
+    (f"no sample for 18m; {MUTED} patch night; {RELEASED}", ["no sample for 18m"], "patch night", True),
+    (MUTED, [], "a maintenance window", False),
+])
+def test_notes_split_into_mute_release_and_the_rest(note, notes, muted_by, released):
+    row = _history_row(_event(note=note), RULE)
+    assert (row["notes"], row["muted_by"], row["released"]) == (notes, muted_by, released)
+
+
+def test_value_and_condition():
+    assert _history_row(_event(), RULE)["value"] == "97.5"
+    assert _history_row(_event(metric_value=None), RULE)["value"] == "no data"
+    assert _history_row(_event(), RULE)["condition"] == "> 90"
+
+
+def test_only_attempted_deliveries_are_listed():
+    row = _history_row(_event(email_sent=False, email_error="timed out", webhook_sent=None), RULE)
+    assert row["deliveries"] == [("email", False, "timed out")]
+    row = _history_row(_event(email_sent=True, webhook_sent=True), RULE)
+    assert [(label, sent) for label, sent, _ in row["deliveries"]] == [("email", True), ("webhook", True)]
+    assert _history_row(_event(), RULE)["deliveries"] == []

tests/integration/test_alert_history_page.py

@@ -0,0 +1,83 @@
+"""Integration: /alerts/history (#5066) against real Postgres. A viewer can
+read it; it shows the transition, value against the condition, delivery
+outcome with its error, and a maintenance mute; ?rule= narrows it to one
+rule, and an HTMX request gets only the rows."""
+from __future__ import annotations
+
+import asyncio
+import os
+import time
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def test_a_viewer_reads_the_alert_history(app):
+    from steward.core.alerts import MUTED
+    from steward.models.alerts import AlertEvent, AlertOperator, AlertRule
+    from steward.models.users import User, UserRole
+
+    async def _go():
+        tag = uuid.uuid4().hex[:8]
+        uid = str(uuid.uuid4())
+        now = datetime.now(timezone.utc)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add(User(id=uid, username=f"hist-{tag}", email=f"hist-{tag}@test",
+                           password_hash="x", role=UserRole.viewer, is_active=True))
+                await s.flush()
+                hot = AlertRule(name=f"hot {tag}", source_module="host_agent", resource_name=f"box-{tag}",
+                                metric_name="cpu_pct", operator=AlertOperator(">"), threshold=90,
+                                created_by=uid)
+                cold = AlertRule(name=f"cold {tag}", source_module="host_agent", resource_name=f"box-{tag}",
+                                 metric_name="cpu_pct", operator=AlertOperator("<"), threshold=1,
+                                 created_by=uid)
+                s.add_all([hot, cold])
+                await s.flush()
+                s.add_all([
+                    AlertEvent(rule_id=hot.id, from_state="inactive", to_state="firing", metric_value=97.5,
+                               transitioned_at=now - timedelta(minutes=5), email_sent=False,
+                               email_error=f"mx-{tag}: timed out", webhook_sent=True),
+                    AlertEvent(rule_id=hot.id, from_state="firing", to_state="resolved", metric_value=None,
+                               transitioned_at=now - timedelta(minutes=2), note=f"{MUTED} patch-{tag}"),
+                    AlertEvent(rule_id=cold.id, from_state="inactive", to_state="firing", metric_value=0.5,
+                               transitioned_at=now - timedelta(minutes=1), note=f"cold-note-{tag}"),
+                    # Outside a 1h range.
+                    AlertEvent(rule_id=hot.id, from_state="pending", to_state="firing", metric_value=95,
+                               transitioned_at=now - timedelta(hours=3), note=f"old-{tag}"),
+                ])
+            hot_id = hot.id
+
+        client = app.test_client()
+        async with client.session_transaction() as sess:
+            sess.update(user_id=uid, user_role="viewer", username=f"hist-{tag}", login_at=time.time())
+
+        resp = await client.get("/alerts/history?range=1h")
+        assert resp.status_code == 200
+        page = await resp.get_data(as_text=True)
+        assert "Alert History" in page and 'href="/alerts/"' in page  # crumbs back to Alerts
+        assert f"hot {tag}" in page and f"cold-note-{tag}" in page and f"old-{tag}" not in page
+        assert "cpu_pct = 97.5" in page and "rule: &gt; 90" in page
+        assert "email failed" in page and f"mx-{tag}: timed out" in page and "webhook sent" in page
+        assert f"muted · patch-{tag}" in page and "cpu_pct = no data" in page
+
+        resp = await client.get(f"/alerts/history?range=24h&rule={hot_id}", headers={"HX-Request": "true"})
+        rows = await resp.get_data(as_text=True)
+        assert "<html" not in rows and "<table" in rows
+        assert f"old-{tag}" in rows and f"cold-note-{tag}" not in rows
+
+        # The rules list links each rule to its history.
+        assert f'/alerts/history?rule={hot_id}' in await (await client.get("/alerts/")).get_data(as_text=True)
+
+    asyncio.run(_go())