fix(alerts): acknowledgements hold until resolve; clear after N OK; rule changes reset (#5059)

#5059 "Alert state machine: acknowledgement holds until resolve; N
consecutive OK to resolve; editing, disabling or deleting a rule resets it".

- The state machine is now next_step(), a pure function of one sample. An
  acknowledged alert stays acknowledged while breached; before, the next
  breaching sample re-fired it. Firing and acknowledged alerts resolve after
  consecutive_ok_required OK checks in a row (new rule column, default 1;
  rule form, rules list and MCP create/update_alert_rule clear_after_checks).
- Changing what a rule watches (source, resource, metric, operator,
  threshold) or disabling it resets its alert to inactive; a firing or
  acknowledged one records a resolved event whose new note says why, and
  nobody is notified. Web and MCP edits share alerts.service.apply_rule_change.
- Migration 0028: consecutive_ok_required, consecutive_ok_count,
  alert_events.note, and metric_value nullable (a reset has no sample).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/alerts.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 import asyncio
 import logging
 import uuid
+from dataclasses import dataclass
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING
 
@@ -151,6 +152,44 @@ def _is_breached(value: float, operator: AlertOperator, threshold: float) -> boo
     return False
 
 
+@dataclass(frozen=True)
+class Step:
+    """One sample's effect on an alert: the state after it, the two run
+    counters, and the event to record ("firing" / "resolved") if any."""
+    state: AlertStateEnum
+    fail_count: int
+    ok_count: int
+    event: str | None = None
+
+
+def next_step(state: AlertStateEnum, fail_count: int, ok_count: int, breached: bool,
+              fail_required: int, ok_required: int) -> Step:
+    """The alert state machine, as a pure function of one sample.
+
+    inactive → pending → firing when `fail_required` breaching checks run in a
+    row; a non-breaching check while pending goes back to inactive.
+    firing and acknowledged resolve (to inactive, with a resolved event) only
+    after `ok_required` non-breaching checks in a row; a breaching check
+    resets that run. An acknowledgement holds while the alert is still
+    breached: only a resolve clears it.
+    """
+    fail_required, ok_required = max(1, fail_required), max(1, ok_required)
+    S = AlertStateEnum
+    if state in (S.firing, S.acknowledged):
+        if breached:
+            return Step(state, fail_count + 1, 0)
+        if ok_count + 1 >= ok_required:
+            return Step(S.inactive, 0, 0, "resolved")
+        return Step(state, fail_count, ok_count + 1)
+    # inactive, pending (and the transient resolved, read as inactive)
+    if not breached:
+        return Step(S.inactive, 0, 0)
+    fails = fail_count + 1 if state == S.pending else 1
+    if fails >= fail_required:
+        return Step(S.firing, fails, 0, "firing")
+    return Step(S.pending, fails, 0)
+
+
 async def _evaluate_rule(
     session: AsyncSession,
     rule: AlertRule,
@@ -158,73 +197,56 @@ async def _evaluate_rule(
     value: float,
     now: datetime,
 ) -> list[tuple[str, str]]:
-    """Evaluate a single rule against a metric value. Returns list of (notif_type, event_id)."""
-    breached = _is_breached(value, rule.operator, rule.threshold)
+    """Apply one sample to a rule's state. Returns [(notif_type, event_id)]."""
     from_state = state.state
+    step = next_step(from_state, state.consecutive_failure_count, state.consecutive_ok_count,
+                     _is_breached(value, rule.operator, rule.threshold),
+                     rule.consecutive_failures_required, rule.consecutive_ok_required)
+    state.consecutive_failure_count = step.fail_count
+    state.consecutive_ok_count = step.ok_count
     notifications: list[tuple[str, str]] = []
-
-    if state.state == AlertStateEnum.inactive:
-        if breached:
-            state.consecutive_failure_count = 1
-            if state.consecutive_failure_count >= rule.consecutive_failures_required:
-                state.state = AlertStateEnum.firing
-                state.last_transition_at = now
-                event_id = _add_event(session, rule.id, from_state.value, "firing", value, now)
-                notifications.append(("firing", event_id))
-            else:
-                state.state = AlertStateEnum.pending
-                state.last_transition_at = now
-
-    elif state.state == AlertStateEnum.pending:
-        if breached:
-            state.consecutive_failure_count += 1
-            if state.consecutive_failure_count >= rule.consecutive_failures_required:
-                state.state = AlertStateEnum.firing
-                state.last_transition_at = now
-                event_id = _add_event(session, rule.id, from_state.value, "firing", value, now)
-                notifications.append(("firing", event_id))
-        else:
-            state.consecutive_failure_count = 0
-            state.state = AlertStateEnum.inactive
-            state.last_transition_at = now
-
-    elif state.state == AlertStateEnum.firing:
-        if not breached:
-            # RESOLVED is transient: record event, notify, then immediately go inactive
-            event_id = _add_event(session, rule.id, from_state.value, "resolved", value, now)
-            notifications.append(("resolved", event_id))
-            state.consecutive_failure_count = 0
-            state.state = AlertStateEnum.inactive
-            state.last_transition_at = now
-
-    elif state.state == AlertStateEnum.acknowledged:
-        if not breached:
-            event_id = _add_event(session, rule.id, from_state.value, "resolved", value, now)
-            notifications.append(("resolved", event_id))
-            state.consecutive_failure_count = 0
-            state.state = AlertStateEnum.inactive
-            state.last_transition_at = now
+    if step.state != from_state:
+        state.state = step.state
+        state.last_transition_at = now
+        if step.state == AlertStateEnum.inactive:
             state.acknowledged_by = None
             state.acknowledged_at = None
-        else:
-            state.consecutive_failure_count += 1
-            if state.consecutive_failure_count >= rule.consecutive_failures_required:
-                state.state = AlertStateEnum.firing
-                state.last_transition_at = now
-                event_id = _add_event(session, rule.id, from_state.value, "firing", value, now)
-                notifications.append(("firing", event_id))
-
+    if step.event:
+        event_id = _add_event(session, rule.id, from_state.value, step.event, value, now)
+        notifications.append((step.event, event_id))
     await session.flush()
     return notifications
 
 
+async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str) -> None:
+    """Put a rule's alert back to inactive because the rule itself changed
+    (what it watches, its threshold, or it was disabled). An alert that was
+    firing or acknowledged gets a resolved event carrying `note`; nobody is
+    notified, since nothing about the watched resource changed."""
+    state = (await session.execute(
+        select(AlertState).where(AlertState.rule_id == rule.id))).scalar_one_or_none()
+    if state is None:
+        return
+    now = datetime.now(timezone.utc)
+    if state.state in (AlertStateEnum.firing, AlertStateEnum.acknowledged):
+        _add_event(session, rule.id, state.state.value, "resolved", None, now, note=note)
+    if state.state != AlertStateEnum.inactive:
+        state.last_transition_at = now
+    state.state = AlertStateEnum.inactive
+    state.consecutive_failure_count = 0
+    state.consecutive_ok_count = 0
+    state.acknowledged_by = None
+    state.acknowledged_at = None
+
+
 def _add_event(
     session: AsyncSession,
     rule_id: str,
     from_state: str,
     to_state: str,
-    value: float,
+    value: float | None,
     now: datetime,
+    note: str | None = None,
 ) -> str:
     """Add an AlertEvent row and return its ID."""
     event_id = str(uuid.uuid4())
@@ -235,6 +257,7 @@ def _add_event(
         to_state=to_state,
         metric_value=value,
         transitioned_at=now,
+        note=note,
     )
     session.add(event)
     return event_id