fix(alerts): acknowledgements hold until resolve; clear after N OK; rule changes reset (#5059)

#5059 "Alert state machine: acknowledgement holds until resolve; N
consecutive OK to resolve; editing, disabling or deleting a rule resets it".

- The state machine is now next_step(), a pure function of one sample. An
  acknowledged alert stays acknowledged while breached; before, the next
  breaching sample re-fired it. Firing and acknowledged alerts resolve after
  consecutive_ok_required OK checks in a row (new rule column, default 1;
  rule form, rules list and MCP create/update_alert_rule clear_after_checks).
- Changing what a rule watches (source, resource, metric, operator,
  threshold) or disabling it resets its alert to inactive; a firing or
  acknowledged one records a resolved event whose new note says why, and
  nobody is notified. Web and MCP edits share alerts.service.apply_rule_change.
- Migration 0028: consecutive_ok_required, consecutive_ok_count,
  alert_events.note, and metric_value nullable (a reset has no sample).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/mcp/tools/alerts.py

@@ -18,8 +18,10 @@ from ._shape import (
 
 def _condition(rule: AlertRule) -> str:
     n = rule.consecutive_failures_required or 1
+    ok = rule.consecutive_ok_required or 1
     tail = f" for {n} consecutive checks" if n > 1 else ""
-    return f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}{tail}"
+    clear = f"; clears after {ok} OK checks" if ok > 1 else ""
+    return f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}{tail}{clear}"
 
 
 def _action(rule: AlertRule) -> dict | None:
@@ -104,6 +106,7 @@ async def get_alert_history(rule: str = "", range: str = "7d", limit: int = 50)
             "from_state": e.from_state,
             "to_state": e.to_state,
             "metric_value": rnd(e.metric_value),
+            "note": e.note,
             "at": iso(e.transitioned_at),
             "age_s": age_s(e.transitioned_at),
             "email_sent": e.email_sent,
@@ -226,7 +229,7 @@ def _rule_row(r: AlertRule) -> dict:
 
 async def create_alert_rule(name: str, source_module: str, resource: str, metric: str,
                             operator: str, threshold: float, consecutive_checks: int = 1,
-                            enabled: bool = True) -> dict:
+                            clear_after_checks: int = 1, enabled: bool = True) -> dict:
     """Add an alert rule: notify when `metric` of `resource` crosses `threshold`.
 
     source_module, resource, metric: get_alert_rule_options lists the valid
@@ -234,6 +237,9 @@ async def create_alert_rule(name: str, source_module: str, resource: str, metric
     operator: ">", "<", ">=", "<=", "==" or "!=".
     consecutive_checks: how many checks in a row must breach before it fires
       (default 1); higher values ride out a single blip.
+    clear_after_checks: how many checks in a row must be OK before a firing or
+      acknowledged alert resolves (default 1). An acknowledgement holds until
+      then, however long the breach lasts.
 
     Notifications go to the channels set in Steward's Settings. The playbook a
     rule runs when it fires is set separately (set_alert_action, admin).
@@ -244,7 +250,8 @@ async def create_alert_rule(name: str, source_module: str, resource: str, metric
         fields = await alert_svc.rule_fields(
             db, name=name, source_module=source_module, resource_name=resource,
             metric_name=metric, operator=operator, threshold=threshold,
-            consecutive_failures_required=consecutive_checks)
+            consecutive_failures_required=consecutive_checks,
+            consecutive_ok_required=clear_after_checks)
         r = await alert_svc.add_rule(db, fields, created_by=caller.user_id, enabled=enabled)
     await audit_mcp("alert_rule.created", "alert_rule", r.name,
                     {"rule_id": r.id, "source": r.source_module, "resource": r.resource_name,
@@ -256,12 +263,15 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s
                             resource: str | None = None, metric: str | None = None,
                             operator: str | None = None, threshold: float | None = None,
                             consecutive_checks: int | None = None,
+                            clear_after_checks: int | None = None,
                             enabled: bool | None = None) -> dict:
     """Change an alert rule. Only the arguments you pass change; the rest stay.
 
     rule: its name or id (list_alert_rules). Pass enabled=false to silence it
     without deleting it. The changed rule is validated as create_alert_rule
-    validates a new one. Recorded in the audit log.
+    validates a new one. Changing what it watches (source, resource, metric,
+    operator, threshold) or disabling it puts its alert back to inactive; a
+    firing alert records a resolved event saying why. Recorded in the audit log.
     """
     async with db_session() as db, db.begin():
         r = await _rule(db, rule)
@@ -274,14 +284,14 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s
             operator=r.operator.value if operator is None else operator,
             threshold=r.threshold if threshold is None else threshold,
             consecutive_failures_required=(r.consecutive_failures_required
-                                           if consecutive_checks is None else consecutive_checks))
-        for k, v in fields.items():
-            setattr(r, k, v)
-        if enabled is not None:
-            r.enabled = bool(enabled)
+                                           if consecutive_checks is None else consecutive_checks),
+            consecutive_ok_required=(r.consecutive_ok_required
+                                     if clear_after_checks is None else clear_after_checks))
+        await alert_svc.apply_rule_change(db, r, fields, enabled=enabled)
     changed = sorted(k for k, v in {"name": name, "source_module": source_module,
                                     "resource": resource, "metric": metric, "operator": operator,
                                     "threshold": threshold, "consecutive_checks": consecutive_checks,
+                                    "clear_after_checks": clear_after_checks,
                                     "enabled": enabled}.items() if v is not None)
     await audit_mcp("alert_rule.updated", "alert_rule", r.name, {"rule_id": r.id, "changed": changed})
     return _rule_row(r)