feat(alerts): a rule can fire when a reading rises, "rose by N" (#6724)

A rule compared a reading with a fixed number, so Synology Disk 1's bad
sectors going from 3,160 to 3,392 could say nothing: a count that only grows
is soon past any line. "rose by" compares each reading with the one before it,
kept on the alert's state and only for such a rule. A fall (a replaced disk)
doesn't fire. On the rule form, through MCP, and worded in emails, the alert
sentence, history and the timeline.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/alerts.py

@@ -506,8 +506,14 @@ def _track_samples(state: AlertState, samples: list[PluginMetric], now: datetime
     return note
 
 
-def _is_breached(value: float, operator: AlertOperator, threshold: float) -> bool:
+def _is_breached(value: float, operator: AlertOperator, threshold: float,
+                 previous: float | None = None) -> bool:
+    """Whether a reading breaks a condition. "rose" needs the reading before
+    it (`previous`): the first has nothing to have risen from, and a fall,
+    such as a replaced disk's count starting over, is no rise."""
     op = operator.value
+    if op == "rose":
+        return previous is not None and value - previous >= threshold
     if op == ">":
         return value > threshold
     if op == "<":
@@ -581,8 +587,10 @@ async def _evaluate_rule(
     """
     from_state = state.state
     step = next_step(from_state, state.consecutive_failure_count, state.consecutive_ok_count,
-                     _is_breached(value, rule.operator, rule.threshold),
+                     _is_breached(value, rule.operator, rule.threshold, state.last_value),
                      rule.consecutive_failures_required, rule.consecutive_ok_required)
+    if rule.operator == AlertOperator.rose:
+        state.last_value = value
     state.consecutive_failure_count = step.fail_count
     state.consecutive_ok_count = step.ok_count
     notifications: list[tuple[str, str, str | None]] = []
@@ -627,6 +635,7 @@ async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str,
     state.last_sample_at = None
     state.sample_interval_s = None
     state.no_data_since = None
+    state.last_value = None
     state.state = AlertStateEnum.inactive
     state.consecutive_failure_count = 0
     state.consecutive_ok_count = 0

steward/core/mail.py

@@ -224,6 +224,8 @@ def alert_facts(alert: dict, tz: tzinfo, show_utc: bool = True) -> dict:
             else _upper_first(f"{counted}{on} again.")
     elif counted:
         headline = _upper_first(f"{counted}{on}.")
+    elif state == "resolved" and alert.get("operator") == "rose":
+        headline = f"{metric_label(metric)}{on} has stopped rising" + (f", at {reading}." if reading else ".")
     elif state == "resolved":
         headline = f"{metric_label(metric)}{on} is back within its limit" + (
             f", at {reading}." if reading else ".")