feat(alerts): a rule can fire when a reading rises, "rose by N" (#6724)

A rule compared a reading with a fixed number, so Synology Disk 1's bad
sectors going from 3,160 to 3,392 could say nothing: a count that only grows
is soon past any line. "rose by" compares each reading with the one before it,
kept on the alert's state and only for such a rule. A fall (a replaced disk)
doesn't fire. On the rule form, through MCP, and worded in emails, the alert
sentence, history and the timeline.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/episodes.py

@@ -15,7 +15,7 @@ QUIET = frozenset({"inactive", "resolved"})
 # The worst thing an episode went through, most severe first.
 _SEVERITY = ("firing", "acknowledged", "no_data", "pending")
 _WORDS = {">": "above", ">=": "at or above", "<": "below", "<=": "at or below",
-          "==": "at", "!=": "other than"}
+          "==": "at", "!=": "other than", "rose": "on a rise of"}
 
 
 def fires_when(rule) -> str:

steward/alerts/routes.py

@@ -15,7 +15,7 @@ from steward.models.users import UserRole
 
 alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")
 
-_OPERATORS = [(op.value, op.value) for op in AlertOperator]
+_OPERATORS = [(op.value, op.said) for op in AlertOperator]
 
 def _form_condition(form) -> dict:
     """The operator and threshold the form asks for. An up/down metric's form

steward/alerts/service.py

@@ -154,7 +154,7 @@ def condition_text(metric: str, operator: AlertOperator | str, threshold: float)
                     if _is_breached(v, operator, threshold)]
         if len(fires_on) == 1:
             return f"is {fires_on[0]}"
-    return f"{metric_label(metric)} {operator.value} {metric_value(metric, threshold)}"
+    return f"{metric_label(metric)} {operator.said} {metric_value(metric, threshold)}"
 
 
 def alert_sentence(rule: AlertRule, label: str, value: float | None,
@@ -175,7 +175,7 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None,
     # not how long ago someone acknowledged it (the page says that apart).
     word = "breaching" if label == "acknowledged" else label
     for_how_long = f", {word} for {held}" if held else f", {word}"
-    condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})"
+    condition = f" ({rule.operator.said} {metric_value(metric, rule.threshold)})"
     counted = count_words(metric, value)
     if counted and rule.resource_name in WHOLE_RESOURCES:
         # The count is the whole of it, and links where the swarm is shown.
@@ -599,6 +599,8 @@ async def rule_fields(db, *, name: str, source_module: str, resource_name: str,
         threshold = float(threshold)
     except (TypeError, ValueError):
         raise ValueError(f"threshold must be a number, got {threshold!r}.") from None
+    if op == AlertOperator.rose and threshold <= 0:
+        raise ValueError("A rule on a rise needs how much it rises by, above 0.")
     try:
         consecutive = (1 if consecutive_failures_required in (None, "")
                        else int(consecutive_failures_required))

steward/core/alerts.py

@@ -506,8 +506,14 @@ def _track_samples(state: AlertState, samples: list[PluginMetric], now: datetime
     return note
 
 
-def _is_breached(value: float, operator: AlertOperator, threshold: float) -> bool:
+def _is_breached(value: float, operator: AlertOperator, threshold: float,
+                 previous: float | None = None) -> bool:
+    """Whether a reading breaks a condition. "rose" needs the reading before
+    it (`previous`): the first has nothing to have risen from, and a fall,
+    such as a replaced disk's count starting over, is no rise."""
     op = operator.value
+    if op == "rose":
+        return previous is not None and value - previous >= threshold
     if op == ">":
         return value > threshold
     if op == "<":
@@ -581,8 +587,10 @@ async def _evaluate_rule(
     """
     from_state = state.state
     step = next_step(from_state, state.consecutive_failure_count, state.consecutive_ok_count,
-                     _is_breached(value, rule.operator, rule.threshold),
+                     _is_breached(value, rule.operator, rule.threshold, state.last_value),
                      rule.consecutive_failures_required, rule.consecutive_ok_required)
+    if rule.operator == AlertOperator.rose:
+        state.last_value = value
     state.consecutive_failure_count = step.fail_count
     state.consecutive_ok_count = step.ok_count
     notifications: list[tuple[str, str, str | None]] = []
@@ -627,6 +635,7 @@ async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str,
     state.last_sample_at = None
     state.sample_interval_s = None
     state.no_data_since = None
+    state.last_value = None
     state.state = AlertStateEnum.inactive
     state.consecutive_failure_count = 0
     state.consecutive_ok_count = 0

steward/core/mail.py

@@ -224,6 +224,8 @@ def alert_facts(alert: dict, tz: tzinfo, show_utc: bool = True) -> dict:
             else _upper_first(f"{counted}{on} again.")
     elif counted:
         headline = _upper_first(f"{counted}{on}.")
+    elif state == "resolved" and alert.get("operator") == "rose":
+        headline = f"{metric_label(metric)}{on} has stopped rising" + (f", at {reading}." if reading else ".")
     elif state == "resolved":
         headline = f"{metric_label(metric)}{on} is back within its limit" + (
             f", at {reading}." if reading else ".")

steward/mcp/tools/alerts.py

@@ -23,7 +23,7 @@ def _condition(rule: AlertRule) -> str:
     ok = rule.consecutive_ok_required or 1
     tail = f" for {n} consecutive checks" if n > 1 else ""
     clear = f"; clears after {ok} OK checks" if ok > 1 else ""
-    return f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}{tail}{clear}"
+    return f"{rule.metric_name} {rule.operator.said} {rule.threshold:g}{tail}{clear}"
 
 
 def _action(rule: AlertRule) -> dict | None:
@@ -288,7 +288,11 @@ async def create_alert_rule(name: str, source_module: str, resource: str, metric
       `resource` as "". Steward then keeps one rule per host, named
       "<name>: <host>", and adds one within a minute for a host that starts
       reporting or joins the group. Only for host_agent.
-    operator: ">", "<", ">=", "<=", "==" or "!=".
+    operator: ">", "<", ">=", "<=", "==" or "!=", or "rose" to fire when a
+      reading is at least `threshold` above the one before it (for a count
+      that only grows). A falling reading doesn't fire, and the next reading
+      that hasn't risen counts as OK: clear_after_checks sets how long the
+      alert stays up, so repeats inside it are one alert.
     consecutive_checks: how many checks in a row must breach before it fires
       (default 1); higher values ride out a single blip.
     clear_after_checks: how many checks in a row must be OK before a firing or

steward/mcp/tools/hosts.py

@@ -190,7 +190,7 @@ async def get_host(host: str, parts_limit: int = 20) -> dict:
         {
             "rule_id": rule.id, "rule_name": rule.name, "state": alert_svc.alert_label(state),
             "resource": rule.resource_name, "metric": rule.metric_name,
-            "condition": f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}",
+            "condition": f"{rule.metric_name} {rule.operator.said} {rule.threshold:g}",
             "since": iso(state.last_transition_at), "age_s": age_s(state.last_transition_at),
         }
         for state, rule in active

steward/mcp/tools/overview.py

@@ -127,7 +127,7 @@ def alert_findings(active, silent: dict[str, list[str]] | None = None) -> list[d
             what += _also_silent(silent.get(rule.resource_name))
         out.append(finding(
             sev, f"alert_{s}", rule.name,
-            f"{rule.metric_name} {rule.operator.value} {rule.threshold:g} on "
+            f"{rule.metric_name} {rule.operator.said} {rule.threshold:g} on "
             f"{rule.source_module}/{rule.resource_name} {what}",
             since=state.no_data_since if s == "no_data" else state.last_transition_at,
             ids={"rule_id": rule.id},

steward/migrations/versions/0049_alert_on_a_rise.py

@@ -0,0 +1,28 @@
+"""Alert rules: a condition that fires on a rise, and the reading before this one.
+
+Revision ID: 0049_alert_on_a_rise
+Revises: 0048_tickets
+Create Date: 2026-10-30
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0049_alert_on_a_rise"
+down_revision: Union[str, None] = "0048_tickets"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ALTER TYPE ... ADD VALUE cannot run inside a transaction block.
+    with op.get_context().autocommit_block():
+        op.execute("ALTER TYPE alertoperator ADD VALUE IF NOT EXISTS 'rose'")
+    op.add_column("alert_states", sa.Column("last_value", sa.Float, nullable=True))
+
+
+def downgrade() -> None:
+    # Postgres can't drop an enum value; 'rose' stays in the type, unused.
+    op.execute("DELETE FROM alert_rules WHERE operator = 'rose'")
+    op.drop_column("alert_states", "last_value")

steward/models/alerts.py

@@ -14,6 +14,14 @@ class AlertOperator(str, enum.Enum):
     lte = "<="
     eq = "=="
     ne = "!="
+    # The reading against the one before it, not against a fixed line: for a
+    # count that only grows, where any fixed line is soon behind it.
+    rose = "rose"
+
+    @property
+    def said(self) -> str:
+        """The operator as a condition reads: "> 90", "rose by 1"."""
+        return "rose by" if self is AlertOperator.rose else self.value
 
 
 class AlertStateEnum(str, enum.Enum):
@@ -96,6 +104,8 @@ class AlertState(Base):
     last_sample_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
     sample_interval_s: Mapped[float | None] = mapped_column(Float, nullable=True)
     no_data_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    # The reading before this one, kept only for a rule that fires on a rise.
+    last_value: Mapped[float | None] = mapped_column(Float, nullable=True)
 
 
 class MaintenanceWindow(Base):

steward/templates/alerts/rules_form.html

@@ -102,12 +102,13 @@
       </div>
       <div data-cond-numeric {% if binary %}hidden{% endif %} class="form-pair">
         <div class="form-group">
-          <label>Fires when the reading is</label>
+          <label>Fires when the reading</label>
           <select name="operator" {% if binary %}disabled{% endif %}>
             {% for val, label in operators %}
             <option value="{{ val }}" {% if rule and rule.operator.value == val %}selected{% endif %}>{{ label }}</option>
             {% endfor %}
           </select>
+          <p class="field-hint">"rose by" compares each reading with the one before it, for a count that only grows. A reading that hasn't risen counts as OK, so "Good checks in a row before it clears" sets how long the alert stays up.</p>
         </div>
         <div class="form-group">
           <label>Threshold <span data-threshold-unit style="color:var(--text-muted);font-weight:normal;">{% if metric_unit(m0) %}({{ metric_unit(m0) }}){% endif %}</span></label>

steward/timeline/record.py

@@ -112,7 +112,7 @@ def alert_summary(type_: str, rule, value: float | None) -> str:
     if metric in _BINARY:
         reading = breach = f"{res} is {metric_value(metric, value)}"
     else:
-        op = getattr(rule.operator, "value", rule.operator)
+        op = getattr(rule.operator, "said", rule.operator)
         reading = f"{said} on {res} is {metric_value(metric, value)}"
         breach = f"{reading} ({op} {metric_value(metric, rule.threshold)})"
     return {