feat(alerts): a rule can fire when a reading rises, "rose by N" (#6724)

A rule compared a reading with a fixed number, so Synology Disk 1's bad
sectors going from 3,160 to 3,392 could say nothing: a count that only grows
is soon past any line. "rose by" compares each reading with the one before it,
kept on the alert's state and only for such a rule. A fall (a replaced disk)
doesn't fire. On the rule form, through MCP, and worded in emails, the alert
sentence, history and the timeline.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-30 19:03:29 -04:00
co-authored by Claude Opus 5.5
parent d83f8ffe9b
commit e51d969c43
15 changed files with 133 additions and 12 deletions
+6 -2
View File
@@ -23,7 +23,7 @@ def _condition(rule: AlertRule) -> str:
ok = rule.consecutive_ok_required or 1
tail = f" for {n} consecutive checks" if n > 1 else ""
clear = f"; clears after {ok} OK checks" if ok > 1 else ""
return f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}{tail}{clear}"
return f"{rule.metric_name} {rule.operator.said} {rule.threshold:g}{tail}{clear}"
def _action(rule: AlertRule) -> dict | None:
@@ -288,7 +288,11 @@ async def create_alert_rule(name: str, source_module: str, resource: str, metric
`resource` as "". Steward then keeps one rule per host, named
"<name>: <host>", and adds one within a minute for a host that starts
reporting or joins the group. Only for host_agent.
operator: ">", "<", ">=", "<=", "==" or "!=".
operator: ">", "<", ">=", "<=", "==" or "!=", or "rose" to fire when a
reading is at least `threshold` above the one before it (for a count
that only grows). A falling reading doesn't fire, and the next reading
that hasn't risen counts as OK: clear_after_checks sets how long the
alert stays up, so repeats inside it are one alert.
consecutive_checks: how many checks in a row must breach before it fires
(default 1); higher values ride out a single blip.
clear_after_checks: how many checks in a row must be OK before a firing or
+1 -1
View File
@@ -190,7 +190,7 @@ async def get_host(host: str, parts_limit: int = 20) -> dict:
{
"rule_id": rule.id, "rule_name": rule.name, "state": alert_svc.alert_label(state),
"resource": rule.resource_name, "metric": rule.metric_name,
"condition": f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}",
"condition": f"{rule.metric_name} {rule.operator.said} {rule.threshold:g}",
"since": iso(state.last_transition_at), "age_s": age_s(state.last_transition_at),
}
for state, rule in active
+1 -1
View File
@@ -127,7 +127,7 @@ def alert_findings(active, silent: dict[str, list[str]] | None = None) -> list[d
what += _also_silent(silent.get(rule.resource_name))
out.append(finding(
sev, f"alert_{s}", rule.name,
f"{rule.metric_name} {rule.operator.value} {rule.threshold:g} on "
f"{rule.metric_name} {rule.operator.said} {rule.threshold:g} on "
f"{rule.source_module}/{rule.resource_name} {what}",
since=state.no_data_since if s == "no_data" else state.last_transition_at,
ids={"rule_id": rule.id},