feat(alerts): a rule can fire when a reading rises, "rose by N" (#6724)
A rule compared a reading with a fixed number, so Synology Disk 1's bad sectors going from 3,160 to 3,392 could say nothing: a count that only grows is soon past any line. "rose by" compares each reading with the one before it, kept on the alert's state and only for such a rule. A fall (a replaced disk) doesn't fire. On the rule form, through MCP, and worded in emails, the alert sentence, history and the timeline. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -23,7 +23,7 @@ def _condition(rule: AlertRule) -> str:
|
||||
ok = rule.consecutive_ok_required or 1
|
||||
tail = f" for {n} consecutive checks" if n > 1 else ""
|
||||
clear = f"; clears after {ok} OK checks" if ok > 1 else ""
|
||||
return f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}{tail}{clear}"
|
||||
return f"{rule.metric_name} {rule.operator.said} {rule.threshold:g}{tail}{clear}"
|
||||
|
||||
|
||||
def _action(rule: AlertRule) -> dict | None:
|
||||
@@ -288,7 +288,11 @@ async def create_alert_rule(name: str, source_module: str, resource: str, metric
|
||||
`resource` as "". Steward then keeps one rule per host, named
|
||||
"<name>: <host>", and adds one within a minute for a host that starts
|
||||
reporting or joins the group. Only for host_agent.
|
||||
operator: ">", "<", ">=", "<=", "==" or "!=".
|
||||
operator: ">", "<", ">=", "<=", "==" or "!=", or "rose" to fire when a
|
||||
reading is at least `threshold` above the one before it (for a count
|
||||
that only grows). A falling reading doesn't fire, and the next reading
|
||||
that hasn't risen counts as OK: clear_after_checks sets how long the
|
||||
alert stays up, so repeats inside it are one alert.
|
||||
consecutive_checks: how many checks in a row must breach before it fires
|
||||
(default 1); higher values ride out a single blip.
|
||||
clear_after_checks: how many checks in a row must be OK before a firing or
|
||||
|
||||
@@ -190,7 +190,7 @@ async def get_host(host: str, parts_limit: int = 20) -> dict:
|
||||
{
|
||||
"rule_id": rule.id, "rule_name": rule.name, "state": alert_svc.alert_label(state),
|
||||
"resource": rule.resource_name, "metric": rule.metric_name,
|
||||
"condition": f"{rule.metric_name} {rule.operator.value} {rule.threshold:g}",
|
||||
"condition": f"{rule.metric_name} {rule.operator.said} {rule.threshold:g}",
|
||||
"since": iso(state.last_transition_at), "age_s": age_s(state.last_transition_at),
|
||||
}
|
||||
for state, rule in active
|
||||
|
||||
@@ -127,7 +127,7 @@ def alert_findings(active, silent: dict[str, list[str]] | None = None) -> list[d
|
||||
what += _also_silent(silent.get(rule.resource_name))
|
||||
out.append(finding(
|
||||
sev, f"alert_{s}", rule.name,
|
||||
f"{rule.metric_name} {rule.operator.value} {rule.threshold:g} on "
|
||||
f"{rule.metric_name} {rule.operator.said} {rule.threshold:g} on "
|
||||
f"{rule.source_module}/{rule.resource_name} {what}",
|
||||
since=state.no_data_since if s == "no_data" else state.last_transition_at,
|
||||
ids={"rule_id": rule.id},
|
||||
|
||||
Reference in New Issue
Block a user