fix(anomalies): a level over its usual hold opens an anomaly only past its usual peak or once held (#6723)

Rows on the live instance were mostly bursts of a few minutes: 28 of 52
peaked within the metric's own usual peak, and 42 of 52 latest occurrences
lasted 30 minutes or less, which is what a burst shorter than the 15-minute
average looks like once its span is back-dated. Nothing asked how high or how
long, as the System 182 ruling says a normal records.

A departure now also has to be past the slot's usual peak, or held: the
median of the last hour over the same line. Those spans were also keeping the
burstiest hours out of learning, lowering the usual lines further.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-30 19:01:55 -04:00
co-authored by Claude Opus 5.5
parent 50b8922895
commit d83f8ffe9b
4 changed files with 96 additions and 15 deletions
+39
View File
@@ -104,6 +104,45 @@ def test_a_level_within_its_normal_is_no_anomaly(app, src):
assert rows == []
def _hour_of(app, src: str, res: str, *, level: float, for_minutes: int, then: float = 10.0):
"""A detection pass after an hour reading `then`, with its last
`for_minutes` at `level`, against the normal that holds 10 and peaks at 20."""
from sqlalchemy import select
from steward.core.anomalies import detect
from steward.models.metrics import PluginMetric, PluginMetricLatest
from steward.models.norms import MetricAnomaly
async def _go():
async with app.db_sessionmaker() as s:
async with s.begin():
s.add_all(_norms(src, res))
s.add_all([PluginMetric(source_module=src, resource_name=res, metric_name=METRIC,
value=level if m < for_minutes else then,
recorded_at=NOW - timedelta(minutes=m)) for m in range(60)])
await s.merge(PluginMetricLatest(source_module=src, resource_name=res, metric_name=METRIC,
value=level, recorded_at=NOW))
async with s.begin():
await detect(s, NOW)
return list((await s.execute(select(MetricAnomaly).where(MetricAnomaly.resource_name == res))).scalars())
return asyncio.run(_go())
def test_a_burst_within_its_usual_peak_is_no_anomaly(app, src):
# 18 is over the line (15) and under the usual peak (20), for 15 minutes of the hour.
assert _hour_of(app, src, _res("burst"), level=18.0, for_minutes=15) == []
def test_a_burst_past_its_usual_peak_is_an_anomaly(app, src):
(a,) = _hour_of(app, src, _res("tall"), level=30.0, for_minutes=15)
assert a.started_at == NOW - timedelta(minutes=15)
def test_a_level_within_its_usual_peak_is_an_anomaly_once_held(app, src):
# The system running at its peaked rate: never past the peak, but not a burst.
(a,) = _hour_of(app, src, _res("held"), level=18.0, for_minutes=45)
assert a.started_at == NOW - timedelta(hours=1) and a.peak_level == 18.0
def test_a_step_change_stays_open_and_to_review(app, src):
from steward.core.anomalies import TO_REVIEW
res = _res("step")