fix(anomalies): one slow reading no longer opens an anomaly (#6793)
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 1m38s
CI / integration (push) Successful in 2m3s
CI / publish (push) Successful in 1m30s

A single tall reading lifted a 15-minute average past the usual height
on its own, so one slow request opened a row that stayed open until the
reading rolled out of the window. Four of the seven rows that occurred
after the 10-30 detection change were this: HA web (one 6.7 s check),
error-pages, codex and mailcow-svc latency.

detect() now opens by height only when the window is still past the
usual height, and still departs, with its highest reading left out. A
window that fails that falls through to the held test, which now needs
at least HELD_FEWEST (3) readings behind its median, so a lone reading
on a quiet router opens nothing either way.

The median and the topless average share one per-key read (_per_key).
The architecture doc and get_metric_normal's description state the
rule.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-02 22:41:29 -05:00
co-authored by Claude Opus 5.5
parent 19f405e55a
commit c3072312a4
4 changed files with 104 additions and 22 deletions
+43
View File
@@ -140,6 +140,49 @@ def test_a_burst_past_its_usual_peak_is_an_anomaly(app, src):
assert a.started_at == NOW - timedelta(minutes=15)
def _window_of(app, src: str, res: str, readings: dict[int, float]):
"""A detection pass at NOW over `readings` ({minutes before NOW: value}),
against the normal that holds 10 and peaks at 20."""
from sqlalchemy import select
from steward.core.anomalies import detect
from steward.models.metrics import PluginMetric, PluginMetricLatest
from steward.models.norms import MetricAnomaly
async def _go():
async with app.db_sessionmaker() as s:
async with s.begin():
s.add_all(_norms(src, res))
s.add_all([PluginMetric(source_module=src, resource_name=res, metric_name=METRIC,
value=v, recorded_at=NOW - timedelta(minutes=m))
for m, v in readings.items()])
await s.merge(PluginMetricLatest(source_module=src, resource_name=res, metric_name=METRIC,
value=readings[min(readings)], recorded_at=NOW))
async with s.begin():
await detect(s, NOW)
return list((await s.execute(select(MetricAnomaly).where(MetricAnomaly.resource_name == res))).scalars())
return asyncio.run(_go())
def _every_3_minutes(value: float = 10.0) -> dict[int, float]:
"""Six readings across the 15-minute window, as a router or monitor reports."""
return {m: value for m in range(0, 15, 3)} | {14: value}
def test_one_slow_reading_is_no_anomaly(app, src):
# One 200 among five 10s averages 41.7, past the peak of 20; without it the window holds 10.
assert _window_of(app, src, _res("one"), _every_3_minutes() | {6: 200.0}) == []
def test_two_slow_readings_past_the_peak_are_an_anomaly(app, src):
(a,) = _window_of(app, src, _res("two"), _every_3_minutes() | {3: 200.0, 6: 200.0})
assert a.started_at == NOW - timedelta(minutes=15)
def test_a_lone_reading_is_no_anomaly(app, src):
# A quiet router's only request in the hour: neither a height nor a held level.
assert _window_of(app, src, _res("lone"), {2: 500.0}) == []
def test_a_burst_past_its_slots_peak_but_within_what_most_hours_reach_is_no_anomaly(app, src):
assert _hour_of(app, src, _res("anywhere"), level=30.0, for_minutes=15, anywhere=40.0) == []