fix(anomalies): one slow reading no longer opens an anomaly (#6793)
A single tall reading lifted a 15-minute average past the usual height on its own, so one slow request opened a row that stayed open until the reading rolled out of the window. Four of the seven rows that occurred after the 10-30 detection change were this: HA web (one 6.7 s check), error-pages, codex and mailcow-svc latency. detect() now opens by height only when the window is still past the usual height, and still departs, with its highest reading left out. A window that fails that falls through to the held test, which now needs at least HELD_FEWEST (3) readings behind its median, so a lone reading on a quiet router opens nothing either way. The median and the topless average share one per-key read (_per_key). The architecture doc and get_metric_normal's description state the rule. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -140,6 +140,49 @@ def test_a_burst_past_its_usual_peak_is_an_anomaly(app, src):
|
||||
assert a.started_at == NOW - timedelta(minutes=15)
|
||||
|
||||
|
||||
def _window_of(app, src: str, res: str, readings: dict[int, float]):
|
||||
"""A detection pass at NOW over `readings` ({minutes before NOW: value}),
|
||||
against the normal that holds 10 and peaks at 20."""
|
||||
from sqlalchemy import select
|
||||
from steward.core.anomalies import detect
|
||||
from steward.models.metrics import PluginMetric, PluginMetricLatest
|
||||
from steward.models.norms import MetricAnomaly
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
s.add_all(_norms(src, res))
|
||||
s.add_all([PluginMetric(source_module=src, resource_name=res, metric_name=METRIC,
|
||||
value=v, recorded_at=NOW - timedelta(minutes=m))
|
||||
for m, v in readings.items()])
|
||||
await s.merge(PluginMetricLatest(source_module=src, resource_name=res, metric_name=METRIC,
|
||||
value=readings[min(readings)], recorded_at=NOW))
|
||||
async with s.begin():
|
||||
await detect(s, NOW)
|
||||
return list((await s.execute(select(MetricAnomaly).where(MetricAnomaly.resource_name == res))).scalars())
|
||||
return asyncio.run(_go())
|
||||
|
||||
|
||||
def _every_3_minutes(value: float = 10.0) -> dict[int, float]:
|
||||
"""Six readings across the 15-minute window, as a router or monitor reports."""
|
||||
return {m: value for m in range(0, 15, 3)} | {14: value}
|
||||
|
||||
|
||||
def test_one_slow_reading_is_no_anomaly(app, src):
|
||||
# One 200 among five 10s averages 41.7, past the peak of 20; without it the window holds 10.
|
||||
assert _window_of(app, src, _res("one"), _every_3_minutes() | {6: 200.0}) == []
|
||||
|
||||
|
||||
def test_two_slow_readings_past_the_peak_are_an_anomaly(app, src):
|
||||
(a,) = _window_of(app, src, _res("two"), _every_3_minutes() | {3: 200.0, 6: 200.0})
|
||||
assert a.started_at == NOW - timedelta(minutes=15)
|
||||
|
||||
|
||||
def test_a_lone_reading_is_no_anomaly(app, src):
|
||||
# A quiet router's only request in the hour: neither a height nor a held level.
|
||||
assert _window_of(app, src, _res("lone"), {2: 500.0}) == []
|
||||
|
||||
|
||||
def test_a_burst_past_its_slots_peak_but_within_what_most_hours_reach_is_no_anomaly(app, src):
|
||||
assert _hour_of(app, src, _res("anywhere"), level=30.0, for_minutes=15, anywhere=40.0) == []
|
||||
|
||||
|
||||
Reference in New Issue
Block a user