feat(anomalies): a metric out of its normal is an anomaly to review (#5740)
#5740 "A metric that departs from its normal is recorded as an anomaly to review, and is findable after it ends", milestone 513. - Every minute each learned key's last 15 minutes is judged against its hour's normal: above the usual hold x1.25 and by the metric's floor opens an occurrence, which ends when the level comes back. - metric_anomalies holds one row per item to review; a repeat while it still waits joins it as an occurrence (metric_anomaly_spans). - An ended row nobody reviews within anomalies.review_days (14, Settings -> Thresholds) lapses; an ongoing one never does. - Learning skips every hour an occurrence covers unless its anomaly was marked normal: no normal is learned without review. - Alerts -> Anomalies lists those to review first, then the reviewed and lapsed under day dividers; marking is operator/admin and audited. - The Overview widget and get_overview warn of open ones and count those to review; the host page shows its own; the weekly report lists the week's and the lapsed; MCP list_anomalies and review_anomaly. - Reviewed and lapsed rows are pruned at data.retention_days, and kept in backups like alert history. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,148 @@
|
||||
"""Metric anomalies (#5740): what counts as a departure, and how one is shown
|
||||
on Alerts → Anomalies, the Overview and the weekly report. The detection,
|
||||
lapsing and learning passes run against Postgres in
|
||||
tests/integration/test_anomalies.py."""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from quart import render_template, session
|
||||
|
||||
from steward.core import mail
|
||||
from steward.core.anomalies import DEPART, AnomalyView, departs
|
||||
from steward.core.norms import Norm
|
||||
from tests.test_style_tokens import hex_colours
|
||||
|
||||
NOW = datetime.now(timezone.utc)
|
||||
NORM = Norm(kind="day", slot=14, usual_hold=10.0, usual_peak=20.0, days_seen=7)
|
||||
|
||||
|
||||
def test_a_departure_is_above_the_usual_hold_by_the_ratio_and_the_floor():
|
||||
assert departs(50.0, NORM, floor=5)
|
||||
assert not departs(10.0 * DEPART, NORM, floor=0) # at the line is not over it
|
||||
assert not departs(14.0, NORM, floor=5) # over ×1.25, but by less than the floor
|
||||
assert departs(15.5, NORM, floor=5)
|
||||
assert not departs(0.3, Norm("day", 1, 0.1, 0.2, 7), floor=2) # 0.1% → 0.3% is never news
|
||||
|
||||
|
||||
def _anomaly(**kw):
|
||||
a = dict(id="a1", source_module="host_agent", resource_name="nas", metric_name="cpu_pct",
|
||||
started_at=NOW - timedelta(hours=3), last_started_at=NOW - timedelta(hours=1),
|
||||
ended_at=None, occurrences=1, peak_level=82.0, peak_at=NOW, usual_hold=30.0,
|
||||
usual_peak=45.0, review="to_review", reviewed_by=None, reviewed_at=None, note=None)
|
||||
a.update(kw)
|
||||
return SimpleNamespace(**a)
|
||||
|
||||
|
||||
def _view(href="/hosts/h1", **kw):
|
||||
return AnomalyView(_anomaly(**kw), None, href)
|
||||
|
||||
|
||||
def test_a_rows_link_opens_its_page_over_a_range_that_reaches_it():
|
||||
assert _view().link == "/hosts/h1?range=24h"
|
||||
assert _view(last_started_at=NOW - timedelta(days=3)).link == "/hosts/h1?range=7d"
|
||||
assert _view(last_started_at=NOW - timedelta(days=12)).link == "/hosts/h1?range=30d"
|
||||
assert _view(href=None).link is None
|
||||
|
||||
|
||||
async def _page(app, role, to_review, past):
|
||||
async with app.test_request_context("/alerts/anomalies"):
|
||||
session["user_id"], session["user_role"] = "u1", role
|
||||
return await render_template("alerts/anomalies.html", review_days=14,
|
||||
to_review=[{"v": v, "opens_group": i == 0, "group": "Today"}
|
||||
for i, v in enumerate(to_review)],
|
||||
past=[{"v": v, "opens_group": i == 0, "group": "Older"}
|
||||
for i, v in enumerate(past)])
|
||||
|
||||
|
||||
async def test_the_page_lists_what_waits_for_review_first(app):
|
||||
waiting = _view(resource_name="waiting-box")
|
||||
done = _view(id="a2", resource_name="done-box", review="normal", reviewed_by="ann",
|
||||
reviewed_at=NOW, ended_at=NOW - timedelta(minutes=5),
|
||||
last_started_at=NOW - timedelta(minutes=30))
|
||||
page = await _page(app, "operator", [waiting], [done])
|
||||
assert "1 to review" in page
|
||||
assert page.index("waiting-box") < page.index("done-box")
|
||||
assert "Metric anomaly" in page and "Marked normal by ann" in page
|
||||
assert "Ongoing since" in page and "usual for that hour" in page
|
||||
assert 'action="/alerts/anomalies/a1/review"' in page # to review: buttons
|
||||
assert 'action="/alerts/anomalies/a2/review"' not in page # reviewed: none
|
||||
assert 'href="/hosts/h1?range=24h"' in page
|
||||
assert ">Today<" in page and ">Older<" in page
|
||||
assert not hex_colours(page), hex_colours(page)
|
||||
|
||||
|
||||
async def test_a_viewer_sees_anomalies_but_cannot_mark_them(app):
|
||||
page = await _page(app, "viewer", [_view()], [])
|
||||
assert "nas" in page and "This is normal" not in page and "Real problem" not in page
|
||||
|
||||
|
||||
async def test_a_repeat_says_how_often_and_when_the_latest_was(app):
|
||||
v = _view(occurrences=5, ended_at=NOW - timedelta(minutes=10))
|
||||
page = await _page(app, "operator", [v], [])
|
||||
assert "5 times since" in page and ", last" in page
|
||||
|
||||
|
||||
async def test_a_quiet_page_says_nothing_waits(app):
|
||||
page = await _page(app, "admin", [], [])
|
||||
assert "Nothing to review" in page and "within 14 days" in page
|
||||
|
||||
|
||||
async def _widget(app, logged_in, anomalies, to_review):
|
||||
async with app.test_request_context("/status/widget"):
|
||||
if logged_in:
|
||||
session["user_id"] = "u1"
|
||||
return await render_template("status/widget.html", entries=[], up=0, down=0, pending=0,
|
||||
total_hosts=1, alerts=[], anomalies=anomalies,
|
||||
to_review=to_review, widget_id="1")
|
||||
|
||||
|
||||
async def test_the_overview_warns_of_an_open_anomaly_and_counts_those_to_review(app):
|
||||
page = await _widget(app, True, [_view()], 3)
|
||||
assert '<span class="badge badge-warn">Metric anomaly</span>' in page
|
||||
assert "nas · CPU" in page
|
||||
assert '<a href="/alerts/anomalies">3 metric anomalies to review</a>' in page
|
||||
assert "No monitors configured" not in page
|
||||
shared = await _widget(app, False, [_view()], 1)
|
||||
assert "1 metric anomaly to review" in shared and 'href="/alerts/anomalies"' not in shared
|
||||
|
||||
|
||||
async def test_a_quiet_overview_has_no_review_line(app):
|
||||
page = await _widget(app, True, [], 0)
|
||||
assert "to review" not in page
|
||||
|
||||
|
||||
REPORT = {"generated_at": datetime(2026, 10, 11, 12, 0, tzinfo=timezone.utc),
|
||||
"uptime_summary": [], "active_alerts": [], "alert_events": {"fired": 0, "resolved": 0},
|
||||
"top_routers": [], "hosts_down": [],
|
||||
"anomalies": {
|
||||
"seen": [{"name": "nas — CPU", "review": "to_review", "occurrences": 2,
|
||||
"started_at": datetime(2026, 10, 6, 7, 0, tzinfo=timezone.utc),
|
||||
"last_started_at": datetime(2026, 10, 9, 7, 10, tzinfo=timezone.utc),
|
||||
"ended_at": datetime(2026, 10, 9, 7, 40, tzinfo=timezone.utc),
|
||||
"level": "held 82%, usual 30%"}],
|
||||
"lapsed": [{"name": "web01 — Load", "review": "lapsed", "occurrences": 1,
|
||||
"started_at": datetime(2026, 9, 20, 3, 0, tzinfo=timezone.utc),
|
||||
"last_started_at": datetime(2026, 9, 20, 3, 0, tzinfo=timezone.utc),
|
||||
"ended_at": datetime(2026, 9, 20, 4, 0, tzinfo=timezone.utc),
|
||||
"level": "held 9.1, usual 2.0"}],
|
||||
"to_review": 1}}
|
||||
|
||||
|
||||
def test_the_weekly_report_lists_the_weeks_anomalies_in_the_readers_zone():
|
||||
to = mail.Recipient("ann@example.com", ZoneInfo("America/New_York"), False, "ann")
|
||||
_, text, html = mail.report_email({"base_url": "https://steward.example"}, REPORT, to)
|
||||
for part in (text, html):
|
||||
assert "nas — CPU" in part and "2 times since Tue 6 Oct 2026 03:00 EDT" in part
|
||||
assert "Fri 9 Oct 2026 03:10 EDT – Fri 9 Oct 2026 03:40 EDT" in part
|
||||
assert "web01 — Load" in part and "lapsed unreviewed" in part
|
||||
assert "1 anomaly waits to be marked normal or real" in part
|
||||
assert "https://steward.example/alerts/anomalies" in part
|
||||
assert "METRIC ANOMALIES THIS WEEK" in text
|
||||
|
||||
|
||||
def test_a_week_without_anomalies_leaves_the_section_out():
|
||||
to = mail.Recipient("ann@example.com", timezone.utc, False, "ann")
|
||||
quiet = {**REPORT, "anomalies": {"seen": [], "lapsed": [], "to_review": 0}}
|
||||
_, text, html = mail.report_email({}, quiet, to)
|
||||
assert "anomal" not in (text + html).lower()
|
||||
@@ -0,0 +1,267 @@
|
||||
"""Integration: a metric out of its normal is an anomaly to review (#5740).
|
||||
|
||||
The operator's rulings (System 182): an anomaly stays "to review" until
|
||||
someone marks it, even after it ends; a repeat joins the row still waiting;
|
||||
an ended one nobody reviews lapses after the review window, an ongoing one
|
||||
never; and only an anomaly marked normal is learned.
|
||||
Requires STEWARD_DATABASE_URL.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
METRIC = "cpu_pct"
|
||||
NOW = datetime(2026, 10, 12, 10, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
if not os.environ.get("STEWARD_DATABASE_URL"):
|
||||
pytest.skip("needs Postgres")
|
||||
from steward.app import create_app
|
||||
return create_app(testing=False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def src():
|
||||
"""A source of the test's own, learned for its run. A detection pass judges
|
||||
every learned key, so another test's keys must not be learned here; within
|
||||
a test, resources are reported at times far enough apart not to overlap."""
|
||||
from steward.core import norms
|
||||
name = f"anom{uuid.uuid4().hex[:8]}"
|
||||
norms.learn_normal(name, METRIC, 5)
|
||||
yield name
|
||||
norms.LEARNED.pop((name, METRIC), None)
|
||||
|
||||
|
||||
def _norms(src: str, res: str, hold: float = 10.0, peak: float = 20.0):
|
||||
"""A learned day normal for every hour: holds `hold`, peaks at `peak`."""
|
||||
from steward.models.norms import MetricNorm
|
||||
return [MetricNorm(source_module=src, resource_name=res, metric_name=METRIC, slot=h,
|
||||
kind="day", usual_hold=hold, usual_peak=peak, days_seen=7, updated_at=NOW)
|
||||
for h in range(24)]
|
||||
|
||||
|
||||
async def _report(s, src: str, res: str, until: datetime, value: float):
|
||||
"""A sample a minute over the window before `until`, at `value`."""
|
||||
from steward.core.anomalies import WINDOW
|
||||
from steward.models.metrics import PluginMetric, PluginMetricLatest
|
||||
n = int(WINDOW.total_seconds() // 60)
|
||||
s.add_all([PluginMetric(source_module=src, resource_name=res, metric_name=METRIC,
|
||||
value=value, recorded_at=until - timedelta(minutes=m))
|
||||
for m in range(n)])
|
||||
await s.merge(PluginMetricLatest(source_module=src, resource_name=res, metric_name=METRIC,
|
||||
value=value, recorded_at=until))
|
||||
|
||||
|
||||
def _run(app, src: str, res: str, steps, *, seed: bool = True):
|
||||
"""Seed the normal (unless `seed` is false), then for each (at, value)
|
||||
report the window before `at` and run one detection pass at `at`. Returns
|
||||
the key's anomalies, oldest first, with their spans."""
|
||||
from sqlalchemy import select
|
||||
from steward.core.anomalies import detect
|
||||
from steward.models.norms import MetricAnomaly, MetricAnomalySpan
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
if seed:
|
||||
async with s.begin():
|
||||
s.add_all(_norms(src, res))
|
||||
for at, value in steps:
|
||||
async with s.begin():
|
||||
await _report(s, src, res, at, value)
|
||||
async with s.begin():
|
||||
await detect(s, at)
|
||||
rows = list((await s.execute(
|
||||
select(MetricAnomaly).where(MetricAnomaly.resource_name == res)
|
||||
.order_by(MetricAnomaly.started_at))).scalars())
|
||||
spans = {a.id: list((await s.execute(
|
||||
select(MetricAnomalySpan).where(MetricAnomalySpan.anomaly_id == a.id)
|
||||
.order_by(MetricAnomalySpan.started_at))).scalars()) for a in rows}
|
||||
return rows, spans
|
||||
|
||||
return asyncio.run(_go())
|
||||
|
||||
|
||||
def _res(tag: str) -> str:
|
||||
return f"{tag}-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
|
||||
def test_a_level_within_its_normal_is_no_anomaly(app, src):
|
||||
# Usual hold 10: 12 is under ×1.25, and 14 is over it but not by the floor of 5.
|
||||
rows, _ = _run(app, src, _res("calm"), [(NOW, 12.0), (NOW + timedelta(minutes=30), 14.0)])
|
||||
assert rows == []
|
||||
|
||||
|
||||
def test_a_step_change_stays_open_and_to_review(app, src):
|
||||
from steward.core.anomalies import TO_REVIEW
|
||||
res = _res("step")
|
||||
rows, spans = _run(app, src, res, [(NOW + timedelta(minutes=m), 50.0) for m in (0, 30, 60, 90)])
|
||||
(a,) = rows
|
||||
assert a.review == TO_REVIEW and a.ended_at is None and a.occurrences == 1
|
||||
assert a.peak_level == 50.0 and a.usual_hold == 10.0 and a.usual_peak == 20.0
|
||||
assert a.started_at == NOW - timedelta(minutes=15)
|
||||
assert len(spans[a.id]) == 1 and spans[a.id][0].ended_at is None
|
||||
|
||||
|
||||
def test_one_that_ended_unwatched_is_still_to_review_and_a_repeat_joins_it(app, src):
|
||||
from steward.core.anomalies import TO_REVIEW
|
||||
res = _res("repeat")
|
||||
back = NOW + timedelta(minutes=30)
|
||||
rows, spans = _run(app, src, res, [
|
||||
(NOW, 60.0), (back, 10.0), # out, then back
|
||||
(NOW + timedelta(hours=3), 40.0), # again, later
|
||||
(NOW + timedelta(hours=3, minutes=30), 10.0),
|
||||
(NOW + timedelta(hours=3, minutes=40), 45.0), # back out within a window
|
||||
])
|
||||
(a,) = rows
|
||||
assert a.review == TO_REVIEW
|
||||
# Two occurrences: the third departure came back within one window of
|
||||
# the second ending, so it continues the second.
|
||||
assert a.occurrences == 2 and len(spans[a.id]) == 2
|
||||
first, second = spans[a.id]
|
||||
assert first.ended_at == back and first.peak_level == 60.0
|
||||
# The continuation's window still held some of the 10s, so the
|
||||
# occurrence's peak stays its first level.
|
||||
assert second.ended_at is None and second.peak_level == 40.0
|
||||
assert a.peak_level == 60.0 and a.last_started_at == second.started_at
|
||||
|
||||
|
||||
def test_an_ended_anomaly_lapses_after_the_window_and_an_ongoing_one_never(app, src):
|
||||
from sqlalchemy import select
|
||||
from steward.core.anomalies import LAPSED, TO_REVIEW, detect, lapse
|
||||
from steward.models.norms import MetricAnomaly
|
||||
ended, going = _res("ended"), _res("going")
|
||||
end = NOW + timedelta(minutes=30)
|
||||
_run(app, src, ended, [(NOW, 60.0), (end, 10.0)])
|
||||
_run(app, src, going, [(NOW + timedelta(days=2), 60.0)])
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
# The ongoing one keeps reporting high, so it stays open.
|
||||
await _report(s, src, going, end + timedelta(days=30), 60.0)
|
||||
await detect(s, end + timedelta(days=30))
|
||||
async with s.begin():
|
||||
await lapse(s, end + timedelta(days=13), 14)
|
||||
mid = {a.resource_name: a.review for a in (await s.execute(
|
||||
select(MetricAnomaly).where(MetricAnomaly.resource_name.in_((ended, going)))
|
||||
)).scalars()}
|
||||
async with s.begin():
|
||||
await lapse(s, end + timedelta(days=30), 14)
|
||||
s.expire_all()
|
||||
late = {a.resource_name: (a.review, a.ended_at) for a in (await s.execute(
|
||||
select(MetricAnomaly).where(MetricAnomaly.resource_name.in_((ended, going)))
|
||||
)).scalars()}
|
||||
return mid, late
|
||||
|
||||
mid, late = asyncio.run(_go())
|
||||
assert mid == {ended: TO_REVIEW, going: TO_REVIEW}
|
||||
assert late[ended][0] == LAPSED
|
||||
assert late[going] == (TO_REVIEW, None)
|
||||
|
||||
|
||||
def test_a_repeat_after_review_is_a_new_row(app, src):
|
||||
from steward.core.anomalies import REAL, TO_REVIEW, mark
|
||||
res = _res("again")
|
||||
rows, _ = _run(app, src, res, [(NOW, 60.0), (NOW + timedelta(minutes=30), 10.0)])
|
||||
|
||||
async def _mark():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await mark(s, rows[0].id, REAL, "tester", "disk filled")
|
||||
|
||||
asyncio.run(_mark())
|
||||
rows, _ = _run(app, src, res, [(NOW + timedelta(hours=5), 60.0)], seed=False)
|
||||
assert [(a.review, a.occurrences) for a in rows] == [(REAL, 1), (TO_REVIEW, 1)]
|
||||
assert rows[0].note == "disk filled" and rows[0].reviewed_by == "tester"
|
||||
|
||||
|
||||
def test_to_review_is_listed_first(app, src):
|
||||
from steward.core.anomalies import NORMAL, TO_REVIEW, list_anomalies, mark
|
||||
older, newer = _res("older"), _res("newer")
|
||||
old_rows, _ = _run(app, src, older, [(NOW, 60.0), (NOW + timedelta(minutes=30), 10.0)])
|
||||
new_rows, _ = _run(app, src, newer, [(NOW + timedelta(hours=2), 60.0),
|
||||
(NOW + timedelta(hours=2, minutes=30), 10.0)])
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await mark(s, new_rows[0].id, NORMAL, "tester")
|
||||
got = await list_anomalies(s, limit=1000)
|
||||
return [a.resource_name for a in got if a.resource_name in (older, newer)]
|
||||
|
||||
assert asyncio.run(_go()) == [older, newer]
|
||||
assert old_rows[0].review == TO_REVIEW
|
||||
|
||||
|
||||
@pytest.mark.parametrize("review,learned_hold", [
|
||||
("normal", 90.0), ("real", 10.0), ("lapsed", 10.0), ("to_review", 10.0)])
|
||||
def test_only_an_anomaly_marked_normal_is_learned(app, src, review, learned_hold):
|
||||
"""Seven days at 10, but 14:00–15:00 held 90 on three of them, each under
|
||||
one anomaly. Only when it is marked normal does 14:00 learn 90."""
|
||||
from steward.core.norms import norms_at, rebuild_norms
|
||||
from steward.models.metrics import PluginMetricHourly
|
||||
from steward.models.norms import MetricAnomaly, MetricAnomalySpan
|
||||
res = _res(f"learn-{review}")
|
||||
hot_days = (1, 2, 3)
|
||||
rows = []
|
||||
for d in range(1, 8):
|
||||
for h in range(24):
|
||||
bucket = NOW.replace(hour=0) - timedelta(days=d) + timedelta(hours=h)
|
||||
v = 90.0 if (d in hot_days and h == 14) else 10.0
|
||||
rows.append(PluginMetricHourly(source_module=src, resource_name=res, metric_name=METRIC,
|
||||
bucket=bucket, value_avg=v, value_max=v, value_p95=v,
|
||||
sample_count=60))
|
||||
first = NOW.replace(hour=14) - timedelta(days=max(hot_days))
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
s.add_all(rows)
|
||||
a = MetricAnomaly(source_module=src, resource_name=res, metric_name=METRIC,
|
||||
started_at=first, last_started_at=first, occurrences=3,
|
||||
ended_at=first + timedelta(hours=1), peak_level=90.0,
|
||||
peak_at=first, usual_hold=10.0, usual_peak=10.0, review=review)
|
||||
s.add(a)
|
||||
await s.flush()
|
||||
s.add_all([MetricAnomalySpan(
|
||||
anomaly_id=a.id, started_at=NOW.replace(hour=14) - timedelta(days=d),
|
||||
ended_at=NOW.replace(hour=15) - timedelta(days=d), peak_level=90.0)
|
||||
for d in hot_days])
|
||||
async with s.begin():
|
||||
await rebuild_norms(s, NOW)
|
||||
got = await norms_at(s, [(src, res, METRIC)], NOW.replace(hour=14))
|
||||
return got[(src, res, METRIC)]
|
||||
|
||||
norm = asyncio.run(_go())
|
||||
assert norm.usual_hold == learned_hold
|
||||
|
||||
|
||||
def test_cleanup_prunes_reviewed_anomalies_and_never_one_to_review(app, src):
|
||||
from sqlalchemy import select
|
||||
from steward.core.anomalies import REAL, mark, prune
|
||||
from steward.models.norms import MetricAnomaly
|
||||
kept, gone = _res("kept"), _res("gone")
|
||||
_run(app, src, kept, [(NOW, 60.0), (NOW + timedelta(minutes=30), 10.0)])
|
||||
later = NOW + timedelta(days=1)
|
||||
rows, _ = _run(app, src, gone, [(later, 60.0), (later + timedelta(minutes=30), 10.0)])
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await mark(s, rows[0].id, REAL, "tester")
|
||||
async with s.begin():
|
||||
await prune(s, NOW + timedelta(days=2))
|
||||
return {a.resource_name for a in (await s.execute(
|
||||
select(MetricAnomaly).where(MetricAnomaly.resource_name.in_((kept, gone)))
|
||||
)).scalars()}
|
||||
|
||||
assert asyncio.run(_go()) == {kept}
|
||||
Reference in New Issue
Block a user