feat(norms): hosts, containers and monitor response times are graded against their own normal (#5742)
Host CPU, memory and load per core, container CPU and memory, and every monitor's response time learn a normal per hour (milestone 513, System 182). A reading past its fixed cutoff but within its normal reads healthy and says what is usual at this hour; one well out of it keeps its grade and is a metric anomaly. Disk, a monitor down and an agent offline keep their cutoffs. - Each normal's floor, the smallest departure that counts, is a setting: Settings -> Thresholds -> Normals (norms.floor.*). Defaults: host CPU 10 points, host memory 5, load per core 10, container CPU 10 points of one core, container memory 5, monitor response 100 ms; Traefik's 2-point 4xx and 50 ms p95 floors move there from constants. learn_normal takes a floor function (floor_setting) read at each pass. - learn_normal(parts=False): a host learns, its cores don't. - Ingest records load_pct (load_1m / cores x 100), the figure pages grade. - Shared grading: anomalies.against / graded, worst_reading(normals=), hosts' host_grades, docker usage(thresholds=, norms=), monitors' latency_grade. Applied to host vitals, the Hosts widget, MCP overview, Docker usage, and the monitor list, widget and page. - The monitor and container pages list their anomalies. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -333,3 +333,45 @@ def test_a_metric_learns_only_from_its_since(app, src):
|
||||
learned, held_back = asyncio.run(_both())
|
||||
assert learned is not None and learned.usual_hold == 90.0
|
||||
assert held_back is None
|
||||
|
||||
|
||||
def test_a_metric_learned_without_its_parts_ignores_them(app, src):
|
||||
"""A host's CPU learns a normal and each of its cores doesn't (#5742):
|
||||
with parts=False, "<host>:core0" is neither learned nor judged, and the
|
||||
floor may be a function read at each pass."""
|
||||
from sqlalchemy import select
|
||||
from steward.core import norms
|
||||
from steward.core.anomalies import detect, levels_now
|
||||
from steward.core.norms import norms_at, rebuild_norms
|
||||
from steward.models.metrics import PluginMetricHourly
|
||||
from steward.models.norms import MetricAnomaly
|
||||
host = _res("whole")
|
||||
part = f"{host}:core0"
|
||||
rows = [PluginMetricHourly(source_module=src, resource_name=r, metric_name=METRIC,
|
||||
bucket=NOW.replace(hour=0) - timedelta(days=d) + timedelta(hours=h),
|
||||
value_avg=10.0, value_max=10.0, value_p95=10.0, sample_count=60)
|
||||
for r in (host, part) for d in range(1, 8) for h in range(24)]
|
||||
at = NOW + timedelta(days=3)
|
||||
|
||||
async def _go():
|
||||
norms.learn_normal(src, METRIC, lambda: 5.0, parts=False)
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
s.add_all(rows)
|
||||
async with s.begin():
|
||||
await rebuild_norms(s, NOW)
|
||||
learned = await norms_at(s, [(src, host, METRIC), (src, part, METRIC)], NOW.replace(hour=14))
|
||||
async with s.begin():
|
||||
await _report(s, src, host, at, 90.0)
|
||||
await _report(s, src, part, at, 90.0)
|
||||
levels = await levels_now(s, at)
|
||||
async with s.begin():
|
||||
await detect(s, at)
|
||||
opened = list((await s.execute(select(MetricAnomaly.resource_name).where(
|
||||
MetricAnomaly.source_module == src))).scalars())
|
||||
return learned, levels, opened
|
||||
|
||||
learned, levels, opened = asyncio.run(_go())
|
||||
assert set(learned) == {(src, host, METRIC)}
|
||||
assert (src, host, METRIC) in levels and (src, part, METRIC) not in levels
|
||||
assert opened == [host] # 90 against a usual 10: the host departs, its core isn't judged
|
||||
|
||||
Reference in New Issue
Block a user