bvandeusenandClaude Opus 5.5
f3433fe3ce
Host CPU, memory and load per core, container CPU and memory, and every monitor's response time learn a normal per hour (milestone 513, System 182). A reading past its fixed cutoff but within its normal reads healthy and says what is usual at this hour; one well out of it keeps its grade and is a metric anomaly. Disk, a monitor down and an agent offline keep their cutoffs. - Each normal's floor, the smallest departure that counts, is a setting: Settings -> Thresholds -> Normals (norms.floor.*). Defaults: host CPU 10 points, host memory 5, load per core 10, container CPU 10 points of one core, container memory 5, monitor response 100 ms; Traefik's 2-point 4xx and 50 ms p95 floors move there from constants. learn_normal takes a floor function (floor_setting) read at each pass. - learn_normal(parts=False): a host learns, its cores don't. - Ingest records load_pct (load_1m / cores x 100), the figure pages grade. - Shared grading: anomalies.against / graded, worst_reading(normals=), hosts' host_grades, docker usage(thresholds=, norms=), monitors' latency_grade. Applied to host vitals, the Hosts widget, MCP overview, Docker usage, and the monitor list, widget and page. - The monitor and container pages list their anomalies. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
136 lines
6.8 KiB
Python
136 lines
6.8 KiB
Python
"""Hosts, containers and monitor response times are graded against their own
|
||
normal (#5742, System 182): a reading past its fixed cutoff but within its
|
||
normal for the hour reads healthy and says what is usual; one well out of it
|
||
keeps its grade. How far out counts is each normal's floor, a setting on
|
||
Settings → Thresholds. Disk keeps its cutoff whatever the history."""
|
||
from types import SimpleNamespace as NS
|
||
|
||
from quart import render_template, session
|
||
|
||
from plugins.docker.service import usage
|
||
from plugins.host_agent.routes import _expand_sample_to_metrics
|
||
from steward.core import norms
|
||
from steward.core.anomalies import graded
|
||
from steward.core.formatting import worst_reading
|
||
from steward.core.norms import Norm, floor_setting
|
||
from steward.core.settings import DEFAULTS, NORM_FLOORS, floor_of, to_thresholds_cfg
|
||
from steward.hosts.service import host_glance, host_grades
|
||
from steward.monitors.service import latency_grade, row_state
|
||
|
||
TH = to_thresholds_cfg({}) # cpu 80/90, mem 80/90, disk 80/90, load 80/100, latency 50/200 ms
|
||
|
||
|
||
def _norm(hold):
|
||
return Norm(kind="day", slot=14, usual_hold=hold, usual_peak=hold * 1.2, days_seen=7)
|
||
|
||
|
||
def test_each_floor_is_a_setting_with_the_chosen_default():
|
||
assert {n: floor_of(TH, n) for n in NORM_FLOORS} == {
|
||
"host_cpu_pct": 10.0, "host_mem_pct": 5.0, "host_load_pct": 10.0,
|
||
"container_cpu_pct": 10.0, "container_mem_pct": 5.0, "monitor_ms": 100.0,
|
||
"router_4xx_pct": 2.0, "router_p95_ms": 50.0}
|
||
assert floor_of(to_thresholds_cfg({"norms.floor.host_cpu_pct": 25}), "host_cpu_pct") == 25.0
|
||
assert floor_of({}, "monitor_ms") == DEFAULTS["norms.floor.monitor_ms"]
|
||
|
||
|
||
def test_a_floor_from_settings_is_read_at_each_pass():
|
||
app = NS(config={"THRESHOLDS": to_thresholds_cfg({})})
|
||
norms.learn_normal("t5742", "x_pct", floor_setting(app, "host_mem_pct"))
|
||
try:
|
||
learned = norms.LEARNED[("t5742", "x_pct")]
|
||
assert learned.least_departure == 5.0
|
||
app.config["THRESHOLDS"] = to_thresholds_cfg({"norms.floor.host_mem_pct": 12})
|
||
assert learned.least_departure == 12.0
|
||
finally:
|
||
norms.LEARNED.pop(("t5742", "x_pct"), None)
|
||
|
||
|
||
def test_past_the_cutoff_but_within_its_normal_reads_healthy():
|
||
assert graded(93.0, "cpu", TH, _norm(88.0), 10.0, "%") == ("ok", "usual 88.0% at this hour")
|
||
assert graded(93.0, "cpu", TH, None, 10.0, "%") == ("crit", None)
|
||
# Out of its normal: a quarter over the usual hold and past the floor.
|
||
assert graded(93.0, "cpu", TH, _norm(40.0), 10.0, "%") == ("crit", None)
|
||
# Under the cutoff there is nothing to quiet.
|
||
assert graded(30.0, "cpu", TH, _norm(5.0), 10.0, "%") == ("ok", None)
|
||
|
||
|
||
def test_a_departure_smaller_than_the_floor_is_still_usual():
|
||
# 1.25 × 70 is 87.5 and 91 is over it, but only 21 over: under a 25-point floor.
|
||
assert graded(91.0, "cpu", TH, _norm(70.0), 25.0, "%")[0] == "ok"
|
||
assert graded(91.0, "cpu", TH, _norm(70.0), 10.0, "%")[0] == "crit"
|
||
|
||
|
||
def test_a_hosts_worst_reading_skips_what_is_usual_for_it():
|
||
readings = (("CPU", 95.0, "cpu"), ("Memory", 85.0, "mem"), ("Disk /", 50.0, "disk"))
|
||
assert worst_reading(readings, TH) == ("CPU", 95.0, "crit")
|
||
usual = {"cpu": (_norm(92.0), 10.0), "mem": (_norm(84.0), 5.0)}
|
||
assert worst_reading(readings, TH, usual) is None
|
||
|
||
|
||
def test_disk_keeps_its_cutoff_whatever_the_history():
|
||
grades = host_grades({"cpu": 95.0, "disk": 95.0}, {"cpu": _norm(93.0)}, TH)
|
||
assert grades["cpu"] == ("ok", "usual 93.0% at this hour")
|
||
assert grades["disk"] == ("crit", None)
|
||
|
||
|
||
def test_the_hosts_glance_grades_each_host_by_its_own_normal():
|
||
hosts = [NS(id="b", name="builder"), NS(id="w", name="web")]
|
||
agent = {"builder": {"cpu_pct": 95.0, "load_per_core": 120, "fresh": True,
|
||
"norms": {"cpu": _norm(90.0), "load": _norm(110.0)}},
|
||
"web": {"cpu_pct": 95.0, "fresh": True, "norms": {"cpu": _norm(20.0)}}}
|
||
rows = {r["host"].name: r for r in host_glance(hosts, {}, agent, TH)}
|
||
assert rows["builder"]["state"] == "up" and rows["builder"]["worst"] is None
|
||
assert rows["builder"]["grades"]["load"] == ("ok", "usual 110.0% at this hour")
|
||
assert rows["web"]["state"] == "crit" and rows["web"]["worst"][0] == "CPU"
|
||
|
||
|
||
def test_a_monitor_slow_as_usual_is_up_not_degraded():
|
||
monitor = NS(enabled=True)
|
||
slow = NS(is_up=True, response_ms=450.0)
|
||
assert row_state(monitor, slow, 100.0, TH) == "degraded"
|
||
assert row_state(monitor, slow, 100.0, TH, _norm(400.0)) == "up"
|
||
assert latency_grade(slow, TH, _norm(400.0)) == ("ok", "usual 400 ms at this hour")
|
||
# 1.25 × 300 = 375, and 150 over is past the 100 ms floor.
|
||
assert row_state(monitor, slow, 100.0, TH, _norm(300.0)) == "degraded"
|
||
|
||
|
||
def test_a_monitor_down_stays_down_whatever_its_normal():
|
||
down = NS(is_up=False, response_ms=None)
|
||
assert row_state(NS(enabled=True), down, 100.0, TH, _norm(400.0)) == "down"
|
||
|
||
|
||
def test_a_containers_cpu_is_learned_per_core_and_said_in_cores():
|
||
# 3.8 cores of a 4-core host is 95% of it: crit, unless 3.6 cores is usual.
|
||
c = NS(cpu_pct=380.0, mem_pct=50.0, mem_usage_bytes=2**30, mem_limit_bytes=2 * 2**30)
|
||
u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"cpu_pct": _norm(360.0)})
|
||
assert u["cpu_grade"] == "ok" and u["cpu_usual"] == "usual 3.60 cores at this hour"
|
||
assert usage(c, 4, 64 * 2**30, thresholds=TH)["cpu_grade"] == "crit"
|
||
assert usage(c, 4, 64 * 2**30)["cpu_grade"] is None # no thresholds: the page grades
|
||
|
||
|
||
def test_a_containers_memory_within_its_normal_reads_healthy():
|
||
c = NS(cpu_pct=1.0, mem_pct=92.0, mem_usage_bytes=2**30, mem_limit_bytes=2 * 2**30)
|
||
u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"mem_pct": _norm(90.0)})
|
||
assert (u["mem_grade"], u["mem_usual"]) == ("ok", "usual 90.0% at this hour")
|
||
assert usage(c, 4, 64 * 2**30, thresholds=TH)["mem_grade"] == "crit"
|
||
|
||
|
||
def test_ingest_records_load_per_core():
|
||
sample = {"load": {"1m": 6.0, "5m": 5.0, "15m": 4.0}, "cpu_cores": [10.0] * 8}
|
||
rows = _expand_sample_to_metrics(sample, "h", None)
|
||
(load_pct,) = [r for r in rows if r.metric_name == "load_pct"]
|
||
assert load_pct.resource_name == "h" and load_pct.value == 75.0
|
||
assert not [r for r in _expand_sample_to_metrics({"load": {"1m": 1.0}}, "h", None)
|
||
if r.metric_name == "load_pct"] # no cores reported: no figure
|
||
|
||
|
||
async def test_the_thresholds_page_offers_each_running_kinds_floor(app):
|
||
async with app.test_request_context("/settings/thresholds/"):
|
||
session["user_role"] = "admin"
|
||
page = await render_template("settings/thresholds.html", settings=dict(DEFAULTS),
|
||
traefik_on=False, plugins_on={"host_agent"})
|
||
assert 'id="normals"' in page
|
||
assert 'name="norms.floor.host_cpu_pct"' in page and 'name="norms.floor.monitor_ms"' in page
|
||
assert 'name="norms.floor.container_cpu_pct"' not in page # docker isn't running
|
||
assert 'name="norms.floor.router_4xx_pct"' not in page
|