"""Hosts, containers and monitor response times are graded against their own normal (#5742, System 182): a reading past its fixed cutoff but within its normal for the hour reads healthy and says what is usual; one well out of it keeps its grade. How far out counts is each normal's floor, a setting on Settings → Thresholds. Disk keeps its cutoff whatever the history.""" from types import SimpleNamespace as NS from quart import render_template, session from plugins.docker.service import usage from plugins.host_agent.routes import _expand_sample_to_metrics from steward.core import norms from steward.core.anomalies import graded from steward.core.formatting import worst_reading from steward.core.norms import Norm, floor_setting from steward.core.settings import DEFAULTS, NORM_FLOORS, floor_of, to_thresholds_cfg from steward.hosts.service import host_glance, host_grades from steward.monitors.service import latency_grade, row_state TH = to_thresholds_cfg({}) # cpu 80/90, mem 80/90, disk 80/90, load 80/100, latency 50/200 ms def _norm(hold): return Norm(kind="day", slot=14, usual_hold=hold, usual_peak=hold * 1.2, days_seen=7) def test_each_floor_is_a_setting_with_the_chosen_default(): assert {n: floor_of(TH, n) for n in NORM_FLOORS} == { "host_cpu_pct": 10.0, "host_mem_pct": 5.0, "host_load_pct": 10.0, "container_cpu_pct": 10.0, "container_mem_pct": 5.0, "monitor_ms": 100.0, "router_4xx_pct": 2.0, "router_p95_ms": 50.0} assert floor_of(to_thresholds_cfg({"norms.floor.host_cpu_pct": 25}), "host_cpu_pct") == 25.0 assert floor_of({}, "monitor_ms") == DEFAULTS["norms.floor.monitor_ms"] def test_a_floor_from_settings_is_read_at_each_pass(): app = NS(config={"THRESHOLDS": to_thresholds_cfg({})}) norms.learn_normal("t5742", "x_pct", floor_setting(app, "host_mem_pct")) try: learned = norms.LEARNED[("t5742", "x_pct")] assert learned.least_departure == 5.0 app.config["THRESHOLDS"] = to_thresholds_cfg({"norms.floor.host_mem_pct": 12}) assert learned.least_departure == 12.0 finally: norms.LEARNED.pop(("t5742", "x_pct"), None) def test_past_the_cutoff_but_within_its_normal_reads_healthy(): assert graded(93.0, "cpu", TH, _norm(88.0), 10.0, "%") == ("ok", "usual 88.0% at this hour") assert graded(93.0, "cpu", TH, None, 10.0, "%") == ("crit", None) # Out of its normal: a quarter over the usual hold and past the floor. assert graded(93.0, "cpu", TH, _norm(40.0), 10.0, "%") == ("crit", None) # Under the cutoff there is nothing to quiet. assert graded(30.0, "cpu", TH, _norm(5.0), 10.0, "%") == ("ok", None) def test_a_departure_smaller_than_the_floor_is_still_usual(): # 1.25 × 70 is 87.5 and 91 is over it, but only 21 over: under a 25-point floor. assert graded(91.0, "cpu", TH, _norm(70.0), 25.0, "%")[0] == "ok" assert graded(91.0, "cpu", TH, _norm(70.0), 10.0, "%")[0] == "crit" def test_a_hosts_worst_reading_skips_what_is_usual_for_it(): readings = (("CPU", 95.0, "cpu"), ("Memory", 85.0, "mem"), ("Disk /", 50.0, "disk")) assert worst_reading(readings, TH) == ("CPU", 95.0, "crit") usual = {"cpu": (_norm(92.0), 10.0), "mem": (_norm(84.0), 5.0)} assert worst_reading(readings, TH, usual) is None def test_disk_keeps_its_cutoff_whatever_the_history(): grades = host_grades({"cpu": 95.0, "disk": 95.0}, {"cpu": _norm(93.0)}, TH) assert grades["cpu"] == ("ok", "usual 93.0% at this hour") assert grades["disk"] == ("crit", None) def test_the_hosts_glance_grades_each_host_by_its_own_normal(): hosts = [NS(id="b", name="builder"), NS(id="w", name="web")] agent = {"builder": {"cpu_pct": 95.0, "load_per_core": 120, "fresh": True, "norms": {"cpu": _norm(90.0), "load": _norm(110.0)}}, "web": {"cpu_pct": 95.0, "fresh": True, "norms": {"cpu": _norm(20.0)}}} rows = {r["host"].name: r for r in host_glance(hosts, {}, agent, TH)} assert rows["builder"]["state"] == "up" and rows["builder"]["worst"] is None assert rows["builder"]["grades"]["load"] == ("ok", "usual 110.0% at this hour") assert rows["web"]["state"] == "crit" and rows["web"]["worst"][0] == "CPU" def test_a_monitor_slow_as_usual_is_up_not_degraded(): monitor = NS(enabled=True) slow = NS(is_up=True, response_ms=450.0) assert row_state(monitor, slow, 100.0, TH) == "degraded" assert row_state(monitor, slow, 100.0, TH, _norm(400.0)) == "up" assert latency_grade(slow, TH, _norm(400.0)) == ("ok", "usual 400 ms at this hour") # 1.25 × 300 = 375, and 150 over is past the 100 ms floor. assert row_state(monitor, slow, 100.0, TH, _norm(300.0)) == "degraded" def test_a_monitor_down_stays_down_whatever_its_normal(): down = NS(is_up=False, response_ms=None) assert row_state(NS(enabled=True), down, 100.0, TH, _norm(400.0)) == "down" def test_a_containers_cpu_is_learned_per_core_and_said_in_cores(): # 3.8 cores of a 4-core host is 95% of it: crit, unless 3.6 cores is usual. c = NS(cpu_pct=380.0, mem_used_pct=50.0, mem_used_bytes=2**30, mem_limit_bytes=2 * 2**30) u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"cpu_pct": _norm(360.0)}) assert u["cpu_grade"] == "ok" and u["cpu_usual"] == "usual 3.60 cores at this hour" assert usage(c, 4, 64 * 2**30, thresholds=TH)["cpu_grade"] == "crit" assert usage(c, 4, 64 * 2**30)["cpu_grade"] is None # no thresholds: the page grades def test_a_containers_memory_within_its_normal_reads_healthy(): c = NS(cpu_pct=1.0, mem_used_pct=92.0, mem_used_bytes=2**30, mem_limit_bytes=2 * 2**30) u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"mem_used_pct": _norm(90.0)}) assert (u["mem_grade"], u["mem_usual"]) == ("ok", "usual 90.0% at this hour") assert usage(c, 4, 64 * 2**30, thresholds=TH)["mem_grade"] == "crit" def test_ingest_records_load_per_core(): sample = {"load": {"1m": 6.0, "5m": 5.0, "15m": 4.0}, "cpu_cores": [10.0] * 8} rows = _expand_sample_to_metrics(sample, "h", None) (load_pct,) = [r for r in rows if r.metric_name == "load_pct"] assert load_pct.resource_name == "h" and load_pct.value == 75.0 assert not [r for r in _expand_sample_to_metrics({"load": {"1m": 1.0}}, "h", None) if r.metric_name == "load_pct"] # no cores reported: no figure async def test_the_thresholds_page_offers_each_running_kinds_floor(app): async with app.test_request_context("/settings/thresholds/"): session["user_role"] = "admin" page = await render_template("settings/thresholds.html", settings=dict(DEFAULTS), traefik_on=False, plugins_on={"host_agent"}) assert 'id="normals"' in page assert 'name="norms.floor.host_cpu_pct"' in page and 'name="norms.floor.monitor_ms"' in page assert 'name="norms.floor.container_cpu_pct"' not in page # docker isn't running assert 'name="norms.floor.router_4xx_pct"' not in page def test_a_host_out_of_inodes_reads_critical_and_says_inodes(): """#5745: space 66%, inodes 100% is a full disk, named as inodes.""" hosts = [NS(id="u", name="util01")] agent = {"util01": {"disk_root": 66.0, "inodes_root": 100.0, "fresh": True}} (row,) = host_glance(hosts, {}, agent, TH) assert row["state"] == "crit" and row["worst"][0] == "Inodes on /" assert row["grades"]["disk"][0] == "crit"