bvandeusenandClaude Opus 5.5
5dd9956f35
VDNT-Util01's / ran out of inodes (956 free of 60.8M) with 274G of space
left; every file create failed and Steward's page read healthy at 66%.
- Agent 1.13.0 reports each mount's inodes (statvfs; a filesystem reporting
none, like btrfs, is left without).
- Ingest records inodes_used_pct per mount, and disk_used_pct_worst is the
fullest mount by space or inodes, so existing disk-full rules and the
overview see it.
- The host's health strip, the Hosts widget and the host glance grade / by
whichever is fuller and name inodes when they are ("Disk / 66% · inodes
100%", "Inodes on / ... past its critical threshold").
Hosts pick up the inode figures once their agent is updated to 1.13.0.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
145 lines
7.3 KiB
Python
145 lines
7.3 KiB
Python
"""Hosts, containers and monitor response times are graded against their own
|
||
normal (#5742, System 182): a reading past its fixed cutoff but within its
|
||
normal for the hour reads healthy and says what is usual; one well out of it
|
||
keeps its grade. How far out counts is each normal's floor, a setting on
|
||
Settings → Thresholds. Disk keeps its cutoff whatever the history."""
|
||
from types import SimpleNamespace as NS
|
||
|
||
from quart import render_template, session
|
||
|
||
from plugins.docker.service import usage
|
||
from plugins.host_agent.routes import _expand_sample_to_metrics
|
||
from steward.core import norms
|
||
from steward.core.anomalies import graded
|
||
from steward.core.formatting import worst_reading
|
||
from steward.core.norms import Norm, floor_setting
|
||
from steward.core.settings import DEFAULTS, NORM_FLOORS, floor_of, to_thresholds_cfg
|
||
from steward.hosts.service import host_glance, host_grades
|
||
from steward.monitors.service import latency_grade, row_state
|
||
|
||
TH = to_thresholds_cfg({}) # cpu 80/90, mem 80/90, disk 80/90, load 80/100, latency 50/200 ms
|
||
|
||
|
||
def _norm(hold):
|
||
return Norm(kind="day", slot=14, usual_hold=hold, usual_peak=hold * 1.2, days_seen=7)
|
||
|
||
|
||
def test_each_floor_is_a_setting_with_the_chosen_default():
|
||
assert {n: floor_of(TH, n) for n in NORM_FLOORS} == {
|
||
"host_cpu_pct": 10.0, "host_mem_pct": 5.0, "host_load_pct": 10.0,
|
||
"container_cpu_pct": 10.0, "container_mem_pct": 5.0, "monitor_ms": 100.0,
|
||
"router_4xx_pct": 2.0, "router_p95_ms": 50.0}
|
||
assert floor_of(to_thresholds_cfg({"norms.floor.host_cpu_pct": 25}), "host_cpu_pct") == 25.0
|
||
assert floor_of({}, "monitor_ms") == DEFAULTS["norms.floor.monitor_ms"]
|
||
|
||
|
||
def test_a_floor_from_settings_is_read_at_each_pass():
|
||
app = NS(config={"THRESHOLDS": to_thresholds_cfg({})})
|
||
norms.learn_normal("t5742", "x_pct", floor_setting(app, "host_mem_pct"))
|
||
try:
|
||
learned = norms.LEARNED[("t5742", "x_pct")]
|
||
assert learned.least_departure == 5.0
|
||
app.config["THRESHOLDS"] = to_thresholds_cfg({"norms.floor.host_mem_pct": 12})
|
||
assert learned.least_departure == 12.0
|
||
finally:
|
||
norms.LEARNED.pop(("t5742", "x_pct"), None)
|
||
|
||
|
||
def test_past_the_cutoff_but_within_its_normal_reads_healthy():
|
||
assert graded(93.0, "cpu", TH, _norm(88.0), 10.0, "%") == ("ok", "usual 88.0% at this hour")
|
||
assert graded(93.0, "cpu", TH, None, 10.0, "%") == ("crit", None)
|
||
# Out of its normal: a quarter over the usual hold and past the floor.
|
||
assert graded(93.0, "cpu", TH, _norm(40.0), 10.0, "%") == ("crit", None)
|
||
# Under the cutoff there is nothing to quiet.
|
||
assert graded(30.0, "cpu", TH, _norm(5.0), 10.0, "%") == ("ok", None)
|
||
|
||
|
||
def test_a_departure_smaller_than_the_floor_is_still_usual():
|
||
# 1.25 × 70 is 87.5 and 91 is over it, but only 21 over: under a 25-point floor.
|
||
assert graded(91.0, "cpu", TH, _norm(70.0), 25.0, "%")[0] == "ok"
|
||
assert graded(91.0, "cpu", TH, _norm(70.0), 10.0, "%")[0] == "crit"
|
||
|
||
|
||
def test_a_hosts_worst_reading_skips_what_is_usual_for_it():
|
||
readings = (("CPU", 95.0, "cpu"), ("Memory", 85.0, "mem"), ("Disk /", 50.0, "disk"))
|
||
assert worst_reading(readings, TH) == ("CPU", 95.0, "crit")
|
||
usual = {"cpu": (_norm(92.0), 10.0), "mem": (_norm(84.0), 5.0)}
|
||
assert worst_reading(readings, TH, usual) is None
|
||
|
||
|
||
def test_disk_keeps_its_cutoff_whatever_the_history():
|
||
grades = host_grades({"cpu": 95.0, "disk": 95.0}, {"cpu": _norm(93.0)}, TH)
|
||
assert grades["cpu"] == ("ok", "usual 93.0% at this hour")
|
||
assert grades["disk"] == ("crit", None)
|
||
|
||
|
||
def test_the_hosts_glance_grades_each_host_by_its_own_normal():
|
||
hosts = [NS(id="b", name="builder"), NS(id="w", name="web")]
|
||
agent = {"builder": {"cpu_pct": 95.0, "load_per_core": 120, "fresh": True,
|
||
"norms": {"cpu": _norm(90.0), "load": _norm(110.0)}},
|
||
"web": {"cpu_pct": 95.0, "fresh": True, "norms": {"cpu": _norm(20.0)}}}
|
||
rows = {r["host"].name: r for r in host_glance(hosts, {}, agent, TH)}
|
||
assert rows["builder"]["state"] == "up" and rows["builder"]["worst"] is None
|
||
assert rows["builder"]["grades"]["load"] == ("ok", "usual 110.0% at this hour")
|
||
assert rows["web"]["state"] == "crit" and rows["web"]["worst"][0] == "CPU"
|
||
|
||
|
||
def test_a_monitor_slow_as_usual_is_up_not_degraded():
|
||
monitor = NS(enabled=True)
|
||
slow = NS(is_up=True, response_ms=450.0)
|
||
assert row_state(monitor, slow, 100.0, TH) == "degraded"
|
||
assert row_state(monitor, slow, 100.0, TH, _norm(400.0)) == "up"
|
||
assert latency_grade(slow, TH, _norm(400.0)) == ("ok", "usual 400 ms at this hour")
|
||
# 1.25 × 300 = 375, and 150 over is past the 100 ms floor.
|
||
assert row_state(monitor, slow, 100.0, TH, _norm(300.0)) == "degraded"
|
||
|
||
|
||
def test_a_monitor_down_stays_down_whatever_its_normal():
|
||
down = NS(is_up=False, response_ms=None)
|
||
assert row_state(NS(enabled=True), down, 100.0, TH, _norm(400.0)) == "down"
|
||
|
||
|
||
def test_a_containers_cpu_is_learned_per_core_and_said_in_cores():
|
||
# 3.8 cores of a 4-core host is 95% of it: crit, unless 3.6 cores is usual.
|
||
c = NS(cpu_pct=380.0, mem_pct=50.0, mem_usage_bytes=2**30, mem_limit_bytes=2 * 2**30)
|
||
u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"cpu_pct": _norm(360.0)})
|
||
assert u["cpu_grade"] == "ok" and u["cpu_usual"] == "usual 3.60 cores at this hour"
|
||
assert usage(c, 4, 64 * 2**30, thresholds=TH)["cpu_grade"] == "crit"
|
||
assert usage(c, 4, 64 * 2**30)["cpu_grade"] is None # no thresholds: the page grades
|
||
|
||
|
||
def test_a_containers_memory_within_its_normal_reads_healthy():
|
||
c = NS(cpu_pct=1.0, mem_pct=92.0, mem_usage_bytes=2**30, mem_limit_bytes=2 * 2**30)
|
||
u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"mem_pct": _norm(90.0)})
|
||
assert (u["mem_grade"], u["mem_usual"]) == ("ok", "usual 90.0% at this hour")
|
||
assert usage(c, 4, 64 * 2**30, thresholds=TH)["mem_grade"] == "crit"
|
||
|
||
|
||
def test_ingest_records_load_per_core():
|
||
sample = {"load": {"1m": 6.0, "5m": 5.0, "15m": 4.0}, "cpu_cores": [10.0] * 8}
|
||
rows = _expand_sample_to_metrics(sample, "h", None)
|
||
(load_pct,) = [r for r in rows if r.metric_name == "load_pct"]
|
||
assert load_pct.resource_name == "h" and load_pct.value == 75.0
|
||
assert not [r for r in _expand_sample_to_metrics({"load": {"1m": 1.0}}, "h", None)
|
||
if r.metric_name == "load_pct"] # no cores reported: no figure
|
||
|
||
|
||
async def test_the_thresholds_page_offers_each_running_kinds_floor(app):
|
||
async with app.test_request_context("/settings/thresholds/"):
|
||
session["user_role"] = "admin"
|
||
page = await render_template("settings/thresholds.html", settings=dict(DEFAULTS),
|
||
traefik_on=False, plugins_on={"host_agent"})
|
||
assert 'id="normals"' in page
|
||
assert 'name="norms.floor.host_cpu_pct"' in page and 'name="norms.floor.monitor_ms"' in page
|
||
assert 'name="norms.floor.container_cpu_pct"' not in page # docker isn't running
|
||
assert 'name="norms.floor.router_4xx_pct"' not in page
|
||
|
||
|
||
def test_a_host_out_of_inodes_reads_critical_and_says_inodes():
|
||
"""#5745: space 66%, inodes 100% is a full disk, named as inodes."""
|
||
hosts = [NS(id="u", name="util01")]
|
||
agent = {"util01": {"disk_root": 66.0, "inodes_root": 100.0, "fresh": True}}
|
||
(row,) = host_glance(hosts, {}, agent, TH)
|
||
assert row["state"] == "crit" and row["worst"][0] == "Inodes on /"
|
||
assert row["grades"]["disk"][0] == "crit"
|