bvandeusenandClaude Opus 5.5
56063f27a4
The agent sent Docker's working set (usage less inactive_file), which still counts recently used file cache. A file-heavy container with no limit read 191 GB while its processes held 2.2 GB, and the host's own figure, which already leaves cache out, disagreed with it. Agent 1.23.0 sends mem_used_bytes / mem_used_pct (anon + shmem, so Postgres's shared buffers count; v1: rss + shmem) and mem_cache_bytes apart. The new names keep the two meanings apart in history and in learned normals: docker_013 renames the columns, clears the old values and adds mem_cache_bytes. The Docker page, swarm by-node sums, Busiest Containers, container detail, normals and the MCP tools all read the new figure. File cache is shown only on the container's page and in get_container. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
145 lines
7.3 KiB
Python
145 lines
7.3 KiB
Python
"""Hosts, containers and monitor response times are graded against their own
|
||
normal (#5742, System 182): a reading past its fixed cutoff but within its
|
||
normal for the hour reads healthy and says what is usual; one well out of it
|
||
keeps its grade. How far out counts is each normal's floor, a setting on
|
||
Settings → Thresholds. Disk keeps its cutoff whatever the history."""
|
||
from types import SimpleNamespace as NS
|
||
|
||
from quart import render_template, session
|
||
|
||
from plugins.docker.service import usage
|
||
from plugins.host_agent.routes import _expand_sample_to_metrics
|
||
from steward.core import norms
|
||
from steward.core.anomalies import graded
|
||
from steward.core.formatting import worst_reading
|
||
from steward.core.norms import Norm, floor_setting
|
||
from steward.core.settings import DEFAULTS, NORM_FLOORS, floor_of, to_thresholds_cfg
|
||
from steward.hosts.service import host_glance, host_grades
|
||
from steward.monitors.service import latency_grade, row_state
|
||
|
||
TH = to_thresholds_cfg({}) # cpu 80/90, mem 80/90, disk 80/90, load 80/100, latency 50/200 ms
|
||
|
||
|
||
def _norm(hold):
|
||
return Norm(kind="day", slot=14, usual_hold=hold, usual_peak=hold * 1.2, days_seen=7)
|
||
|
||
|
||
def test_each_floor_is_a_setting_with_the_chosen_default():
|
||
assert {n: floor_of(TH, n) for n in NORM_FLOORS} == {
|
||
"host_cpu_pct": 10.0, "host_mem_pct": 5.0, "host_load_pct": 10.0,
|
||
"container_cpu_pct": 10.0, "container_mem_pct": 5.0, "monitor_ms": 100.0,
|
||
"router_4xx_pct": 2.0, "router_p95_ms": 50.0}
|
||
assert floor_of(to_thresholds_cfg({"norms.floor.host_cpu_pct": 25}), "host_cpu_pct") == 25.0
|
||
assert floor_of({}, "monitor_ms") == DEFAULTS["norms.floor.monitor_ms"]
|
||
|
||
|
||
def test_a_floor_from_settings_is_read_at_each_pass():
|
||
app = NS(config={"THRESHOLDS": to_thresholds_cfg({})})
|
||
norms.learn_normal("t5742", "x_pct", floor_setting(app, "host_mem_pct"))
|
||
try:
|
||
learned = norms.LEARNED[("t5742", "x_pct")]
|
||
assert learned.least_departure == 5.0
|
||
app.config["THRESHOLDS"] = to_thresholds_cfg({"norms.floor.host_mem_pct": 12})
|
||
assert learned.least_departure == 12.0
|
||
finally:
|
||
norms.LEARNED.pop(("t5742", "x_pct"), None)
|
||
|
||
|
||
def test_past_the_cutoff_but_within_its_normal_reads_healthy():
|
||
assert graded(93.0, "cpu", TH, _norm(88.0), 10.0, "%") == ("ok", "usual 88.0% at this hour")
|
||
assert graded(93.0, "cpu", TH, None, 10.0, "%") == ("crit", None)
|
||
# Out of its normal: a quarter over the usual hold and past the floor.
|
||
assert graded(93.0, "cpu", TH, _norm(40.0), 10.0, "%") == ("crit", None)
|
||
# Under the cutoff there is nothing to quiet.
|
||
assert graded(30.0, "cpu", TH, _norm(5.0), 10.0, "%") == ("ok", None)
|
||
|
||
|
||
def test_a_departure_smaller_than_the_floor_is_still_usual():
|
||
# 1.25 × 70 is 87.5 and 91 is over it, but only 21 over: under a 25-point floor.
|
||
assert graded(91.0, "cpu", TH, _norm(70.0), 25.0, "%")[0] == "ok"
|
||
assert graded(91.0, "cpu", TH, _norm(70.0), 10.0, "%")[0] == "crit"
|
||
|
||
|
||
def test_a_hosts_worst_reading_skips_what_is_usual_for_it():
|
||
readings = (("CPU", 95.0, "cpu"), ("Memory", 85.0, "mem"), ("Disk /", 50.0, "disk"))
|
||
assert worst_reading(readings, TH) == ("CPU", 95.0, "crit")
|
||
usual = {"cpu": (_norm(92.0), 10.0), "mem": (_norm(84.0), 5.0)}
|
||
assert worst_reading(readings, TH, usual) is None
|
||
|
||
|
||
def test_disk_keeps_its_cutoff_whatever_the_history():
|
||
grades = host_grades({"cpu": 95.0, "disk": 95.0}, {"cpu": _norm(93.0)}, TH)
|
||
assert grades["cpu"] == ("ok", "usual 93.0% at this hour")
|
||
assert grades["disk"] == ("crit", None)
|
||
|
||
|
||
def test_the_hosts_glance_grades_each_host_by_its_own_normal():
|
||
hosts = [NS(id="b", name="builder"), NS(id="w", name="web")]
|
||
agent = {"builder": {"cpu_pct": 95.0, "load_per_core": 120, "fresh": True,
|
||
"norms": {"cpu": _norm(90.0), "load": _norm(110.0)}},
|
||
"web": {"cpu_pct": 95.0, "fresh": True, "norms": {"cpu": _norm(20.0)}}}
|
||
rows = {r["host"].name: r for r in host_glance(hosts, {}, agent, TH)}
|
||
assert rows["builder"]["state"] == "up" and rows["builder"]["worst"] is None
|
||
assert rows["builder"]["grades"]["load"] == ("ok", "usual 110.0% at this hour")
|
||
assert rows["web"]["state"] == "crit" and rows["web"]["worst"][0] == "CPU"
|
||
|
||
|
||
def test_a_monitor_slow_as_usual_is_up_not_degraded():
|
||
monitor = NS(enabled=True)
|
||
slow = NS(is_up=True, response_ms=450.0)
|
||
assert row_state(monitor, slow, 100.0, TH) == "degraded"
|
||
assert row_state(monitor, slow, 100.0, TH, _norm(400.0)) == "up"
|
||
assert latency_grade(slow, TH, _norm(400.0)) == ("ok", "usual 400 ms at this hour")
|
||
# 1.25 × 300 = 375, and 150 over is past the 100 ms floor.
|
||
assert row_state(monitor, slow, 100.0, TH, _norm(300.0)) == "degraded"
|
||
|
||
|
||
def test_a_monitor_down_stays_down_whatever_its_normal():
|
||
down = NS(is_up=False, response_ms=None)
|
||
assert row_state(NS(enabled=True), down, 100.0, TH, _norm(400.0)) == "down"
|
||
|
||
|
||
def test_a_containers_cpu_is_learned_per_core_and_said_in_cores():
|
||
# 3.8 cores of a 4-core host is 95% of it: crit, unless 3.6 cores is usual.
|
||
c = NS(cpu_pct=380.0, mem_used_pct=50.0, mem_used_bytes=2**30, mem_limit_bytes=2 * 2**30)
|
||
u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"cpu_pct": _norm(360.0)})
|
||
assert u["cpu_grade"] == "ok" and u["cpu_usual"] == "usual 3.60 cores at this hour"
|
||
assert usage(c, 4, 64 * 2**30, thresholds=TH)["cpu_grade"] == "crit"
|
||
assert usage(c, 4, 64 * 2**30)["cpu_grade"] is None # no thresholds: the page grades
|
||
|
||
|
||
def test_a_containers_memory_within_its_normal_reads_healthy():
|
||
c = NS(cpu_pct=1.0, mem_used_pct=92.0, mem_used_bytes=2**30, mem_limit_bytes=2 * 2**30)
|
||
u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"mem_used_pct": _norm(90.0)})
|
||
assert (u["mem_grade"], u["mem_usual"]) == ("ok", "usual 90.0% at this hour")
|
||
assert usage(c, 4, 64 * 2**30, thresholds=TH)["mem_grade"] == "crit"
|
||
|
||
|
||
def test_ingest_records_load_per_core():
|
||
sample = {"load": {"1m": 6.0, "5m": 5.0, "15m": 4.0}, "cpu_cores": [10.0] * 8}
|
||
rows = _expand_sample_to_metrics(sample, "h", None)
|
||
(load_pct,) = [r for r in rows if r.metric_name == "load_pct"]
|
||
assert load_pct.resource_name == "h" and load_pct.value == 75.0
|
||
assert not [r for r in _expand_sample_to_metrics({"load": {"1m": 1.0}}, "h", None)
|
||
if r.metric_name == "load_pct"] # no cores reported: no figure
|
||
|
||
|
||
async def test_the_thresholds_page_offers_each_running_kinds_floor(app):
|
||
async with app.test_request_context("/settings/thresholds/"):
|
||
session["user_role"] = "admin"
|
||
page = await render_template("settings/thresholds.html", settings=dict(DEFAULTS),
|
||
traefik_on=False, plugins_on={"host_agent"})
|
||
assert 'id="normals"' in page
|
||
assert 'name="norms.floor.host_cpu_pct"' in page and 'name="norms.floor.monitor_ms"' in page
|
||
assert 'name="norms.floor.container_cpu_pct"' not in page # docker isn't running
|
||
assert 'name="norms.floor.router_4xx_pct"' not in page
|
||
|
||
|
||
def test_a_host_out_of_inodes_reads_critical_and_says_inodes():
|
||
"""#5745: space 66%, inodes 100% is a full disk, named as inodes."""
|
||
hosts = [NS(id="u", name="util01")]
|
||
agent = {"util01": {"disk_root": 66.0, "inodes_root": 100.0, "fresh": True}}
|
||
(row,) = host_glance(hosts, {}, agent, TH)
|
||
assert row["state"] == "crit" and row["worst"][0] == "Inodes on /"
|
||
assert row["grades"]["disk"][0] == "crit"
|