bvandeusenandClaude Opus 5.5
3f9aae7f46
Each sample carries each NFS mount's requests since the last one: how many
a second, their round trip and execute time (READ and WRITE on their own),
and how many were sent again or hit a major timeout. mountstats is the
kernel's own count and never touches the share, so it reads while one is
stuck. Steward stores them per mount and the slowest on the host as
nfs_exe_max_ms, for an "NFS slow" rule and the host page's NFS chart beside
mounts_stalled; each mount's bar says what a request took.
Agent 1.22.0 (which also carries the backlog order from 41fc807).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
117 lines
6.5 KiB
Python
117 lines
6.5 KiB
Python
"""The host page says how the host is in one header, and grades every reading by
|
|
Settings → Thresholds, nowhere else (#5122)."""
|
|
import re
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from steward.core.formatting import worst_reading
|
|
from steward.core.settings import to_thresholds_cfg
|
|
from tests._templates import render_plugin_template
|
|
from tests.test_style_tokens import hex_colours
|
|
|
|
TH = to_thresholds_cfg({}) # cpu/mem/disk warn 80 crit 90; load warn 80 crit 100
|
|
|
|
|
|
@pytest.mark.parametrize("readings, worst", [
|
|
([("CPU", 40, "cpu"), ("Memory", 50, "mem")], None),
|
|
([("CPU", None, "cpu")], None), # no reading is not a problem
|
|
([("CPU", 85, "cpu"), ("Memory", 95, "mem")], ("Memory", 95, "crit")), # crit beats warn
|
|
([("CPU", 92, "cpu"), ("Memory", 95, "mem")], ("CPU", 92, "crit")), # first of equals
|
|
([("Disk /", 81, "disk")], ("Disk /", 81, "warn")),
|
|
])
|
|
def test_worst_reading(readings, worst):
|
|
assert worst_reading(readings, TH) == worst
|
|
|
|
|
|
async def _vitals(app, *, cpu=40.0, mem=50.0, stale=False, seen_min=1):
|
|
readings = (("CPU", cpu, "cpu"), ("Memory", mem, "mem"), ("Disk /", 30.0, "disk"),
|
|
("Load per core", 20, "load"))
|
|
async with app.test_request_context("/plugins/host_agent/vitals/h1"):
|
|
page = await render_plugin_template(
|
|
app, "host_agent", "_host_vitals.html", reporting=True, stale=stale,
|
|
reg=SimpleNamespace(last_seen_at=datetime.now(timezone.utc) - timedelta(minutes=seen_min),
|
|
agent_version="1.9.0"),
|
|
cpu=cpu, mem=mem, disk_root=30.0, load_per_core=20,
|
|
sparks={"cpu": "", "mem": "", "disk": "", "load": ""},
|
|
psi={"cpu": None, "mem": None, "io": None}, worst=worst_reading(readings, TH))
|
|
return re.sub(r">\s+<", "><", " ".join(page.split()))
|
|
|
|
|
|
async def test_a_host_over_a_threshold_says_which_reading(app):
|
|
page = await _vitals(app, cpu=40.0, mem=93.0)
|
|
assert '<span class="badge badge-ok">Live</span>' in page
|
|
assert '<span class="tone-crit">Memory 93%</span> is past its critical threshold.' in page
|
|
assert 'class="stat-tile stat-crit"><span class="stat-tile-label">Memory</span>' in page
|
|
assert 'class="stat-tile stat-ok"><span class="stat-tile-label">CPU</span>' in page
|
|
assert "seen <time" in page and "agent v1.9.0" in page
|
|
assert not hex_colours(page), hex_colours(page)
|
|
|
|
|
|
async def test_a_quiet_host_says_so(app):
|
|
assert "Every reading is within its thresholds." in await _vitals(app)
|
|
|
|
|
|
async def test_a_stale_host_says_since_when_before_any_reading(app):
|
|
page = await _vitals(app, mem=93.0, stale=True, seen_min=20)
|
|
assert '<span class="badge badge-warn">Stale</span>' in page
|
|
assert "No report since <time" in page and ">20m ago</time>; the readings below are from then." in page
|
|
assert "past its" not in page # an old reading is not news
|
|
|
|
|
|
async def test_the_metrics_page_grades_by_the_same_cutoffs(app):
|
|
# 75 % was amber under the old hard-coded 70/90; Settings → Thresholds says warn at 80.
|
|
async with app.test_request_context("/plugins/host_agent/h1/metrics"):
|
|
page = await render_plugin_template(
|
|
app, "host_agent", "_host_metrics.html", host=SimpleNamespace(address="10.0.0.2"), stale=False,
|
|
reg=SimpleNamespace(distro=None, kernel=None, arch=None, agent_version=None,
|
|
last_seen_at=datetime.now(timezone.utc)),
|
|
hostlvl={"cpu_pct": 75.0, "mem_used_pct": 95.0, "temp_c_max": 72.0},
|
|
cores=[(0, 85.0)], nets={}, disks_io={}, temps={"cpu": 90.0},
|
|
mounts={"/": {"disk_used_pct": 50.0, "disk_used_bytes": 1, "disk_total_bytes": 2}})
|
|
assert 'class="stat-val tone-ok" style="font-size:1.6rem;">75%' in page
|
|
assert 'class="stat-val tone-crit" style="font-size:1.6rem;">95%' in page
|
|
assert 'class="meter-fill meter-ok" style="width:75.0%;"' in page
|
|
assert 'class="meter-fill meter-warn" style="width:85.0%;"' in page # the core
|
|
assert 'class="stat-val tone-warn" style="font-size:1.6rem;">72°C' in page
|
|
assert '<span class="tone-crit" style="font-variant-numeric:tabular-nums;">90°C' in page
|
|
assert not hex_colours(page), hex_colours(page)
|
|
|
|
|
|
async def test_sensors_not_counted_are_listed_once_per_reason(app):
|
|
"""Util01's five open inputs read as one list with its reason, not five
|
|
repeats of it; a sensor judged against the CPU carries its own (#5783)."""
|
|
held = "reads a value no wired sensor gives"
|
|
falls = "falls as the CPU warms (r -0.81 over 168 h)"
|
|
async with app.test_request_context("/plugins/host_agent/h1/metrics"):
|
|
page = await render_plugin_template(
|
|
app, "host_agent", "_host_metrics.html", host=SimpleNamespace(address="10.0.0.2"), stale=False,
|
|
reg=SimpleNamespace(distro=None, kernel=None, arch=None, agent_version=None,
|
|
last_seen_at=datetime.now(timezone.utc)),
|
|
hostlvl={"temp_c_max": 52.0}, cores=[], nets={}, disks_io={}, temps={"Package id 0": 52.0},
|
|
unwired_temps={"SYSTIN": (115.0, held), "AUXTIN0": (110.0, held), "temp3": (62.0, falls)},
|
|
mounts={})
|
|
page = " ".join(page.split())
|
|
assert (f"Not counted, likely nothing wired to them: temp3 62°C ({falls}); "
|
|
f"SYSTIN 115°C, AUXTIN0 110°C ({held}).") in page
|
|
|
|
|
|
async def test_an_nfs_mount_says_what_a_request_took_and_a_local_one_says_nothing(app):
|
|
"""#5996: the delay under each NFS mount's bar, graded at 100 and 1000 ms."""
|
|
async with app.test_request_context("/plugins/host_agent/h1/metrics"):
|
|
page = await render_plugin_template(
|
|
app, "host_agent", "_host_metrics.html", host=SimpleNamespace(address="10.0.0.2"), stale=False,
|
|
reg=SimpleNamespace(distro=None, kernel=None, arch=None, agent_version=None,
|
|
last_seen_at=datetime.now(timezone.utc)),
|
|
hostlvl={}, cores=[], nets={}, disks_io={}, temps={},
|
|
mounts={"/nfs/data": {"disk_used_pct": 40.0, "nfs_ops_per_sec": 4.0, "nfs_exe_ms": 250.0,
|
|
"nfs_retrans": 3.0},
|
|
"/nfs/media": {"disk_used_pct": 30.0, "nfs_ops_per_sec": 0.0},
|
|
"/": {"disk_used_pct": 20.0}})
|
|
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
|
assert '<span class="tone-warn">250.0 ms a request</span> · 4/s · <span class="tone-warn">3 sent again</span>' in page
|
|
assert "NFS idle" in page and page.count("NFS ") == 2
|
|
assert "timed out" not in page
|
|
assert not hex_colours(page), hex_colours(page)
|