bvandeusenandClaude Opus 5.5
3f9aae7f46
Each sample carries each NFS mount's requests since the last one: how many
a second, their round trip and execute time (READ and WRITE on their own),
and how many were sent again or hit a major timeout. mountstats is the
kernel's own count and never touches the share, so it reads while one is
stuck. Steward stores them per mount and the slowest on the host as
nfs_exe_max_ms, for an "NFS slow" rule and the host page's NFS chart beside
mounts_stalled; each mount's bar says what a request took.
Agent 1.22.0 (which also carries the backlog order from 41fc807).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
206 lines
11 KiB
Python
206 lines
11 KiB
Python
"""Unit tests for _expand_sample_to_metrics — sample dict → PluginMetric rows.
|
|
|
|
No DB: PluginMetric objects are constructed in-memory, so we assert the
|
|
(metric_name, resource_name) → value contract directly.
|
|
"""
|
|
from datetime import datetime, timezone
|
|
|
|
from plugins.host_agent.routes import _expand_sample_to_metrics
|
|
|
|
|
|
def _index(rows):
|
|
return {(r.metric_name, r.resource_name): r.value for r in rows}
|
|
|
|
|
|
def test_expand_emits_all_new_metric_families():
|
|
sample = {
|
|
"ts": "2026-06-16T00:00:00+00:00",
|
|
"cpu_pct": 12.5,
|
|
"cpu_cores": [10.0, 15.0],
|
|
"mem": {"total_bytes": 100, "available_bytes": 40, "swap_used_bytes": 5,
|
|
"cached_bytes": 20, "buffers_bytes": 3},
|
|
"load": {"1m": 0.1, "5m": 0.2, "15m": 0.3},
|
|
"uptime_secs": 1000,
|
|
"storage": [{"mount": "/", "total_bytes": 100, "used_bytes": 50}],
|
|
"net": [{"iface": "eth0", "rx_bps": 1000.0, "tx_bps": 500.0}],
|
|
"diskio": [{"device": "sda", "read_bps": 2000.0, "write_bps": 1000.0}],
|
|
"temps": [{"label": "Package", "celsius": 45.0}, {"label": "Core0", "celsius": 50.0}],
|
|
"psi": {"mem_some_avg10": 1.5, "cpu_some_avg10": 0.0},
|
|
}
|
|
idx = _index(_expand_sample_to_metrics(sample, "alpha", datetime.now(timezone.utc)))
|
|
|
|
# per-core CPU as sub-resources
|
|
assert idx[("cpu_pct", "alpha:core0")] == 10.0
|
|
assert idx[("cpu_pct", "alpha:core1")] == 15.0
|
|
# richer memory
|
|
assert idx[("mem_cached_bytes", "alpha")] == 20
|
|
assert idx[("mem_buffers_bytes", "alpha")] == 3
|
|
# network: per-iface + host total
|
|
assert idx[("net_rx_bps", "alpha:net:eth0")] == 1000.0
|
|
assert idx[("net_rx_bps", "alpha")] == 1000.0
|
|
# disk I/O: per-device + host total
|
|
assert idx[("disk_read_bps", "alpha:diskio:sda")] == 2000.0
|
|
assert idx[("disk_write_bps", "alpha")] == 1000.0
|
|
# temps: per-sensor + host max
|
|
assert idx[("temp_c", "alpha:temp:Package")] == 45.0
|
|
assert idx[("temp_c_max", "alpha")] == 50.0
|
|
# PSI
|
|
assert idx[("psi_mem_some_avg10", "alpha")] == 1.5
|
|
|
|
|
|
def test_expand_is_backcompat_with_old_agent_sample():
|
|
"""An old agent (no new keys) must still expand cleanly to legacy metrics."""
|
|
sample = {"ts": "x", "cpu_pct": 5.0, "mem": {"total_bytes": 10, "available_bytes": 5}}
|
|
rows = _expand_sample_to_metrics(sample, "h", datetime.now(timezone.utc))
|
|
names = {r.metric_name for r in rows}
|
|
assert "cpu_pct" in names
|
|
assert not any(
|
|
n.startswith(("net_", "disk_read", "disk_write", "psi_", "temp_", "mem_cached"))
|
|
for n in names
|
|
)
|
|
|
|
|
|
def test_every_resource_is_the_host_or_a_host_colon_part():
|
|
"""A host rename carries "<host>" and "<host>:…" (steward/core/renames.py
|
|
HOST_SOURCES); a resource named any other way would stay behind."""
|
|
from steward.core.renames import HOST_SOURCES
|
|
sample = {
|
|
"ts": "x", "cpu_pct": 1.0, "cpu_cores": [1.0],
|
|
"mem": {"total_bytes": 10, "available_bytes": 5, "cached_bytes": 1, "buffers_bytes": 1},
|
|
"load": {"1m": 0.1}, "uptime_secs": 1,
|
|
"storage": [{"mount": "/", "total_bytes": 10, "used_bytes": 5}],
|
|
"net": [{"iface": "eth0", "rx_bps": 1.0, "tx_bps": 1.0}],
|
|
"diskio": [{"device": "sda", "read_bps": 1.0, "write_bps": 1.0}],
|
|
"temps": [{"label": "Package", "celsius": 40.0}],
|
|
"psi": {"cpu_some_avg10": 0.0},
|
|
}
|
|
sep = dict(HOST_SOURCES)["host_agent"]
|
|
resources = {r.resource_name for r in _expand_sample_to_metrics(sample, "web01", datetime.now(timezone.utc))}
|
|
assert len(resources) > 5
|
|
assert {r for r in resources if r != "web01" and not r.startswith("web01" + sep)} == set()
|
|
|
|
|
|
def test_a_sensor_with_nothing_wired_to_it_does_not_set_the_host_s_temperature():
|
|
"""#5759: VDNT-Util01's AUXTIN0 read 110 °C flat while its CPU package
|
|
moved 49-87, so the host read critical all week. Every sensor keeps its
|
|
own row; only readings that can be real set the max."""
|
|
sample = {"ts": "2026-10-14T21:00:00+00:00",
|
|
"temps": [{"label": "SYSTIN", "celsius": 114.0},
|
|
{"label": "AUXTIN0", "celsius": 110.0},
|
|
{"label": "PCH_CHIP_CPU_MAX_TEMP", "celsius": 0.0},
|
|
{"label": "AUXTIN3", "celsius": -1.0},
|
|
{"label": "Package id 0", "celsius": 60.0}]}
|
|
idx = _index(_expand_sample_to_metrics(sample, "util", datetime.now(timezone.utc)))
|
|
assert idx[("temp_c_max", "util")] == 60.0
|
|
assert idx[("temp_c", "util:temp:AUXTIN0")] == 110.0
|
|
assert idx[("temp_c", "util:temp:PCH_CHIP_CPU_MAX_TEMP")] == 0.0
|
|
|
|
|
|
def test_stalled_mounts_are_counted_and_have_no_figures():
|
|
"""#5760: a mount the agent couldn't read in time."""
|
|
sample = {"ts": "2026-10-14T21:00:00+00:00",
|
|
"storage": [{"mount": "/", "total_bytes": 100, "used_bytes": 50},
|
|
{"mount": "/nfs/media", "stalled": True}]}
|
|
idx = _index(_expand_sample_to_metrics(sample, "d3", datetime.now(timezone.utc)))
|
|
assert idx[("mounts_stalled", "d3")] == 1
|
|
assert idx[("disk_used_pct_worst", "d3")] == 50.0
|
|
assert ("disk_used_pct", "d3:/nfs/media") not in idx
|
|
|
|
|
|
def test_a_wired_board_input_still_counts():
|
|
"""#5759: below 100 °C a board input is a real reading."""
|
|
from plugins.host_agent.routes import counts_as_hottest
|
|
assert counts_as_hottest("SYSTIN", 38.0) and counts_as_hottest("AUXTIN1", 60.0)
|
|
assert not counts_as_hottest("SYSTIN", 114.0) and not counts_as_hottest("PCH_CHIP_TEMP", 0.0)
|
|
assert counts_as_hottest("Package id 0", 101.0) # a CPU past 100 is a hot CPU
|
|
|
|
|
|
def test_the_agent_s_own_footprint_is_recorded():
|
|
"""#5764: a runaway agent shows on the record before it goes silent."""
|
|
sample = {"ts": "2026-10-14T22:00:00+00:00",
|
|
"agent": {"rss_bytes": 18_000_000, "cpu_pct": 1.5}}
|
|
idx = _index(_expand_sample_to_metrics(sample, "d3", datetime.now(timezone.utc)))
|
|
assert idx[("agent_rss_bytes", "d3")] == 18_000_000
|
|
assert idx[("agent_cpu_pct", "d3")] == 1.5
|
|
|
|
|
|
def test_pending_updates_and_a_needed_reboot_are_metrics():
|
|
"""#5765: rules can watch what a host has waiting."""
|
|
sample = {"ts": "2026-10-14T22:00:00+00:00",
|
|
"updates": {"manager": "apt", "pending": 12, "security": 3, "checked_at": "2026-10-14T21:30:00+00:00"},
|
|
"reboot": {"required": True, "since": "2026-10-14T06:00:00+00:00", "packages": ["libc6"]}}
|
|
idx = _index(_expand_sample_to_metrics(sample, "d3", datetime.now(timezone.utc)))
|
|
assert idx[("updates_pending", "d3")] == 12
|
|
assert idx[("updates_security", "d3")] == 3
|
|
assert idx[("reboot_required", "d3")] == 1
|
|
|
|
|
|
def test_the_registration_keeps_the_host_s_updates_and_reboot():
|
|
from types import SimpleNamespace
|
|
from plugins.host_agent.routes import apply_updates
|
|
now = datetime(2026, 10, 14, 22, tzinfo=timezone.utc)
|
|
reg = SimpleNamespace(updates_manager=None, updates_pending=None, updates_security=None,
|
|
updates_checked_at=None, updates_error=None, reboot_required=None,
|
|
reboot_required_since=None, reboot_packages=None)
|
|
sample = {"updates": {"manager": "apt", "pending": 4, "security": 1,
|
|
"checked_at": "2026-10-14T21:30:00+00:00"},
|
|
"reboot": {"required": True, "since": "2026-10-14T06:00:00Z",
|
|
"packages": ["libc6", "linux-image-6.8.0-146-generic"]}}
|
|
assert apply_updates(reg, sample, now) is True
|
|
assert (reg.updates_pending, reg.updates_security, reg.reboot_required) == (4, 1, True)
|
|
assert reg.reboot_required_since.hour == 6
|
|
assert reg.reboot_packages == "libc6, linux-image-6.8.0-146-generic"
|
|
assert apply_updates(reg, sample, now) is False # nothing new
|
|
# A failed check keeps the last counts and says why; the reboot clears.
|
|
apply_updates(reg, {"updates": {"error": "apt is locked", "checked_at": "2026-10-14T22:30:00Z"},
|
|
"reboot": {"required": False}}, now)
|
|
assert reg.updates_pending == 4 and reg.updates_error == "apt is locked"
|
|
assert reg.reboot_required is False and reg.reboot_required_since is None
|
|
|
|
|
|
def test_the_page_lists_unwired_sensors_apart_hottest_first():
|
|
"""VDNT-Util01's sensor list read SYSTIN 115 °C in red at the top, though
|
|
the host's temperature already left it out (#5759)."""
|
|
from plugins.host_agent.routes import split_temps
|
|
real, unwired = split_temps("h", {"SYSTIN": 115.0, "AUXTIN0": 110.0, "Package id 0": 52.0,
|
|
"Composite": 48.9, "PCH_CHIP_TEMP": 0.0, "CPUTIN": 47.0, "gone": None})
|
|
assert list(real) == ["Package id 0", "Composite", "CPUTIN", "gone"]
|
|
assert list(unwired) == ["SYSTIN", "AUXTIN0", "PCH_CHIP_TEMP"]
|
|
assert unwired["SYSTIN"] == (115.0, "reads a value no wired sensor gives")
|
|
|
|
|
|
def test_the_host_page_lists_filesystems_fullest_first():
|
|
"""#5810: by the worse of space and inodes, so a full /var/lib/docker
|
|
doesn't sort after /boot; ties by name."""
|
|
from plugins.host_agent.routes import _split_host_metrics
|
|
latest = {"h": {"cpu_pct": 1.0},
|
|
"h:/": {"disk_used_pct": 40.0, "inodes_used_pct": 5.0},
|
|
"h:/boot": {"disk_used_pct": 30.0},
|
|
"h:/var/lib/docker": {"disk_used_pct": 91.0},
|
|
"h:/srv": {"disk_used_pct": 20.0, "inodes_used_pct": 97.0},
|
|
"h:/data": {"disk_used_pct": 30.0}}
|
|
*_, mounts = _split_host_metrics("h", latest)
|
|
assert list(mounts) == ["/srv", "/var/lib/docker", "/", "/boot", "/data"]
|
|
|
|
|
|
def test_nfs_delay_is_recorded_per_mount_and_the_slowest_on_the_host():
|
|
"""#5996: what each NFS mount's requests took, and the host's slowest for a rule."""
|
|
sample = {"ts": "2026-10-17T12:00:00+00:00",
|
|
"nfs": [{"mount": "/nfs/data", "ops_per_sec": 4.0, "retrans": 2, "timeouts": 0,
|
|
"rtt_ms": 3.5, "exe_ms": 1250.0, "write_ms": 1400.0},
|
|
{"mount": "/nfs/media", "ops_per_sec": 1.0, "retrans": 0, "timeouts": 0,
|
|
"rtt_ms": 1.0, "exe_ms": 2.0, "read_ms": 2.0}]}
|
|
idx = _index(_expand_sample_to_metrics(sample, "d3", datetime.now(timezone.utc)))
|
|
assert idx[("nfs_exe_ms", "d3:/nfs/data")] == 1250.0
|
|
assert idx[("nfs_write_ms", "d3:/nfs/data")] == 1400.0 and ("nfs_read_ms", "d3:/nfs/data") not in idx
|
|
assert idx[("nfs_retrans", "d3:/nfs/data")] == 2 and idx[("nfs_ops_per_sec", "d3:/nfs/media")] == 1.0
|
|
assert idx[("nfs_exe_max_ms", "d3")] == 1250.0
|
|
|
|
|
|
def test_a_host_whose_nfs_mounts_finished_nothing_has_no_nfs_delay():
|
|
sample = {"ts": "2026-10-17T12:00:00+00:00",
|
|
"nfs": [{"mount": "/nfs/data", "ops_per_sec": 0.0, "retrans": 0, "timeouts": 0}]}
|
|
idx = _index(_expand_sample_to_metrics(sample, "d3", datetime.now(timezone.utc)))
|
|
assert idx[("nfs_ops_per_sec", "d3:/nfs/data")] == 0.0
|
|
assert ("nfs_exe_max_ms", "d3") not in idx and ("nfs_exe_ms", "d3:/nfs/data") not in idx
|