Files
FabledSteward/tests/plugins/host_agent/test_expand_metrics.py
T
bvandeusenandClaude Opus 5.5 a805e1d539
CI / lint (push) Successful in 2s
CI / integration (push) Successful in 1m20s
CI / unit (push) Failing after 1m32s
CI / publish (push) Skipped
fix(host_agent): every Docker read is bounded, and the agent reports its own footprint (#5764)
#5764 "The agent reads a container's whole log backlog into memory, on its
main loop, with no overall deadline". Docker03's 1.13.0 agent went silent
for a day at 2.1 GB and 46 h of CPU, then ignored SIGTERM.

- One shared reader, _docker_get: each answer has an overall deadline and a
  byte cap, not only a per-recv timeout. A trickling or endless body is cut
  off (15 s / 32 MiB for JSON, 5 s / 4 MiB for a log read).
- Container logs: since the cursor but only the newest 2,000 lines, all
  within 10 s. A note says when older lines were skipped, and a container
  whose answer is too big skips ahead to now instead of re-reading its
  backlog every interval.
- The agent reports its own RSS and CPU; Steward records agent_rss_bytes
  and agent_cpu_pct, so a runaway agent shows before it goes silent.

Agent 1.15.0.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 18:01:45 -04:00

125 lines
5.9 KiB
Python

"""Unit tests for _expand_sample_to_metrics — sample dict → PluginMetric rows.
No DB: PluginMetric objects are constructed in-memory, so we assert the
(metric_name, resource_name) → value contract directly.
"""
from datetime import datetime, timezone
from plugins.host_agent.routes import _expand_sample_to_metrics
def _index(rows):
return {(r.metric_name, r.resource_name): r.value for r in rows}
def test_expand_emits_all_new_metric_families():
sample = {
"ts": "2026-06-16T00:00:00+00:00",
"cpu_pct": 12.5,
"cpu_cores": [10.0, 15.0],
"mem": {"total_bytes": 100, "available_bytes": 40, "swap_used_bytes": 5,
"cached_bytes": 20, "buffers_bytes": 3},
"load": {"1m": 0.1, "5m": 0.2, "15m": 0.3},
"uptime_secs": 1000,
"storage": [{"mount": "/", "total_bytes": 100, "used_bytes": 50}],
"net": [{"iface": "eth0", "rx_bps": 1000.0, "tx_bps": 500.0}],
"diskio": [{"device": "sda", "read_bps": 2000.0, "write_bps": 1000.0}],
"temps": [{"label": "Package", "celsius": 45.0}, {"label": "Core0", "celsius": 50.0}],
"psi": {"mem_some_avg10": 1.5, "cpu_some_avg10": 0.0},
}
idx = _index(_expand_sample_to_metrics(sample, "alpha", datetime.now(timezone.utc)))
# per-core CPU as sub-resources
assert idx[("cpu_pct", "alpha:core0")] == 10.0
assert idx[("cpu_pct", "alpha:core1")] == 15.0
# richer memory
assert idx[("mem_cached_bytes", "alpha")] == 20
assert idx[("mem_buffers_bytes", "alpha")] == 3
# network: per-iface + host total
assert idx[("net_rx_bps", "alpha:net:eth0")] == 1000.0
assert idx[("net_rx_bps", "alpha")] == 1000.0
# disk I/O: per-device + host total
assert idx[("disk_read_bps", "alpha:diskio:sda")] == 2000.0
assert idx[("disk_write_bps", "alpha")] == 1000.0
# temps: per-sensor + host max
assert idx[("temp_c", "alpha:temp:Package")] == 45.0
assert idx[("temp_c_max", "alpha")] == 50.0
# PSI
assert idx[("psi_mem_some_avg10", "alpha")] == 1.5
def test_expand_is_backcompat_with_old_agent_sample():
"""An old agent (no new keys) must still expand cleanly to legacy metrics."""
sample = {"ts": "x", "cpu_pct": 5.0, "mem": {"total_bytes": 10, "available_bytes": 5}}
rows = _expand_sample_to_metrics(sample, "h", datetime.now(timezone.utc))
names = {r.metric_name for r in rows}
assert "cpu_pct" in names
assert not any(
n.startswith(("net_", "disk_read", "disk_write", "psi_", "temp_", "mem_cached"))
for n in names
)
def test_every_resource_is_the_host_or_a_host_colon_part():
"""A host rename carries "<host>" and "<host>:…" (steward/core/renames.py
HOST_SOURCES); a resource named any other way would stay behind."""
from steward.core.renames import HOST_SOURCES
sample = {
"ts": "x", "cpu_pct": 1.0, "cpu_cores": [1.0],
"mem": {"total_bytes": 10, "available_bytes": 5, "cached_bytes": 1, "buffers_bytes": 1},
"load": {"1m": 0.1}, "uptime_secs": 1,
"storage": [{"mount": "/", "total_bytes": 10, "used_bytes": 5}],
"net": [{"iface": "eth0", "rx_bps": 1.0, "tx_bps": 1.0}],
"diskio": [{"device": "sda", "read_bps": 1.0, "write_bps": 1.0}],
"temps": [{"label": "Package", "celsius": 40.0}],
"psi": {"cpu_some_avg10": 0.0},
}
sep = dict(HOST_SOURCES)["host_agent"]
resources = {r.resource_name for r in _expand_sample_to_metrics(sample, "web01", datetime.now(timezone.utc))}
assert len(resources) > 5
assert {r for r in resources if r != "web01" and not r.startswith("web01" + sep)} == set()
def test_a_sensor_with_nothing_wired_to_it_does_not_set_the_host_s_temperature():
"""#5759: VDNT-Util01's AUXTIN0 read 110 °C flat while its CPU package
moved 49-87, so the host read critical all week. Every sensor keeps its
own row; only readings that can be real set the max."""
sample = {"ts": "2026-10-14T21:00:00+00:00",
"temps": [{"label": "SYSTIN", "celsius": 114.0},
{"label": "AUXTIN0", "celsius": 110.0},
{"label": "PCH_CHIP_CPU_MAX_TEMP", "celsius": 0.0},
{"label": "AUXTIN3", "celsius": -1.0},
{"label": "Package id 0", "celsius": 60.0}]}
idx = _index(_expand_sample_to_metrics(sample, "util", datetime.now(timezone.utc)))
assert idx[("temp_c_max", "util")] == 60.0
assert idx[("temp_c", "util:temp:AUXTIN0")] == 110.0
assert idx[("temp_c", "util:temp:PCH_CHIP_CPU_MAX_TEMP")] == 0.0
def test_stalled_mounts_are_counted_and_have_no_figures():
"""#5760: a mount the agent couldn't read in time."""
sample = {"ts": "2026-10-14T21:00:00+00:00",
"storage": [{"mount": "/", "total_bytes": 100, "used_bytes": 50},
{"mount": "/nfs/media", "stalled": True}]}
idx = _index(_expand_sample_to_metrics(sample, "d3", datetime.now(timezone.utc)))
assert idx[("mounts_stalled", "d3")] == 1
assert idx[("disk_used_pct_worst", "d3")] == 50.0
assert ("disk_used_pct", "d3:/nfs/media") not in idx
def test_a_wired_board_input_still_counts():
"""#5759: below 100 °C a board input is a real reading."""
from plugins.host_agent.routes import counts_as_hottest
assert counts_as_hottest("SYSTIN", 38.0) and counts_as_hottest("AUXTIN1", 60.0)
assert not counts_as_hottest("SYSTIN", 114.0) and not counts_as_hottest("PCH_CHIP_TEMP", 0.0)
assert counts_as_hottest("Package id 0", 101.0) # a CPU past 100 is a hot CPU
def test_the_agent_s_own_footprint_is_recorded():
"""#5764: a runaway agent shows on the record before it goes silent."""
sample = {"ts": "2026-10-14T22:00:00+00:00",
"agent": {"rss_bytes": 18_000_000, "cpu_pct": 1.5}}
idx = _index(_expand_sample_to_metrics(sample, "d3", datetime.now(timezone.utc)))
assert idx[("agent_rss_bytes", "d3")] == 18_000_000
assert idx[("agent_cpu_pct", "d3")] == 1.5