On 2026-10-17 the NFS under Postgres stalled for ~15 minutes. Two things went wrong for the readings that would have explained it: - The SNMP poll stored before it polled again, so while the database hung it stopped polling. It now collects on schedule and hands each round to a Holding (steward/core/holding.py): one writer stores what is held, oldest first, each reading at the time it was collected. Bounded at two hours of polls, with a 30-minute deadline on a store; the device list read falls back to the last one read after 10 s. - "Mount not answering" (10 checks in a row) went pending on Docker02 and Docker03 and never fired: the agent sent each post's backlog with its newest, healthy sample, and that sample ended every post's run. Agent 1.22.0 queues the new sample behind the backlog, so Steward receives a host's samples in the order they were taken. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
131 lines
5.2 KiB
Python
131 lines
5.2 KiB
Python
import urllib.error
|
|
from unittest.mock import patch
|
|
from plugins.host_agent import agent as a
|
|
|
|
|
|
def test_build_sample_shape():
|
|
fake_mem = {"total_bytes": 100, "used_bytes": 40,
|
|
"available_bytes": 60, "swap_used_bytes": 0}
|
|
fake_load = {"1m": 0.1, "5m": 0.2, "15m": 0.3}
|
|
fake_storage = [{"mount": "/", "total_bytes": 1000, "used_bytes": 400}]
|
|
|
|
with patch.object(a, "collect_cpu", return_value=(12.5, [12.0, 13.0])), \
|
|
patch.object(a, "collect_memory", return_value=fake_mem), \
|
|
patch.object(a, "collect_load", return_value=fake_load), \
|
|
patch.object(a, "collect_uptime", return_value=1234), \
|
|
patch.object(a, "collect_storage", return_value=fake_storage), \
|
|
patch.object(a, "collect_temps", return_value=[]), \
|
|
patch.object(a, "collect_psi", return_value={}), \
|
|
patch.object(a, "collect_net_raw", return_value={}), \
|
|
patch.object(a, "collect_diskio_raw", return_value={}):
|
|
sample = a.build_sample(["/"], {})
|
|
|
|
assert sample["cpu_pct"] == 12.5
|
|
assert sample["cpu_cores"] == [12.0, 13.0]
|
|
assert sample["mem"]["used_bytes"] == 40
|
|
assert sample["load"]["1m"] == 0.1
|
|
assert sample["uptime_secs"] == 1234
|
|
assert sample["storage"][0]["mount"] == "/"
|
|
assert "ts" in sample
|
|
# ISO-8601 UTC string
|
|
assert "T" in sample["ts"]
|
|
|
|
|
|
def test_build_sample_tolerates_collector_failure():
|
|
# If a collector raises, the sample should still be built with None for that field.
|
|
def _boom():
|
|
raise RuntimeError("no /proc")
|
|
|
|
with patch.object(a, "collect_cpu", side_effect=_boom), \
|
|
patch.object(a, "collect_memory", return_value={"total_bytes": 1, "used_bytes": 0,
|
|
"available_bytes": 1, "swap_used_bytes": 0}), \
|
|
patch.object(a, "collect_load", return_value={"1m": 0.0, "5m": 0.0, "15m": 0.0}), \
|
|
patch.object(a, "collect_uptime", return_value=0), \
|
|
patch.object(a, "collect_storage", return_value=[]), \
|
|
patch.object(a, "collect_temps", return_value=[]), \
|
|
patch.object(a, "collect_psi", return_value={}), \
|
|
patch.object(a, "collect_net_raw", return_value={}), \
|
|
patch.object(a, "collect_diskio_raw", return_value={}):
|
|
sample = a.build_sample(["/"], {})
|
|
assert sample["cpu_pct"] is None
|
|
assert sample["mem"]["total_bytes"] == 1
|
|
|
|
|
|
def test_build_payload_wraps_samples():
|
|
metadata = {"kernel": "6.8", "distro": "Ubuntu", "arch": "x86_64"}
|
|
payload = a.build_payload(
|
|
samples=[{"ts": "2026-04-14T00:00:00+00:00", "cpu_pct": 5.0}],
|
|
hostname="myhost",
|
|
metadata=metadata,
|
|
)
|
|
assert payload["agent_version"] == a.AGENT_VERSION
|
|
assert payload["hostname"] == "myhost"
|
|
assert payload["metadata"] == metadata
|
|
assert len(payload["samples"]) == 1
|
|
assert payload["samples"][0]["cpu_pct"] == 5.0
|
|
|
|
|
|
def test_post_payload_success():
|
|
from plugins.host_agent import agent as a
|
|
captured = {}
|
|
|
|
class FakeResp:
|
|
status = 200
|
|
def __enter__(self): return self
|
|
def __exit__(self, *a): return False
|
|
def read(self): return b'{"ok":true}'
|
|
|
|
def fake_urlopen(req, timeout=10):
|
|
captured["url"] = req.full_url
|
|
captured["headers"] = dict(req.header_items())
|
|
captured["body"] = req.data
|
|
return FakeResp()
|
|
|
|
with patch.object(a.urllib.request, "urlopen", side_effect=fake_urlopen):
|
|
ok, status = a.post_payload("https://x", "tok", {"hello": "world"})
|
|
assert ok is True
|
|
assert status == 200
|
|
assert captured["url"] == "https://x/plugins/host_agent/ingest"
|
|
assert captured["headers"]["Authorization"] == "Bearer tok"
|
|
|
|
|
|
def test_post_payload_http_error_returns_status():
|
|
from plugins.host_agent import agent as a
|
|
|
|
def raise_401(req, timeout=10):
|
|
raise urllib.error.HTTPError(req.full_url, 401, "unauthorized", {}, None)
|
|
|
|
with patch.object(a.urllib.request, "urlopen", side_effect=raise_401):
|
|
ok, status = a.post_payload("https://x", "tok", {})
|
|
assert ok is False
|
|
assert status == 401
|
|
|
|
|
|
def test_post_payload_network_error_returns_none_status():
|
|
from plugins.host_agent import agent as a
|
|
|
|
def raise_url(req, timeout=10):
|
|
raise urllib.error.URLError("conn refused")
|
|
|
|
with patch.object(a.urllib.request, "urlopen", side_effect=raise_url):
|
|
ok, status = a.post_payload("https://x", "tok", {})
|
|
assert ok is False
|
|
assert status is None
|
|
|
|
|
|
def test_backoff_schedule_stays_inside_the_stale_window():
|
|
from plugins.host_agent.agent import next_backoff
|
|
assert next_backoff(0) == 30
|
|
assert next_backoff(30) == 60
|
|
assert next_backoff(60) == 60 # capped below Steward's 180 s stale window (#5154)
|
|
|
|
|
|
def test_a_backlog_drains_a_few_samples_a_post_oldest_first_and_the_new_one_waits_its_turn():
|
|
from plugins.host_agent.agent import MAX_SAMPLES_PER_POST, split_backlog
|
|
queued = list(range(20)) + ["new"]
|
|
send, held = split_backlog(queued)
|
|
assert send == [0, 1, 2, 3, 4] and len(send) == MAX_SAMPLES_PER_POST
|
|
assert held == list(range(5, 20)) + ["new"] # never ahead of an older sample
|
|
assert split_backlog(["new"]) == (["new"], [])
|
|
assert split_backlog([7, "new"]) == ([7, "new"], [])
|