Files
FabledSteward/tests/plugins/host_agent/test_agent_build_payload.py
T
bvandeusenandClaude Opus 5.5 41fc8073db
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 1m39s
CI / integration (push) Successful in 1m49s
CI / publish (push) Successful in 1m49s
feat: readings kept through a storage stall; an agent's backlog arrives in order (#5995)
On 2026-10-17 the NFS under Postgres stalled for ~15 minutes. Two things
went wrong for the readings that would have explained it:

- The SNMP poll stored before it polled again, so while the database hung
  it stopped polling. It now collects on schedule and hands each round to a
  Holding (steward/core/holding.py): one writer stores what is held, oldest
  first, each reading at the time it was collected. Bounded at two hours of
  polls, with a 30-minute deadline on a store; the device list read falls
  back to the last one read after 10 s.
- "Mount not answering" (10 checks in a row) went pending on Docker02 and
  Docker03 and never fired: the agent sent each post's backlog with its
  newest, healthy sample, and that sample ended every post's run. Agent
  1.22.0 queues the new sample behind the backlog, so Steward receives a
  host's samples in the order they were taken.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-17 09:36:47 -04:00

131 lines
5.2 KiB
Python

import urllib.error
from unittest.mock import patch
from plugins.host_agent import agent as a
def test_build_sample_shape():
fake_mem = {"total_bytes": 100, "used_bytes": 40,
"available_bytes": 60, "swap_used_bytes": 0}
fake_load = {"1m": 0.1, "5m": 0.2, "15m": 0.3}
fake_storage = [{"mount": "/", "total_bytes": 1000, "used_bytes": 400}]
with patch.object(a, "collect_cpu", return_value=(12.5, [12.0, 13.0])), \
patch.object(a, "collect_memory", return_value=fake_mem), \
patch.object(a, "collect_load", return_value=fake_load), \
patch.object(a, "collect_uptime", return_value=1234), \
patch.object(a, "collect_storage", return_value=fake_storage), \
patch.object(a, "collect_temps", return_value=[]), \
patch.object(a, "collect_psi", return_value={}), \
patch.object(a, "collect_net_raw", return_value={}), \
patch.object(a, "collect_diskio_raw", return_value={}):
sample = a.build_sample(["/"], {})
assert sample["cpu_pct"] == 12.5
assert sample["cpu_cores"] == [12.0, 13.0]
assert sample["mem"]["used_bytes"] == 40
assert sample["load"]["1m"] == 0.1
assert sample["uptime_secs"] == 1234
assert sample["storage"][0]["mount"] == "/"
assert "ts" in sample
# ISO-8601 UTC string
assert "T" in sample["ts"]
def test_build_sample_tolerates_collector_failure():
# If a collector raises, the sample should still be built with None for that field.
def _boom():
raise RuntimeError("no /proc")
with patch.object(a, "collect_cpu", side_effect=_boom), \
patch.object(a, "collect_memory", return_value={"total_bytes": 1, "used_bytes": 0,
"available_bytes": 1, "swap_used_bytes": 0}), \
patch.object(a, "collect_load", return_value={"1m": 0.0, "5m": 0.0, "15m": 0.0}), \
patch.object(a, "collect_uptime", return_value=0), \
patch.object(a, "collect_storage", return_value=[]), \
patch.object(a, "collect_temps", return_value=[]), \
patch.object(a, "collect_psi", return_value={}), \
patch.object(a, "collect_net_raw", return_value={}), \
patch.object(a, "collect_diskio_raw", return_value={}):
sample = a.build_sample(["/"], {})
assert sample["cpu_pct"] is None
assert sample["mem"]["total_bytes"] == 1
def test_build_payload_wraps_samples():
metadata = {"kernel": "6.8", "distro": "Ubuntu", "arch": "x86_64"}
payload = a.build_payload(
samples=[{"ts": "2026-04-14T00:00:00+00:00", "cpu_pct": 5.0}],
hostname="myhost",
metadata=metadata,
)
assert payload["agent_version"] == a.AGENT_VERSION
assert payload["hostname"] == "myhost"
assert payload["metadata"] == metadata
assert len(payload["samples"]) == 1
assert payload["samples"][0]["cpu_pct"] == 5.0
def test_post_payload_success():
from plugins.host_agent import agent as a
captured = {}
class FakeResp:
status = 200
def __enter__(self): return self
def __exit__(self, *a): return False
def read(self): return b'{"ok":true}'
def fake_urlopen(req, timeout=10):
captured["url"] = req.full_url
captured["headers"] = dict(req.header_items())
captured["body"] = req.data
return FakeResp()
with patch.object(a.urllib.request, "urlopen", side_effect=fake_urlopen):
ok, status = a.post_payload("https://x", "tok", {"hello": "world"})
assert ok is True
assert status == 200
assert captured["url"] == "https://x/plugins/host_agent/ingest"
assert captured["headers"]["Authorization"] == "Bearer tok"
def test_post_payload_http_error_returns_status():
from plugins.host_agent import agent as a
def raise_401(req, timeout=10):
raise urllib.error.HTTPError(req.full_url, 401, "unauthorized", {}, None)
with patch.object(a.urllib.request, "urlopen", side_effect=raise_401):
ok, status = a.post_payload("https://x", "tok", {})
assert ok is False
assert status == 401
def test_post_payload_network_error_returns_none_status():
from plugins.host_agent import agent as a
def raise_url(req, timeout=10):
raise urllib.error.URLError("conn refused")
with patch.object(a.urllib.request, "urlopen", side_effect=raise_url):
ok, status = a.post_payload("https://x", "tok", {})
assert ok is False
assert status is None
def test_backoff_schedule_stays_inside_the_stale_window():
from plugins.host_agent.agent import next_backoff
assert next_backoff(0) == 30
assert next_backoff(30) == 60
assert next_backoff(60) == 60 # capped below Steward's 180 s stale window (#5154)
def test_a_backlog_drains_a_few_samples_a_post_oldest_first_and_the_new_one_waits_its_turn():
from plugins.host_agent.agent import MAX_SAMPLES_PER_POST, split_backlog
queued = list(range(20)) + ["new"]
send, held = split_backlog(queued)
assert send == [0, 1, 2, 3, 4] and len(send) == MAX_SAMPLES_PER_POST
assert held == list(range(5, 20)) + ["new"] # never ahead of an older sample
assert split_backlog(["new"]) == (["new"], [])
assert split_backlog([7, "new"]) == ([7, "new"], [])