bvandeusen
82c3d2cf36
First slice of milestone 77 (Docker monitoring depth). Surfaces real per-container
stats beyond basic state, all read-only on the existing push model.
- agent (→1.4.0): collect_docker now inspects each container (health, restart
count, exit code, OOM) and reads net + block I/O from the stats payload; pulls
compose project + swarm service/task/node from container labels. Per-container
inspect+stats calls run over a small bounded ThreadPool so the ~1s-per-stats
blocking doesn't stretch the sample on a busy host.
- schema (docker_003): additive columns on docker_containers — health, exit_code,
oom_killed, compose_project, service_name, task_id, node_id, and BigInteger
net/blk byte counters.
- ingest: persists the enrichment + restart_count (.get keeps older agents working).
- ui: Docker page rows now show health badge, uptime ("up 3d 4h"), restart count,
exit code (+OOM) for stopped containers, and compose/service grouping label.
- tests: agent helpers (grouping, inspect fields, net/IO sum) + collect_docker
assembly incl. inspect; integration asserts enrichment round-trips.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC
225 lines
9.3 KiB
Python
225 lines
9.3 KiB
Python
# tests/plugins/host_agent/test_agent_docker.py
|
||
"""Unit tests for the agent's stdlib Docker collector.
|
||
|
||
No socket / no network — the UDS request layer is monkeypatched so we exercise
|
||
the cpu/mem math, chunked-body decoding, sample assembly, and the silent-skip
|
||
contract in isolation.
|
||
"""
|
||
from plugins.host_agent import agent as a
|
||
|
||
|
||
# ── chunked transfer decoding ─────────────────────────────────────────────────
|
||
|
||
def test_dechunk_reassembles_body():
|
||
chunked = b"4\r\nWiki\r\n5\r\npedia\r\n0\r\n\r\n"
|
||
assert a._dechunk(chunked) == b"Wikipedia"
|
||
|
||
|
||
def test_dechunk_ignores_chunk_extensions():
|
||
assert a._dechunk(b"3;name=v\r\nabc\r\n0\r\n\r\n") == b"abc"
|
||
|
||
|
||
# ── cpu / mem math (ported from the old central scraper) ──────────────────────
|
||
|
||
def test_docker_cpu_pct():
|
||
stats = {
|
||
"cpu_stats": {
|
||
"cpu_usage": {"total_usage": 200_000_000},
|
||
"system_cpu_usage": 2_000_000_000,
|
||
"online_cpus": 4,
|
||
},
|
||
"precpu_stats": {
|
||
"cpu_usage": {"total_usage": 100_000_000},
|
||
"system_cpu_usage": 1_000_000_000,
|
||
},
|
||
}
|
||
# cpu_delta=1e8, sys_delta=1e9 → 0.1 * 4 cpus * 100 = 40.0%
|
||
assert a._docker_cpu_pct(stats) == 40.0
|
||
|
||
|
||
def test_docker_cpu_pct_guards_bad_stats():
|
||
assert a._docker_cpu_pct({}) == 0.0
|
||
# No forward progress in system time → 0, never a divide error.
|
||
assert a._docker_cpu_pct({
|
||
"cpu_stats": {"cpu_usage": {"total_usage": 5}, "system_cpu_usage": 10},
|
||
"precpu_stats": {"cpu_usage": {"total_usage": 5}, "system_cpu_usage": 10},
|
||
}) == 0.0
|
||
|
||
|
||
def test_docker_mem_subtracts_cache():
|
||
stats = {"memory_stats": {
|
||
"usage": 200 * 1024 * 1024,
|
||
"limit": 1000 * 1024 * 1024,
|
||
"stats": {"cache": 50 * 1024 * 1024},
|
||
}}
|
||
usage, limit, pct = a._docker_mem(stats)
|
||
assert usage == 150 * 1024 * 1024 # working set = usage − cache
|
||
assert limit == 1000 * 1024 * 1024
|
||
assert pct == 15.0
|
||
|
||
|
||
def test_docker_ports_keeps_only_published():
|
||
ports = [
|
||
{"PublicPort": 8080, "PrivatePort": 80, "Type": "tcp"},
|
||
{"PrivatePort": 5432, "Type": "tcp"}, # unpublished → dropped
|
||
]
|
||
assert a._docker_ports(ports) == [
|
||
{"host_port": 8080, "container_port": 80, "protocol": "tcp"},
|
||
]
|
||
|
||
|
||
# ── enrichment helpers ────────────────────────────────────────────────────────
|
||
|
||
def test_docker_grouping_from_labels():
|
||
g = a._docker_grouping({
|
||
"com.docker.compose.project": "myproj",
|
||
"com.docker.swarm.service.name": "web",
|
||
"com.docker.swarm.task.id": "t123",
|
||
"com.docker.swarm.node.id": "n456",
|
||
})
|
||
assert g == {"compose_project": "myproj", "service_name": "web",
|
||
"task_id": "t123", "node_id": "n456"}
|
||
|
||
|
||
def test_docker_grouping_empty():
|
||
assert a._docker_grouping(None) == {
|
||
"compose_project": None, "service_name": None, "task_id": None, "node_id": None}
|
||
|
||
|
||
def test_docker_inspect_fields():
|
||
insp = {"RestartCount": 3,
|
||
"State": {"Health": {"Status": "unhealthy"}, "ExitCode": 137, "OOMKilled": True}}
|
||
assert a._docker_inspect_fields(insp) == ("unhealthy", 3, 137, True)
|
||
|
||
|
||
def test_docker_inspect_fields_no_healthcheck():
|
||
# Containers without a HEALTHCHECK have no State.Health → health is None.
|
||
health, restarts, exit_code, oom = a._docker_inspect_fields(
|
||
{"RestartCount": 0, "State": {"ExitCode": 0, "OOMKilled": False}})
|
||
assert health is None and restarts == 0 and exit_code == 0 and oom is False
|
||
|
||
|
||
def test_docker_net_io_sums_counters():
|
||
stats = {
|
||
"networks": {"eth0": {"rx_bytes": 1000, "tx_bytes": 2000},
|
||
"eth1": {"rx_bytes": 5, "tx_bytes": 7}},
|
||
"blkio_stats": {"io_service_bytes_recursive": [
|
||
{"op": "Read", "value": 4096}, {"op": "Write", "value": 8192}]},
|
||
}
|
||
assert a._docker_net_io(stats) == (1005, 2007, 4096, 8192)
|
||
|
||
|
||
# ── collect_docker assembly + silent skip ─────────────────────────────────────
|
||
|
||
def test_collect_docker_silent_skip_when_no_socket():
|
||
# Absent socket path must degrade to [] (zero-config on Docker-less hosts).
|
||
assert a.collect_docker("/nonexistent/does-not-exist.sock") == []
|
||
|
||
|
||
def test_collect_docker_assembles_container(monkeypatch):
|
||
container = {
|
||
"Id": "abc123def4567890",
|
||
"Names": ["/web"],
|
||
"Image": "nginx:latest",
|
||
"State": "running",
|
||
"Created": 1_700_000_000,
|
||
"Ports": [{"PublicPort": 8080, "PrivatePort": 80, "Type": "tcp"}],
|
||
"Labels": {"com.docker.compose.project": "myproj",
|
||
"com.docker.swarm.service.name": "web"},
|
||
}
|
||
inspect = {"RestartCount": 2,
|
||
"State": {"Health": {"Status": "healthy"}, "ExitCode": 0, "OOMKilled": False}}
|
||
stats = {
|
||
"cpu_stats": {"cpu_usage": {"total_usage": 200_000_000},
|
||
"system_cpu_usage": 2_000_000_000, "online_cpus": 4},
|
||
"precpu_stats": {"cpu_usage": {"total_usage": 100_000_000},
|
||
"system_cpu_usage": 1_000_000_000},
|
||
"memory_stats": {"usage": 200 * 1024 * 1024, "limit": 1000 * 1024 * 1024,
|
||
"stats": {"cache": 50 * 1024 * 1024}},
|
||
"networks": {"eth0": {"rx_bytes": 1000, "tx_bytes": 2000}},
|
||
"blkio_stats": {"io_service_bytes_recursive": [
|
||
{"op": "Read", "value": 4096}, {"op": "Write", "value": 8192}]},
|
||
}
|
||
|
||
def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
|
||
if path.startswith("/containers/json"):
|
||
return [container]
|
||
if "/stats" in path:
|
||
return stats
|
||
if path.endswith("/json"): # inspect
|
||
return inspect
|
||
raise AssertionError(f"unexpected path {path}")
|
||
|
||
monkeypatch.setattr(a, "_docker_request", fake_request)
|
||
out = a.collect_docker("/var/run/docker.sock")
|
||
assert len(out) == 1
|
||
c = out[0]
|
||
assert c["name"] == "web"
|
||
assert c["container_id"] == "abc123def456" # 12-char short id
|
||
assert c["image"] == "nginx:latest"
|
||
assert c["status"] == "running"
|
||
assert c["cpu_pct"] == 40.0
|
||
assert c["mem_pct"] == 15.0
|
||
assert c["ports"] == [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}]
|
||
assert c["started_at"].startswith("2023-11-14T") # epoch → ISO-8601
|
||
# enrichment
|
||
assert c["health"] == "healthy" and c["restart_count"] == 2
|
||
assert c["exit_code"] == 0 and c["oom_killed"] is False
|
||
assert c["net_rx_bytes"] == 1000 and c["net_tx_bytes"] == 2000
|
||
assert c["blk_read_bytes"] == 4096 and c["blk_write_bytes"] == 8192
|
||
assert c["compose_project"] == "myproj" and c["service_name"] == "web"
|
||
|
||
|
||
def test_collect_docker_skips_stats_for_stopped(monkeypatch):
|
||
container = {"Id": "x" * 16, "Names": ["/db"], "Image": "pg", "State": "exited",
|
||
"Created": None, "Ports": [], "Labels": {}}
|
||
inspect = {"RestartCount": 0,
|
||
"State": {"ExitCode": 137, "OOMKilled": True}} # no Health block
|
||
|
||
def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
|
||
if path.startswith("/containers/json"):
|
||
return [container]
|
||
if path.endswith("/json"): # inspect — fetched for stopped too (exit code)
|
||
return inspect
|
||
raise AssertionError("stats must not be fetched for a stopped container")
|
||
|
||
monkeypatch.setattr(a, "_docker_request", fake_request)
|
||
out = a.collect_docker("/var/run/docker.sock")
|
||
assert out[0]["status"] == "exited"
|
||
assert out[0]["cpu_pct"] is None and out[0]["started_at"] is None
|
||
# inspect still gives us why it died
|
||
assert out[0]["exit_code"] == 137 and out[0]["oom_killed"] is True
|
||
assert out[0]["health"] is None
|
||
|
||
|
||
# ── build_sample wiring ───────────────────────────────────────────────────────
|
||
|
||
def test_build_sample_includes_docker_when_present(monkeypatch):
|
||
monkeypatch.setattr(a, "collect_docker", lambda _s: [{"name": "web", "status": "running"}])
|
||
sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
|
||
assert sample["docker"] == [{"name": "web", "status": "running"}]
|
||
|
||
|
||
def test_build_sample_omits_docker_when_empty(monkeypatch):
|
||
monkeypatch.setattr(a, "collect_docker", lambda _s: [])
|
||
sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
|
||
assert "docker" not in sample
|
||
|
||
|
||
# ── config ────────────────────────────────────────────────────────────────────
|
||
|
||
def test_read_config_defaults_docker_socket(tmp_path):
|
||
cfg_file = tmp_path / "agent.conf"
|
||
cfg_file.write_text("url = https://steward.example\ntoken = abc\n")
|
||
cfg = a.read_config(str(cfg_file))
|
||
assert cfg["docker_socket"] == a.DEFAULT_DOCKER_SOCKET
|
||
|
||
|
||
def test_read_config_honours_explicit_docker_socket(tmp_path):
|
||
cfg_file = tmp_path / "agent.conf"
|
||
cfg_file.write_text(
|
||
"url = https://steward.example\ntoken = abc\n"
|
||
"docker_socket = /run/user/1000/docker.sock\n")
|
||
cfg = a.read_config(str(cfg_file))
|
||
assert cfg["docker_socket"] == "/run/user/1000/docker.sock"
|