bvandeusenandClaude Opus 5.5
56063f27a4
The agent sent Docker's working set (usage less inactive_file), which still counts recently used file cache. A file-heavy container with no limit read 191 GB while its processes held 2.2 GB, and the host's own figure, which already leaves cache out, disagreed with it. Agent 1.23.0 sends mem_used_bytes / mem_used_pct (anon + shmem, so Postgres's shared buffers count; v1: rss + shmem) and mem_cache_bytes apart. The new names keep the two meanings apart in history and in learned normals: docker_013 renames the columns, clears the old values and adds mem_cache_bytes. The Docker page, swarm by-node sums, Busiest Containers, container detail, normals and the MCP tools all read the new figure. File cache is shown only on the container's page and in get_container. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
844 lines
38 KiB
Python
844 lines
38 KiB
Python
# tests/plugins/host_agent/test_agent_docker.py
|
||
"""Unit tests for the agent's stdlib Docker collector.
|
||
|
||
No socket / no network — the UDS request layer is monkeypatched so we exercise
|
||
the cpu/mem math, chunked-body decoding, sample assembly, and what the agent
|
||
says when Docker cannot be asked, in isolation.
|
||
"""
|
||
import socket
|
||
|
||
import pytest
|
||
|
||
from plugins.host_agent import agent as a
|
||
|
||
|
||
# ── chunked transfer decoding ─────────────────────────────────────────────────
|
||
|
||
def test_dechunk_reassembles_body():
|
||
chunked = b"4\r\nWiki\r\n5\r\npedia\r\n0\r\n\r\n"
|
||
assert a._dechunk(chunked) == b"Wikipedia"
|
||
|
||
|
||
def test_dechunk_ignores_chunk_extensions():
|
||
assert a._dechunk(b"3;name=v\r\nabc\r\n0\r\n\r\n") == b"abc"
|
||
|
||
|
||
# ── cpu / mem math (ported from the old central scraper) ──────────────────────
|
||
|
||
def test_docker_cpu_pct():
|
||
stats = {
|
||
"cpu_stats": {
|
||
"cpu_usage": {"total_usage": 200_000_000},
|
||
"system_cpu_usage": 2_000_000_000,
|
||
"online_cpus": 4,
|
||
},
|
||
"precpu_stats": {
|
||
"cpu_usage": {"total_usage": 100_000_000},
|
||
"system_cpu_usage": 1_000_000_000,
|
||
},
|
||
}
|
||
# cpu_delta=1e8, sys_delta=1e9 → 0.1 * 4 cpus * 100 = 40.0%
|
||
assert a._docker_cpu_pct(stats) == 40.0
|
||
|
||
|
||
def test_docker_cpu_pct_guards_bad_stats():
|
||
assert a._docker_cpu_pct({}) == 0.0
|
||
# No forward progress in system time → 0, never a divide error.
|
||
assert a._docker_cpu_pct({
|
||
"cpu_stats": {"cpu_usage": {"total_usage": 5}, "system_cpu_usage": 10},
|
||
"precpu_stats": {"cpu_usage": {"total_usage": 5}, "system_cpu_usage": 10},
|
||
}) == 0.0
|
||
|
||
|
||
def test_docker_mem_is_process_memory_not_file_cache():
|
||
# cgroup v2, as curator_app read on 2026-10-17 (#6030): 315 GB charged to
|
||
# it, 2.2 GB its own; Docker's working set (usage − inactive_file) said 191 GB.
|
||
stats = {"memory_stats": {
|
||
"usage": 314_896_379_904,
|
||
"limit": 405_493_768_192,
|
||
"stats": {"anon": 2_241_888_256, "shmem": 106_496, "file": 312_003_436_544,
|
||
"active_file": 188_779_880_448, "inactive_file": 123_223_449_600},
|
||
}}
|
||
used, limit, pct, cache = a._docker_mem(stats)
|
||
assert used == 2_241_888_256 + 106_496
|
||
assert limit == 405_493_768_192
|
||
assert pct == 0.55
|
||
assert cache == 312_003_436_544 - 106_496 # shmem is in file; counted as used, not cache
|
||
|
||
|
||
def test_docker_mem_counts_shared_memory_as_used():
|
||
# Postgres keeps its shared buffers in shmem: not anon, but not reclaimable.
|
||
mib = 1024 * 1024
|
||
stats = {"memory_stats": {"usage": 900 * mib, "limit": 1000 * mib,
|
||
"stats": {"anon": 100 * mib, "shmem": 400 * mib, "file": 700 * mib}}}
|
||
assert a._docker_mem(stats) == (500 * mib, 1000 * mib, 50.0, 300 * mib)
|
||
|
||
|
||
def test_docker_mem_reads_cgroup_v1():
|
||
# v1's rss leaves shmem out and its cache has it in.
|
||
mib = 1024 * 1024
|
||
stats = {"memory_stats": {"usage": 200 * mib, "limit": 1000 * mib,
|
||
"stats": {"total_rss": 120 * mib, "total_shmem": 30 * mib,
|
||
"total_cache": 80 * mib, "rss": 1, "cache": 1}}}
|
||
assert a._docker_mem(stats) == (150 * mib, 1000 * mib, 15.0, 50 * mib)
|
||
|
||
|
||
def test_docker_mem_without_stats_is_zero():
|
||
assert a._docker_mem({}) == (0, 0, 0.0, 0)
|
||
|
||
|
||
def test_docker_ports_keeps_only_published():
|
||
ports = [
|
||
{"PublicPort": 8080, "PrivatePort": 80, "Type": "tcp"},
|
||
{"PrivatePort": 5432, "Type": "tcp"}, # unpublished → dropped
|
||
]
|
||
assert a._docker_ports(ports) == [
|
||
{"host_port": 8080, "container_port": 80, "protocol": "tcp"},
|
||
]
|
||
|
||
|
||
# ── enrichment helpers ────────────────────────────────────────────────────────
|
||
|
||
def test_docker_grouping_from_labels():
|
||
g = a._docker_grouping({
|
||
"com.docker.compose.project": "myproj",
|
||
"com.docker.swarm.service.name": "web",
|
||
"com.docker.swarm.task.id": "t123",
|
||
"com.docker.swarm.node.id": "n456",
|
||
})
|
||
assert g == {"compose_project": "myproj", "service_name": "web",
|
||
"task_id": "t123", "node_id": "n456"}
|
||
|
||
|
||
def test_docker_grouping_empty():
|
||
assert a._docker_grouping(None) == {
|
||
"compose_project": None, "service_name": None, "task_id": None, "node_id": None}
|
||
|
||
|
||
def test_docker_inspect_fields():
|
||
insp = {"RestartCount": 3,
|
||
"State": {"Health": {"Status": "unhealthy"}, "ExitCode": 137, "OOMKilled": True}}
|
||
assert a._docker_inspect_fields(insp) == ("unhealthy", 3, 137, True)
|
||
|
||
|
||
def test_docker_inspect_fields_no_healthcheck():
|
||
# Containers without a HEALTHCHECK have no State.Health → health is None.
|
||
health, restarts, exit_code, oom = a._docker_inspect_fields(
|
||
{"RestartCount": 0, "State": {"ExitCode": 0, "OOMKilled": False}})
|
||
assert health is None and restarts == 0 and exit_code == 0 and oom is False
|
||
|
||
|
||
def test_docker_net_io_sums_counters():
|
||
stats = {
|
||
"networks": {"eth0": {"rx_bytes": 1000, "tx_bytes": 2000},
|
||
"eth1": {"rx_bytes": 5, "tx_bytes": 7}},
|
||
"blkio_stats": {"io_service_bytes_recursive": [
|
||
{"op": "Read", "value": 4096}, {"op": "Write", "value": 8192}]},
|
||
}
|
||
assert a._docker_net_io(stats) == (1005, 2007, 4096, 8192)
|
||
|
||
|
||
# ── collect_docker assembly + silent skip ─────────────────────────────────────
|
||
|
||
def test_collect_docker_says_unavailable_when_no_socket():
|
||
# A Docker-less host needs no configuration: it reports there is no Docker.
|
||
assert a.collect_docker("/nonexistent/does-not-exist.sock") == (
|
||
"unavailable", "no Docker socket at /nonexistent/does-not-exist.sock", [])
|
||
|
||
|
||
@pytest.mark.parametrize("exc, status, detail", [
|
||
(FileNotFoundError(2, "x"), "unavailable", "no Docker socket at /s"),
|
||
(ConnectionRefusedError(111, "x"), "unavailable", "Docker is not accepting connections on /s"),
|
||
(PermissionError(13, "x"), "error", "permission denied on /s"),
|
||
(socket.timeout("timed out"), "error", "Docker did not answer within 5 s"),
|
||
(OSError("docker API /containers/json → 500"), "error", "docker API /containers/json → 500"),
|
||
(ValueError("Expecting value"), "error", "Expecting value"),
|
||
])
|
||
def test_collect_docker_says_why_it_could_not_list(monkeypatch, exc, status, detail):
|
||
def fail(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
|
||
raise exc
|
||
|
||
monkeypatch.setattr(a, "_docker_request", fail)
|
||
assert a.collect_docker("/s") == (status, detail, [])
|
||
|
||
|
||
def test_collect_docker_with_no_containers_is_ok_and_empty(monkeypatch):
|
||
monkeypatch.setattr(a, "_docker_request", lambda s, p, timeout=a.DOCKER_API_TIMEOUT: [])
|
||
assert a.collect_docker("/s") == ("ok", None, [])
|
||
|
||
|
||
def test_collect_docker_assembles_container(monkeypatch):
|
||
container = {
|
||
"Id": "abc123def4567890",
|
||
"Names": ["/web"],
|
||
"Image": "nginx:latest",
|
||
"State": "running",
|
||
"Created": 1_600_000_000,
|
||
"Ports": [{"PublicPort": 8080, "PrivatePort": 80, "Type": "tcp"}],
|
||
"Labels": {"com.docker.compose.project": "myproj",
|
||
"com.docker.swarm.service.name": "web"},
|
||
}
|
||
inspect = {"RestartCount": 2,
|
||
"State": {"Health": {"Status": "healthy"}, "ExitCode": 0, "OOMKilled": False,
|
||
"StartedAt": "2023-11-14T22:13:20.123456789Z"}}
|
||
stats = {
|
||
"cpu_stats": {"cpu_usage": {"total_usage": 200_000_000},
|
||
"system_cpu_usage": 2_000_000_000, "online_cpus": 4},
|
||
"precpu_stats": {"cpu_usage": {"total_usage": 100_000_000},
|
||
"system_cpu_usage": 1_000_000_000},
|
||
"memory_stats": {"usage": 200 * 1024 * 1024, "limit": 1000 * 1024 * 1024,
|
||
"stats": {"anon": 140 * 1024 * 1024, "shmem": 10 * 1024 * 1024,
|
||
"file": 60 * 1024 * 1024}},
|
||
"networks": {"eth0": {"rx_bytes": 1000, "tx_bytes": 2000}},
|
||
"blkio_stats": {"io_service_bytes_recursive": [
|
||
{"op": "Read", "value": 4096}, {"op": "Write", "value": 8192}]},
|
||
}
|
||
|
||
def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
|
||
if path.startswith("/containers/json"):
|
||
return [container]
|
||
if "/stats" in path:
|
||
return stats
|
||
if path.endswith("/json"): # inspect
|
||
return inspect
|
||
raise AssertionError(f"unexpected path {path}")
|
||
|
||
monkeypatch.setattr(a, "_docker_request", fake_request)
|
||
status, detail, out = a.collect_docker("/var/run/docker.sock")
|
||
assert (status, detail) == ("ok", None)
|
||
assert len(out) == 1
|
||
c = out[0]
|
||
assert c["name"] == "web"
|
||
assert c["container_id"] == "abc123def456" # 12-char short id
|
||
assert c["image"] == "nginx:latest"
|
||
assert c["status"] == "running"
|
||
assert c["cpu_pct"] == 40.0
|
||
assert c["mem_used_bytes"] == 150 * 1024 * 1024 and c["mem_used_pct"] == 15.0
|
||
assert c["mem_cache_bytes"] == 50 * 1024 * 1024
|
||
assert c["ports"] == [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}]
|
||
# When it last started (inspect), not when it was created (2020).
|
||
assert c["started_at"] == "2023-11-14T22:13:20.123456+00:00"
|
||
# enrichment
|
||
assert c["health"] == "healthy" and c["restart_count"] == 2
|
||
assert c["exit_code"] == 0 and c["oom_killed"] is False
|
||
assert c["net_rx_bytes"] == 1000 and c["net_tx_bytes"] == 2000
|
||
assert c["blk_read_bytes"] == 4096 and c["blk_write_bytes"] == 8192
|
||
assert c["compose_project"] == "myproj" and c["service_name"] == "web"
|
||
|
||
|
||
def test_collect_docker_skips_stats_for_stopped(monkeypatch):
|
||
container = {"Id": "x" * 16, "Names": ["/db"], "Image": "pg", "State": "exited",
|
||
"Created": None, "Ports": [], "Labels": {}}
|
||
inspect = {"RestartCount": 0,
|
||
"State": {"ExitCode": 137, "OOMKilled": True}} # no Health block
|
||
|
||
def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
|
||
if path.startswith("/containers/json"):
|
||
return [container]
|
||
if path.endswith("/json"): # inspect — fetched for stopped too (exit code)
|
||
return inspect
|
||
raise AssertionError("stats must not be fetched for a stopped container")
|
||
|
||
monkeypatch.setattr(a, "_docker_request", fake_request)
|
||
_, _, out = a.collect_docker("/var/run/docker.sock")
|
||
assert out[0]["status"] == "exited"
|
||
assert out[0]["cpu_pct"] is None and out[0]["started_at"] is None
|
||
# inspect still gives us why it died
|
||
assert out[0]["exit_code"] == 137 and out[0]["oom_killed"] is True
|
||
assert out[0]["health"] is None
|
||
|
||
|
||
# ── swarm (manager-only) ──────────────────────────────────────────────────────
|
||
|
||
def test_swarm_is_manager():
|
||
assert a._swarm_is_manager({"Swarm": {"ControlAvailable": True}}) is True
|
||
# Worker: in a swarm but no control plane.
|
||
assert a._swarm_is_manager({"Swarm": {"ControlAvailable": False}}) is False
|
||
# Non-swarm daemon: no Swarm block at all.
|
||
assert a._swarm_is_manager({}) is False
|
||
|
||
|
||
def test_swarm_services_rolls_up_replicas():
|
||
services = [
|
||
{"ID": "svc1", "Spec": {
|
||
"Name": "web", "Mode": {"Replicated": {"Replicas": 3}},
|
||
"TaskTemplate": {"ContainerSpec": {"Image": "nginx:1@sha256:deadbeef"}}}},
|
||
{"ID": "svc2", "Spec": {
|
||
"Name": "agent", "Mode": {"Global": {}},
|
||
"TaskTemplate": {"ContainerSpec": {"Image": "agent:latest"}}}},
|
||
]
|
||
tasks = [
|
||
# web: 2 of 3 desired replicas actually running, both on node n1
|
||
{"ServiceID": "svc1", "NodeID": "n1", "DesiredState": "running",
|
||
"Status": {"State": "running"}},
|
||
{"ServiceID": "svc1", "NodeID": "n1", "DesiredState": "running",
|
||
"Status": {"State": "running"}},
|
||
{"ServiceID": "svc1", "NodeID": "n2", "DesiredState": "running",
|
||
"Status": {"State": "failed"}},
|
||
# agent: global, 2 tasks desired+running across 2 nodes
|
||
{"ServiceID": "svc2", "NodeID": "n1", "DesiredState": "running",
|
||
"Status": {"State": "running"}},
|
||
{"ServiceID": "svc2", "NodeID": "n2", "DesiredState": "running",
|
||
"Status": {"State": "running"}},
|
||
]
|
||
out = {s["service_name"]: s for s in a._swarm_services(services, tasks)}
|
||
assert [t["state"] for t in out["web"].pop("tasks")] == ["running", "running", "failed"]
|
||
assert len(out["agent"].pop("tasks")) == 2
|
||
assert out["web"] == {"service_name": "web", "mode": "replicated",
|
||
"desired": 3, "running": 2, "image": "nginx:1",
|
||
"placement": [{"node_id": "n1", "running": 2}]}
|
||
# Global mode has no fixed replica count → desired derived from desired tasks.
|
||
assert out["agent"] == {"service_name": "agent", "mode": "global",
|
||
"desired": 2, "running": 2, "image": "agent:latest",
|
||
"placement": [{"node_id": "n1", "running": 1},
|
||
{"node_id": "n2", "running": 1}]}
|
||
|
||
|
||
def test_swarm_task_keeps_why_it_stopped():
|
||
t = a._swarm_task({
|
||
"ID": "t1", "Slot": 1, "NodeID": "n2", "DesiredState": "Shutdown",
|
||
"Status": {"State": "failed", "Timestamp": "2026-10-06T13:19:51.5Z",
|
||
"Message": "started", "Err": "task: non-zero exit (1)" + "x" * 400,
|
||
"ContainerStatus": {"ContainerID": "c1", "ExitCode": 1}}})
|
||
assert t["task_id"] == "t1" and t["slot"] == 1 and t["node_id"] == "n2"
|
||
assert t["desired_state"] == "shutdown" and t["state"] == "failed" and t["exit_code"] == 1
|
||
assert t["at"] == "2026-10-06T13:19:51.5Z" and t["message"] == "started"
|
||
assert t["error"].startswith("task: non-zero exit (1)") and len(t["error"]) == 300
|
||
# A task that never got a container has no exit code.
|
||
bare = a._swarm_task({"ID": "t2", "DesiredState": "running",
|
||
"Status": {"State": "rejected", "Err": "No such image: x"}})
|
||
assert bare["exit_code"] is None and bare["slot"] is None and bare["error"] == "No such image: x"
|
||
|
||
|
||
def test_swarm_services_keeps_the_newest_tasks():
|
||
services = [{"ID": "s", "Spec": {"Name": "db", "Mode": {"Replicated": {"Replicas": 1}}}}]
|
||
tasks = [{"ID": f"t{i}", "ServiceID": "s", "Slot": 1, "DesiredState": "shutdown",
|
||
"Status": {"State": "failed", "Timestamp": f"2026-10-06T13:{i:02d}:00Z"}}
|
||
for i in range(a.SWARM_TASKS_PER_SERVICE + 5)]
|
||
(svc,) = a._swarm_services(services, tasks)
|
||
ids = [t["task_id"] for t in svc["tasks"]]
|
||
newest = a.SWARM_TASKS_PER_SERVICE + 4
|
||
assert len(ids) == a.SWARM_TASKS_PER_SERVICE and ids[0] == f"t{newest}"
|
||
|
||
|
||
def test_swarm_nodes_normalises_fields():
|
||
nodes = [
|
||
{"ID": "n1", "Description": {"Hostname": "mgr"},
|
||
"Spec": {"Role": "manager", "Availability": "active"},
|
||
"Status": {"State": "ready"}, "ManagerStatus": {"Leader": True, "Reachability": "Reachable"}},
|
||
{"ID": "n2", "Description": {"Hostname": "wrk"},
|
||
"Spec": {"Role": "worker", "Availability": "drain"},
|
||
"Status": {"State": "down"}},
|
||
]
|
||
out = {n["node_id"]: n for n in a._swarm_nodes(nodes)}
|
||
assert out["n1"] == {"node_id": "n1", "hostname": "mgr", "role": "manager",
|
||
"availability": "active", "status": "ready", "leader": True,
|
||
"reachability": "reachable"}
|
||
assert out["n2"] == {"node_id": "n2", "hostname": "wrk", "role": "worker",
|
||
"availability": "drain", "status": "down", "leader": False,
|
||
"reachability": ""}
|
||
|
||
|
||
def test_collect_swarm_skips_on_worker(monkeypatch):
|
||
# A worker reports ControlAvailable=False → None, and never hits /services.
|
||
def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
|
||
if path == "/info":
|
||
return {"Swarm": {"ControlAvailable": False}}
|
||
raise AssertionError("manager-only endpoint queried on a worker")
|
||
|
||
monkeypatch.setattr(a, "_docker_request", fake_request)
|
||
assert a.collect_swarm("/var/run/docker.sock") == (False, None)
|
||
|
||
|
||
def test_collect_swarm_assembles_on_manager(monkeypatch):
|
||
services = [{"ID": "svc1", "Spec": {
|
||
"Name": "web", "Mode": {"Replicated": {"Replicas": 1}},
|
||
"TaskTemplate": {"ContainerSpec": {"Image": "nginx"}}}}]
|
||
tasks = [{"ServiceID": "svc1", "DesiredState": "running",
|
||
"Status": {"State": "running"}}]
|
||
nodes = [{"ID": "n1", "Description": {"Hostname": "mgr"},
|
||
"Spec": {"Role": "manager", "Availability": "active"},
|
||
"Status": {"State": "ready"}, "ManagerStatus": {"Leader": True}}]
|
||
|
||
def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
|
||
return {
|
||
"/info": {"Swarm": {"ControlAvailable": True}},
|
||
"/services": services, "/tasks": tasks, "/nodes": nodes,
|
||
}[path]
|
||
|
||
monkeypatch.setattr(a, "_docker_request", fake_request)
|
||
is_manager, out = a.collect_swarm("/var/run/docker.sock")
|
||
assert is_manager is True
|
||
assert out["services"][0]["service_name"] == "web"
|
||
assert out["services"][0]["running"] == 1 and out["services"][0]["desired"] == 1
|
||
assert out["nodes"][0]["hostname"] == "mgr" and out["nodes"][0]["leader"] is True
|
||
|
||
|
||
def test_collect_swarm_cannot_say_when_info_fails():
|
||
# Not knowing is not "no longer a manager": Steward must keep the rows.
|
||
assert a.collect_swarm("/nonexistent/does-not-exist.sock") == (None, None)
|
||
|
||
|
||
def test_collect_swarm_manager_whose_listing_fails_sends_nothing(monkeypatch):
|
||
def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
|
||
if path == "/info":
|
||
return {"Swarm": {"ControlAvailable": True}}
|
||
raise OSError("docker API /services → 503")
|
||
|
||
monkeypatch.setattr(a, "_docker_request", fake_request)
|
||
assert a.collect_swarm("/var/run/docker.sock") == (True, None)
|
||
|
||
|
||
# ── build_sample wiring ───────────────────────────────────────────────────────
|
||
|
||
def test_build_sample_includes_docker_when_present(monkeypatch):
|
||
monkeypatch.setattr(a, "collect_docker",
|
||
lambda _s: ("ok", None, [{"name": "web", "status": "running"}]))
|
||
monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
|
||
monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
|
||
sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
|
||
assert sample["docker"] == [{"name": "web", "status": "running"}]
|
||
assert sample["docker_status"] == "ok" and "docker_detail" not in sample
|
||
|
||
|
||
def test_build_sample_sends_an_empty_list_when_docker_has_no_containers(monkeypatch):
|
||
# "No containers" must reach Steward, so it can clear the host's list (#5152).
|
||
monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
|
||
monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
|
||
monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
|
||
sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
|
||
assert sample["docker"] == [] and sample["docker_status"] == "ok"
|
||
|
||
|
||
def test_build_sample_says_why_docker_could_not_be_asked(monkeypatch):
|
||
monkeypatch.setattr(a, "collect_docker",
|
||
lambda _s: ("error", "permission denied on /var/run/docker.sock", []))
|
||
monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
|
||
|
||
def no_df(_s):
|
||
raise AssertionError("/system/df must not be asked when Docker did not answer")
|
||
|
||
monkeypatch.setattr(a, "collect_disk_usage", no_df)
|
||
sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
|
||
assert "docker" not in sample # not "no containers": unknown
|
||
assert sample["docker_status"] == "error"
|
||
assert sample["docker_detail"] == "permission denied on /var/run/docker.sock"
|
||
|
||
|
||
def test_build_sample_includes_swarm_on_manager(monkeypatch):
|
||
monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
|
||
monkeypatch.setattr(a, "collect_swarm",
|
||
lambda _s: (True, {"services": [], "nodes": [{"node_id": "n1"}]}))
|
||
monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
|
||
sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
|
||
assert sample["swarm"]["nodes"] == [{"node_id": "n1"}]
|
||
assert "swarm_manager" not in sample
|
||
|
||
|
||
@pytest.mark.parametrize("swarm, flag", [
|
||
((False, None), False), # /info says not a manager: say so
|
||
((None, None), None), # /info failed: say nothing
|
||
((True, None), None), # a manager whose listing failed: say nothing
|
||
])
|
||
def test_build_sample_says_when_a_node_is_not_a_manager(monkeypatch, swarm, flag):
|
||
monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
|
||
monkeypatch.setattr(a, "collect_swarm", lambda _s: swarm)
|
||
monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
|
||
sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
|
||
assert sample.get("swarm_manager") is flag and "swarm" not in sample
|
||
|
||
|
||
def test_build_sample_omits_swarm_off_manager(monkeypatch):
|
||
monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
|
||
monkeypatch.setattr(a, "collect_swarm", lambda _s: (False, None))
|
||
monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
|
||
sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
|
||
assert "swarm" not in sample
|
||
|
||
|
||
# ── disk usage (/system/df) ─────────────────────────────────────────────────
|
||
|
||
def test_disk_images_normalises_and_caps():
|
||
images = [
|
||
{"Id": "sha256:aaaaaaaaaaaa1111", "RepoTags": ["nginx:latest"],
|
||
"Size": 200, "SharedSize": 50, "Containers": 2},
|
||
{"Id": "sha256:bbbbbbbbbbbb2222", "RepoTags": ["<none>:<none>"],
|
||
"Size": 500, "SharedSize": -1, "Containers": 0},
|
||
]
|
||
out = a._disk_images(images)
|
||
# Sorted largest-first; dangling tag normalised; short id; SharedSize<0 → 0.
|
||
assert out[0]["repo_tag"] == "<none>" and out[0]["size"] == 500
|
||
assert out[0]["shared_size"] == 0 and out[0]["containers"] == 0
|
||
assert out[1]["repo_tag"] == "nginx:latest" and out[1]["image_id"] == "aaaaaaaaaaaa"
|
||
|
||
|
||
def test_collect_disk_usage_computes_reclaimable(monkeypatch):
|
||
df = {
|
||
"LayersSize": 1000,
|
||
"Images": [
|
||
{"Id": "sha256:a", "RepoTags": ["app:1"], "Size": 300, "Containers": 1},
|
||
{"Id": "sha256:b", "RepoTags": None, "Size": 700, "Containers": 0},
|
||
],
|
||
"Containers": [{"Id": "c1", "SizeRw": 25}, {"Id": "c2", "SizeRw": 75}],
|
||
"Volumes": [{"Name": "v1", "UsageData": {"Size": 40}},
|
||
{"Name": "v2", "UsageData": {"Size": -1}}],
|
||
"BuildCache": [{"Size": 10}, {"Size": 5}],
|
||
}
|
||
monkeypatch.setattr(a, "_docker_request",
|
||
lambda s, p, timeout=a.DOCKER_API_TIMEOUT: df)
|
||
out = a.collect_disk_usage("/var/run/docker.sock")
|
||
assert out["layers_size"] == 1000
|
||
assert out["images_total"] == 2 and out["images_active"] == 1
|
||
assert out["images_size"] == 1000
|
||
assert out["images_reclaimable"] == 700 # only the unreferenced image
|
||
assert out["containers_count"] == 2 and out["containers_size"] == 100
|
||
assert out["volumes_count"] == 2 and out["volumes_size"] == 40 # negative skipped
|
||
assert out["build_cache_size"] == 15
|
||
assert len(out["images"]) == 2
|
||
|
||
|
||
def test_collect_disk_usage_silent_skip_when_no_socket():
|
||
assert a.collect_disk_usage("/nonexistent/does-not-exist.sock") is None
|
||
|
||
|
||
def test_build_sample_includes_disk_when_docker_present(monkeypatch):
|
||
monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, [{"name": "web", "status": "running"}]))
|
||
monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
|
||
monkeypatch.setattr(a, "collect_disk_usage", lambda _s: {"images_reclaimable": 700})
|
||
sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
|
||
assert sample["docker_disk"] == {"images_reclaimable": 700}
|
||
|
||
|
||
def test_build_sample_omits_disk_when_no_docker(monkeypatch):
|
||
# No Docker → the /system/df call is skipped entirely (cost guard). The
|
||
# sentinel would land under docker_disk if the gate ever called it.
|
||
monkeypatch.setattr(a, "collect_docker",
|
||
lambda _s: ("unavailable", "no Docker socket at /s", []))
|
||
monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
|
||
monkeypatch.setattr(a, "collect_disk_usage", lambda _s: {"sentinel": True})
|
||
sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
|
||
assert "docker_disk" not in sample
|
||
|
||
|
||
# ── config ────────────────────────────────────────────────────────────────────
|
||
|
||
def test_read_config_defaults_docker_socket(tmp_path):
|
||
cfg_file = tmp_path / "agent.conf"
|
||
cfg_file.write_text("url = https://steward.example\ntoken = abc\n")
|
||
cfg = a.read_config(str(cfg_file))
|
||
assert cfg["docker_socket"] == a.DEFAULT_DOCKER_SOCKET
|
||
|
||
|
||
def test_read_config_honours_explicit_docker_socket(tmp_path):
|
||
cfg_file = tmp_path / "agent.conf"
|
||
cfg_file.write_text(
|
||
"url = https://steward.example\ntoken = abc\n"
|
||
"docker_socket = /run/user/1000/docker.sock\n")
|
||
cfg = a.read_config(str(cfg_file))
|
||
assert cfg["docker_socket"] == "/run/user/1000/docker.sock"
|
||
|
||
|
||
# ── container logs (m79) ──────────────────────────────────────────────────────
|
||
|
||
def _frame(stream: int, payload: bytes) -> bytes:
|
||
"""Build one Docker multiplexed-log frame (8-byte header + payload)."""
|
||
return bytes([stream, 0, 0, 0]) + len(payload).to_bytes(4, "big") + payload
|
||
|
||
|
||
def test_demux_docker_logs_framed():
|
||
raw = _frame(1, b"out line\n") + _frame(2, b"err line\n")
|
||
assert a._demux_docker_logs(raw) == [
|
||
("stdout", b"out line\n"), ("stderr", b"err line\n")]
|
||
|
||
|
||
def test_demux_docker_logs_tty_fallback():
|
||
# A TTY container's stream isn't framed — the leading byte ('2') is not a
|
||
# valid stream id, so the whole blob is treated as one stdout payload.
|
||
raw = b"2023-11-14T12:00:00.000000000Z hi\n"
|
||
assert a._demux_docker_logs(raw) == [("stdout", raw)]
|
||
|
||
|
||
def test_demux_docker_logs_empty():
|
||
assert a._demux_docker_logs(b"") == []
|
||
|
||
|
||
def test_parse_log_ts_handles_nanoseconds_and_z():
|
||
dt = a._parse_log_ts("2023-11-14T12:00:00.123456789Z")
|
||
assert dt is not None
|
||
assert dt.year == 2023 and dt.microsecond == 123456 # ns trimmed to µs
|
||
assert dt.utcoffset().total_seconds() == 0
|
||
assert a._parse_log_ts("not-a-time") is None
|
||
assert a._parse_log_ts("") is None
|
||
|
||
|
||
def test_parse_container_logs_splits_streams_and_ts():
|
||
raw = (_frame(1, b"2023-11-14T12:00:00.000000001Z hello world\n")
|
||
+ _frame(2, b"2023-11-14T12:00:01.000000000Z oops\n"))
|
||
parsed = a._parse_container_logs(raw)
|
||
assert len(parsed) == 2
|
||
dt0, s0, l0 = parsed[0]
|
||
dt1, s1, l1 = parsed[1]
|
||
assert (s0, l0) == ("stdout", "hello world")
|
||
assert (s1, l1) == ("stderr", "oops")
|
||
assert dt1 > dt0
|
||
|
||
|
||
def test_parse_container_logs_keeps_unparseable_line():
|
||
# No timestamp prefix (e.g. a partial write) → dt is None, full line kept.
|
||
parsed = a._parse_container_logs(_frame(1, b"no-timestamp-here\n"))
|
||
assert parsed == [(None, "stdout", "no-timestamp-here")]
|
||
|
||
|
||
def test_collect_docker_logs_since_cursor_and_dedup(monkeypatch):
|
||
containers = [{"name": "web", "status": "running"}]
|
||
calls = []
|
||
t1 = "2023-11-14T12:00:00.000000000Z"
|
||
t2 = "2023-11-14T12:00:01.000000000Z"
|
||
t3 = "2023-11-14T12:00:02.000000000Z"
|
||
|
||
def fake_raw(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
|
||
calls.append(path)
|
||
if "since=" not in path: # first interval seeds from a tail alone
|
||
return _frame(1, f"{t1} a\n".encode()) + _frame(1, f"{t2} b\n".encode())
|
||
# second interval: daemon re-returns the boundary line (t2) + a new one
|
||
return _frame(1, f"{t2} b\n".encode()) + _frame(1, f"{t3} c\n".encode())
|
||
|
||
monkeypatch.setattr(a, "_docker_request_raw", fake_raw)
|
||
state: dict = {}
|
||
first = a.collect_docker_logs("/sock", containers, state)
|
||
assert [r["line"] for r in first] == ["a", "b"]
|
||
assert "tail=" in calls[0]
|
||
|
||
second = a.collect_docker_logs("/sock", containers, state)
|
||
# boundary line b (t2) deduped against the cursor; only c (t3) is new
|
||
assert [r["line"] for r in second] == ["c"]
|
||
assert "since=" in calls[1]
|
||
assert f"tail={a.DOCKER_LOG_MAX_LINES}" in calls[1] # bounded even since the cursor (#5764)
|
||
|
||
|
||
def test_collect_docker_logs_excludes_and_skips_non_running(monkeypatch):
|
||
containers = [
|
||
{"name": "web", "status": "running"},
|
||
{"name": "noisy", "status": "running"},
|
||
{"name": "db", "status": "exited"},
|
||
]
|
||
seen = []
|
||
|
||
def fake_raw(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
|
||
seen.append(path)
|
||
return _frame(1, b"2023-11-14T12:00:00.000000000Z x\n")
|
||
|
||
monkeypatch.setattr(a, "_docker_request_raw", fake_raw)
|
||
out = a.collect_docker_logs("/sock", containers, {}, exclude=["noisy"])
|
||
# only web is fetched: noisy is excluded, db isn't running
|
||
assert seen and all("/containers/web/" in p for p in seen)
|
||
assert {r["container"] for r in out} == {"web"}
|
||
|
||
|
||
def test_collect_docker_logs_byte_cap_truncates(monkeypatch):
|
||
containers = [{"name": "web", "status": "running"}]
|
||
blob = b"".join(
|
||
_frame(1, f"2023-11-14T12:00:{i:02d}.000000000Z {'x' * 20}\n".encode())
|
||
for i in range(10))
|
||
monkeypatch.setattr(a, "_docker_request_raw",
|
||
lambda s, p, timeout=a.DOCKER_API_TIMEOUT: blob)
|
||
out = a.collect_docker_logs("/sock", containers, {}, max_bytes=50)
|
||
# Once the cap is crossed a single marker line is appended and we stop.
|
||
assert out[-1]["container"] == "_steward"
|
||
assert "truncated" in out[-1]["line"]
|
||
real = [r for r in out if r["container"] == "web"]
|
||
assert 0 < len(real) < 10
|
||
|
||
|
||
def test_collect_docker_logs_forgets_gone_container_cursors(monkeypatch):
|
||
monkeypatch.setattr(
|
||
a, "_docker_request_raw",
|
||
lambda s, p, timeout=a.DOCKER_API_TIMEOUT:
|
||
_frame(1, b"2023-11-14T12:00:00.000000000Z x\n"))
|
||
state: dict = {}
|
||
a.collect_docker_logs("/sock", [{"name": "web", "status": "running"}], state)
|
||
assert "web" in state["docker_log_cursors"]
|
||
# web is gone next interval → its cursor is pruned so state can't grow forever
|
||
a.collect_docker_logs("/sock", [{"name": "db", "status": "running"}], state)
|
||
assert "web" not in state["docker_log_cursors"]
|
||
assert "db" in state["docker_log_cursors"]
|
||
|
||
|
||
def _final_raw(seen):
|
||
def fake_raw(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
|
||
seen.append(path)
|
||
return (_frame(1, b"2026-10-06T12:00:00.000000000Z booting\n")
|
||
+ _frame(2, b"2026-10-06T12:00:01.000000000Z fatal: no config\n"))
|
||
return fake_raw
|
||
|
||
|
||
def test_collect_docker_logs_reads_a_stopped_container_once(monkeypatch):
|
||
# #5049: a container that was being read and has stopped is read one last
|
||
# time since its cursor, then not again until it stops again.
|
||
seen: list = []
|
||
monkeypatch.setattr(a, "_docker_request_raw", _final_raw(seen))
|
||
state = {"docker_log_cursors": {"web": 1791288000.5}} # 2026-10-06T12:00:00.5Z
|
||
down = [{"name": "web", "status": "exited", "exit_code": 1,
|
||
"finished_at": "2026-10-06T12:00:02+00:00"}]
|
||
out = a.collect_docker_logs("/sock", down, state)
|
||
assert [r["line"] for r in out] == ["fatal: no config"] # since the cursor
|
||
assert "since=" in seen[0]
|
||
assert a.collect_docker_logs("/sock", down, state) == [] # same stop: not again
|
||
assert len(seen) == 1
|
||
again = [{**down[0], "finished_at": "2026-10-06T12:05:00+00:00"}]
|
||
a.collect_docker_logs("/sock", again, state) # a new stop is read
|
||
assert len(seen) == 2
|
||
|
||
|
||
@pytest.mark.parametrize("exit_code,stopped_ago,read", [
|
||
(1, 30, True), # crashed between two samples: its last lines
|
||
(0, 30, False), # finished cleanly: nothing to explain
|
||
(None, 30, False), # inspect failed: no claim
|
||
(1, 3600, False), # an old crash already on the host when the agent came up
|
||
])
|
||
def test_collect_docker_logs_tails_a_container_first_seen_crashed(monkeypatch, exit_code,
|
||
stopped_ago, read):
|
||
seen: list = []
|
||
monkeypatch.setattr(a, "_docker_request_raw", _final_raw(seen))
|
||
stopped = 1791288000.0
|
||
down = [{"name": "codex_db.1.x", "status": "exited", "exit_code": exit_code,
|
||
"finished_at": "2026-10-06T12:00:00+00:00"}]
|
||
out = a.collect_docker_logs("/sock", down, {}, now=stopped + stopped_ago)
|
||
assert bool(out) is read
|
||
if read:
|
||
assert f"tail={a.DOCKER_LOG_FINAL_TAIL}" in seen[0]
|
||
assert out[-1]["line"] == "fatal: no config"
|
||
|
||
|
||
def test_collect_docker_logs_forgets_finals_of_unlisted_containers(monkeypatch):
|
||
seen: list = []
|
||
monkeypatch.setattr(a, "_docker_request_raw", _final_raw(seen))
|
||
state: dict = {}
|
||
down = [{"name": "job", "status": "exited", "exit_code": 2,
|
||
"finished_at": "2026-10-06T12:00:00+00:00"}]
|
||
a.collect_docker_logs("/sock", down, state, now=1791288010.0)
|
||
assert "job" in state["docker_log_finals"]
|
||
a.collect_docker_logs("/sock", [], state, now=1791288020.0)
|
||
assert state["docker_log_finals"] == {} and state["docker_log_cursors"] == {}
|
||
|
||
|
||
def test_finished_at_only_for_a_stopped_container(monkeypatch):
|
||
insp = {"State": {"StartedAt": "2026-10-06T11:00:00Z", "FinishedAt": "2026-10-06T12:00:00Z",
|
||
"ExitCode": 1}}
|
||
monkeypatch.setattr(a, "_docker_request", lambda s, p, timeout=a.DOCKER_API_TIMEOUT: insp)
|
||
rec = a._collect_one_container("/sock", {"Id": "abc", "Names": ["/job"], "State": "exited"})
|
||
assert rec["finished_at"].startswith("2026-10-06T12:00:00")
|
||
insp["State"]["FinishedAt"] = "0001-01-01T00:00:00Z" # never stopped
|
||
rec = a._collect_one_container("/sock", {"Id": "abc", "Names": ["/job"], "State": "exited"})
|
||
assert rec["finished_at"] is None
|
||
|
||
|
||
def test_build_sample_includes_logs_when_present(monkeypatch):
|
||
monkeypatch.setattr(a, "collect_docker",
|
||
lambda _s: ("ok", None, [{"name": "web", "status": "running"}]))
|
||
monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
|
||
monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
|
||
monkeypatch.setattr(
|
||
a, "collect_docker_logs",
|
||
lambda *args, **kw: [{"container": "web", "stream": "stdout",
|
||
"ts": "t", "line": "hello"}])
|
||
sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
|
||
assert sample["docker_logs"][0]["line"] == "hello"
|
||
|
||
|
||
def test_build_sample_omits_logs_when_disabled(monkeypatch):
|
||
monkeypatch.setattr(a, "collect_docker",
|
||
lambda _s: ("ok", None, [{"name": "web", "status": "running"}]))
|
||
monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
|
||
monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
|
||
called = {"logs": False}
|
||
|
||
def _logs(*args, **kw):
|
||
called["logs"] = True
|
||
return [{"container": "web", "line": "x"}]
|
||
|
||
monkeypatch.setattr(a, "collect_docker_logs", _logs)
|
||
sample = a.build_sample(["/"], {}, "/var/run/docker.sock",
|
||
docker_logs_enabled=False)
|
||
assert "docker_logs" not in sample
|
||
assert called["logs"] is False # collection skipped entirely, not just dropped
|
||
|
||
|
||
def test_drop_logs_strips_only_logs():
|
||
sample = {"ts": "t", "cpu_pct": 5.0,
|
||
"docker": [{"name": "web"}], "docker_logs": [{"line": "x"}]}
|
||
out = a._drop_logs(sample)
|
||
assert "docker_logs" not in out
|
||
assert out["docker"] == [{"name": "web"}] and out["cpu_pct"] == 5.0 # metrics kept
|
||
assert a._drop_logs(out) is out # idempotent
|
||
|
||
|
||
def test_read_config_docker_logs_default_on(tmp_path):
|
||
p = tmp_path / "agent.conf"
|
||
p.write_text("url = x\ntoken = y\n")
|
||
cfg = a.read_config(str(p))
|
||
assert cfg["docker_logs_enabled"] is True
|
||
assert cfg["docker_log_exclude"] == []
|
||
|
||
|
||
def test_read_config_disables_docker_logs(tmp_path):
|
||
p = tmp_path / "agent.conf"
|
||
p.write_text("url = x\ntoken = y\ndocker_logs_enabled = false\n")
|
||
cfg = a.read_config(str(p))
|
||
assert cfg["docker_logs_enabled"] is False
|
||
|
||
|
||
def test_read_config_parses_docker_log_exclude(tmp_path):
|
||
p = tmp_path / "agent.conf"
|
||
p.write_text("url = x\ntoken = y\ndocker_log_exclude = watchtower, foo\n")
|
||
cfg = a.read_config(str(p))
|
||
assert cfg["docker_log_exclude"] == ["watchtower", "foo"]
|
||
|
||
|
||
@pytest.mark.parametrize("insp, started", [
|
||
({"State": {"StartedAt": "2026-10-06T12:00:01.5Z"}}, "2026-10-06T12:00:01.500000+00:00"),
|
||
({"State": {"StartedAt": "2026-10-06T12:00:01Z"}}, "2026-10-06T12:00:01+00:00"),
|
||
({"State": {"StartedAt": "0001-01-01T00:00:00Z"}}, None), # never started
|
||
({"State": {}}, None),
|
||
(None, None), # inspect failed
|
||
({"State": {"StartedAt": "yesterday"}}, None),
|
||
])
|
||
def test_docker_started_at(insp, started):
|
||
assert a._docker_started_at(insp) == started
|
||
|
||
|
||
def test_collect_docker_past_its_deadline_says_so_and_lists_nothing(monkeypatch):
|
||
# A partial list would read as removals; a record without inspect as restarts.
|
||
import threading
|
||
release = threading.Event()
|
||
|
||
def one(socket_path, c):
|
||
if c["Names"] == ["/stuck"]:
|
||
release.wait(2)
|
||
return {"name": c["Names"][0].lstrip("/")}
|
||
|
||
listing = [{"Id": "a" * 16, "Names": ["/fine"]}, {"Id": "b" * 16, "Names": ["/stuck"]}]
|
||
monkeypatch.setattr(a, "_docker_request", lambda s, p, timeout=a.DOCKER_API_TIMEOUT: listing)
|
||
monkeypatch.setattr(a, "_collect_one_container", one)
|
||
monkeypatch.setattr(a, "DOCKER_DEADLINE_SECONDS", 0.05)
|
||
try:
|
||
assert a.collect_docker("/s") == ("error", "1 of 2 containers not read within 0.05 s", [])
|
||
finally:
|
||
release.set()
|
||
|
||
|
||
def test_disk_usage_is_asked_on_its_own_cadence(monkeypatch):
|
||
calls = []
|
||
clock = {"t": 1000.0}
|
||
monkeypatch.setattr(a.time, "monotonic", lambda: clock["t"])
|
||
monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
|
||
monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
|
||
monkeypatch.setattr(a, "collect_disk_usage", lambda _s: calls.append(clock["t"]) or {"images_total": 1})
|
||
state = {}
|
||
seen = []
|
||
for t in (1000.0, 1030.0, 1000.0 + a.DOCKER_DF_INTERVAL_SECONDS):
|
||
clock["t"] = t
|
||
seen.append("docker_disk" in a.build_sample(["/"], state, "/s"))
|
||
assert seen == [True, False, True]
|
||
assert calls == [1000.0, 1000.0 + a.DOCKER_DF_INTERVAL_SECONDS]
|