# tests/plugins/host_agent/test_agent_docker.py """Unit tests for the agent's stdlib Docker collector. No socket / no network — the UDS request layer is monkeypatched so we exercise the cpu/mem math, chunked-body decoding, sample assembly, and what the agent says when Docker cannot be asked, in isolation. """ import socket import pytest from plugins.host_agent import agent as a # ── chunked transfer decoding ───────────────────────────────────────────────── def test_dechunk_reassembles_body(): chunked = b"4\r\nWiki\r\n5\r\npedia\r\n0\r\n\r\n" assert a._dechunk(chunked) == b"Wikipedia" def test_dechunk_ignores_chunk_extensions(): assert a._dechunk(b"3;name=v\r\nabc\r\n0\r\n\r\n") == b"abc" # ── cpu / mem math (ported from the old central scraper) ────────────────────── def test_docker_cpu_pct(): stats = { "cpu_stats": { "cpu_usage": {"total_usage": 200_000_000}, "system_cpu_usage": 2_000_000_000, "online_cpus": 4, }, "precpu_stats": { "cpu_usage": {"total_usage": 100_000_000}, "system_cpu_usage": 1_000_000_000, }, } # cpu_delta=1e8, sys_delta=1e9 → 0.1 * 4 cpus * 100 = 40.0% assert a._docker_cpu_pct(stats) == 40.0 def test_docker_cpu_pct_guards_bad_stats(): assert a._docker_cpu_pct({}) == 0.0 # No forward progress in system time → 0, never a divide error. assert a._docker_cpu_pct({ "cpu_stats": {"cpu_usage": {"total_usage": 5}, "system_cpu_usage": 10}, "precpu_stats": {"cpu_usage": {"total_usage": 5}, "system_cpu_usage": 10}, }) == 0.0 def test_docker_mem_subtracts_cache(): stats = {"memory_stats": { "usage": 200 * 1024 * 1024, "limit": 1000 * 1024 * 1024, "stats": {"cache": 50 * 1024 * 1024}, }} usage, limit, pct = a._docker_mem(stats) assert usage == 150 * 1024 * 1024 # working set = usage − cache assert limit == 1000 * 1024 * 1024 assert pct == 15.0 def test_docker_ports_keeps_only_published(): ports = [ {"PublicPort": 8080, "PrivatePort": 80, "Type": "tcp"}, {"PrivatePort": 5432, "Type": "tcp"}, # unpublished → dropped ] assert a._docker_ports(ports) == [ {"host_port": 8080, "container_port": 80, "protocol": "tcp"}, ] # ── enrichment helpers ──────────────────────────────────────────────────────── def test_docker_grouping_from_labels(): g = a._docker_grouping({ "com.docker.compose.project": "myproj", "com.docker.swarm.service.name": "web", "com.docker.swarm.task.id": "t123", "com.docker.swarm.node.id": "n456", }) assert g == {"compose_project": "myproj", "service_name": "web", "task_id": "t123", "node_id": "n456"} def test_docker_grouping_empty(): assert a._docker_grouping(None) == { "compose_project": None, "service_name": None, "task_id": None, "node_id": None} def test_docker_inspect_fields(): insp = {"RestartCount": 3, "State": {"Health": {"Status": "unhealthy"}, "ExitCode": 137, "OOMKilled": True}} assert a._docker_inspect_fields(insp) == ("unhealthy", 3, 137, True) def test_docker_inspect_fields_no_healthcheck(): # Containers without a HEALTHCHECK have no State.Health → health is None. health, restarts, exit_code, oom = a._docker_inspect_fields( {"RestartCount": 0, "State": {"ExitCode": 0, "OOMKilled": False}}) assert health is None and restarts == 0 and exit_code == 0 and oom is False def test_docker_net_io_sums_counters(): stats = { "networks": {"eth0": {"rx_bytes": 1000, "tx_bytes": 2000}, "eth1": {"rx_bytes": 5, "tx_bytes": 7}}, "blkio_stats": {"io_service_bytes_recursive": [ {"op": "Read", "value": 4096}, {"op": "Write", "value": 8192}]}, } assert a._docker_net_io(stats) == (1005, 2007, 4096, 8192) # ── collect_docker assembly + silent skip ───────────────────────────────────── def test_collect_docker_says_unavailable_when_no_socket(): # A Docker-less host needs no configuration: it reports there is no Docker. assert a.collect_docker("/nonexistent/does-not-exist.sock") == ( "unavailable", "no Docker socket at /nonexistent/does-not-exist.sock", []) @pytest.mark.parametrize("exc, status, detail", [ (FileNotFoundError(2, "x"), "unavailable", "no Docker socket at /s"), (ConnectionRefusedError(111, "x"), "unavailable", "Docker is not accepting connections on /s"), (PermissionError(13, "x"), "error", "permission denied on /s"), (socket.timeout("timed out"), "error", "Docker did not answer within 5 s"), (OSError("docker API /containers/json → 500"), "error", "docker API /containers/json → 500"), (ValueError("Expecting value"), "error", "Expecting value"), ]) def test_collect_docker_says_why_it_could_not_list(monkeypatch, exc, status, detail): def fail(socket_path, path, timeout=a.DOCKER_API_TIMEOUT): raise exc monkeypatch.setattr(a, "_docker_request", fail) assert a.collect_docker("/s") == (status, detail, []) def test_collect_docker_with_no_containers_is_ok_and_empty(monkeypatch): monkeypatch.setattr(a, "_docker_request", lambda s, p, timeout=a.DOCKER_API_TIMEOUT: []) assert a.collect_docker("/s") == ("ok", None, []) def test_collect_docker_assembles_container(monkeypatch): container = { "Id": "abc123def4567890", "Names": ["/web"], "Image": "nginx:latest", "State": "running", "Created": 1_600_000_000, "Ports": [{"PublicPort": 8080, "PrivatePort": 80, "Type": "tcp"}], "Labels": {"com.docker.compose.project": "myproj", "com.docker.swarm.service.name": "web"}, } inspect = {"RestartCount": 2, "State": {"Health": {"Status": "healthy"}, "ExitCode": 0, "OOMKilled": False, "StartedAt": "2023-11-14T22:13:20.123456789Z"}} stats = { "cpu_stats": {"cpu_usage": {"total_usage": 200_000_000}, "system_cpu_usage": 2_000_000_000, "online_cpus": 4}, "precpu_stats": {"cpu_usage": {"total_usage": 100_000_000}, "system_cpu_usage": 1_000_000_000}, "memory_stats": {"usage": 200 * 1024 * 1024, "limit": 1000 * 1024 * 1024, "stats": {"cache": 50 * 1024 * 1024}}, "networks": {"eth0": {"rx_bytes": 1000, "tx_bytes": 2000}}, "blkio_stats": {"io_service_bytes_recursive": [ {"op": "Read", "value": 4096}, {"op": "Write", "value": 8192}]}, } def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT): if path.startswith("/containers/json"): return [container] if "/stats" in path: return stats if path.endswith("/json"): # inspect return inspect raise AssertionError(f"unexpected path {path}") monkeypatch.setattr(a, "_docker_request", fake_request) status, detail, out = a.collect_docker("/var/run/docker.sock") assert (status, detail) == ("ok", None) assert len(out) == 1 c = out[0] assert c["name"] == "web" assert c["container_id"] == "abc123def456" # 12-char short id assert c["image"] == "nginx:latest" assert c["status"] == "running" assert c["cpu_pct"] == 40.0 assert c["mem_pct"] == 15.0 assert c["ports"] == [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}] # When it last started (inspect), not when it was created (2020). assert c["started_at"] == "2023-11-14T22:13:20.123456+00:00" # enrichment assert c["health"] == "healthy" and c["restart_count"] == 2 assert c["exit_code"] == 0 and c["oom_killed"] is False assert c["net_rx_bytes"] == 1000 and c["net_tx_bytes"] == 2000 assert c["blk_read_bytes"] == 4096 and c["blk_write_bytes"] == 8192 assert c["compose_project"] == "myproj" and c["service_name"] == "web" def test_collect_docker_skips_stats_for_stopped(monkeypatch): container = {"Id": "x" * 16, "Names": ["/db"], "Image": "pg", "State": "exited", "Created": None, "Ports": [], "Labels": {}} inspect = {"RestartCount": 0, "State": {"ExitCode": 137, "OOMKilled": True}} # no Health block def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT): if path.startswith("/containers/json"): return [container] if path.endswith("/json"): # inspect — fetched for stopped too (exit code) return inspect raise AssertionError("stats must not be fetched for a stopped container") monkeypatch.setattr(a, "_docker_request", fake_request) _, _, out = a.collect_docker("/var/run/docker.sock") assert out[0]["status"] == "exited" assert out[0]["cpu_pct"] is None and out[0]["started_at"] is None # inspect still gives us why it died assert out[0]["exit_code"] == 137 and out[0]["oom_killed"] is True assert out[0]["health"] is None # ── swarm (manager-only) ────────────────────────────────────────────────────── def test_swarm_is_manager(): assert a._swarm_is_manager({"Swarm": {"ControlAvailable": True}}) is True # Worker: in a swarm but no control plane. assert a._swarm_is_manager({"Swarm": {"ControlAvailable": False}}) is False # Non-swarm daemon: no Swarm block at all. assert a._swarm_is_manager({}) is False def test_swarm_services_rolls_up_replicas(): services = [ {"ID": "svc1", "Spec": { "Name": "web", "Mode": {"Replicated": {"Replicas": 3}}, "TaskTemplate": {"ContainerSpec": {"Image": "nginx:1@sha256:deadbeef"}}}}, {"ID": "svc2", "Spec": { "Name": "agent", "Mode": {"Global": {}}, "TaskTemplate": {"ContainerSpec": {"Image": "agent:latest"}}}}, ] tasks = [ # web: 2 of 3 desired replicas actually running, both on node n1 {"ServiceID": "svc1", "NodeID": "n1", "DesiredState": "running", "Status": {"State": "running"}}, {"ServiceID": "svc1", "NodeID": "n1", "DesiredState": "running", "Status": {"State": "running"}}, {"ServiceID": "svc1", "NodeID": "n2", "DesiredState": "running", "Status": {"State": "failed"}}, # agent: global, 2 tasks desired+running across 2 nodes {"ServiceID": "svc2", "NodeID": "n1", "DesiredState": "running", "Status": {"State": "running"}}, {"ServiceID": "svc2", "NodeID": "n2", "DesiredState": "running", "Status": {"State": "running"}}, ] out = {s["service_name"]: s for s in a._swarm_services(services, tasks)} assert [t["state"] for t in out["web"].pop("tasks")] == ["running", "running", "failed"] assert len(out["agent"].pop("tasks")) == 2 assert out["web"] == {"service_name": "web", "mode": "replicated", "desired": 3, "running": 2, "image": "nginx:1", "placement": [{"node_id": "n1", "running": 2}]} # Global mode has no fixed replica count → desired derived from desired tasks. assert out["agent"] == {"service_name": "agent", "mode": "global", "desired": 2, "running": 2, "image": "agent:latest", "placement": [{"node_id": "n1", "running": 1}, {"node_id": "n2", "running": 1}]} def test_swarm_task_keeps_why_it_stopped(): t = a._swarm_task({ "ID": "t1", "Slot": 1, "NodeID": "n2", "DesiredState": "Shutdown", "Status": {"State": "failed", "Timestamp": "2026-10-06T13:19:51.5Z", "Message": "started", "Err": "task: non-zero exit (1)" + "x" * 400, "ContainerStatus": {"ContainerID": "c1", "ExitCode": 1}}}) assert t["task_id"] == "t1" and t["slot"] == 1 and t["node_id"] == "n2" assert t["desired_state"] == "shutdown" and t["state"] == "failed" and t["exit_code"] == 1 assert t["at"] == "2026-10-06T13:19:51.5Z" and t["message"] == "started" assert t["error"].startswith("task: non-zero exit (1)") and len(t["error"]) == 300 # A task that never got a container has no exit code. bare = a._swarm_task({"ID": "t2", "DesiredState": "running", "Status": {"State": "rejected", "Err": "No such image: x"}}) assert bare["exit_code"] is None and bare["slot"] is None and bare["error"] == "No such image: x" def test_swarm_services_keeps_the_newest_tasks(): services = [{"ID": "s", "Spec": {"Name": "db", "Mode": {"Replicated": {"Replicas": 1}}}}] tasks = [{"ID": f"t{i}", "ServiceID": "s", "Slot": 1, "DesiredState": "shutdown", "Status": {"State": "failed", "Timestamp": f"2026-10-06T13:{i:02d}:00Z"}} for i in range(a.SWARM_TASKS_PER_SERVICE + 5)] (svc,) = a._swarm_services(services, tasks) ids = [t["task_id"] for t in svc["tasks"]] newest = a.SWARM_TASKS_PER_SERVICE + 4 assert len(ids) == a.SWARM_TASKS_PER_SERVICE and ids[0] == f"t{newest}" def test_swarm_nodes_normalises_fields(): nodes = [ {"ID": "n1", "Description": {"Hostname": "mgr"}, "Spec": {"Role": "manager", "Availability": "active"}, "Status": {"State": "ready"}, "ManagerStatus": {"Leader": True}}, {"ID": "n2", "Description": {"Hostname": "wrk"}, "Spec": {"Role": "worker", "Availability": "drain"}, "Status": {"State": "down"}}, ] out = {n["node_id"]: n for n in a._swarm_nodes(nodes)} assert out["n1"] == {"node_id": "n1", "hostname": "mgr", "role": "manager", "availability": "active", "status": "ready", "leader": True} assert out["n2"] == {"node_id": "n2", "hostname": "wrk", "role": "worker", "availability": "drain", "status": "down", "leader": False} def test_collect_swarm_skips_on_worker(monkeypatch): # A worker reports ControlAvailable=False → None, and never hits /services. def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT): if path == "/info": return {"Swarm": {"ControlAvailable": False}} raise AssertionError("manager-only endpoint queried on a worker") monkeypatch.setattr(a, "_docker_request", fake_request) assert a.collect_swarm("/var/run/docker.sock") == (False, None) def test_collect_swarm_assembles_on_manager(monkeypatch): services = [{"ID": "svc1", "Spec": { "Name": "web", "Mode": {"Replicated": {"Replicas": 1}}, "TaskTemplate": {"ContainerSpec": {"Image": "nginx"}}}}] tasks = [{"ServiceID": "svc1", "DesiredState": "running", "Status": {"State": "running"}}] nodes = [{"ID": "n1", "Description": {"Hostname": "mgr"}, "Spec": {"Role": "manager", "Availability": "active"}, "Status": {"State": "ready"}, "ManagerStatus": {"Leader": True}}] def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT): return { "/info": {"Swarm": {"ControlAvailable": True}}, "/services": services, "/tasks": tasks, "/nodes": nodes, }[path] monkeypatch.setattr(a, "_docker_request", fake_request) is_manager, out = a.collect_swarm("/var/run/docker.sock") assert is_manager is True assert out["services"][0]["service_name"] == "web" assert out["services"][0]["running"] == 1 and out["services"][0]["desired"] == 1 assert out["nodes"][0]["hostname"] == "mgr" and out["nodes"][0]["leader"] is True def test_collect_swarm_cannot_say_when_info_fails(): # Not knowing is not "no longer a manager": Steward must keep the rows. assert a.collect_swarm("/nonexistent/does-not-exist.sock") == (None, None) def test_collect_swarm_manager_whose_listing_fails_sends_nothing(monkeypatch): def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT): if path == "/info": return {"Swarm": {"ControlAvailable": True}} raise OSError("docker API /services → 503") monkeypatch.setattr(a, "_docker_request", fake_request) assert a.collect_swarm("/var/run/docker.sock") == (True, None) # ── build_sample wiring ─────────────────────────────────────────────────────── def test_build_sample_includes_docker_when_present(monkeypatch): monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, [{"name": "web", "status": "running"}])) monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None)) monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None) sample = a.build_sample(["/"], {}, "/var/run/docker.sock") assert sample["docker"] == [{"name": "web", "status": "running"}] assert sample["docker_status"] == "ok" and "docker_detail" not in sample def test_build_sample_sends_an_empty_list_when_docker_has_no_containers(monkeypatch): # "No containers" must reach Steward, so it can clear the host's list (#5152). monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, [])) monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None)) monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None) sample = a.build_sample(["/"], {}, "/var/run/docker.sock") assert sample["docker"] == [] and sample["docker_status"] == "ok" def test_build_sample_says_why_docker_could_not_be_asked(monkeypatch): monkeypatch.setattr(a, "collect_docker", lambda _s: ("error", "permission denied on /var/run/docker.sock", [])) monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None)) def no_df(_s): raise AssertionError("/system/df must not be asked when Docker did not answer") monkeypatch.setattr(a, "collect_disk_usage", no_df) sample = a.build_sample(["/"], {}, "/var/run/docker.sock") assert "docker" not in sample # not "no containers": unknown assert sample["docker_status"] == "error" assert sample["docker_detail"] == "permission denied on /var/run/docker.sock" def test_build_sample_includes_swarm_on_manager(monkeypatch): monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, [])) monkeypatch.setattr(a, "collect_swarm", lambda _s: (True, {"services": [], "nodes": [{"node_id": "n1"}]})) monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None) sample = a.build_sample(["/"], {}, "/var/run/docker.sock") assert sample["swarm"]["nodes"] == [{"node_id": "n1"}] assert "swarm_manager" not in sample @pytest.mark.parametrize("swarm, flag", [ ((False, None), False), # /info says not a manager: say so ((None, None), None), # /info failed: say nothing ((True, None), None), # a manager whose listing failed: say nothing ]) def test_build_sample_says_when_a_node_is_not_a_manager(monkeypatch, swarm, flag): monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, [])) monkeypatch.setattr(a, "collect_swarm", lambda _s: swarm) monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None) sample = a.build_sample(["/"], {}, "/var/run/docker.sock") assert sample.get("swarm_manager") is flag and "swarm" not in sample def test_build_sample_omits_swarm_off_manager(monkeypatch): monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, [])) monkeypatch.setattr(a, "collect_swarm", lambda _s: (False, None)) monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None) sample = a.build_sample(["/"], {}, "/var/run/docker.sock") assert "swarm" not in sample # ── disk usage (/system/df) ───────────────────────────────────────────────── def test_disk_images_normalises_and_caps(): images = [ {"Id": "sha256:aaaaaaaaaaaa1111", "RepoTags": ["nginx:latest"], "Size": 200, "SharedSize": 50, "Containers": 2}, {"Id": "sha256:bbbbbbbbbbbb2222", "RepoTags": [":"], "Size": 500, "SharedSize": -1, "Containers": 0}, ] out = a._disk_images(images) # Sorted largest-first; dangling tag normalised; short id; SharedSize<0 → 0. assert out[0]["repo_tag"] == "" and out[0]["size"] == 500 assert out[0]["shared_size"] == 0 and out[0]["containers"] == 0 assert out[1]["repo_tag"] == "nginx:latest" and out[1]["image_id"] == "aaaaaaaaaaaa" def test_collect_disk_usage_computes_reclaimable(monkeypatch): df = { "LayersSize": 1000, "Images": [ {"Id": "sha256:a", "RepoTags": ["app:1"], "Size": 300, "Containers": 1}, {"Id": "sha256:b", "RepoTags": None, "Size": 700, "Containers": 0}, ], "Containers": [{"Id": "c1", "SizeRw": 25}, {"Id": "c2", "SizeRw": 75}], "Volumes": [{"Name": "v1", "UsageData": {"Size": 40}}, {"Name": "v2", "UsageData": {"Size": -1}}], "BuildCache": [{"Size": 10}, {"Size": 5}], } monkeypatch.setattr(a, "_docker_request", lambda s, p, timeout=a.DOCKER_API_TIMEOUT: df) out = a.collect_disk_usage("/var/run/docker.sock") assert out["layers_size"] == 1000 assert out["images_total"] == 2 and out["images_active"] == 1 assert out["images_size"] == 1000 assert out["images_reclaimable"] == 700 # only the unreferenced image assert out["containers_count"] == 2 and out["containers_size"] == 100 assert out["volumes_count"] == 2 and out["volumes_size"] == 40 # negative skipped assert out["build_cache_size"] == 15 assert len(out["images"]) == 2 def test_collect_disk_usage_silent_skip_when_no_socket(): assert a.collect_disk_usage("/nonexistent/does-not-exist.sock") is None def test_build_sample_includes_disk_when_docker_present(monkeypatch): monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, [{"name": "web", "status": "running"}])) monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None)) monkeypatch.setattr(a, "collect_disk_usage", lambda _s: {"images_reclaimable": 700}) sample = a.build_sample(["/"], {}, "/var/run/docker.sock") assert sample["docker_disk"] == {"images_reclaimable": 700} def test_build_sample_omits_disk_when_no_docker(monkeypatch): # No Docker → the /system/df call is skipped entirely (cost guard). The # sentinel would land under docker_disk if the gate ever called it. monkeypatch.setattr(a, "collect_docker", lambda _s: ("unavailable", "no Docker socket at /s", [])) monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None)) monkeypatch.setattr(a, "collect_disk_usage", lambda _s: {"sentinel": True}) sample = a.build_sample(["/"], {}, "/var/run/docker.sock") assert "docker_disk" not in sample # ── config ──────────────────────────────────────────────────────────────────── def test_read_config_defaults_docker_socket(tmp_path): cfg_file = tmp_path / "agent.conf" cfg_file.write_text("url = https://steward.example\ntoken = abc\n") cfg = a.read_config(str(cfg_file)) assert cfg["docker_socket"] == a.DEFAULT_DOCKER_SOCKET def test_read_config_honours_explicit_docker_socket(tmp_path): cfg_file = tmp_path / "agent.conf" cfg_file.write_text( "url = https://steward.example\ntoken = abc\n" "docker_socket = /run/user/1000/docker.sock\n") cfg = a.read_config(str(cfg_file)) assert cfg["docker_socket"] == "/run/user/1000/docker.sock" # ── container logs (m79) ────────────────────────────────────────────────────── def _frame(stream: int, payload: bytes) -> bytes: """Build one Docker multiplexed-log frame (8-byte header + payload).""" return bytes([stream, 0, 0, 0]) + len(payload).to_bytes(4, "big") + payload def test_demux_docker_logs_framed(): raw = _frame(1, b"out line\n") + _frame(2, b"err line\n") assert a._demux_docker_logs(raw) == [ ("stdout", b"out line\n"), ("stderr", b"err line\n")] def test_demux_docker_logs_tty_fallback(): # A TTY container's stream isn't framed — the leading byte ('2') is not a # valid stream id, so the whole blob is treated as one stdout payload. raw = b"2023-11-14T12:00:00.000000000Z hi\n" assert a._demux_docker_logs(raw) == [("stdout", raw)] def test_demux_docker_logs_empty(): assert a._demux_docker_logs(b"") == [] def test_parse_log_ts_handles_nanoseconds_and_z(): dt = a._parse_log_ts("2023-11-14T12:00:00.123456789Z") assert dt is not None assert dt.year == 2023 and dt.microsecond == 123456 # ns trimmed to µs assert dt.utcoffset().total_seconds() == 0 assert a._parse_log_ts("not-a-time") is None assert a._parse_log_ts("") is None def test_parse_container_logs_splits_streams_and_ts(): raw = (_frame(1, b"2023-11-14T12:00:00.000000001Z hello world\n") + _frame(2, b"2023-11-14T12:00:01.000000000Z oops\n")) parsed = a._parse_container_logs(raw) assert len(parsed) == 2 dt0, s0, l0 = parsed[0] dt1, s1, l1 = parsed[1] assert (s0, l0) == ("stdout", "hello world") assert (s1, l1) == ("stderr", "oops") assert dt1 > dt0 def test_parse_container_logs_keeps_unparseable_line(): # No timestamp prefix (e.g. a partial write) → dt is None, full line kept. parsed = a._parse_container_logs(_frame(1, b"no-timestamp-here\n")) assert parsed == [(None, "stdout", "no-timestamp-here")] def test_collect_docker_logs_since_cursor_and_dedup(monkeypatch): containers = [{"name": "web", "status": "running"}] calls = [] t1 = "2023-11-14T12:00:00.000000000Z" t2 = "2023-11-14T12:00:01.000000000Z" t3 = "2023-11-14T12:00:02.000000000Z" def fake_raw(socket_path, path, timeout=a.DOCKER_API_TIMEOUT): calls.append(path) if "since=" not in path: # first interval seeds from a tail alone return _frame(1, f"{t1} a\n".encode()) + _frame(1, f"{t2} b\n".encode()) # second interval: daemon re-returns the boundary line (t2) + a new one return _frame(1, f"{t2} b\n".encode()) + _frame(1, f"{t3} c\n".encode()) monkeypatch.setattr(a, "_docker_request_raw", fake_raw) state: dict = {} first = a.collect_docker_logs("/sock", containers, state) assert [r["line"] for r in first] == ["a", "b"] assert "tail=" in calls[0] second = a.collect_docker_logs("/sock", containers, state) # boundary line b (t2) deduped against the cursor; only c (t3) is new assert [r["line"] for r in second] == ["c"] assert "since=" in calls[1] assert f"tail={a.DOCKER_LOG_MAX_LINES}" in calls[1] # bounded even since the cursor (#5764) def test_collect_docker_logs_excludes_and_skips_non_running(monkeypatch): containers = [ {"name": "web", "status": "running"}, {"name": "noisy", "status": "running"}, {"name": "db", "status": "exited"}, ] seen = [] def fake_raw(socket_path, path, timeout=a.DOCKER_API_TIMEOUT): seen.append(path) return _frame(1, b"2023-11-14T12:00:00.000000000Z x\n") monkeypatch.setattr(a, "_docker_request_raw", fake_raw) out = a.collect_docker_logs("/sock", containers, {}, exclude=["noisy"]) # only web is fetched: noisy is excluded, db isn't running assert seen and all("/containers/web/" in p for p in seen) assert {r["container"] for r in out} == {"web"} def test_collect_docker_logs_byte_cap_truncates(monkeypatch): containers = [{"name": "web", "status": "running"}] blob = b"".join( _frame(1, f"2023-11-14T12:00:{i:02d}.000000000Z {'x' * 20}\n".encode()) for i in range(10)) monkeypatch.setattr(a, "_docker_request_raw", lambda s, p, timeout=a.DOCKER_API_TIMEOUT: blob) out = a.collect_docker_logs("/sock", containers, {}, max_bytes=50) # Once the cap is crossed a single marker line is appended and we stop. assert out[-1]["container"] == "_steward" assert "truncated" in out[-1]["line"] real = [r for r in out if r["container"] == "web"] assert 0 < len(real) < 10 def test_collect_docker_logs_forgets_gone_container_cursors(monkeypatch): monkeypatch.setattr( a, "_docker_request_raw", lambda s, p, timeout=a.DOCKER_API_TIMEOUT: _frame(1, b"2023-11-14T12:00:00.000000000Z x\n")) state: dict = {} a.collect_docker_logs("/sock", [{"name": "web", "status": "running"}], state) assert "web" in state["docker_log_cursors"] # web is gone next interval → its cursor is pruned so state can't grow forever a.collect_docker_logs("/sock", [{"name": "db", "status": "running"}], state) assert "web" not in state["docker_log_cursors"] assert "db" in state["docker_log_cursors"] def _final_raw(seen): def fake_raw(socket_path, path, timeout=a.DOCKER_API_TIMEOUT): seen.append(path) return (_frame(1, b"2026-10-06T12:00:00.000000000Z booting\n") + _frame(2, b"2026-10-06T12:00:01.000000000Z fatal: no config\n")) return fake_raw def test_collect_docker_logs_reads_a_stopped_container_once(monkeypatch): # #5049: a container that was being read and has stopped is read one last # time since its cursor, then not again until it stops again. seen: list = [] monkeypatch.setattr(a, "_docker_request_raw", _final_raw(seen)) state = {"docker_log_cursors": {"web": 1791288000.5}} # 2026-10-06T12:00:00.5Z down = [{"name": "web", "status": "exited", "exit_code": 1, "finished_at": "2026-10-06T12:00:02+00:00"}] out = a.collect_docker_logs("/sock", down, state) assert [r["line"] for r in out] == ["fatal: no config"] # since the cursor assert "since=" in seen[0] assert a.collect_docker_logs("/sock", down, state) == [] # same stop: not again assert len(seen) == 1 again = [{**down[0], "finished_at": "2026-10-06T12:05:00+00:00"}] a.collect_docker_logs("/sock", again, state) # a new stop is read assert len(seen) == 2 @pytest.mark.parametrize("exit_code,stopped_ago,read", [ (1, 30, True), # crashed between two samples: its last lines (0, 30, False), # finished cleanly: nothing to explain (None, 30, False), # inspect failed: no claim (1, 3600, False), # an old crash already on the host when the agent came up ]) def test_collect_docker_logs_tails_a_container_first_seen_crashed(monkeypatch, exit_code, stopped_ago, read): seen: list = [] monkeypatch.setattr(a, "_docker_request_raw", _final_raw(seen)) stopped = 1791288000.0 down = [{"name": "codex_db.1.x", "status": "exited", "exit_code": exit_code, "finished_at": "2026-10-06T12:00:00+00:00"}] out = a.collect_docker_logs("/sock", down, {}, now=stopped + stopped_ago) assert bool(out) is read if read: assert f"tail={a.DOCKER_LOG_FINAL_TAIL}" in seen[0] assert out[-1]["line"] == "fatal: no config" def test_collect_docker_logs_forgets_finals_of_unlisted_containers(monkeypatch): seen: list = [] monkeypatch.setattr(a, "_docker_request_raw", _final_raw(seen)) state: dict = {} down = [{"name": "job", "status": "exited", "exit_code": 2, "finished_at": "2026-10-06T12:00:00+00:00"}] a.collect_docker_logs("/sock", down, state, now=1791288010.0) assert "job" in state["docker_log_finals"] a.collect_docker_logs("/sock", [], state, now=1791288020.0) assert state["docker_log_finals"] == {} and state["docker_log_cursors"] == {} def test_finished_at_only_for_a_stopped_container(monkeypatch): insp = {"State": {"StartedAt": "2026-10-06T11:00:00Z", "FinishedAt": "2026-10-06T12:00:00Z", "ExitCode": 1}} monkeypatch.setattr(a, "_docker_request", lambda s, p, timeout=a.DOCKER_API_TIMEOUT: insp) rec = a._collect_one_container("/sock", {"Id": "abc", "Names": ["/job"], "State": "exited"}) assert rec["finished_at"].startswith("2026-10-06T12:00:00") insp["State"]["FinishedAt"] = "0001-01-01T00:00:00Z" # never stopped rec = a._collect_one_container("/sock", {"Id": "abc", "Names": ["/job"], "State": "exited"}) assert rec["finished_at"] is None def test_build_sample_includes_logs_when_present(monkeypatch): monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, [{"name": "web", "status": "running"}])) monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None)) monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None) monkeypatch.setattr( a, "collect_docker_logs", lambda *args, **kw: [{"container": "web", "stream": "stdout", "ts": "t", "line": "hello"}]) sample = a.build_sample(["/"], {}, "/var/run/docker.sock") assert sample["docker_logs"][0]["line"] == "hello" def test_build_sample_omits_logs_when_disabled(monkeypatch): monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, [{"name": "web", "status": "running"}])) monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None)) monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None) called = {"logs": False} def _logs(*args, **kw): called["logs"] = True return [{"container": "web", "line": "x"}] monkeypatch.setattr(a, "collect_docker_logs", _logs) sample = a.build_sample(["/"], {}, "/var/run/docker.sock", docker_logs_enabled=False) assert "docker_logs" not in sample assert called["logs"] is False # collection skipped entirely, not just dropped def test_drop_logs_strips_only_logs(): sample = {"ts": "t", "cpu_pct": 5.0, "docker": [{"name": "web"}], "docker_logs": [{"line": "x"}]} out = a._drop_logs(sample) assert "docker_logs" not in out assert out["docker"] == [{"name": "web"}] and out["cpu_pct"] == 5.0 # metrics kept assert a._drop_logs(out) is out # idempotent def test_read_config_docker_logs_default_on(tmp_path): p = tmp_path / "agent.conf" p.write_text("url = x\ntoken = y\n") cfg = a.read_config(str(p)) assert cfg["docker_logs_enabled"] is True assert cfg["docker_log_exclude"] == [] def test_read_config_disables_docker_logs(tmp_path): p = tmp_path / "agent.conf" p.write_text("url = x\ntoken = y\ndocker_logs_enabled = false\n") cfg = a.read_config(str(p)) assert cfg["docker_logs_enabled"] is False def test_read_config_parses_docker_log_exclude(tmp_path): p = tmp_path / "agent.conf" p.write_text("url = x\ntoken = y\ndocker_log_exclude = watchtower, foo\n") cfg = a.read_config(str(p)) assert cfg["docker_log_exclude"] == ["watchtower", "foo"] @pytest.mark.parametrize("insp, started", [ ({"State": {"StartedAt": "2026-10-06T12:00:01.5Z"}}, "2026-10-06T12:00:01.500000+00:00"), ({"State": {"StartedAt": "2026-10-06T12:00:01Z"}}, "2026-10-06T12:00:01+00:00"), ({"State": {"StartedAt": "0001-01-01T00:00:00Z"}}, None), # never started ({"State": {}}, None), (None, None), # inspect failed ({"State": {"StartedAt": "yesterday"}}, None), ]) def test_docker_started_at(insp, started): assert a._docker_started_at(insp) == started def test_collect_docker_past_its_deadline_says_so_and_lists_nothing(monkeypatch): # A partial list would read as removals; a record without inspect as restarts. import threading release = threading.Event() def one(socket_path, c): if c["Names"] == ["/stuck"]: release.wait(2) return {"name": c["Names"][0].lstrip("/")} listing = [{"Id": "a" * 16, "Names": ["/fine"]}, {"Id": "b" * 16, "Names": ["/stuck"]}] monkeypatch.setattr(a, "_docker_request", lambda s, p, timeout=a.DOCKER_API_TIMEOUT: listing) monkeypatch.setattr(a, "_collect_one_container", one) monkeypatch.setattr(a, "DOCKER_DEADLINE_SECONDS", 0.05) try: assert a.collect_docker("/s") == ("error", "1 of 2 containers not read within 0.05 s", []) finally: release.set() def test_disk_usage_is_asked_on_its_own_cadence(monkeypatch): calls = [] clock = {"t": 1000.0} monkeypatch.setattr(a.time, "monotonic", lambda: clock["t"]) monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, [])) monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None)) monkeypatch.setattr(a, "collect_disk_usage", lambda _s: calls.append(clock["t"]) or {"images_total": 1}) state = {} seen = [] for t in (1000.0, 1030.0, 1000.0 + a.DOCKER_DF_INTERVAL_SECONDS): clock["t"] = t seen.append("docker_disk" in a.build_sample(["/"], state, "/s")) assert seen == [True, False, True] assert calls == [1000.0, 1000.0 + a.DOCKER_DF_INTERVAL_SECONDS]