feat(docker): uptime from StartedAt, a restart is an event, host totals count physical devices only (#5153)
- Agent: started_at comes from inspect's State.StartedAt (nanoseconds
trimmed; Docker's zero time is "never started"), not the list's Created,
so a restarted container's uptime counts from its restart.
- A rising restart_count between two snapshots emits a "restart" event
("restart #3", or "3 restarts, now #5"), said instead of a start; a
restart then a death gives both. A count that falls (recreated under the
same name) says nothing. The timeline draws it as ↻.
- Network and disk I/O count what the kernel calls physical: interfaces
not under /sys/devices/virtual/net (bridges, docker0, veth, vxlan, tun),
block devices with a device behind them (not dm, md, zram), so a byte is
not counted twice. Where /sys calls everything virtual (an agent in LXC)
the name-filtered set is kept rather than reporting no traffic.
- Agent tests updated for #5152's (status, detail, containers) and
(is_manager, topology) shapes; the no-Docker disk test now states the
case it was passing for by accident.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -4,6 +4,8 @@ _derive_events is a pure function (no DB), and ingest.py imports its models
|
||||
lazily inside functions, so importing it here doesn't register ORM tables —
|
||||
safe for the no-DB unit lane.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from plugins.docker.ingest import _derive_events
|
||||
|
||||
|
||||
@@ -69,6 +71,27 @@ def test_no_event_when_unchanged():
|
||||
|
||||
# ── container-log row shaping (m79, pure) ─────────────────────────────────────
|
||||
|
||||
@pytest.mark.parametrize("old_status, old_n, new, events", [
|
||||
# Docker restarted it between samples: a restart, not silence (#5153).
|
||||
("running", 2, _c("web", "running", restart_count=3), [("web", "restart", "restart #3")]),
|
||||
# Caught restarting, now back up: a restart, said instead of a start.
|
||||
("restarting", 2, _c("web", "running", restart_count=3), [("web", "restart", "restart #3")]),
|
||||
("running", 2, _c("web", "running", restart_count=5), [("web", "restart", "3 restarts, now #5")]),
|
||||
# Restarted, then died for good: both happened.
|
||||
("running", 3, _c("web", "exited", restart_count=4, exit_code=1),
|
||||
[("web", "restart", "restart #4"), ("web", "die", "exit 1")]),
|
||||
# Started by hand (the count did not move): a start.
|
||||
("exited", 2, _c("web", "running", restart_count=2, image="nginx"), [("web", "start", "nginx")]),
|
||||
# Recreated under the same name, so the count reset: no restart.
|
||||
("running", 5, _c("web", "running", restart_count=0), []),
|
||||
("running", 2, _c("web", "running", restart_count=2), []),
|
||||
])
|
||||
def test_restart_events(old_status, old_n, new, events):
|
||||
old = {"web": {"status": old_status, "health": None, "oom_killed": False, "exit_code": None,
|
||||
"restart_count": old_n}}
|
||||
assert _derive_events(old, [new]) == events
|
||||
|
||||
|
||||
def test_log_rows_shapes_and_filters():
|
||||
from datetime import datetime, timezone
|
||||
from plugins.docker.ingest import _log_rows
|
||||
|
||||
@@ -148,13 +148,14 @@ def test_collect_docker_assembles_container(monkeypatch):
|
||||
"Names": ["/web"],
|
||||
"Image": "nginx:latest",
|
||||
"State": "running",
|
||||
"Created": 1_700_000_000,
|
||||
"Created": 1_600_000_000,
|
||||
"Ports": [{"PublicPort": 8080, "PrivatePort": 80, "Type": "tcp"}],
|
||||
"Labels": {"com.docker.compose.project": "myproj",
|
||||
"com.docker.swarm.service.name": "web"},
|
||||
}
|
||||
inspect = {"RestartCount": 2,
|
||||
"State": {"Health": {"Status": "healthy"}, "ExitCode": 0, "OOMKilled": False}}
|
||||
"State": {"Health": {"Status": "healthy"}, "ExitCode": 0, "OOMKilled": False,
|
||||
"StartedAt": "2023-11-14T22:13:20.123456789Z"}}
|
||||
stats = {
|
||||
"cpu_stats": {"cpu_usage": {"total_usage": 200_000_000},
|
||||
"system_cpu_usage": 2_000_000_000, "online_cpus": 4},
|
||||
@@ -188,7 +189,8 @@ def test_collect_docker_assembles_container(monkeypatch):
|
||||
assert c["cpu_pct"] == 40.0
|
||||
assert c["mem_pct"] == 15.0
|
||||
assert c["ports"] == [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}]
|
||||
assert c["started_at"].startswith("2023-11-14T") # epoch → ISO-8601
|
||||
# When it last started (inspect), not when it was created (2020).
|
||||
assert c["started_at"] == "2023-11-14T22:13:20.123456+00:00"
|
||||
# enrichment
|
||||
assert c["health"] == "healthy" and c["restart_count"] == 2
|
||||
assert c["exit_code"] == 0 and c["oom_killed"] is False
|
||||
@@ -418,8 +420,9 @@ def test_build_sample_says_when_a_node_is_not_a_manager(monkeypatch, swarm, flag
|
||||
|
||||
|
||||
def test_build_sample_omits_swarm_off_manager(monkeypatch):
|
||||
monkeypatch.setattr(a, "collect_docker", lambda _s: [])
|
||||
monkeypatch.setattr(a, "collect_swarm", lambda _s: None)
|
||||
monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
|
||||
monkeypatch.setattr(a, "collect_swarm", lambda _s: (False, None))
|
||||
monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
|
||||
sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
|
||||
assert "swarm" not in sample
|
||||
|
||||
@@ -470,18 +473,19 @@ def test_collect_disk_usage_silent_skip_when_no_socket():
|
||||
|
||||
|
||||
def test_build_sample_includes_disk_when_docker_present(monkeypatch):
|
||||
monkeypatch.setattr(a, "collect_docker", lambda _s: [{"name": "web", "status": "running"}])
|
||||
monkeypatch.setattr(a, "collect_swarm", lambda _s: None)
|
||||
monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, [{"name": "web", "status": "running"}]))
|
||||
monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
|
||||
monkeypatch.setattr(a, "collect_disk_usage", lambda _s: {"images_reclaimable": 700})
|
||||
sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
|
||||
assert sample["docker_disk"] == {"images_reclaimable": 700}
|
||||
|
||||
|
||||
def test_build_sample_omits_disk_when_no_docker(monkeypatch):
|
||||
# No containers → the /system/df call is skipped entirely (cost guard). The
|
||||
# No Docker → the /system/df call is skipped entirely (cost guard). The
|
||||
# sentinel would land under docker_disk if the gate ever called it.
|
||||
monkeypatch.setattr(a, "collect_docker", lambda _s: [])
|
||||
monkeypatch.setattr(a, "collect_swarm", lambda _s: None)
|
||||
monkeypatch.setattr(a, "collect_docker",
|
||||
lambda _s: ("unavailable", "no Docker socket at /s", []))
|
||||
monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
|
||||
monkeypatch.setattr(a, "collect_disk_usage", lambda _s: {"sentinel": True})
|
||||
sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
|
||||
assert "docker_disk" not in sample
|
||||
@@ -632,8 +636,8 @@ def test_collect_docker_logs_forgets_gone_container_cursors(monkeypatch):
|
||||
|
||||
def test_build_sample_includes_logs_when_present(monkeypatch):
|
||||
monkeypatch.setattr(a, "collect_docker",
|
||||
lambda _s: [{"name": "web", "status": "running"}])
|
||||
monkeypatch.setattr(a, "collect_swarm", lambda _s: None)
|
||||
lambda _s: ("ok", None, [{"name": "web", "status": "running"}]))
|
||||
monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
|
||||
monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
|
||||
monkeypatch.setattr(
|
||||
a, "collect_docker_logs",
|
||||
@@ -645,8 +649,8 @@ def test_build_sample_includes_logs_when_present(monkeypatch):
|
||||
|
||||
def test_build_sample_omits_logs_when_disabled(monkeypatch):
|
||||
monkeypatch.setattr(a, "collect_docker",
|
||||
lambda _s: [{"name": "web", "status": "running"}])
|
||||
monkeypatch.setattr(a, "collect_swarm", lambda _s: None)
|
||||
lambda _s: ("ok", None, [{"name": "web", "status": "running"}]))
|
||||
monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
|
||||
monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
|
||||
called = {"logs": False}
|
||||
|
||||
@@ -690,3 +694,15 @@ def test_read_config_parses_docker_log_exclude(tmp_path):
|
||||
p.write_text("url = x\ntoken = y\ndocker_log_exclude = watchtower, foo\n")
|
||||
cfg = a.read_config(str(p))
|
||||
assert cfg["docker_log_exclude"] == ["watchtower", "foo"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("insp, started", [
|
||||
({"State": {"StartedAt": "2026-10-06T12:00:01.5Z"}}, "2026-10-06T12:00:01.500000+00:00"),
|
||||
({"State": {"StartedAt": "2026-10-06T12:00:01Z"}}, "2026-10-06T12:00:01+00:00"),
|
||||
({"State": {"StartedAt": "0001-01-01T00:00:00Z"}}, None), # never started
|
||||
({"State": {}}, None),
|
||||
(None, None), # inspect failed
|
||||
({"State": {"StartedAt": "yesterday"}}, None),
|
||||
])
|
||||
def test_docker_started_at(insp, started):
|
||||
assert a._docker_started_at(insp) == started
|
||||
|
||||
@@ -54,7 +54,9 @@ def test_rates_skips_names_missing_from_prev():
|
||||
assert agent._rates({"new": (5, 5)}, {}, dt=1.0) == {}
|
||||
|
||||
|
||||
def test_collect_net_raw_skips_lo_and_veth(monkeypatch):
|
||||
def test_collect_net_raw_skips_lo_and_veth(monkeypatch, tmp_path):
|
||||
# /sys can't say (no such tree): the name filter alone decides.
|
||||
monkeypatch.setattr(agent, "SYS_VIRTUAL_NET_DIR", str(tmp_path / "absent"))
|
||||
fixture = (
|
||||
"Inter-| Receive\n"
|
||||
" face |bytes\n"
|
||||
@@ -66,7 +68,8 @@ def test_collect_net_raw_skips_lo_and_veth(monkeypatch):
|
||||
assert agent.collect_net_raw() == {"eth0": (1000, 500)}
|
||||
|
||||
|
||||
def test_collect_diskio_raw_whole_disks_only(monkeypatch):
|
||||
def test_collect_diskio_raw_whole_disks_only(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(agent, "SYS_BLOCK_DIR", str(tmp_path / "absent"))
|
||||
# fields: major minor name reads rd_merged sectors_read ms ... sectors_written ...
|
||||
fixture = (
|
||||
" 8 0 sda 1 0 10 0 1 0 20 0 0 0 0\n"
|
||||
@@ -79,6 +82,55 @@ def test_collect_diskio_raw_whole_disks_only(monkeypatch):
|
||||
assert out == {"sda": (10 * 512, 20 * 512), "nvme0n1": (100 * 512, 200 * 512)}
|
||||
|
||||
|
||||
_NETDEV = (
|
||||
"Inter-| Receive\n"
|
||||
" face |bytes\n"
|
||||
" lo: 100 1 0 0 0 0 0 0 200 2 0 0 0 0 0 0\n"
|
||||
" eno1: 1000 5 0 0 0 0 0 0 500 4 0 0 0 0 0 0\n"
|
||||
"docker0: 70 1 0 0 0 0 0 0 80 1 0 0 0 0 0 0\n"
|
||||
"br-1a2b: 30 1 0 0 0 0 0 0 40 1 0 0 0 0 0 0\n"
|
||||
"vxlan0: 10 1 0 0 0 0 0 0 20 1 0 0 0 0 0 0\n"
|
||||
)
|
||||
|
||||
|
||||
def _sys_net(tmp_path, virtual):
|
||||
d = tmp_path / "virtual-net"
|
||||
for name in virtual:
|
||||
(d / name).mkdir(parents=True)
|
||||
return str(d)
|
||||
|
||||
|
||||
def test_collect_net_raw_counts_physical_interfaces_only(monkeypatch, tmp_path):
|
||||
# Bridge, docker0 and vxlan traffic also crosses eno1: counting them too
|
||||
# would count those bytes twice in the host total (#5153).
|
||||
monkeypatch.setattr(agent, "SYS_VIRTUAL_NET_DIR",
|
||||
_sys_net(tmp_path, ["lo", "docker0", "br-1a2b", "vxlan0"]))
|
||||
monkeypatch.setattr(agent, "_read_file", lambda p: _NETDEV)
|
||||
assert agent.collect_net_raw() == {"eno1": (1000, 500)}
|
||||
|
||||
|
||||
def test_collect_net_raw_keeps_everything_when_sys_calls_it_all_virtual(monkeypatch, tmp_path):
|
||||
# Inside LXC eth0 is a veth peer: all virtual. No traffic at all would be wrong.
|
||||
netdev = "h\nh\n eth0: 1000 5 0 0 0 0 0 0 500 4 0 0 0 0 0 0\n"
|
||||
monkeypatch.setattr(agent, "SYS_VIRTUAL_NET_DIR", _sys_net(tmp_path, ["lo", "eth0"]))
|
||||
monkeypatch.setattr(agent, "_read_file", lambda p: netdev)
|
||||
assert agent.collect_net_raw() == {"eth0": (1000, 500)}
|
||||
|
||||
|
||||
def test_collect_diskio_raw_counts_physical_disks_only(monkeypatch, tmp_path):
|
||||
# dm-0 (LVM on sda) and md0 (RAID over sdb, sdc) repeat their members' I/O.
|
||||
block = tmp_path / "block"
|
||||
for name in ("sda", "sdb", "nvme0n1"):
|
||||
(block / name / "device").mkdir(parents=True)
|
||||
for name in ("dm-0", "md0", "zram0"):
|
||||
(block / name).mkdir(parents=True)
|
||||
diskstats = "".join(f" 8 0 {n} 1 0 10 0 1 0 20 0 0 0 0\n"
|
||||
for n in ("sda", "sdb", "nvme0n1", "dm-0", "md0", "zram0"))
|
||||
monkeypatch.setattr(agent, "SYS_BLOCK_DIR", str(block))
|
||||
monkeypatch.setattr(agent, "_read_file", lambda p: diskstats)
|
||||
assert sorted(agent.collect_diskio_raw()) == ["nvme0n1", "sda", "sdb"]
|
||||
|
||||
|
||||
def test_build_sample_first_call_has_no_rates_but_keeps_state(monkeypatch):
|
||||
monkeypatch.setattr(agent, "collect_cpu", lambda: (5.0, [5.0]))
|
||||
monkeypatch.setattr(agent, "collect_memory", lambda: {"total_bytes": 10, "available_bytes": 5})
|
||||
|
||||
Reference in New Issue
Block a user