feat(traffic): each container's network rate, and each NFS mount's bytes, so a host's traffic has a sender (#6115)
Docker02's outbound traffic tops out the internet connection and Steward couldn't say who sends it: a container's counters were kept only as a running total, and NFS traffic belongs to no container. - Docker ingest turns each running container's cumulative network counters into net_rx_bps / net_tx_bps on "<host>/<container>" (net_rates), counting from the stored row so a restart of Steward loses nothing; a container new, recreated or restarted has no rate until its next snapshot. - Agent 1.25.0 reads the bytes sent and received per NFS mount from mountstats (already parsed for the delays) as tx_bps / rx_bps; Steward stores them as nfs_tx_bps / nfs_rx_bps on "<host>:<mount>". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,37 @@
|
||||
"""net_rates: a container's bytes a second, from its cumulative network
|
||||
counters against the last ones seen for it (#6115). Pure; no DB."""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from plugins.docker.ingest import net_rates
|
||||
|
||||
T0 = datetime(2026, 10, 19, 18, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _c(name, rx, tx, cid="abc", status="running"):
|
||||
return {"name": name, "container_id": cid, "status": status,
|
||||
"net_rx_bytes": rx, "net_tx_bytes": tx}
|
||||
|
||||
|
||||
def test_rates_count_from_the_stored_row_and_then_from_each_snapshot():
|
||||
before = {"web": (T0, "abc", 1_000, 2_000)}
|
||||
snaps = [(T0 + timedelta(seconds=10), [_c("web", 11_000, 52_000)]),
|
||||
(T0 + timedelta(seconds=20), [_c("web", 11_000, 62_000)])]
|
||||
assert net_rates(before, snaps) == [
|
||||
(T0 + timedelta(seconds=10), "web", 1_000.0, 5_000.0),
|
||||
(T0 + timedelta(seconds=20), "web", 0.0, 1_000.0),
|
||||
]
|
||||
|
||||
|
||||
def test_no_rate_for_a_new_recreated_or_restarted_container_until_its_next_snapshot():
|
||||
before = {"db": (T0, "old", 100, 100), "app": (T0, "abc", 5_000, 5_000)}
|
||||
t1, t2 = T0 + timedelta(seconds=10), T0 + timedelta(seconds=20)
|
||||
snaps = [(t1, [_c("new", 10, 10), _c("db", 200, 200, cid="fresh"), _c("app", 10, 10)]),
|
||||
(t2, [_c("new", 110, 210), _c("db", 300, 300, cid="fresh"), _c("app", 20, 30)])]
|
||||
assert net_rates(before, snaps) == [
|
||||
(t2, "new", 10.0, 20.0), (t2, "db", 10.0, 10.0), (t2, "app", 1.0, 2.0)]
|
||||
|
||||
|
||||
def test_stopped_containers_and_missing_counters_are_skipped():
|
||||
before = {"web": (T0, "abc", 0, 0), "old": (T0, "abc", 0, 0)}
|
||||
snaps = [(T0 + timedelta(seconds=10), [_c("web", None, None), _c("old", 9, 9, status="exited")])]
|
||||
assert net_rates(before, snaps) == []
|
||||
@@ -174,9 +174,9 @@ MOUNTSTATS = (
|
||||
def test_mountstats_reads_each_nfs_mount_s_ops_and_skips_the_rest():
|
||||
got = a.read_mountstats(MOUNTSTATS)
|
||||
assert set(got) == {"/nfs/data", "/nfs/my media"}
|
||||
assert got["/nfs/data"]["READ"] == (10, 10, 0, 5, 30, 40)
|
||||
assert got["/nfs/data"]["NULL"] == (0, 0, 0, 0, 0, 0)
|
||||
assert got["/nfs/my media"] == {"GETATTR": (3, 3, 0, 0, 3, 3)}
|
||||
assert got["/nfs/data"]["READ"] == (10, 10, 0, 5, 30, 40, 1200, 40960)
|
||||
assert got["/nfs/data"]["NULL"] == (0, 0, 0, 0, 0, 0, 0, 0)
|
||||
assert got["/nfs/my media"] == {"GETATTR": (3, 3, 0, 0, 3, 3, 300, 300)}
|
||||
|
||||
|
||||
def test_mountstats_that_cannot_be_read_is_no_mounts(tmp_path):
|
||||
@@ -185,18 +185,23 @@ def test_mountstats_that_cannot_be_read_is_no_mounts(tmp_path):
|
||||
|
||||
|
||||
def test_nfs_delay_is_what_the_requests_since_the_last_sample_took():
|
||||
prev = {"/nfs/data": {"READ": (10, 10, 0, 5, 30, 40), "WRITE": (4, 4, 0, 2, 20, 24)},
|
||||
"/nfs/idle": {"GETATTR": (3, 3, 0, 0, 3, 3)},
|
||||
"/nfs/remounted": {"READ": (50, 50, 0, 0, 50, 50)}}
|
||||
cur = {"/nfs/data": {"READ": (20, 21, 1, 7, 130, 240), "WRITE": (4, 4, 0, 2, 20, 24)},
|
||||
"/nfs/idle": {"GETATTR": (3, 3, 0, 0, 3, 3)},
|
||||
"/nfs/remounted": {"READ": (2, 2, 0, 0, 2, 2)},
|
||||
"/nfs/new": {"READ": (5, 5, 0, 0, 5, 5)}}
|
||||
prev = {"/nfs/data": {"READ": (10, 10, 0, 5, 30, 40, 1200, 40960),
|
||||
"WRITE": (4, 4, 0, 2, 20, 24, 8192, 512)},
|
||||
"/nfs/idle": {"GETATTR": (3, 3, 0, 0, 3, 3, 300, 300)},
|
||||
"/nfs/remounted": {"READ": (50, 50, 0, 0, 50, 50, 5000, 5000)}}
|
||||
cur = {"/nfs/data": {"READ": (20, 21, 1, 7, 130, 240, 21680, 450560),
|
||||
"WRITE": (4, 4, 0, 2, 20, 24, 8192, 512)},
|
||||
"/nfs/idle": {"GETATTR": (3, 3, 0, 0, 3, 3, 300, 300)},
|
||||
"/nfs/remounted": {"READ": (2, 2, 0, 0, 2, 2, 200, 200)},
|
||||
"/nfs/new": {"READ": (5, 5, 0, 0, 5, 5, 500, 500)}}
|
||||
got = {r["mount"]: r for r in a.nfs_delays(cur, prev, 10.0)}
|
||||
# ten READs in 10 s, 100 ms of round trip and 200 ms of waiting between
|
||||
# them, one sent twice and one that hit a major timeout; no WRITE finished
|
||||
# them, one sent twice and one that hit a major timeout; no WRITE finished.
|
||||
# 20,480 bytes sent and 409,600 received on the wire.
|
||||
assert got["/nfs/data"] == {"mount": "/nfs/data", "ops_per_sec": 1.0, "retrans": 1, "timeouts": 1,
|
||||
"tx_bps": 2048.0, "rx_bps": 40960.0,
|
||||
"rtt_ms": 10.0, "exe_ms": 20.0, "read_ms": 20.0}
|
||||
assert got["/nfs/idle"] == {"mount": "/nfs/idle", "ops_per_sec": 0.0, "retrans": 0, "timeouts": 0}
|
||||
assert got["/nfs/idle"] == {"mount": "/nfs/idle", "ops_per_sec": 0.0, "retrans": 0, "timeouts": 0,
|
||||
"tx_bps": 0.0, "rx_bps": 0.0}
|
||||
assert set(got) == {"/nfs/data", "/nfs/idle"} # counters gone back, or no baseline yet
|
||||
assert a.nfs_delays(cur, prev, 0) == []
|
||||
|
||||
Reference in New Issue
Block a user