feat(traffic): each container's network rate, and each NFS mount's bytes, so a host's traffic has a sender (#6115)
CI / lint (push) Successful in 3s
CI / unit (push) Failing after 1m30s
CI / integration (push) Successful in 1m38s
CI / publish (push) Skipped

Docker02's outbound traffic tops out the internet connection and Steward
couldn't say who sends it: a container's counters were kept only as a
running total, and NFS traffic belongs to no container.

- Docker ingest turns each running container's cumulative network counters
  into net_rx_bps / net_tx_bps on "<host>/<container>" (net_rates), counting
  from the stored row so a restart of Steward loses nothing; a container
  new, recreated or restarted has no rate until its next snapshot.
- Agent 1.25.0 reads the bytes sent and received per NFS mount from
  mountstats (already parsed for the delays) as tx_bps / rx_bps; Steward
  stores them as nfs_tx_bps / nfs_rx_bps on "<host>:<mount>".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-19 14:35:24 -04:00
co-authored by Claude Opus 5.5
parent 18032a4782
commit 613ed4e78b
8 changed files with 123 additions and 32 deletions
+37
View File
@@ -0,0 +1,37 @@
"""net_rates: a container's bytes a second, from its cumulative network
counters against the last ones seen for it (#6115). Pure; no DB."""
from datetime import datetime, timedelta, timezone
from plugins.docker.ingest import net_rates
T0 = datetime(2026, 10, 19, 18, 0, tzinfo=timezone.utc)
def _c(name, rx, tx, cid="abc", status="running"):
return {"name": name, "container_id": cid, "status": status,
"net_rx_bytes": rx, "net_tx_bytes": tx}
def test_rates_count_from_the_stored_row_and_then_from_each_snapshot():
before = {"web": (T0, "abc", 1_000, 2_000)}
snaps = [(T0 + timedelta(seconds=10), [_c("web", 11_000, 52_000)]),
(T0 + timedelta(seconds=20), [_c("web", 11_000, 62_000)])]
assert net_rates(before, snaps) == [
(T0 + timedelta(seconds=10), "web", 1_000.0, 5_000.0),
(T0 + timedelta(seconds=20), "web", 0.0, 1_000.0),
]
def test_no_rate_for_a_new_recreated_or_restarted_container_until_its_next_snapshot():
before = {"db": (T0, "old", 100, 100), "app": (T0, "abc", 5_000, 5_000)}
t1, t2 = T0 + timedelta(seconds=10), T0 + timedelta(seconds=20)
snaps = [(t1, [_c("new", 10, 10), _c("db", 200, 200, cid="fresh"), _c("app", 10, 10)]),
(t2, [_c("new", 110, 210), _c("db", 300, 300, cid="fresh"), _c("app", 20, 30)])]
assert net_rates(before, snaps) == [
(t2, "new", 10.0, 20.0), (t2, "db", 10.0, 10.0), (t2, "app", 1.0, 2.0)]
def test_stopped_containers_and_missing_counters_are_skipped():
before = {"web": (T0, "abc", 0, 0), "old": (T0, "abc", 0, 0)}
snaps = [(T0 + timedelta(seconds=10), [_c("web", None, None), _c("old", 9, 9, status="exited")])]
assert net_rates(before, snaps) == []
@@ -174,9 +174,9 @@ MOUNTSTATS = (
def test_mountstats_reads_each_nfs_mount_s_ops_and_skips_the_rest():
got = a.read_mountstats(MOUNTSTATS)
assert set(got) == {"/nfs/data", "/nfs/my media"}
assert got["/nfs/data"]["READ"] == (10, 10, 0, 5, 30, 40)
assert got["/nfs/data"]["NULL"] == (0, 0, 0, 0, 0, 0)
assert got["/nfs/my media"] == {"GETATTR": (3, 3, 0, 0, 3, 3)}
assert got["/nfs/data"]["READ"] == (10, 10, 0, 5, 30, 40, 1200, 40960)
assert got["/nfs/data"]["NULL"] == (0, 0, 0, 0, 0, 0, 0, 0)
assert got["/nfs/my media"] == {"GETATTR": (3, 3, 0, 0, 3, 3, 300, 300)}
def test_mountstats_that_cannot_be_read_is_no_mounts(tmp_path):
@@ -185,18 +185,23 @@ def test_mountstats_that_cannot_be_read_is_no_mounts(tmp_path):
def test_nfs_delay_is_what_the_requests_since_the_last_sample_took():
prev = {"/nfs/data": {"READ": (10, 10, 0, 5, 30, 40), "WRITE": (4, 4, 0, 2, 20, 24)},
"/nfs/idle": {"GETATTR": (3, 3, 0, 0, 3, 3)},
"/nfs/remounted": {"READ": (50, 50, 0, 0, 50, 50)}}
cur = {"/nfs/data": {"READ": (20, 21, 1, 7, 130, 240), "WRITE": (4, 4, 0, 2, 20, 24)},
"/nfs/idle": {"GETATTR": (3, 3, 0, 0, 3, 3)},
"/nfs/remounted": {"READ": (2, 2, 0, 0, 2, 2)},
"/nfs/new": {"READ": (5, 5, 0, 0, 5, 5)}}
prev = {"/nfs/data": {"READ": (10, 10, 0, 5, 30, 40, 1200, 40960),
"WRITE": (4, 4, 0, 2, 20, 24, 8192, 512)},
"/nfs/idle": {"GETATTR": (3, 3, 0, 0, 3, 3, 300, 300)},
"/nfs/remounted": {"READ": (50, 50, 0, 0, 50, 50, 5000, 5000)}}
cur = {"/nfs/data": {"READ": (20, 21, 1, 7, 130, 240, 21680, 450560),
"WRITE": (4, 4, 0, 2, 20, 24, 8192, 512)},
"/nfs/idle": {"GETATTR": (3, 3, 0, 0, 3, 3, 300, 300)},
"/nfs/remounted": {"READ": (2, 2, 0, 0, 2, 2, 200, 200)},
"/nfs/new": {"READ": (5, 5, 0, 0, 5, 5, 500, 500)}}
got = {r["mount"]: r for r in a.nfs_delays(cur, prev, 10.0)}
# ten READs in 10 s, 100 ms of round trip and 200 ms of waiting between
# them, one sent twice and one that hit a major timeout; no WRITE finished
# them, one sent twice and one that hit a major timeout; no WRITE finished.
# 20,480 bytes sent and 409,600 received on the wire.
assert got["/nfs/data"] == {"mount": "/nfs/data", "ops_per_sec": 1.0, "retrans": 1, "timeouts": 1,
"tx_bps": 2048.0, "rx_bps": 40960.0,
"rtt_ms": 10.0, "exe_ms": 20.0, "read_ms": 20.0}
assert got["/nfs/idle"] == {"mount": "/nfs/idle", "ops_per_sec": 0.0, "retrans": 0, "timeouts": 0}
assert got["/nfs/idle"] == {"mount": "/nfs/idle", "ops_per_sec": 0.0, "retrans": 0, "timeouts": 0,
"tx_bps": 0.0, "rx_bps": 0.0}
assert set(got) == {"/nfs/data", "/nfs/idle"} # counters gone back, or no baseline yet
assert a.nfs_delays(cur, prev, 0) == []