feat(traffic): each container's network rate, and each NFS mount's bytes, so a host's traffic has a sender (#6115)
CI / lint (push) Successful in 3s
CI / unit (push) Failing after 1m30s
CI / integration (push) Successful in 1m38s
CI / publish (push) Skipped

Docker02's outbound traffic tops out the internet connection and Steward
couldn't say who sends it: a container's counters were kept only as a
running total, and NFS traffic belongs to no container.

- Docker ingest turns each running container's cumulative network counters
  into net_rx_bps / net_tx_bps on "<host>/<container>" (net_rates), counting
  from the stored row so a restart of Steward loses nothing; a container
  new, recreated or restarted has no rate until its next snapshot.
- Agent 1.25.0 reads the bytes sent and received per NFS mount from
  mountstats (already parsed for the delays) as tx_bps / rx_bps; Steward
  stores them as nfs_tx_bps / nfs_rx_bps on "<host>:<mount>".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-19 14:35:24 -04:00
co-authored by Claude Opus 5.5
parent 18032a4782
commit 613ed4e78b
8 changed files with 123 additions and 32 deletions
@@ -174,9 +174,9 @@ MOUNTSTATS = (
def test_mountstats_reads_each_nfs_mount_s_ops_and_skips_the_rest():
got = a.read_mountstats(MOUNTSTATS)
assert set(got) == {"/nfs/data", "/nfs/my media"}
assert got["/nfs/data"]["READ"] == (10, 10, 0, 5, 30, 40)
assert got["/nfs/data"]["NULL"] == (0, 0, 0, 0, 0, 0)
assert got["/nfs/my media"] == {"GETATTR": (3, 3, 0, 0, 3, 3)}
assert got["/nfs/data"]["READ"] == (10, 10, 0, 5, 30, 40, 1200, 40960)
assert got["/nfs/data"]["NULL"] == (0, 0, 0, 0, 0, 0, 0, 0)
assert got["/nfs/my media"] == {"GETATTR": (3, 3, 0, 0, 3, 3, 300, 300)}
def test_mountstats_that_cannot_be_read_is_no_mounts(tmp_path):
@@ -185,18 +185,23 @@ def test_mountstats_that_cannot_be_read_is_no_mounts(tmp_path):
def test_nfs_delay_is_what_the_requests_since_the_last_sample_took():
prev = {"/nfs/data": {"READ": (10, 10, 0, 5, 30, 40), "WRITE": (4, 4, 0, 2, 20, 24)},
"/nfs/idle": {"GETATTR": (3, 3, 0, 0, 3, 3)},
"/nfs/remounted": {"READ": (50, 50, 0, 0, 50, 50)}}
cur = {"/nfs/data": {"READ": (20, 21, 1, 7, 130, 240), "WRITE": (4, 4, 0, 2, 20, 24)},
"/nfs/idle": {"GETATTR": (3, 3, 0, 0, 3, 3)},
"/nfs/remounted": {"READ": (2, 2, 0, 0, 2, 2)},
"/nfs/new": {"READ": (5, 5, 0, 0, 5, 5)}}
prev = {"/nfs/data": {"READ": (10, 10, 0, 5, 30, 40, 1200, 40960),
"WRITE": (4, 4, 0, 2, 20, 24, 8192, 512)},
"/nfs/idle": {"GETATTR": (3, 3, 0, 0, 3, 3, 300, 300)},
"/nfs/remounted": {"READ": (50, 50, 0, 0, 50, 50, 5000, 5000)}}
cur = {"/nfs/data": {"READ": (20, 21, 1, 7, 130, 240, 21680, 450560),
"WRITE": (4, 4, 0, 2, 20, 24, 8192, 512)},
"/nfs/idle": {"GETATTR": (3, 3, 0, 0, 3, 3, 300, 300)},
"/nfs/remounted": {"READ": (2, 2, 0, 0, 2, 2, 200, 200)},
"/nfs/new": {"READ": (5, 5, 0, 0, 5, 5, 500, 500)}}
got = {r["mount"]: r for r in a.nfs_delays(cur, prev, 10.0)}
# ten READs in 10 s, 100 ms of round trip and 200 ms of waiting between
# them, one sent twice and one that hit a major timeout; no WRITE finished
# them, one sent twice and one that hit a major timeout; no WRITE finished.
# 20,480 bytes sent and 409,600 received on the wire.
assert got["/nfs/data"] == {"mount": "/nfs/data", "ops_per_sec": 1.0, "retrans": 1, "timeouts": 1,
"tx_bps": 2048.0, "rx_bps": 40960.0,
"rtt_ms": 10.0, "exe_ms": 20.0, "read_ms": 20.0}
assert got["/nfs/idle"] == {"mount": "/nfs/idle", "ops_per_sec": 0.0, "retrans": 0, "timeouts": 0}
assert got["/nfs/idle"] == {"mount": "/nfs/idle", "ops_per_sec": 0.0, "retrans": 0, "timeouts": 0,
"tx_bps": 0.0, "rx_bps": 0.0}
assert set(got) == {"/nfs/data", "/nfs/idle"} # counters gone back, or no baseline yet
assert a.nfs_delays(cur, prev, 0) == []