feat(traffic): each container's network rate, and each NFS mount's bytes, so a host's traffic has a sender (#6115)
Docker02's outbound traffic tops out the internet connection and Steward couldn't say who sends it: a container's counters were kept only as a running total, and NFS traffic belongs to no container. - Docker ingest turns each running container's cumulative network counters into net_rx_bps / net_tx_bps on "<host>/<container>" (net_rates), counting from the stored row so a restart of Steward loses nothing; a container new, recreated or restarted has no rate until its next snapshot. - Agent 1.25.0 reads the bytes sent and received per NFS mount from mountstats (already parsed for the delays) as tx_bps / rx_bps; Steward stores them as nfs_tx_bps / nfs_rx_bps on "<host>:<mount>". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -22,7 +22,7 @@ from collections import deque
|
||||
from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
|
||||
from datetime import datetime, timezone
|
||||
|
||||
AGENT_VERSION = "1.24.0"
|
||||
AGENT_VERSION = "1.25.0"
|
||||
|
||||
# Default path to the local Docker Engine socket. Overridable via the
|
||||
# `docker_socket` config key; collection is silently skipped if it's absent or
|
||||
@@ -408,7 +408,8 @@ def collect_diskio_raw() -> dict:
|
||||
|
||||
def read_mountstats(text: str) -> dict:
|
||||
"""{mount point: {op: (ops, transmissions, timeouts, queue_ms, rtt_ms,
|
||||
execute_ms)}} for each NFS mount in a mountstats text."""
|
||||
execute_ms, bytes_sent, bytes_received)}} for each NFS mount in a
|
||||
mountstats text."""
|
||||
out: dict = {}
|
||||
mount = None
|
||||
in_ops = False
|
||||
@@ -432,8 +433,8 @@ def read_mountstats(text: str) -> dict:
|
||||
name, sep, rest = line.strip().partition(":")
|
||||
nums = rest.split()
|
||||
if sep and len(nums) >= 8 and all(n.isdigit() for n in nums[:8]):
|
||||
ops, trans, timeouts, _sent, _recv, queue, rtt, exe = (int(n) for n in nums[:8])
|
||||
out[mount][name] = (ops, trans, timeouts, queue, rtt, exe)
|
||||
ops, trans, timeouts, sent, recv, queue, rtt, exe = (int(n) for n in nums[:8])
|
||||
out[mount][name] = (ops, trans, timeouts, queue, rtt, exe, sent, recv)
|
||||
return out
|
||||
|
||||
|
||||
@@ -448,8 +449,9 @@ def nfs_delays(cur: dict, prev: dict, dt: float) -> list:
|
||||
"""Each NFS mount's requests since the last sample: how many a second,
|
||||
their average round trip and execute time (queueing included, what a
|
||||
program waits), READ and WRITE on their own, and the retransmissions and
|
||||
major timeouts among them. A mount new since the last sample, or whose
|
||||
counters went back (remounted), is left for the next one."""
|
||||
major timeouts among them, and the bytes a second it sent and received
|
||||
on the wire (#6115). A mount new since the last sample, or whose counters
|
||||
went back (remounted), is left for the next one."""
|
||||
out: list = []
|
||||
if dt <= 0:
|
||||
return out
|
||||
@@ -457,13 +459,15 @@ def nfs_delays(cur: dict, prev: dict, dt: float) -> list:
|
||||
before = prev.get(mount)
|
||||
if before is None:
|
||||
continue
|
||||
d = {op: tuple(a - b for a, b in zip(v, before.get(op, (0,) * 6))) for op, v in ops.items()}
|
||||
d = {op: tuple(a - b for a, b in zip(v, before.get(op, (0,) * len(v)))) for op, v in ops.items()}
|
||||
if any(x < 0 for v in d.values() for x in v):
|
||||
continue
|
||||
n = sum(v[0] for v in d.values())
|
||||
row = {"mount": mount, "ops_per_sec": round(n / dt, 2),
|
||||
"retrans": max(0, sum(v[1] - v[0] for v in d.values())),
|
||||
"timeouts": sum(v[2] for v in d.values())}
|
||||
"timeouts": sum(v[2] for v in d.values()),
|
||||
"tx_bps": round(sum(v[6] for v in d.values()) / dt, 1),
|
||||
"rx_bps": round(sum(v[7] for v in d.values()) / dt, 1)}
|
||||
if n:
|
||||
row["rtt_ms"] = round(sum(v[4] for v in d.values()) / n, 2)
|
||||
row["exe_ms"] = round(sum(v[5] for v in d.values()) / n, 2)
|
||||
|
||||
Reference in New Issue
Block a user