feat(traffic): each container's network rate, and each NFS mount's bytes, so a host's traffic has a sender (#6115)

Docker02's outbound traffic tops out the internet connection and Steward
couldn't say who sends it: a container's counters were kept only as a
running total, and NFS traffic belongs to no container.

- Docker ingest turns each running container's cumulative network counters
  into net_rx_bps / net_tx_bps on "<host>/<container>" (net_rates), counting
  from the stored row so a restart of Steward loses nothing; a container
  new, recreated or restarted has no rate until its next snapshot.
- Agent 1.25.0 reads the bytes sent and received per NFS mount from
  mountstats (already parsed for the delays) as tx_bps / rx_bps; Steward
  stores them as nfs_tx_bps / nfs_rx_bps on "<host>:<mount>".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/__init__.py

@@ -96,7 +96,9 @@ def get_alert_metrics() -> dict[str, list[str]]:
     average time a program waited on one (queueing included), nfs_rtt_ms the
     server's round trip, nfs_read_ms / nfs_write_ms the same for reads and
     writes alone, nfs_ops_per_sec how many, and nfs_retrans / nfs_timeouts the
-    retransmissions and major timeouts among them. nfs_exe_max_ms is the
+    retransmissions and major timeouts among them; nfs_tx_bps / nfs_rx_bps
+    (agent ≥ 1.25.0, #6115) are the bytes a second it sent and received, its
+    share of the host's network traffic. nfs_exe_max_ms is the
     host's slowest mount, for a covering rule: "NFS slow" is > 1000 for 3
     checks. A healthy share answers in a few ms.
 
@@ -125,7 +127,7 @@ def get_alert_metrics() -> dict[str, list[str]]:
         "updates_pending", "updates_security", "reboot_required", "maintenance_failed",
         "temp_c_max", "temp_c", "net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
         "nfs_exe_max_ms", "nfs_exe_ms", "nfs_rtt_ms", "nfs_read_ms", "nfs_write_ms",
-        "nfs_ops_per_sec", "nfs_retrans", "nfs_timeouts",
+        "nfs_ops_per_sec", "nfs_retrans", "nfs_timeouts", "nfs_tx_bps", "nfs_rx_bps",
         *DRIVE_METRICS,
     ]}
 

plugins/host_agent/agent.py

@@ -22,7 +22,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.24.0"
+AGENT_VERSION = "1.25.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -408,7 +408,8 @@ def collect_diskio_raw() -> dict:
 
 def read_mountstats(text: str) -> dict:
     """{mount point: {op: (ops, transmissions, timeouts, queue_ms, rtt_ms,
-    execute_ms)}} for each NFS mount in a mountstats text."""
+    execute_ms, bytes_sent, bytes_received)}} for each NFS mount in a
+    mountstats text."""
     out: dict = {}
     mount = None
     in_ops = False
@@ -432,8 +433,8 @@ def read_mountstats(text: str) -> dict:
             name, sep, rest = line.strip().partition(":")
             nums = rest.split()
             if sep and len(nums) >= 8 and all(n.isdigit() for n in nums[:8]):
-                ops, trans, timeouts, _sent, _recv, queue, rtt, exe = (int(n) for n in nums[:8])
-                out[mount][name] = (ops, trans, timeouts, queue, rtt, exe)
+                ops, trans, timeouts, sent, recv, queue, rtt, exe = (int(n) for n in nums[:8])
+                out[mount][name] = (ops, trans, timeouts, queue, rtt, exe, sent, recv)
     return out
 
 
@@ -448,8 +449,9 @@ def nfs_delays(cur: dict, prev: dict, dt: float) -> list:
     """Each NFS mount's requests since the last sample: how many a second,
     their average round trip and execute time (queueing included, what a
     program waits), READ and WRITE on their own, and the retransmissions and
-    major timeouts among them. A mount new since the last sample, or whose
-    counters went back (remounted), is left for the next one."""
+    major timeouts among them, and the bytes a second it sent and received
+    on the wire (#6115). A mount new since the last sample, or whose counters
+    went back (remounted), is left for the next one."""
     out: list = []
     if dt <= 0:
         return out
@@ -457,13 +459,15 @@ def nfs_delays(cur: dict, prev: dict, dt: float) -> list:
         before = prev.get(mount)
         if before is None:
             continue
-        d = {op: tuple(a - b for a, b in zip(v, before.get(op, (0,) * 6))) for op, v in ops.items()}
+        d = {op: tuple(a - b for a, b in zip(v, before.get(op, (0,) * len(v)))) for op, v in ops.items()}
         if any(x < 0 for v in d.values() for x in v):
             continue
         n = sum(v[0] for v in d.values())
         row = {"mount": mount, "ops_per_sec": round(n / dt, 2),
                "retrans": max(0, sum(v[1] - v[0] for v in d.values())),
-               "timeouts": sum(v[2] for v in d.values())}
+               "timeouts": sum(v[2] for v in d.values()),
+               "tx_bps": round(sum(v[6] for v in d.values()) / dt, 1),
+               "rx_bps": round(sum(v[7] for v in d.values()) / dt, 1)}
         if n:
             row["rtt_ms"] = round(sum(v[4] for v in d.values()) / n, 2)
             row["exe_ms"] = round(sum(v[5] for v in d.values()) / n, 2)

plugins/host_agent/routes.py

@@ -292,7 +292,8 @@ def _expand_sample_to_metrics(
         row("mounts_stalled", host_name, stalled)
 
     # NFS delay per mount (agent ≥ 1.22.0, #5996): what its requests took
-    # since the last sample. The host's slowest mount is its own metric, for
+    # since the last sample, and (agent ≥ 1.25.0, #6115) the bytes a second
+    # it sent and received. The host's slowest mount is its own metric, for
     # a rule; recorded only when some mount finished a request, since a mount
     # with none, idle or stuck, has no delay to give (mounts_stalled covers
     # stuck).
@@ -302,7 +303,8 @@ def _expand_sample_to_metrics(
         for key, metric in (("ops_per_sec", "nfs_ops_per_sec"), ("rtt_ms", "nfs_rtt_ms"),
                             ("exe_ms", "nfs_exe_ms"), ("read_ms", "nfs_read_ms"),
                             ("write_ms", "nfs_write_ms"), ("retrans", "nfs_retrans"),
-                            ("timeouts", "nfs_timeouts")):
+                            ("timeouts", "nfs_timeouts"), ("tx_bps", "nfs_tx_bps"),
+                            ("rx_bps", "nfs_rx_bps")):
             if isinstance(nfs.get(key), (int, float)):
                 row(metric, mount_res, nfs[key])
         if isinstance(nfs.get("exe_ms"), (int, float)):