feat(traffic): each container's network rate, and each NFS mount's bytes, so a host's traffic has a sender (#6115)

Docker02's outbound traffic tops out the internet connection and Steward
couldn't say who sends it: a container's counters were kept only as a
running total, and NFS traffic belongs to no container.

- Docker ingest turns each running container's cumulative network counters
  into net_rx_bps / net_tx_bps on "<host>/<container>" (net_rates), counting
  from the stored row so a restart of Steward loses nothing; a container
  new, recreated or restarted has no rate until its next snapshot.
- Agent 1.25.0 reads the bytes sent and received per NFS mount from
  mountstats (already parsed for the delays) as tx_bps / rx_bps; Steward
  stores them as nfs_tx_bps / nfs_rx_bps on "<host>:<mount>".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/ingest.py

@@ -30,6 +30,32 @@ def _parse_started_at(value) -> datetime | None:
         return None
 
 
+def net_rates(before: dict, snapshots: list) -> list:
+    """(recorded_at, name, rx_bps, tx_bps) for each running container in each
+    snapshot, from its cumulative network counters against the last ones seen
+    for it: `before` is {name: (at, container_id, rx, tx)} from the stored rows,
+    and each snapshot moves it on. A container new since then, recreated under
+    the same name, or whose counters went back (restarted) has no rate until
+    its next snapshot (#6115)."""
+    last = dict(before)
+    out = []
+    for at, containers in snapshots:
+        for c in containers:
+            name, rx, tx = c.get("name"), c.get("net_rx_bytes"), c.get("net_tx_bytes")
+            if not name or c.get("status") != "running" or rx is None or tx is None:
+                continue
+            cid = c.get("container_id") or ""
+            prev = last.get(name)
+            last[name] = (at, cid, rx, tx)
+            if prev is None or prev[1] != cid or prev[2] is None or prev[3] is None:
+                continue
+            dt = (at - prev[0]).total_seconds()
+            if dt <= 0 or rx < prev[2] or tx < prev[3]:
+                continue
+            out.append((at, name, (rx - prev[2]) / dt, (tx - prev[3]) / dt))
+    return out
+
+
 def _derive_events(old_state: dict, new_containers: list) -> list:
     """Diff a fresh snapshot against stored per-container state → lifecycle events.
 
@@ -346,7 +372,8 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
     one entry per ingested sample carrying docker data (usually one; more when
     the agent flushes a backlog). Every snapshot's running containers record
     cpu_pct, mem_used_pct, mem_used_bytes and mem_cache_bytes into the one
-    metric store, at the snapshot's time; the newest snapshot drives current container state and
+    metric store, at the snapshot's time, and net_rx_bps / net_tx_bps from
+    their counters (net_rates); the newest snapshot drives current container state and
     lifecycle-event derivation. `swarm` is the newest
     sample's swarm object (or None off managers) — persisted when present.
     `disk` is the newest sample's /system/df summary (or None on Docker-less
@@ -394,6 +421,14 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
     # the one metric store (#5752): resource "<host>/<container>", so
     # containers of the same name on different hosts don't collide. Recorded
     # with the alert batch below, at each snapshot's own time.
+    # Snapshot of stored per-container state BEFORE the upsert overwrites it —
+    # the network counters each rate counts from, and the lifecycle-event diff.
+    # Skip event derivation on the host's first snapshot (empty state) so we
+    # don't emit a start per pre-existing container.
+    old_rows = (await session.execute(
+        select(DockerContainer).where(DockerContainer.host_id == host.id)
+    )).scalars().all()
+
     batch = MetricBatch()
     for recorded_at, containers in ordered:
         for c in containers:
@@ -404,14 +439,16 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
                 if c.get(metric) is not None:
                     batch.add(source_module="docker", resource_name=resource,
                               metric_name=metric, value=c[metric], recorded_at=recorded_at)
+    # Bytes a second each container receives and sends, so a host's traffic
+    # can be put down to the container behind it (#6115).
+    before = {r.name: (r.scraped_at, r.container_id, r.net_rx_bytes, r.net_tx_bytes) for r in old_rows}
+    for recorded_at, name, rx_bps, tx_bps in net_rates(before, ordered):
+        resource = f"{host.name}/{name}"
+        batch.add(source_module="docker", resource_name=resource,
+                  metric_name="net_rx_bps", value=round(rx_bps, 1), recorded_at=recorded_at)
+        batch.add(source_module="docker", resource_name=resource,
+                  metric_name="net_tx_bps", value=round(tx_bps, 1), recorded_at=recorded_at)
     await mark("metrics")
-
-    # Snapshot of stored per-container state BEFORE the upsert overwrites it —
-    # used to diff lifecycle events. Skip event derivation on the host's first
-    # snapshot (empty state) so we don't emit a start per pre-existing container.
-    old_rows = (await session.execute(
-        select(DockerContainer).where(DockerContainer.host_id == host.id)
-    )).scalars().all()
     old_state = {
         r.name: {"status": r.status, "health": r.health,
                  "oom_killed": r.oom_killed, "exit_code": r.exit_code,

plugins/host_agent/__init__.py

@@ -96,7 +96,9 @@ def get_alert_metrics() -> dict[str, list[str]]:
     average time a program waited on one (queueing included), nfs_rtt_ms the
     server's round trip, nfs_read_ms / nfs_write_ms the same for reads and
     writes alone, nfs_ops_per_sec how many, and nfs_retrans / nfs_timeouts the
-    retransmissions and major timeouts among them. nfs_exe_max_ms is the
+    retransmissions and major timeouts among them; nfs_tx_bps / nfs_rx_bps
+    (agent ≥ 1.25.0, #6115) are the bytes a second it sent and received, its
+    share of the host's network traffic. nfs_exe_max_ms is the
     host's slowest mount, for a covering rule: "NFS slow" is > 1000 for 3
     checks. A healthy share answers in a few ms.
 
@@ -125,7 +127,7 @@ def get_alert_metrics() -> dict[str, list[str]]:
         "updates_pending", "updates_security", "reboot_required", "maintenance_failed",
         "temp_c_max", "temp_c", "net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
         "nfs_exe_max_ms", "nfs_exe_ms", "nfs_rtt_ms", "nfs_read_ms", "nfs_write_ms",
-        "nfs_ops_per_sec", "nfs_retrans", "nfs_timeouts",
+        "nfs_ops_per_sec", "nfs_retrans", "nfs_timeouts", "nfs_tx_bps", "nfs_rx_bps",
         *DRIVE_METRICS,
     ]}
 

plugins/host_agent/agent.py

@@ -22,7 +22,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.24.0"
+AGENT_VERSION = "1.25.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -408,7 +408,8 @@ def collect_diskio_raw() -> dict:
 
 def read_mountstats(text: str) -> dict:
     """{mount point: {op: (ops, transmissions, timeouts, queue_ms, rtt_ms,
-    execute_ms)}} for each NFS mount in a mountstats text."""
+    execute_ms, bytes_sent, bytes_received)}} for each NFS mount in a
+    mountstats text."""
     out: dict = {}
     mount = None
     in_ops = False
@@ -432,8 +433,8 @@ def read_mountstats(text: str) -> dict:
             name, sep, rest = line.strip().partition(":")
             nums = rest.split()
             if sep and len(nums) >= 8 and all(n.isdigit() for n in nums[:8]):
-                ops, trans, timeouts, _sent, _recv, queue, rtt, exe = (int(n) for n in nums[:8])
-                out[mount][name] = (ops, trans, timeouts, queue, rtt, exe)
+                ops, trans, timeouts, sent, recv, queue, rtt, exe = (int(n) for n in nums[:8])
+                out[mount][name] = (ops, trans, timeouts, queue, rtt, exe, sent, recv)
     return out
 
 
@@ -448,8 +449,9 @@ def nfs_delays(cur: dict, prev: dict, dt: float) -> list:
     """Each NFS mount's requests since the last sample: how many a second,
     their average round trip and execute time (queueing included, what a
     program waits), READ and WRITE on their own, and the retransmissions and
-    major timeouts among them. A mount new since the last sample, or whose
-    counters went back (remounted), is left for the next one."""
+    major timeouts among them, and the bytes a second it sent and received
+    on the wire (#6115). A mount new since the last sample, or whose counters
+    went back (remounted), is left for the next one."""
     out: list = []
     if dt <= 0:
         return out
@@ -457,13 +459,15 @@ def nfs_delays(cur: dict, prev: dict, dt: float) -> list:
         before = prev.get(mount)
         if before is None:
             continue
-        d = {op: tuple(a - b for a, b in zip(v, before.get(op, (0,) * 6))) for op, v in ops.items()}
+        d = {op: tuple(a - b for a, b in zip(v, before.get(op, (0,) * len(v)))) for op, v in ops.items()}
         if any(x < 0 for v in d.values() for x in v):
             continue
         n = sum(v[0] for v in d.values())
         row = {"mount": mount, "ops_per_sec": round(n / dt, 2),
                "retrans": max(0, sum(v[1] - v[0] for v in d.values())),
-               "timeouts": sum(v[2] for v in d.values())}
+               "timeouts": sum(v[2] for v in d.values()),
+               "tx_bps": round(sum(v[6] for v in d.values()) / dt, 1),
+               "rx_bps": round(sum(v[7] for v in d.values()) / dt, 1)}
         if n:
             row["rtt_ms"] = round(sum(v[4] for v in d.values()) / n, 2)
             row["exe_ms"] = round(sum(v[5] for v in d.values()) / n, 2)

plugins/host_agent/routes.py

@@ -292,7 +292,8 @@ def _expand_sample_to_metrics(
         row("mounts_stalled", host_name, stalled)
 
     # NFS delay per mount (agent ≥ 1.22.0, #5996): what its requests took
-    # since the last sample. The host's slowest mount is its own metric, for
+    # since the last sample, and (agent ≥ 1.25.0, #6115) the bytes a second
+    # it sent and received. The host's slowest mount is its own metric, for
     # a rule; recorded only when some mount finished a request, since a mount
     # with none, idle or stuck, has no delay to give (mounts_stalled covers
     # stuck).
@@ -302,7 +303,8 @@ def _expand_sample_to_metrics(
         for key, metric in (("ops_per_sec", "nfs_ops_per_sec"), ("rtt_ms", "nfs_rtt_ms"),
                             ("exe_ms", "nfs_exe_ms"), ("read_ms", "nfs_read_ms"),
                             ("write_ms", "nfs_write_ms"), ("retrans", "nfs_retrans"),
-                            ("timeouts", "nfs_timeouts")):
+                            ("timeouts", "nfs_timeouts"), ("tx_bps", "nfs_tx_bps"),
+                            ("rx_bps", "nfs_rx_bps")):
             if isinstance(nfs.get(key), (int, float)):
                 row(metric, mount_res, nfs[key])
         if isinstance(nfs.get("exe_ms"), (int, float)):