feat(host_agent): NFS delay per mount, read from /proc/self/mountstats (#5996)

Each sample carries each NFS mount's requests since the last one: how many
a second, their round trip and execute time (READ and WRITE on their own),
and how many were sent again or hit a major timeout. mountstats is the
kernel's own count and never touches the share, so it reads while one is
stuck. Steward stores them per mount and the slowest on the host as
nfs_exe_max_ms, for an "NFS slow" rule and the host page's NFS chart beside
mounts_stalled; each mount's bar says what a request took.

Agent 1.22.0 (which also carries the backlog order from 41fc807).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/agent.py

@@ -92,6 +92,7 @@ UPTIME_PATH = "/proc/uptime"
 OS_RELEASE_PATH = "/etc/os-release"
 NETDEV_PATH = "/proc/net/dev"
 DISKSTATS_PATH = "/proc/diskstats"
+MOUNTSTATS_PATH = "/proc/self/mountstats"
 HWMON_DIR = "/sys/class/hwmon"
 # Where the kernel lists virtual network devices (bridges, veth, docker0,
 # vxlan, tun, VLANs, bonds) and the block devices (a disk has a `device`).
@@ -394,6 +395,86 @@ def collect_diskio_raw() -> dict:
     return _physical_only(out, _virtual_disk)
 
 
+# ─── NFS delay per mount (#5996) ─────────────────────────────────────────────
+#
+# The kernel's own count of every NFS request each mount has finished, read
+# from /proc/self/mountstats. Reading it never touches the share, so it answers
+# while one is stuck. Each op's line is cumulative: ops, transmissions, major
+# timeouts, bytes sent, bytes received, then queue, round-trip and execute
+# time in ms. A request is counted when it finishes, so a share that is stuck
+# shows nothing until it comes back, and then its long requests at once
+# (mounts_stalled says it is stuck meanwhile).
+
+
+def read_mountstats(text: str) -> dict:
+    """{mount point: {op: (ops, transmissions, timeouts, queue_ms, rtt_ms,
+    execute_ms)}} for each NFS mount in a mountstats text."""
+    out: dict = {}
+    mount = None
+    in_ops = False
+    for line in text.splitlines():
+        if line.startswith("device "):
+            mount, in_ops = None, False
+            parts = line.split()
+            try:
+                at = parts.index("on")
+                fstype = parts[parts.index("fstype") + 1]
+            except (ValueError, IndexError):
+                continue
+            if fstype.startswith("nfs") and at + 1 < len(parts):
+                mount = parts[at + 1].replace("\\040", " ")
+                out[mount] = {}
+        elif mount is None:
+            continue
+        elif line.strip() == "per-op statistics":
+            in_ops = True
+        elif in_ops:
+            name, sep, rest = line.strip().partition(":")
+            nums = rest.split()
+            if sep and len(nums) >= 8 and all(n.isdigit() for n in nums[:8]):
+                ops, trans, timeouts, _sent, _recv, queue, rtt, exe = (int(n) for n in nums[:8])
+                out[mount][name] = (ops, trans, timeouts, queue, rtt, exe)
+    return out
+
+
+def collect_nfs_raw() -> dict:
+    try:
+        return read_mountstats(_read_file(MOUNTSTATS_PATH))
+    except OSError:
+        return {}
+
+
+def nfs_delays(cur: dict, prev: dict, dt: float) -> list:
+    """Each NFS mount's requests since the last sample: how many a second,
+    their average round trip and execute time (queueing included, what a
+    program waits), READ and WRITE on their own, and the retransmissions and
+    major timeouts among them. A mount new since the last sample, or whose
+    counters went back (remounted), is left for the next one."""
+    out: list = []
+    if dt <= 0:
+        return out
+    for mount, ops in cur.items():
+        before = prev.get(mount)
+        if before is None:
+            continue
+        d = {op: tuple(a - b for a, b in zip(v, before.get(op, (0,) * 6))) for op, v in ops.items()}
+        if any(x < 0 for v in d.values() for x in v):
+            continue
+        n = sum(v[0] for v in d.values())
+        row = {"mount": mount, "ops_per_sec": round(n / dt, 2),
+               "retrans": max(0, sum(v[1] - v[0] for v in d.values())),
+               "timeouts": sum(v[2] for v in d.values())}
+        if n:
+            row["rtt_ms"] = round(sum(v[4] for v in d.values()) / n, 2)
+            row["exe_ms"] = round(sum(v[5] for v in d.values()) / n, 2)
+        for op, key in (("READ", "read_ms"), ("WRITE", "write_ms")):
+            v = d.get(op)
+            if v and v[0]:
+                row[key] = round(v[5] / v[0], 2)
+        out.append(row)
+    return out
+
+
 def collect_temps() -> list:
     """Return [{label, celsius}] from /sys/class/hwmon (best-effort, may be empty)."""
     out: list = []
@@ -1789,7 +1870,13 @@ def build_sample(mounts: list[str], state: dict,
         {"device": n, "read_bps": round(rd, 1), "write_bps": round(wr, 1)}
         for n, (rd, wr) in _rates(disk_raw, state.get("disk", {}), dt).items()
     ]
-    state["net"], state["disk"], state["mono"] = net_raw, disk_raw, now_mono
+    try:
+        nfs_raw = collect_nfs_raw()
+    except Exception:
+        nfs_raw = {}
+    if nfs_raw:
+        sample["nfs"] = nfs_delays(nfs_raw, state.get("nfs", {}), dt)
+    state["net"], state["disk"], state["nfs"], state["mono"] = net_raw, disk_raw, nfs_raw, now_mono
 
     # Docker's status rides every sample, so Steward can tell "no containers"
     # (status ok, docker: []) from "couldn't ask" (unavailable / error).