feat(host_agent): NFS delay per mount, read from /proc/self/mountstats (#5996)

Each sample carries each NFS mount's requests since the last one: how many
a second, their round trip and execute time (READ and WRITE on their own),
and how many were sent again or hit a major timeout. mountstats is the
kernel's own count and never touches the share, so it reads while one is
stuck. Steward stores them per mount and the slowest on the host as
nfs_exe_max_ms, for an "NFS slow" rule and the host page's NFS chart beside
mounts_stalled; each mount's bar says what a request took.

Agent 1.22.0 (which also carries the backlog order from 41fc807).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/__init__.py

@@ -91,6 +91,15 @@ def get_alert_metrics() -> dict[str, list[str]]:
     the whole host ("<host>") or one interface ("<host>:net:<iface>") or disk
     ("<host>:diskio:<device>").
 
+    NFS delay (agent ≥ 1.22.0, #5996), per NFS mount on "<host>:<mount>",
+    over the requests it finished since the last sample: nfs_exe_ms is the
+    average time a program waited on one (queueing included), nfs_rtt_ms the
+    server's round trip, nfs_read_ms / nfs_write_ms the same for reads and
+    writes alone, nfs_ops_per_sec how many, and nfs_retrans / nfs_timeouts the
+    retransmissions and major timeouts among them. nfs_exe_max_ms is the
+    host's slowest mount, for a covering rule: "NFS slow" is > 1000 for 3
+    checks. A healthy share answers in a few ms.
+
     Drive health (agent ≥ 1.20.0, #5789), read hourly with smartctl by the
     root helper and judged in drives.py, per drive on "<host>:drive:<device>":
     drive_failing is 1 when the drive fails its own self-assessment, raises
@@ -115,6 +124,8 @@ def get_alert_metrics() -> dict[str, list[str]]:
         "agent_rss_bytes", "agent_cpu_pct",
         "updates_pending", "updates_security", "reboot_required", "maintenance_failed",
         "temp_c_max", "temp_c", "net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
+        "nfs_exe_max_ms", "nfs_exe_ms", "nfs_rtt_ms", "nfs_read_ms", "nfs_write_ms",
+        "nfs_ops_per_sec", "nfs_retrans", "nfs_timeouts",
         *DRIVE_METRICS,
     ]}
 

plugins/host_agent/agent.py

@@ -92,6 +92,7 @@ UPTIME_PATH = "/proc/uptime"
 OS_RELEASE_PATH = "/etc/os-release"
 NETDEV_PATH = "/proc/net/dev"
 DISKSTATS_PATH = "/proc/diskstats"
+MOUNTSTATS_PATH = "/proc/self/mountstats"
 HWMON_DIR = "/sys/class/hwmon"
 # Where the kernel lists virtual network devices (bridges, veth, docker0,
 # vxlan, tun, VLANs, bonds) and the block devices (a disk has a `device`).
@@ -394,6 +395,86 @@ def collect_diskio_raw() -> dict:
     return _physical_only(out, _virtual_disk)
 
 
+# ─── NFS delay per mount (#5996) ─────────────────────────────────────────────
+#
+# The kernel's own count of every NFS request each mount has finished, read
+# from /proc/self/mountstats. Reading it never touches the share, so it answers
+# while one is stuck. Each op's line is cumulative: ops, transmissions, major
+# timeouts, bytes sent, bytes received, then queue, round-trip and execute
+# time in ms. A request is counted when it finishes, so a share that is stuck
+# shows nothing until it comes back, and then its long requests at once
+# (mounts_stalled says it is stuck meanwhile).
+
+
+def read_mountstats(text: str) -> dict:
+    """{mount point: {op: (ops, transmissions, timeouts, queue_ms, rtt_ms,
+    execute_ms)}} for each NFS mount in a mountstats text."""
+    out: dict = {}
+    mount = None
+    in_ops = False
+    for line in text.splitlines():
+        if line.startswith("device "):
+            mount, in_ops = None, False
+            parts = line.split()
+            try:
+                at = parts.index("on")
+                fstype = parts[parts.index("fstype") + 1]
+            except (ValueError, IndexError):
+                continue
+            if fstype.startswith("nfs") and at + 1 < len(parts):
+                mount = parts[at + 1].replace("\\040", " ")
+                out[mount] = {}
+        elif mount is None:
+            continue
+        elif line.strip() == "per-op statistics":
+            in_ops = True
+        elif in_ops:
+            name, sep, rest = line.strip().partition(":")
+            nums = rest.split()
+            if sep and len(nums) >= 8 and all(n.isdigit() for n in nums[:8]):
+                ops, trans, timeouts, _sent, _recv, queue, rtt, exe = (int(n) for n in nums[:8])
+                out[mount][name] = (ops, trans, timeouts, queue, rtt, exe)
+    return out
+
+
+def collect_nfs_raw() -> dict:
+    try:
+        return read_mountstats(_read_file(MOUNTSTATS_PATH))
+    except OSError:
+        return {}
+
+
+def nfs_delays(cur: dict, prev: dict, dt: float) -> list:
+    """Each NFS mount's requests since the last sample: how many a second,
+    their average round trip and execute time (queueing included, what a
+    program waits), READ and WRITE on their own, and the retransmissions and
+    major timeouts among them. A mount new since the last sample, or whose
+    counters went back (remounted), is left for the next one."""
+    out: list = []
+    if dt <= 0:
+        return out
+    for mount, ops in cur.items():
+        before = prev.get(mount)
+        if before is None:
+            continue
+        d = {op: tuple(a - b for a, b in zip(v, before.get(op, (0,) * 6))) for op, v in ops.items()}
+        if any(x < 0 for v in d.values() for x in v):
+            continue
+        n = sum(v[0] for v in d.values())
+        row = {"mount": mount, "ops_per_sec": round(n / dt, 2),
+               "retrans": max(0, sum(v[1] - v[0] for v in d.values())),
+               "timeouts": sum(v[2] for v in d.values())}
+        if n:
+            row["rtt_ms"] = round(sum(v[4] for v in d.values()) / n, 2)
+            row["exe_ms"] = round(sum(v[5] for v in d.values()) / n, 2)
+        for op, key in (("READ", "read_ms"), ("WRITE", "write_ms")):
+            v = d.get(op)
+            if v and v[0]:
+                row[key] = round(v[5] / v[0], 2)
+        out.append(row)
+    return out
+
+
 def collect_temps() -> list:
     """Return [{label, celsius}] from /sys/class/hwmon (best-effort, may be empty)."""
     out: list = []
@@ -1789,7 +1870,13 @@ def build_sample(mounts: list[str], state: dict,
         {"device": n, "read_bps": round(rd, 1), "write_bps": round(wr, 1)}
         for n, (rd, wr) in _rates(disk_raw, state.get("disk", {}), dt).items()
     ]
-    state["net"], state["disk"], state["mono"] = net_raw, disk_raw, now_mono
+    try:
+        nfs_raw = collect_nfs_raw()
+    except Exception:
+        nfs_raw = {}
+    if nfs_raw:
+        sample["nfs"] = nfs_delays(nfs_raw, state.get("nfs", {}), dt)
+    state["net"], state["disk"], state["nfs"], state["mono"] = net_raw, disk_raw, nfs_raw, now_mono
 
     # Docker's status rides every sample, so Steward can tell "no containers"
     # (status ok, docker: []) from "couldn't ask" (unavailable / error).

plugins/host_agent/metrics_query.py

@@ -21,10 +21,20 @@ SOURCE_MODULE = "host_agent"
 HISTORY_METRICS = (
     "cpu_pct", "mem_used_pct", "disk_used_pct_worst", "load_1m",
     "net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
-    "temp_c_max", "psi_mem_some_avg10",
+    "temp_c_max", "psi_mem_some_avg10", "nfs_exe_max_ms", "mounts_stalled",
 )
 
 
+async def has_nfs(session, host_name: str) -> bool:
+    """Whether the host's agent has reported an NFS mount's delay (#5996)."""
+    from steward.models.metrics import PluginMetricLatest
+    return (await session.execute(
+        select(PluginMetricLatest.metric_name).where(
+            PluginMetricLatest.source_module == SOURCE_MODULE,
+            PluginMetricLatest.resource_name.startswith(host_name + ":"),
+            PluginMetricLatest.metric_name == "nfs_ops_per_sec").limit(1))).first() is not None
+
+
 async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[str, float]]:
     """{resource_name: {metric: value}} — latest sample per (resource, metric) for
     a host + its sub-resources, from plugin_metric_latest (#5208). A host whose

plugins/host_agent/routes.py

@@ -36,6 +36,7 @@ from .metrics_query import (
     SOURCE_MODULE,
     _history_for_host,
     _latest_metrics_for_host,
+    has_nfs,
 )
 
 host_agent_bp = Blueprint("host_agent", __name__, template_folder="templates")
@@ -290,6 +291,25 @@ def _expand_sample_to_metrics(
             row("disk_used_pct_worst", host_name, worst_pct)
         row("mounts_stalled", host_name, stalled)
 
+    # NFS delay per mount (agent ≥ 1.22.0, #5996): what its requests took
+    # since the last sample. The host's slowest mount is its own metric, for
+    # a rule; recorded only when some mount finished a request, since a mount
+    # with none, idle or stuck, has no delay to give (mounts_stalled covers
+    # stuck).
+    slowest = None
+    for nfs in sample.get("nfs") or []:
+        mount_res = f"{host_name}:{nfs['mount']}"
+        for key, metric in (("ops_per_sec", "nfs_ops_per_sec"), ("rtt_ms", "nfs_rtt_ms"),
+                            ("exe_ms", "nfs_exe_ms"), ("read_ms", "nfs_read_ms"),
+                            ("write_ms", "nfs_write_ms"), ("retrans", "nfs_retrans"),
+                            ("timeouts", "nfs_timeouts")):
+            if isinstance(nfs.get(key), (int, float)):
+                row(metric, mount_res, nfs[key])
+        if isinstance(nfs.get("exe_ms"), (int, float)):
+            slowest = max(slowest or 0.0, nfs["exe_ms"])
+    if slowest is not None:
+        row("nfs_exe_max_ms", host_name, slowest)
+
     # Per-core CPU. Sub-resources carry a ':' so the fleet widget filters them out.
     for i, core_pct in enumerate(sample.get("cpu_cores") or []):
         if core_pct is not None:
@@ -883,9 +903,10 @@ async def host_detail(host_id: str):
             select(Host).where(Host.id == host_id))).scalar_one_or_none()
         if host is None:
             return _error(404, "not_found")
+        nfs = await has_nfs(session, host.name)
     return await render_template(
         "host_detail.html",
-        host=host, current_range=range_key, range_options=RANGE_OPTIONS,
+        host=host, current_range=range_key, range_options=RANGE_OPTIONS, has_nfs=nfs,
         poll_seconds=_DETAIL_POLL_SECONDS, chart_poll_seconds=_CHART_POLL_SECONDS,
     )
 

plugins/host_agent/templates/_host_metrics.html

@@ -114,6 +114,17 @@
       <span style="color:var(--text-muted);">{{ m.get('disk_used_bytes')|bytes }} / {{ m.get('disk_total_bytes')|bytes }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %}</span>
     </div>
     {{ bar(m.get('disk_used_pct'), 'disk') }}
+    {% if m.get('nfs_ops_per_sec') is not none %}
+    {# NFS delay (#5996): what a request took since the agent's last sample. #}
+    {% set exe = m.get('nfs_exe_ms') %}
+    <div style="font-size:0.72rem;color:var(--text-dim);margin-top:0.2rem;">
+      NFS {% if m.get('nfs_ops_per_sec') > 0 and exe is not none -%}
+      <span class="{{ 'tone-crit' if exe >= 1000 else ('tone-warn' if exe >= 100 else '') }}">{{ "%.1f"|format(exe) }} ms a request</span> · {{ "%.3g"|format(m.get('nfs_ops_per_sec')) }}/s
+      {%- else %}idle{% endif %}
+      {%- if m.get('nfs_retrans') %} · <span class="tone-warn">{{ m.get('nfs_retrans')|int }} sent again</span>{% endif %}
+      {%- if m.get('nfs_timeouts') %} · <span class="tone-crit">{{ m.get('nfs_timeouts')|int }} timed out</span>{% endif %}
+    </div>
+    {% endif %}
   </div>
   {% endfor %}
 </div>

plugins/host_agent/templates/host_detail.html

@@ -35,6 +35,12 @@
     <div class="section-title" style="margin-bottom:0.5rem;">Load &amp; pressure — last <span class="hm-chart-range">{{ current_range }}</span></div>
     <div style="height:220px;"><canvas id="chart-pressure"></canvas></div>
   </div>
+  {% if has_nfs %}
+  <div class="card" style="margin-bottom:0;">
+    <div class="section-title" style="margin-bottom:0.5rem;">NFS delay (ms) — last <span class="hm-chart-range">{{ current_range }}</span></div>
+    <div style="height:220px;"><canvas id="chart-nfs"></canvas></div>
+  </div>
+  {% endif %}
 </div>
 {# Hidden poller: fetches the history series and applies it to the charts above
    (no visible swap). Lazy on load → never blocks paint; refreshed slowly + on
@@ -79,6 +85,10 @@
       { key: "psi_mem_some_avg10", label: "Mem PSI %", color: stewardColor("--red") },
       { key: "temp_c_max", label: "Temp °C", color: stewardColor("--yellow") },
     ] },
+    "chart-nfs": { ymax: undefined, ds: [
+      { key: "nfs_exe_max_ms", label: "Slowest mount, ms a request", color: stewardColor("--blue") },
+      { key: "mounts_stalled", label: "Mounts not answering", color: stewardColor("--red") },
+    ] },
   };
   const charts = {};
   for (const id of Object.keys(defs)) {