feat(host_agent): NFS delay per mount, read from /proc/self/mountstats (#5996)
CI / lint (push) Successful in 2s
CI / unit (push) Failing after 1m32s
CI / integration (push) Successful in 1m31s
CI / publish (push) Skipped

Each sample carries each NFS mount's requests since the last one: how many
a second, their round trip and execute time (READ and WRITE on their own),
and how many were sent again or hit a major timeout. mountstats is the
kernel's own count and never touches the share, so it reads while one is
stuck. Steward stores them per mount and the slowest on the host as
nfs_exe_max_ms, for an "NFS slow" rule and the host page's NFS chart beside
mounts_stalled; each mount's bar says what a request took.

Agent 1.22.0 (which also carries the backlog order from 41fc807).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-17 09:46:14 -04:00
co-authored by Claude Opus 5.5
parent 2df8431de9
commit 3f9aae7f46
12 changed files with 282 additions and 13 deletions
+22 -1
View File
@@ -36,6 +36,7 @@ from .metrics_query import (
SOURCE_MODULE,
_history_for_host,
_latest_metrics_for_host,
has_nfs,
)
host_agent_bp = Blueprint("host_agent", __name__, template_folder="templates")
@@ -290,6 +291,25 @@ def _expand_sample_to_metrics(
row("disk_used_pct_worst", host_name, worst_pct)
row("mounts_stalled", host_name, stalled)
# NFS delay per mount (agent ≥ 1.22.0, #5996): what its requests took
# since the last sample. The host's slowest mount is its own metric, for
# a rule; recorded only when some mount finished a request, since a mount
# with none, idle or stuck, has no delay to give (mounts_stalled covers
# stuck).
slowest = None
for nfs in sample.get("nfs") or []:
mount_res = f"{host_name}:{nfs['mount']}"
for key, metric in (("ops_per_sec", "nfs_ops_per_sec"), ("rtt_ms", "nfs_rtt_ms"),
("exe_ms", "nfs_exe_ms"), ("read_ms", "nfs_read_ms"),
("write_ms", "nfs_write_ms"), ("retrans", "nfs_retrans"),
("timeouts", "nfs_timeouts")):
if isinstance(nfs.get(key), (int, float)):
row(metric, mount_res, nfs[key])
if isinstance(nfs.get("exe_ms"), (int, float)):
slowest = max(slowest or 0.0, nfs["exe_ms"])
if slowest is not None:
row("nfs_exe_max_ms", host_name, slowest)
# Per-core CPU. Sub-resources carry a ':' so the fleet widget filters them out.
for i, core_pct in enumerate(sample.get("cpu_cores") or []):
if core_pct is not None:
@@ -883,9 +903,10 @@ async def host_detail(host_id: str):
select(Host).where(Host.id == host_id))).scalar_one_or_none()
if host is None:
return _error(404, "not_found")
nfs = await has_nfs(session, host.name)
return await render_template(
"host_detail.html",
host=host, current_range=range_key, range_options=RANGE_OPTIONS,
host=host, current_range=range_key, range_options=RANGE_OPTIONS, has_nfs=nfs,
poll_seconds=_DETAIL_POLL_SECONDS, chart_poll_seconds=_CHART_POLL_SECONDS,
)