fix(host_agent): a stalled NFS mount no longer freezes the agent (#5760)

#5760 "One stalled NFS mount freezes the whole agent: statvfs has no
deadline, so the host goes silent while it still answers ping". The agent
read every mount inline in its only loop; a hard-mounted share whose server
stops blocks statvfs in the kernel with no timeout. Each mount is now read
in a daemon thread under one 5 s deadline. One that misses it is sent as
stalled, and isn't probed again while its last probe is still blocked. A
mount that errors (a stale handle) is left out instead of emptying the list.

Steward counts them as mounts_stalled (alertable: "an NFS share stopped"),
and skips the fullest-disk figure when every mount stalled.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/__init__.py

@@ -63,10 +63,12 @@ def get_alert_metrics() -> dict[str, list[str]]:
     clock_off_secs is how far the host's clock is from Steward's, either way
     (agent ≥ 1.12.0): "clock drifting" is > 30. load_pct is load_1m as a %
     of the host's cores. inodes_used_pct is per mount (agent ≥ 1.13.0), and
-    disk_used_pct_worst is the fullest mount by space or inodes."""
+    disk_used_pct_worst is the fullest mount by space or inodes.
+    mounts_stalled counts the mounts that didn't answer the agent in time
+    (agent ≥ 1.14.0): "an NFS share stopped" is > 0."""
     return {"host_agent": [
         "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
         "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m",
         "load_pct",
-        "uptime_secs", "seconds_since_seen", "clock_off_secs",
+        "uptime_secs", "seconds_since_seen", "clock_off_secs", "mounts_stalled",
     ]}

plugins/host_agent/agent.py

@@ -14,6 +14,7 @@ import signal
 import socket
 import subprocess
 import sys
+import threading
 import time
 import urllib.error
 import urllib.request
@@ -182,25 +183,64 @@ def collect_memory() -> dict:
     }
 
 
-def collect_storage(mounts: list[str]) -> list[dict]:
+# A mount that hasn't answered in this long is reported stalled (#5760): a
+# hard-mounted NFS share whose server stops blocks statvfs in the kernel with
+# no timeout, and read inline that froze the whole agent, silent for a day on
+# a host that still answered ping.
+MOUNT_DEADLINE_SECONDS = 5.0
+# Mounts whose last probe is still blocked. They aren't probed again until it
+# returns, so a dead share costs one stuck thread, not one per sample.
+_stuck_mounts: dict = {}
+
+
+def _read_mount(m: str) -> dict:
+    u = shutil.disk_usage(m)
+    row = {"mount": m, "total_bytes": u.total, "used_bytes": u.used}
+    # Inodes too (1.13.0): a filesystem can run out of them with space
+    # to spare, and every create then fails with ENOSPC. Some (btrfs)
+    # report none, and are left without.
+    try:
+        st = os.statvfs(m)
+    except OSError:
+        st = None
+    if st is not None and st.f_files:
+        row["inodes_total"] = st.f_files
+        row["inodes_used"] = st.f_files - st.f_ffree
+    return row
+
+
+def collect_storage(mounts: list[str], deadline: float = MOUNT_DEADLINE_SECONDS) -> list[dict]:
+    """Each mount's space and inodes, read side by side in daemon threads under
+    one deadline; a mount that doesn't answer in time is {"mount", "stalled"}."""
+    probes: dict = {}
+    for m in mounts:
+        stuck = _stuck_mounts.get(m)
+        if stuck is not None and stuck.is_alive():
+            continue
+        _stuck_mounts.pop(m, None)
+        box: dict = {}
+
+        def probe(m=m, box=box):
+            try:
+                box["row"] = _read_mount(m)
+            except OSError:
+                pass       # gone, not ours to read, or a stale handle: left out
+        t = threading.Thread(target=probe, name=f"mount {m}", daemon=True)
+        t.start()
+        probes[m] = (t, box)
+    end = time.monotonic() + deadline
     out: list[dict] = []
     for m in mounts:
-        try:
-            u = shutil.disk_usage(m)
-        except (FileNotFoundError, PermissionError):
+        if m not in probes:
+            out.append({"mount": m, "stalled": True})
             continue
-        row = {"mount": m, "total_bytes": u.total, "used_bytes": u.used}
-        # Inodes too (1.13.0): a filesystem can run out of them with space
-        # to spare, and every create then fails with ENOSPC. Some (btrfs)
-        # report none, and are left without.
-        try:
-            st = os.statvfs(m)
-        except OSError:
-            st = None
-        if st is not None and st.f_files:
-            row["inodes_total"] = st.f_files
-            row["inodes_used"] = st.f_files - st.f_ffree
-        out.append(row)
+        t, box = probes[m]
+        t.join(max(0.0, end - time.monotonic()))
+        if t.is_alive():
+            _stuck_mounts[m] = t
+            out.append({"mount": m, "stalled": True})
+        elif "row" in box:
+            out.append(box["row"])
     return out
 
 

plugins/host_agent/routes.py

@@ -143,7 +143,13 @@ def _expand_sample_to_metrics(
     # inodes, every file create fails with space to spare, so the disk rules
     # and the overview, which read disk_used_pct_worst, see either.
     worst_pct = 0.0
+    stalled = 0
     for disk in sample.get("storage") or []:
+        # A mount that didn't answer in time has no figures (#5760); how many
+        # did is its own metric, so a rule can say "an NFS share stopped".
+        if disk.get("stalled"):
+            stalled += 1
+            continue
         total = disk.get("total_bytes", 0)
         used = disk.get("used_bytes", 0)
         pct = 100.0 * used / total if total else 0.0
@@ -158,7 +164,10 @@ def _expand_sample_to_metrics(
         if pct > worst_pct:
             worst_pct = pct
     if sample.get("storage"):
-        row("disk_used_pct_worst", host_name, worst_pct)
+        # Every mount stalled says nothing about how full they are.
+        if stalled < len(sample["storage"]):
+            row("disk_used_pct_worst", host_name, worst_pct)
+        row("mounts_stalled", host_name, stalled)
 
     # Per-core CPU. Sub-resources carry a ':' so the fleet widget filters them out.
     for i, core_pct in enumerate(sample.get("cpu_cores") or []):