fix(host_agent): a stalled NFS mount no longer freezes the agent (#5760)

#5760 "One stalled NFS mount freezes the whole agent: statvfs has no
deadline, so the host goes silent while it still answers ping". The agent
read every mount inline in its only loop; a hard-mounted share whose server
stops blocks statvfs in the kernel with no timeout. Each mount is now read
in a daemon thread under one 5 s deadline. One that misses it is sent as
stalled, and isn't probed again while its last probe is still blocked. A
mount that errors (a stale handle) is left out instead of emptying the list.

Steward counts them as mounts_stalled (alertable: "an NFS share stopped"),
and skips the fullest-disk figure when every mount stalled.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/agent.py

@@ -14,6 +14,7 @@ import signal
 import socket
 import subprocess
 import sys
+import threading
 import time
 import urllib.error
 import urllib.request
@@ -182,25 +183,64 @@ def collect_memory() -> dict:
     }
 
 
-def collect_storage(mounts: list[str]) -> list[dict]:
+# A mount that hasn't answered in this long is reported stalled (#5760): a
+# hard-mounted NFS share whose server stops blocks statvfs in the kernel with
+# no timeout, and read inline that froze the whole agent, silent for a day on
+# a host that still answered ping.
+MOUNT_DEADLINE_SECONDS = 5.0
+# Mounts whose last probe is still blocked. They aren't probed again until it
+# returns, so a dead share costs one stuck thread, not one per sample.
+_stuck_mounts: dict = {}
+
+
+def _read_mount(m: str) -> dict:
+    u = shutil.disk_usage(m)
+    row = {"mount": m, "total_bytes": u.total, "used_bytes": u.used}
+    # Inodes too (1.13.0): a filesystem can run out of them with space
+    # to spare, and every create then fails with ENOSPC. Some (btrfs)
+    # report none, and are left without.
+    try:
+        st = os.statvfs(m)
+    except OSError:
+        st = None
+    if st is not None and st.f_files:
+        row["inodes_total"] = st.f_files
+        row["inodes_used"] = st.f_files - st.f_ffree
+    return row
+
+
+def collect_storage(mounts: list[str], deadline: float = MOUNT_DEADLINE_SECONDS) -> list[dict]:
+    """Each mount's space and inodes, read side by side in daemon threads under
+    one deadline; a mount that doesn't answer in time is {"mount", "stalled"}."""
+    probes: dict = {}
+    for m in mounts:
+        stuck = _stuck_mounts.get(m)
+        if stuck is not None and stuck.is_alive():
+            continue
+        _stuck_mounts.pop(m, None)
+        box: dict = {}
+
+        def probe(m=m, box=box):
+            try:
+                box["row"] = _read_mount(m)
+            except OSError:
+                pass       # gone, not ours to read, or a stale handle: left out
+        t = threading.Thread(target=probe, name=f"mount {m}", daemon=True)
+        t.start()
+        probes[m] = (t, box)
+    end = time.monotonic() + deadline
     out: list[dict] = []
     for m in mounts:
-        try:
-            u = shutil.disk_usage(m)
-        except (FileNotFoundError, PermissionError):
+        if m not in probes:
+            out.append({"mount": m, "stalled": True})
             continue
-        row = {"mount": m, "total_bytes": u.total, "used_bytes": u.used}
-        # Inodes too (1.13.0): a filesystem can run out of them with space
-        # to spare, and every create then fails with ENOSPC. Some (btrfs)
-        # report none, and are left without.
-        try:
-            st = os.statvfs(m)
-        except OSError:
-            st = None
-        if st is not None and st.f_files:
-            row["inodes_total"] = st.f_files
-            row["inodes_used"] = st.f_files - st.f_ffree
-        out.append(row)
+        t, box = probes[m]
+        t.join(max(0.0, end - time.monotonic()))
+        if t.is_alive():
+            _stuck_mounts[m] = t
+            out.append({"mount": m, "stalled": True})
+        elif "row" in box:
+            out.append(box["row"])
     return out