fix(host_agent): a stalled NFS mount no longer freezes the agent (#5760)

#5760 "One stalled NFS mount freezes the whole agent: statvfs has no
deadline, so the host goes silent while it still answers ping". The agent
read every mount inline in its only loop; a hard-mounted share whose server
stops blocks statvfs in the kernel with no timeout. Each mount is now read
in a daemon thread under one 5 s deadline. One that misses it is sent as
stalled, and isn't probed again while its last probe is still blocked. A
mount that errors (a stale handle) is left out instead of emptying the list.

Steward counts them as mounts_stalled (alertable: "an NFS share stopped"),
and skips the fullest-disk figure when every mount stalled.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/plugins/host-agent-design.md

@@ -116,6 +116,10 @@ Python 3.8+ stdlib only. `urllib.request` for the POST, plus `json`, `os`, `shut
 - **SIGTERM** (from systemd) → finish current poll cycle → flush buffer one last time, with a `stopping` event saying whether the host is going down too (`systemctl is-system-running` says `stopping`) → exit.
 - **Systemd restart policy** → `Restart=on-failure`, `RestartSec=10`.
 
+### Mounts that don't answer (1.14.0, #5760)
+
+Each mount is read in a daemon thread under one 5 s deadline. A hard-mounted NFS share whose server stops blocks `statvfs` in the kernel with no timeout; read inline, that froze the whole agent. A mount that misses the deadline is sent as `{"mount", "stalled": true}`, and isn't probed again while its last probe is still blocked. Steward counts them as `mounts_stalled`.
+
 ### What happened to the host (1.14.0, #5756)
 
 A post may carry `events`, said once each, with the next post that gets through:

plugins/host_agent/__init__.py

@@ -63,10 +63,12 @@ def get_alert_metrics() -> dict[str, list[str]]:
     clock_off_secs is how far the host's clock is from Steward's, either way
     (agent ≥ 1.12.0): "clock drifting" is > 30. load_pct is load_1m as a %
     of the host's cores. inodes_used_pct is per mount (agent ≥ 1.13.0), and
-    disk_used_pct_worst is the fullest mount by space or inodes."""
+    disk_used_pct_worst is the fullest mount by space or inodes.
+    mounts_stalled counts the mounts that didn't answer the agent in time
+    (agent ≥ 1.14.0): "an NFS share stopped" is > 0."""
     return {"host_agent": [
         "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
         "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m",
         "load_pct",
-        "uptime_secs", "seconds_since_seen", "clock_off_secs",
+        "uptime_secs", "seconds_since_seen", "clock_off_secs", "mounts_stalled",
     ]}

plugins/host_agent/agent.py

@@ -14,6 +14,7 @@ import signal
 import socket
 import subprocess
 import sys
+import threading
 import time
 import urllib.error
 import urllib.request
@@ -182,25 +183,64 @@ def collect_memory() -> dict:
     }
 
 
-def collect_storage(mounts: list[str]) -> list[dict]:
+# A mount that hasn't answered in this long is reported stalled (#5760): a
+# hard-mounted NFS share whose server stops blocks statvfs in the kernel with
+# no timeout, and read inline that froze the whole agent, silent for a day on
+# a host that still answered ping.
+MOUNT_DEADLINE_SECONDS = 5.0
+# Mounts whose last probe is still blocked. They aren't probed again until it
+# returns, so a dead share costs one stuck thread, not one per sample.
+_stuck_mounts: dict = {}
+
+
+def _read_mount(m: str) -> dict:
+    u = shutil.disk_usage(m)
+    row = {"mount": m, "total_bytes": u.total, "used_bytes": u.used}
+    # Inodes too (1.13.0): a filesystem can run out of them with space
+    # to spare, and every create then fails with ENOSPC. Some (btrfs)
+    # report none, and are left without.
+    try:
+        st = os.statvfs(m)
+    except OSError:
+        st = None
+    if st is not None and st.f_files:
+        row["inodes_total"] = st.f_files
+        row["inodes_used"] = st.f_files - st.f_ffree
+    return row
+
+
+def collect_storage(mounts: list[str], deadline: float = MOUNT_DEADLINE_SECONDS) -> list[dict]:
+    """Each mount's space and inodes, read side by side in daemon threads under
+    one deadline; a mount that doesn't answer in time is {"mount", "stalled"}."""
+    probes: dict = {}
+    for m in mounts:
+        stuck = _stuck_mounts.get(m)
+        if stuck is not None and stuck.is_alive():
+            continue
+        _stuck_mounts.pop(m, None)
+        box: dict = {}
+
+        def probe(m=m, box=box):
+            try:
+                box["row"] = _read_mount(m)
+            except OSError:
+                pass       # gone, not ours to read, or a stale handle: left out
+        t = threading.Thread(target=probe, name=f"mount {m}", daemon=True)
+        t.start()
+        probes[m] = (t, box)
+    end = time.monotonic() + deadline
     out: list[dict] = []
     for m in mounts:
-        try:
-            u = shutil.disk_usage(m)
-        except (FileNotFoundError, PermissionError):
+        if m not in probes:
+            out.append({"mount": m, "stalled": True})
             continue
-        row = {"mount": m, "total_bytes": u.total, "used_bytes": u.used}
-        # Inodes too (1.13.0): a filesystem can run out of them with space
-        # to spare, and every create then fails with ENOSPC. Some (btrfs)
-        # report none, and are left without.
-        try:
-            st = os.statvfs(m)
-        except OSError:
-            st = None
-        if st is not None and st.f_files:
-            row["inodes_total"] = st.f_files
-            row["inodes_used"] = st.f_files - st.f_ffree
-        out.append(row)
+        t, box = probes[m]
+        t.join(max(0.0, end - time.monotonic()))
+        if t.is_alive():
+            _stuck_mounts[m] = t
+            out.append({"mount": m, "stalled": True})
+        elif "row" in box:
+            out.append(box["row"])
     return out
 
 

plugins/host_agent/routes.py

@@ -143,7 +143,13 @@ def _expand_sample_to_metrics(
     # inodes, every file create fails with space to spare, so the disk rules
     # and the overview, which read disk_used_pct_worst, see either.
     worst_pct = 0.0
+    stalled = 0
     for disk in sample.get("storage") or []:
+        # A mount that didn't answer in time has no figures (#5760); how many
+        # did is its own metric, so a rule can say "an NFS share stopped".
+        if disk.get("stalled"):
+            stalled += 1
+            continue
         total = disk.get("total_bytes", 0)
         used = disk.get("used_bytes", 0)
         pct = 100.0 * used / total if total else 0.0
@@ -158,7 +164,10 @@ def _expand_sample_to_metrics(
         if pct > worst_pct:
             worst_pct = pct
     if sample.get("storage"):
-        row("disk_used_pct_worst", host_name, worst_pct)
+        # Every mount stalled says nothing about how full they are.
+        if stalled < len(sample["storage"]):
+            row("disk_used_pct_worst", host_name, worst_pct)
+        row("mounts_stalled", host_name, stalled)
 
     # Per-core CPU. Sub-resources carry a ':' so the fleet widget filters them out.
     for i, core_pct in enumerate(sample.get("cpu_cores") or []):

steward/core/formatting.py

@@ -339,6 +339,7 @@ METRICS: dict[str, tuple[str, str]] = {
     "uptime_secs": ("Uptime", "time since the host booted"),
     "seconds_since_seen": ("Time since the agent reported", "over 3 minutes means the agent is offline"),
     "clock_off_secs": ("Clock off by", "how far the host's clock is from Steward's, either way"),
+    "mounts_stalled": ("Mounts not answering", "mounts the agent couldn't read in 5 s, e.g. an NFS share whose server stopped"),
 }
 
 # Where a rule's metric comes from, as the picker and the rules list say it.

tests/plugins/host_agent/test_agent_collectors.py

@@ -109,3 +109,45 @@ def test_collect_metadata(tmp_path):
     assert meta["kernel"] == "6.8.0-45-generic"
     assert meta["arch"] == "x86_64"
     assert "Ubuntu 24.04" in meta["distro"]
+
+
+def test_a_mount_that_does_not_answer_is_reported_stalled_and_not_probed_again():
+    """#5760: statvfs on a stalled NFS share blocks with no timeout, and read
+    inline that froze the whole agent. The sample goes on without it, and a
+    probe still blocked isn't stacked on by the next sample."""
+    import threading
+    release = threading.Event()
+    calls: list[str] = []
+
+    def _usage(path):
+        calls.append(path)
+        if path == "/nfs/media":
+            release.wait(5)
+        return type("R", (), {"total": 1000, "used": 400, "free": 600})()
+
+    a._stuck_mounts.clear()
+    try:
+        with patch.object(a.shutil, "disk_usage", side_effect=_usage), \
+                patch.object(a.os, "statvfs", side_effect=OSError):
+            first = a.collect_storage(["/", "/nfs/media"], deadline=0.2)
+            second = a.collect_storage(["/", "/nfs/media"], deadline=0.2)
+        assert first == second == [
+            {"mount": "/", "total_bytes": 1000, "used_bytes": 400},
+            {"mount": "/nfs/media", "stalled": True},
+        ]
+        assert calls.count("/nfs/media") == 1
+    finally:
+        release.set()
+        a._stuck_mounts.clear()
+
+
+def test_a_mount_that_errors_is_left_out_not_the_whole_list():
+    def _usage(path):
+        if path == "/nfs/gone":
+            raise OSError(116, "Stale file handle")
+        return type("R", (), {"total": 10, "used": 1, "free": 9})()
+
+    with patch.object(a.shutil, "disk_usage", side_effect=_usage), \
+            patch.object(a.os, "statvfs", side_effect=OSError):
+        out = a.collect_storage(["/", "/nfs/gone"])
+    assert out == [{"mount": "/", "total_bytes": 10, "used_bytes": 1}]

tests/plugins/host_agent/test_expand_metrics.py

@@ -93,3 +93,14 @@ def test_a_sensor_with_nothing_wired_to_it_does_not_set_the_host_s_temperature()
     assert idx[("temp_c_max", "util")] == 60.0
     assert idx[("temp_c", "util:temp:AUXTIN0")] == 110.0
     assert idx[("temp_c", "util:temp:PCH_CHIP_CPU_MAX_TEMP")] == 0.0
+
+
+def test_stalled_mounts_are_counted_and_have_no_figures():
+    """#5760: a mount the agent couldn't read in time."""
+    sample = {"ts": "2026-10-14T21:00:00+00:00",
+              "storage": [{"mount": "/", "total_bytes": 100, "used_bytes": 50},
+                          {"mount": "/nfs/media", "stalled": True}]}
+    idx = _index(_expand_sample_to_metrics(sample, "d3", datetime.now(timezone.utc)))
+    assert idx[("mounts_stalled", "d3")] == 1
+    assert idx[("disk_used_pct_worst", "d3")] == 50.0
+    assert ("disk_used_pct", "d3:/nfs/media") not in idx