diff --git a/docs/plugins/host-agent-design.md b/docs/plugins/host-agent-design.md index 3487554..18f068d 100644 --- a/docs/plugins/host-agent-design.md +++ b/docs/plugins/host-agent-design.md @@ -116,6 +116,10 @@ Python 3.8+ stdlib only. `urllib.request` for the POST, plus `json`, `os`, `shut - **SIGTERM** (from systemd) → finish current poll cycle → flush buffer one last time, with a `stopping` event saying whether the host is going down too (`systemctl is-system-running` says `stopping`) → exit. - **Systemd restart policy** → `Restart=on-failure`, `RestartSec=10`. +### Mounts that don't answer (1.14.0, #5760) + +Each mount is read in a daemon thread under one 5 s deadline. A hard-mounted NFS share whose server stops blocks `statvfs` in the kernel with no timeout; read inline, that froze the whole agent. A mount that misses the deadline is sent as `{"mount", "stalled": true}`, and isn't probed again while its last probe is still blocked. Steward counts them as `mounts_stalled`. + ### What happened to the host (1.14.0, #5756) A post may carry `events`, said once each, with the next post that gets through: diff --git a/plugins/host_agent/__init__.py b/plugins/host_agent/__init__.py index 5892c19..cbe7510 100644 --- a/plugins/host_agent/__init__.py +++ b/plugins/host_agent/__init__.py @@ -63,10 +63,12 @@ def get_alert_metrics() -> dict[str, list[str]]: clock_off_secs is how far the host's clock is from Steward's, either way (agent ≥ 1.12.0): "clock drifting" is > 30. load_pct is load_1m as a % of the host's cores. inodes_used_pct is per mount (agent ≥ 1.13.0), and - disk_used_pct_worst is the fullest mount by space or inodes.""" + disk_used_pct_worst is the fullest mount by space or inodes. + mounts_stalled counts the mounts that didn't answer the agent in time + (agent ≥ 1.14.0): "an NFS share stopped" is > 0.""" return {"host_agent": [ "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes", "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m", "load_pct", - "uptime_secs", "seconds_since_seen", "clock_off_secs", + "uptime_secs", "seconds_since_seen", "clock_off_secs", "mounts_stalled", ]} diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py index 1237756..0de5d39 100644 --- a/plugins/host_agent/agent.py +++ b/plugins/host_agent/agent.py @@ -14,6 +14,7 @@ import signal import socket import subprocess import sys +import threading import time import urllib.error import urllib.request @@ -182,25 +183,64 @@ def collect_memory() -> dict: } -def collect_storage(mounts: list[str]) -> list[dict]: +# A mount that hasn't answered in this long is reported stalled (#5760): a +# hard-mounted NFS share whose server stops blocks statvfs in the kernel with +# no timeout, and read inline that froze the whole agent, silent for a day on +# a host that still answered ping. +MOUNT_DEADLINE_SECONDS = 5.0 +# Mounts whose last probe is still blocked. They aren't probed again until it +# returns, so a dead share costs one stuck thread, not one per sample. +_stuck_mounts: dict = {} + + +def _read_mount(m: str) -> dict: + u = shutil.disk_usage(m) + row = {"mount": m, "total_bytes": u.total, "used_bytes": u.used} + # Inodes too (1.13.0): a filesystem can run out of them with space + # to spare, and every create then fails with ENOSPC. Some (btrfs) + # report none, and are left without. + try: + st = os.statvfs(m) + except OSError: + st = None + if st is not None and st.f_files: + row["inodes_total"] = st.f_files + row["inodes_used"] = st.f_files - st.f_ffree + return row + + +def collect_storage(mounts: list[str], deadline: float = MOUNT_DEADLINE_SECONDS) -> list[dict]: + """Each mount's space and inodes, read side by side in daemon threads under + one deadline; a mount that doesn't answer in time is {"mount", "stalled"}.""" + probes: dict = {} + for m in mounts: + stuck = _stuck_mounts.get(m) + if stuck is not None and stuck.is_alive(): + continue + _stuck_mounts.pop(m, None) + box: dict = {} + + def probe(m=m, box=box): + try: + box["row"] = _read_mount(m) + except OSError: + pass # gone, not ours to read, or a stale handle: left out + t = threading.Thread(target=probe, name=f"mount {m}", daemon=True) + t.start() + probes[m] = (t, box) + end = time.monotonic() + deadline out: list[dict] = [] for m in mounts: - try: - u = shutil.disk_usage(m) - except (FileNotFoundError, PermissionError): + if m not in probes: + out.append({"mount": m, "stalled": True}) continue - row = {"mount": m, "total_bytes": u.total, "used_bytes": u.used} - # Inodes too (1.13.0): a filesystem can run out of them with space - # to spare, and every create then fails with ENOSPC. Some (btrfs) - # report none, and are left without. - try: - st = os.statvfs(m) - except OSError: - st = None - if st is not None and st.f_files: - row["inodes_total"] = st.f_files - row["inodes_used"] = st.f_files - st.f_ffree - out.append(row) + t, box = probes[m] + t.join(max(0.0, end - time.monotonic())) + if t.is_alive(): + _stuck_mounts[m] = t + out.append({"mount": m, "stalled": True}) + elif "row" in box: + out.append(box["row"]) return out diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 510f361..039002f 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -143,7 +143,13 @@ def _expand_sample_to_metrics( # inodes, every file create fails with space to spare, so the disk rules # and the overview, which read disk_used_pct_worst, see either. worst_pct = 0.0 + stalled = 0 for disk in sample.get("storage") or []: + # A mount that didn't answer in time has no figures (#5760); how many + # did is its own metric, so a rule can say "an NFS share stopped". + if disk.get("stalled"): + stalled += 1 + continue total = disk.get("total_bytes", 0) used = disk.get("used_bytes", 0) pct = 100.0 * used / total if total else 0.0 @@ -158,7 +164,10 @@ def _expand_sample_to_metrics( if pct > worst_pct: worst_pct = pct if sample.get("storage"): - row("disk_used_pct_worst", host_name, worst_pct) + # Every mount stalled says nothing about how full they are. + if stalled < len(sample["storage"]): + row("disk_used_pct_worst", host_name, worst_pct) + row("mounts_stalled", host_name, stalled) # Per-core CPU. Sub-resources carry a ':' so the fleet widget filters them out. for i, core_pct in enumerate(sample.get("cpu_cores") or []): diff --git a/steward/core/formatting.py b/steward/core/formatting.py index ec0b119..74868e4 100644 --- a/steward/core/formatting.py +++ b/steward/core/formatting.py @@ -339,6 +339,7 @@ METRICS: dict[str, tuple[str, str]] = { "uptime_secs": ("Uptime", "time since the host booted"), "seconds_since_seen": ("Time since the agent reported", "over 3 minutes means the agent is offline"), "clock_off_secs": ("Clock off by", "how far the host's clock is from Steward's, either way"), + "mounts_stalled": ("Mounts not answering", "mounts the agent couldn't read in 5 s, e.g. an NFS share whose server stopped"), } # Where a rule's metric comes from, as the picker and the rules list say it. diff --git a/tests/plugins/host_agent/test_agent_collectors.py b/tests/plugins/host_agent/test_agent_collectors.py index 57a52db..8a9f238 100644 --- a/tests/plugins/host_agent/test_agent_collectors.py +++ b/tests/plugins/host_agent/test_agent_collectors.py @@ -109,3 +109,45 @@ def test_collect_metadata(tmp_path): assert meta["kernel"] == "6.8.0-45-generic" assert meta["arch"] == "x86_64" assert "Ubuntu 24.04" in meta["distro"] + + +def test_a_mount_that_does_not_answer_is_reported_stalled_and_not_probed_again(): + """#5760: statvfs on a stalled NFS share blocks with no timeout, and read + inline that froze the whole agent. The sample goes on without it, and a + probe still blocked isn't stacked on by the next sample.""" + import threading + release = threading.Event() + calls: list[str] = [] + + def _usage(path): + calls.append(path) + if path == "/nfs/media": + release.wait(5) + return type("R", (), {"total": 1000, "used": 400, "free": 600})() + + a._stuck_mounts.clear() + try: + with patch.object(a.shutil, "disk_usage", side_effect=_usage), \ + patch.object(a.os, "statvfs", side_effect=OSError): + first = a.collect_storage(["/", "/nfs/media"], deadline=0.2) + second = a.collect_storage(["/", "/nfs/media"], deadline=0.2) + assert first == second == [ + {"mount": "/", "total_bytes": 1000, "used_bytes": 400}, + {"mount": "/nfs/media", "stalled": True}, + ] + assert calls.count("/nfs/media") == 1 + finally: + release.set() + a._stuck_mounts.clear() + + +def test_a_mount_that_errors_is_left_out_not_the_whole_list(): + def _usage(path): + if path == "/nfs/gone": + raise OSError(116, "Stale file handle") + return type("R", (), {"total": 10, "used": 1, "free": 9})() + + with patch.object(a.shutil, "disk_usage", side_effect=_usage), \ + patch.object(a.os, "statvfs", side_effect=OSError): + out = a.collect_storage(["/", "/nfs/gone"]) + assert out == [{"mount": "/", "total_bytes": 10, "used_bytes": 1}] diff --git a/tests/plugins/host_agent/test_expand_metrics.py b/tests/plugins/host_agent/test_expand_metrics.py index 5c75780..22a3b57 100644 --- a/tests/plugins/host_agent/test_expand_metrics.py +++ b/tests/plugins/host_agent/test_expand_metrics.py @@ -93,3 +93,14 @@ def test_a_sensor_with_nothing_wired_to_it_does_not_set_the_host_s_temperature() assert idx[("temp_c_max", "util")] == 60.0 assert idx[("temp_c", "util:temp:AUXTIN0")] == 110.0 assert idx[("temp_c", "util:temp:PCH_CHIP_CPU_MAX_TEMP")] == 0.0 + + +def test_stalled_mounts_are_counted_and_have_no_figures(): + """#5760: a mount the agent couldn't read in time.""" + sample = {"ts": "2026-10-14T21:00:00+00:00", + "storage": [{"mount": "/", "total_bytes": 100, "used_bytes": 50}, + {"mount": "/nfs/media", "stalled": True}]} + idx = _index(_expand_sample_to_metrics(sample, "d3", datetime.now(timezone.utc))) + assert idx[("mounts_stalled", "d3")] == 1 + assert idx[("disk_used_pct_worst", "d3")] == 50.0 + assert ("disk_used_pct", "d3:/nfs/media") not in idx