feat(host_agent): a disk out of inodes reads as full (#5745)

VDNT-Util01's / ran out of inodes (956 free of 60.8M) with 274G of space
left; every file create failed and Steward's page read healthy at 66%.

- Agent 1.13.0 reports each mount's inodes (statvfs; a filesystem reporting
  none, like btrfs, is left without).
- Ingest records inodes_used_pct per mount, and disk_used_pct_worst is the
  fullest mount by space or inodes, so existing disk-full rules and the
  overview see it.
- The host's health strip, the Hosts widget and the host glance grade / by
  whichever is fuller and name inodes when they are ("Disk / 66% · inodes
  100%", "Inodes on / ... past its critical threshold").

Hosts pick up the inode figures once their agent is updated to 1.13.0.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/__init__.py

@@ -53,9 +53,11 @@ def get_alert_metrics() -> dict[str, list[str]]:
     comes from the scheduler, not the agent: "agent offline" is > 180.
     clock_off_secs is how far the host's clock is from Steward's, either way
     (agent ≥ 1.12.0): "clock drifting" is > 30. load_pct is load_1m as a %
-    of the host's cores."""
+    of the host's cores. inodes_used_pct is per mount (agent ≥ 1.13.0), and
+    disk_used_pct_worst is the fullest mount by space or inodes."""
     return {"host_agent": [
         "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
-        "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m", "load_pct",
+        "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m",
+        "load_pct",
         "uptime_secs", "seconds_since_seen", "clock_off_secs",
     ]}

plugins/host_agent/agent.py

@@ -20,7 +20,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.12.0"
+AGENT_VERSION = "1.13.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -188,7 +188,18 @@ def collect_storage(mounts: list[str]) -> list[dict]:
             u = shutil.disk_usage(m)
         except (FileNotFoundError, PermissionError):
             continue
-        out.append({"mount": m, "total_bytes": u.total, "used_bytes": u.used})
+        row = {"mount": m, "total_bytes": u.total, "used_bytes": u.used}
+        # Inodes too (1.13.0): a filesystem can run out of them with space
+        # to spare, and every create then fails with ENOSPC. Some (btrfs)
+        # report none, and are left without.
+        try:
+            st = os.statvfs(m)
+        except OSError:
+            st = None
+        if st is not None and st.f_files:
+            row["inodes_total"] = st.f_files
+            row["inodes_used"] = st.f_files - st.f_ffree
+        out.append(row)
     return out
 
 

plugins/host_agent/routes.py

@@ -127,6 +127,9 @@ def _expand_sample_to_metrics(
     if sample.get("uptime_secs") is not None:
         row("uptime_secs", host_name, sample["uptime_secs"])
 
+    # The fullest disk is the fullest by space or by inodes (#5745): out of
+    # inodes, every file create fails with space to spare, so the disk rules
+    # and the overview, which read disk_used_pct_worst, see either.
     worst_pct = 0.0
     for disk in sample.get("storage") or []:
         total = disk.get("total_bytes", 0)
@@ -136,6 +139,10 @@ def _expand_sample_to_metrics(
         row("disk_used_pct", mount_res, pct)
         row("disk_used_bytes", mount_res, used)
         row("disk_total_bytes", mount_res, total)
+        if disk.get("inodes_total"):
+            inode_pct = 100.0 * disk.get("inodes_used", 0) / disk["inodes_total"]
+            row("inodes_used_pct", mount_res, inode_pct)
+            pct = max(pct, inode_pct)
         if pct > worst_pct:
             worst_pct = pct
     if sample.get("storage"):
@@ -750,6 +757,8 @@ async def host_vitals(host_id: str):
     # At-a-glance disk = the root filesystem (what people actually care about),
     # not the "worst" mount which is opaque. Worst stays on the full-metrics page.
     disk_root = (latest.get(host.name + ":/", {}) or {}).get("disk_used_pct")
+    # Out of inodes, "/" is full with space to spare (#5745): named when worse.
+    inodes_root = (latest.get(host.name + ":/", {}) or {}).get("inodes_used_pct")
     sparks = {k: sparkline_svg(v, height=24, fill_width=True) for k, v in series_raw.items()}
     # Normalize load by core count so it's comparable across hardware (load/core
     # as %). Cores derived from the per-core CPU sub-resources already collected.
@@ -767,12 +776,12 @@ async def host_vitals(host_id: str):
     cpu, mem = hostlvl.get("cpu_pct"), hostlvl.get("mem_used_pct")
     thresholds = current_app.config.get("THRESHOLDS", {})
     worst = worst_reading((("CPU", cpu, "cpu"), ("Memory", mem, "mem"), ("Disk /", disk_root, "disk"),
-                           ("Load per core", load_per_core, "load")),
+                           ("Inodes on /", inodes_root, "disk"), ("Load per core", load_per_core, "load")),
                           thresholds, host_svc.host_normals(norms, thresholds))
     # Each tile's grade; CPU, memory and load within the host's own normal
     # for the hour read healthy, and say what is usual (#5742).
-    grades = host_svc.host_grades({"cpu": cpu, "mem": mem, "disk": disk_root, "load": load_per_core},
-                                  norms, thresholds)
+    grades = host_svc.host_grades({"cpu": cpu, "mem": mem, "disk": host_svc.fuller(disk_root, inodes_root),
+                                   "load": load_per_core}, norms, thresholds)
     usual = [f"{label} {value:.0f}% is {grades[kind][1]}"
              for label, value, kind in (("CPU", cpu, "cpu"), ("Memory", mem, "mem"),
                                         ("load per core", load_per_core, "load"))
@@ -780,6 +789,7 @@ async def host_vitals(host_id: str):
     return await render_template(
         "_host_vitals.html",
         reg=reg, cpu=cpu, mem=mem, disk_root=disk_root, load_per_core=load_per_core,
+        inodes_root=inodes_root,
         sparks=sparks, psi=psi, reporting=reporting, stale=stale, worst=worst,
         grades=grades, usual=usual,
         clock=clock_words(reg.clock_offset_s if reg else None, current_app.config.get("THRESHOLDS", {})),

plugins/host_agent/templates/_host_vitals.html

@@ -25,7 +25,9 @@
   <div class="stat-strip host-health-tiles" style="margin-bottom:0;">
     {{ stat_tile("CPU", pct(cpu), state=(grades.cpu or [status_class(cpu, 'cpu')])[0], hint=sparks.cpu|safe) }}
     {{ stat_tile("Memory", pct(mem), state=(grades.mem or [status_class(mem, 'mem')])[0], hint=sparks.mem|safe) }}
-    {{ stat_tile("Disk /", pct(disk_root), state=(grades.disk or [status_class(disk_root, 'disk')])[0], hint=sparks.disk|safe) }}
+    {%- set inodes_worse = inodes_root is defined and inodes_root is not none and (disk_root is none or inodes_root > disk_root) %}
+    {{ stat_tile("Disk /", pct(disk_root) ~ (" · inodes " ~ pct(inodes_root) if inodes_worse else ""),
+                 state=(grades.disk or [status_class(disk_root, 'disk')])[0], hint=sparks.disk|safe) }}
     {{ stat_tile("Load per core", pct(load_per_core), state=(grades.load or [status_class(load_per_core, 'load')])[0], hint=sparks.load|safe) }}
   </div>
   {% if usual and not stale %}