feat(host_agent): a disk out of inodes reads as full (#5745)

VDNT-Util01's / ran out of inodes (956 free of 60.8M) with 274G of space
left; every file create failed and Steward's page read healthy at 66%.

- Agent 1.13.0 reports each mount's inodes (statvfs; a filesystem reporting
  none, like btrfs, is left without).
- Ingest records inodes_used_pct per mount, and disk_used_pct_worst is the
  fullest mount by space or inodes, so existing disk-full rules and the
  overview see it.
- The host's health strip, the Hosts widget and the host glance grade / by
  whichever is fuller and name inodes when they are ("Disk / 66% · inodes
  100%", "Inodes on / ... past its critical threshold").

Hosts pick up the inode figures once their agent is updated to 1.13.0.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/formatting.py

@@ -329,8 +329,9 @@ METRICS: dict[str, tuple[str, str]] = {
     "mem_used_pct": ("Memory used", "memory in use"),
     "mem_available_bytes": ("Memory available", "memory free for new work"),
     "swap_used_bytes": ("Swap used", "memory paged out to disk"),
-    "disk_used_pct_worst": ("Fullest disk", "the most-full mount on the host"),
+    "disk_used_pct_worst": ("Fullest disk", "the most-full mount on the host, by space or inodes"),
     "disk_used_pct": ("Disk used", "one mount (resource host:mount)"),
+    "inodes_used_pct": ("Inodes used", "one mount's file slots in use (resource host:mount); at 100% no file can be created"),
     "load_1m": ("Load (1 min)", "runnable processes, averaged over a minute"),
     "load_5m": ("Load (5 min)", "runnable processes, averaged over 5 minutes"),
     "load_15m": ("Load (15 min)", "runnable processes, averaged over 15 minutes"),

steward/hosts/service.py

@@ -269,7 +269,7 @@ async def agent_overview_by_host(
     stale_after = int(stale_after_seconds)
     latest = await latest_by_resource(
         db, "host_agent", list(host_names) + [h + ":/" for h in host_names],
-        set(AGENT_GLANCE_METRICS) | {"disk_used_pct", "clock_off_secs"})
+        set(AGENT_GLANCE_METRICS) | {"disk_used_pct", "inodes_used_pct", "clock_off_secs"})
     from steward.models.metrics import PluginMetricLatest
     cores: dict[str, int] = {}
     for res in (await db.execute(
@@ -289,6 +289,8 @@ async def agent_overview_by_host(
         d = {m: r.value for m, r in own.items()}
         if root is not None:
             d["disk_root"] = root.value
+        inodes = latest.get(name + ":/", {}).get("inodes_used_pct")
+        d["inodes_root"] = inodes.value if inodes is not None else None
         ts = max((r.recorded_at for r in own.values()), default=None)
         d["last_at"] = ts
         d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
@@ -314,6 +316,11 @@ HOST_NORMED = {"cpu": ("cpu_pct", "host_cpu_pct"), "mem": ("mem_used_pct", "host
 _KIND_OF = {metric: kind for kind, (metric, _f) in HOST_NORMED.items()}
 
 
+def fuller(space: float | None, inodes: float | None) -> float | None:
+    """How full a mount is: by space or by inodes, whichever is fuller (#5745)."""
+    return max((v for v in (space, inodes) if v is not None), default=None)
+
+
 def host_normals(norms: dict, thresholds: dict) -> dict:
     """{kind: (Norm, floor)} for worst_reading and anomalies.graded, from an
     agent glance's `norms`."""
@@ -381,11 +388,13 @@ def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict
         worst, grades = None, {}
         if a and a.get("fresh"):
             grades = host_grades({"cpu": a.get("cpu_pct"), "mem": a.get("mem_used_pct"),
-                                  "disk": a.get("disk_root"), "load": a.get("load_per_core")},
+                                  "disk": fuller(a.get("disk_root"), a.get("inodes_root")),
+                                  "load": a.get("load_per_core")},
                                  a.get("norms"), thresholds)
             worst = worst_reading((
                 ("CPU", a.get("cpu_pct"), "cpu"), ("Memory", a.get("mem_used_pct"), "mem"),
-                ("Disk /", a.get("disk_root"), "disk"), ("Load", a.get("load_per_core"), "load")),
+                ("Disk /", a.get("disk_root"), "disk"), ("Inodes on /", a.get("inodes_root"), "disk"),
+                ("Load", a.get("load_per_core"), "load")),
                 thresholds, host_normals(a.get("norms"), thresholds))
         if st and st["state"] == "down":
             state = "down"

steward/mcp/tools/overview.py

@@ -127,7 +127,7 @@ def alert_findings(active, silent: dict[str, list[str]] | None = None) -> list[d
 
 
 _AGENT_CHECKS = (("cpu_pct", "cpu", "CPU"), ("mem_used_pct", "mem", "memory"),
-                 ("disk_used_pct_worst", "disk", "fullest disk"))
+                 ("disk_used_pct_worst", "disk", "fullest disk (space or inodes)"))
 
 
 def agent_findings(hosts, agent: dict, thresholds: dict,

steward/templates/hosts/overview_widget.html

@@ -33,7 +33,8 @@
     </td>
     {% for v, kind in ((a.get('cpu_pct'), 'cpu'), (a.get('mem_used_pct'), 'mem'), (a.get('disk_root'), 'disk')) %}
     {% set g = (r.grades or {}).get(kind) or (none, none) %}
-    <td style="{{ grade_style(g[0]) }}"{% if g[1] %} title="{{ g[1]|capitalize }}"{% endif %}>{% if v is none %}—{% else %}{{ "%.0f"|format(v) }}%{% endif %}</td>
+    {% set inodes = a.get('inodes_root') if kind == 'disk' else none %}
+    <td style="{{ grade_style(g[0]) }}"{% if g[1] %} title="{{ g[1]|capitalize }}"{% elif inodes is not none and (v is none or inodes > v) %} title="Inodes {{ '%.0f'|format(inodes) }}% used: no file can be created once they run out"{% endif %}>{% if v is none %}—{% else %}{{ "%.0f"|format(v) }}%{% endif %}{% if inodes is not none and (v is none or inodes > v) and inodes >= 80 %} <span class="w-sub">inodes {{ "%.0f"|format(inodes) }}%</span>{% endif %}</td>
     {% endfor %}
     {% set g = (r.grades or {}).get('load') or (none, none) %}
     <td style="{{ grade_style(g[0]) }}"