From 5dd9956f3597fe7c3b10053b12b99f2bbfdbf52b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 14 Oct 2026 16:51:46 -0400 Subject: [PATCH] feat(host_agent): a disk out of inodes reads as full (#5745) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VDNT-Util01's / ran out of inodes (956 free of 60.8M) with 274G of space left; every file create failed and Steward's page read healthy at 66%. - Agent 1.13.0 reports each mount's inodes (statvfs; a filesystem reporting none, like btrfs, is left without). - Ingest records inodes_used_pct per mount, and disk_used_pct_worst is the fullest mount by space or inodes, so existing disk-full rules and the overview see it. - The host's health strip, the Hosts widget and the host glance grade / by whichever is fuller and name inodes when they are ("Disk / 66% · inodes 100%", "Inodes on / ... past its critical threshold"). Hosts pick up the inode figures once their agent is updated to 1.13.0. Co-Authored-By: Claude Opus 5.5 --- plugins/host_agent/__init__.py | 6 ++++-- plugins/host_agent/agent.py | 15 +++++++++++++-- plugins/host_agent/routes.py | 16 +++++++++++++--- plugins/host_agent/templates/_host_vitals.html | 4 +++- steward/core/formatting.py | 3 ++- steward/hosts/service.py | 15 ++++++++++++--- steward/mcp/tools/overview.py | 2 +- steward/templates/hosts/overview_widget.html | 3 ++- tests/core/test_norm_grading.py | 9 +++++++++ .../plugins/host_agent/test_agent_collectors.py | 13 +++++++++++-- .../host_agent/test_routes_expand_metrics.py | 15 +++++++++++++++ 11 files changed, 85 insertions(+), 16 deletions(-) diff --git a/plugins/host_agent/__init__.py b/plugins/host_agent/__init__.py index 07692a5..fa50043 100644 --- a/plugins/host_agent/__init__.py +++ b/plugins/host_agent/__init__.py @@ -53,9 +53,11 @@ def get_alert_metrics() -> dict[str, list[str]]: comes from the scheduler, not the agent: "agent offline" is > 180. clock_off_secs is how far the host's clock is from Steward's, either way (agent ≥ 1.12.0): "clock drifting" is > 30. load_pct is load_1m as a % - of the host's cores.""" + of the host's cores. inodes_used_pct is per mount (agent ≥ 1.13.0), and + disk_used_pct_worst is the fullest mount by space or inodes.""" return {"host_agent": [ "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes", - "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m", "load_pct", + "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m", + "load_pct", "uptime_secs", "seconds_since_seen", "clock_off_secs", ]} diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py index 6e5160c..2024604 100644 --- a/plugins/host_agent/agent.py +++ b/plugins/host_agent/agent.py @@ -20,7 +20,7 @@ from collections import deque from concurrent.futures import ThreadPoolExecutor, wait as wait_futures from datetime import datetime, timezone -AGENT_VERSION = "1.12.0" +AGENT_VERSION = "1.13.0" # Default path to the local Docker Engine socket. Overridable via the # `docker_socket` config key; collection is silently skipped if it's absent or @@ -188,7 +188,18 @@ def collect_storage(mounts: list[str]) -> list[dict]: u = shutil.disk_usage(m) except (FileNotFoundError, PermissionError): continue - out.append({"mount": m, "total_bytes": u.total, "used_bytes": u.used}) + row = {"mount": m, "total_bytes": u.total, "used_bytes": u.used} + # Inodes too (1.13.0): a filesystem can run out of them with space + # to spare, and every create then fails with ENOSPC. Some (btrfs) + # report none, and are left without. + try: + st = os.statvfs(m) + except OSError: + st = None + if st is not None and st.f_files: + row["inodes_total"] = st.f_files + row["inodes_used"] = st.f_files - st.f_ffree + out.append(row) return out diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index fa40caa..ea693a4 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -127,6 +127,9 @@ def _expand_sample_to_metrics( if sample.get("uptime_secs") is not None: row("uptime_secs", host_name, sample["uptime_secs"]) + # The fullest disk is the fullest by space or by inodes (#5745): out of + # inodes, every file create fails with space to spare, so the disk rules + # and the overview, which read disk_used_pct_worst, see either. worst_pct = 0.0 for disk in sample.get("storage") or []: total = disk.get("total_bytes", 0) @@ -136,6 +139,10 @@ def _expand_sample_to_metrics( row("disk_used_pct", mount_res, pct) row("disk_used_bytes", mount_res, used) row("disk_total_bytes", mount_res, total) + if disk.get("inodes_total"): + inode_pct = 100.0 * disk.get("inodes_used", 0) / disk["inodes_total"] + row("inodes_used_pct", mount_res, inode_pct) + pct = max(pct, inode_pct) if pct > worst_pct: worst_pct = pct if sample.get("storage"): @@ -750,6 +757,8 @@ async def host_vitals(host_id: str): # At-a-glance disk = the root filesystem (what people actually care about), # not the "worst" mount which is opaque. Worst stays on the full-metrics page. disk_root = (latest.get(host.name + ":/", {}) or {}).get("disk_used_pct") + # Out of inodes, "/" is full with space to spare (#5745): named when worse. + inodes_root = (latest.get(host.name + ":/", {}) or {}).get("inodes_used_pct") sparks = {k: sparkline_svg(v, height=24, fill_width=True) for k, v in series_raw.items()} # Normalize load by core count so it's comparable across hardware (load/core # as %). Cores derived from the per-core CPU sub-resources already collected. @@ -767,12 +776,12 @@ async def host_vitals(host_id: str): cpu, mem = hostlvl.get("cpu_pct"), hostlvl.get("mem_used_pct") thresholds = current_app.config.get("THRESHOLDS", {}) worst = worst_reading((("CPU", cpu, "cpu"), ("Memory", mem, "mem"), ("Disk /", disk_root, "disk"), - ("Load per core", load_per_core, "load")), + ("Inodes on /", inodes_root, "disk"), ("Load per core", load_per_core, "load")), thresholds, host_svc.host_normals(norms, thresholds)) # Each tile's grade; CPU, memory and load within the host's own normal # for the hour read healthy, and say what is usual (#5742). - grades = host_svc.host_grades({"cpu": cpu, "mem": mem, "disk": disk_root, "load": load_per_core}, - norms, thresholds) + grades = host_svc.host_grades({"cpu": cpu, "mem": mem, "disk": host_svc.fuller(disk_root, inodes_root), + "load": load_per_core}, norms, thresholds) usual = [f"{label} {value:.0f}% is {grades[kind][1]}" for label, value, kind in (("CPU", cpu, "cpu"), ("Memory", mem, "mem"), ("load per core", load_per_core, "load")) @@ -780,6 +789,7 @@ async def host_vitals(host_id: str): return await render_template( "_host_vitals.html", reg=reg, cpu=cpu, mem=mem, disk_root=disk_root, load_per_core=load_per_core, + inodes_root=inodes_root, sparks=sparks, psi=psi, reporting=reporting, stale=stale, worst=worst, grades=grades, usual=usual, clock=clock_words(reg.clock_offset_s if reg else None, current_app.config.get("THRESHOLDS", {})), diff --git a/plugins/host_agent/templates/_host_vitals.html b/plugins/host_agent/templates/_host_vitals.html index c5f3528..ccd02b8 100644 --- a/plugins/host_agent/templates/_host_vitals.html +++ b/plugins/host_agent/templates/_host_vitals.html @@ -25,7 +25,9 @@
{{ stat_tile("CPU", pct(cpu), state=(grades.cpu or [status_class(cpu, 'cpu')])[0], hint=sparks.cpu|safe) }} {{ stat_tile("Memory", pct(mem), state=(grades.mem or [status_class(mem, 'mem')])[0], hint=sparks.mem|safe) }} - {{ stat_tile("Disk /", pct(disk_root), state=(grades.disk or [status_class(disk_root, 'disk')])[0], hint=sparks.disk|safe) }} + {%- set inodes_worse = inodes_root is defined and inodes_root is not none and (disk_root is none or inodes_root > disk_root) %} + {{ stat_tile("Disk /", pct(disk_root) ~ (" · inodes " ~ pct(inodes_root) if inodes_worse else ""), + state=(grades.disk or [status_class(disk_root, 'disk')])[0], hint=sparks.disk|safe) }} {{ stat_tile("Load per core", pct(load_per_core), state=(grades.load or [status_class(load_per_core, 'load')])[0], hint=sparks.load|safe) }}
{% if usual and not stale %} diff --git a/steward/core/formatting.py b/steward/core/formatting.py index 1101555..ec0b119 100644 --- a/steward/core/formatting.py +++ b/steward/core/formatting.py @@ -329,8 +329,9 @@ METRICS: dict[str, tuple[str, str]] = { "mem_used_pct": ("Memory used", "memory in use"), "mem_available_bytes": ("Memory available", "memory free for new work"), "swap_used_bytes": ("Swap used", "memory paged out to disk"), - "disk_used_pct_worst": ("Fullest disk", "the most-full mount on the host"), + "disk_used_pct_worst": ("Fullest disk", "the most-full mount on the host, by space or inodes"), "disk_used_pct": ("Disk used", "one mount (resource host:mount)"), + "inodes_used_pct": ("Inodes used", "one mount's file slots in use (resource host:mount); at 100% no file can be created"), "load_1m": ("Load (1 min)", "runnable processes, averaged over a minute"), "load_5m": ("Load (5 min)", "runnable processes, averaged over 5 minutes"), "load_15m": ("Load (15 min)", "runnable processes, averaged over 15 minutes"), diff --git a/steward/hosts/service.py b/steward/hosts/service.py index b5a4864..3565e90 100644 --- a/steward/hosts/service.py +++ b/steward/hosts/service.py @@ -269,7 +269,7 @@ async def agent_overview_by_host( stale_after = int(stale_after_seconds) latest = await latest_by_resource( db, "host_agent", list(host_names) + [h + ":/" for h in host_names], - set(AGENT_GLANCE_METRICS) | {"disk_used_pct", "clock_off_secs"}) + set(AGENT_GLANCE_METRICS) | {"disk_used_pct", "inodes_used_pct", "clock_off_secs"}) from steward.models.metrics import PluginMetricLatest cores: dict[str, int] = {} for res in (await db.execute( @@ -289,6 +289,8 @@ async def agent_overview_by_host( d = {m: r.value for m, r in own.items()} if root is not None: d["disk_root"] = root.value + inodes = latest.get(name + ":/", {}).get("inodes_used_pct") + d["inodes_root"] = inodes.value if inodes is not None else None ts = max((r.recorded_at for r in own.values()), default=None) d["last_at"] = ts d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after) @@ -314,6 +316,11 @@ HOST_NORMED = {"cpu": ("cpu_pct", "host_cpu_pct"), "mem": ("mem_used_pct", "host _KIND_OF = {metric: kind for kind, (metric, _f) in HOST_NORMED.items()} +def fuller(space: float | None, inodes: float | None) -> float | None: + """How full a mount is: by space or by inodes, whichever is fuller (#5745).""" + return max((v for v in (space, inodes) if v is not None), default=None) + + def host_normals(norms: dict, thresholds: dict) -> dict: """{kind: (Norm, floor)} for worst_reading and anomalies.graded, from an agent glance's `norms`.""" @@ -381,11 +388,13 @@ def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict worst, grades = None, {} if a and a.get("fresh"): grades = host_grades({"cpu": a.get("cpu_pct"), "mem": a.get("mem_used_pct"), - "disk": a.get("disk_root"), "load": a.get("load_per_core")}, + "disk": fuller(a.get("disk_root"), a.get("inodes_root")), + "load": a.get("load_per_core")}, a.get("norms"), thresholds) worst = worst_reading(( ("CPU", a.get("cpu_pct"), "cpu"), ("Memory", a.get("mem_used_pct"), "mem"), - ("Disk /", a.get("disk_root"), "disk"), ("Load", a.get("load_per_core"), "load")), + ("Disk /", a.get("disk_root"), "disk"), ("Inodes on /", a.get("inodes_root"), "disk"), + ("Load", a.get("load_per_core"), "load")), thresholds, host_normals(a.get("norms"), thresholds)) if st and st["state"] == "down": state = "down" diff --git a/steward/mcp/tools/overview.py b/steward/mcp/tools/overview.py index 8d3b703..905630e 100644 --- a/steward/mcp/tools/overview.py +++ b/steward/mcp/tools/overview.py @@ -127,7 +127,7 @@ def alert_findings(active, silent: dict[str, list[str]] | None = None) -> list[d _AGENT_CHECKS = (("cpu_pct", "cpu", "CPU"), ("mem_used_pct", "mem", "memory"), - ("disk_used_pct_worst", "disk", "fullest disk")) + ("disk_used_pct_worst", "disk", "fullest disk (space or inodes)")) def agent_findings(hosts, agent: dict, thresholds: dict, diff --git a/steward/templates/hosts/overview_widget.html b/steward/templates/hosts/overview_widget.html index 1fe2dcb..bfb4efb 100644 --- a/steward/templates/hosts/overview_widget.html +++ b/steward/templates/hosts/overview_widget.html @@ -33,7 +33,8 @@ {% for v, kind in ((a.get('cpu_pct'), 'cpu'), (a.get('mem_used_pct'), 'mem'), (a.get('disk_root'), 'disk')) %} {% set g = (r.grades or {}).get(kind) or (none, none) %} - {% if v is none %}—{% else %}{{ "%.0f"|format(v) }}%{% endif %} + {% set inodes = a.get('inodes_root') if kind == 'disk' else none %} + v) %} title="Inodes {{ '%.0f'|format(inodes) }}% used: no file can be created once they run out"{% endif %}>{% if v is none %}—{% else %}{{ "%.0f"|format(v) }}%{% endif %}{% if inodes is not none and (v is none or inodes > v) and inodes >= 80 %} inodes {{ "%.0f"|format(inodes) }}%{% endif %} {% endfor %} {% set g = (r.grades or {}).get('load') or (none, none) %} 99.99 + (worst,) = _by_metric(rows, "disk_used_pct_worst") + assert worst.value == inodes.value + # A filesystem reporting no inodes (btrfs) records none. + assert len(_by_metric(_expand_sample_to_metrics(_sample(), "testhost", NOW), "inodes_used_pct")) == 0