feat(host_agent): a disk out of inodes reads as full (#5745)
VDNT-Util01's / ran out of inodes (956 free of 60.8M) with 274G of space
left; every file create failed and Steward's page read healthy at 66%.
- Agent 1.13.0 reports each mount's inodes (statvfs; a filesystem reporting
none, like btrfs, is left without).
- Ingest records inodes_used_pct per mount, and disk_used_pct_worst is the
fullest mount by space or inodes, so existing disk-full rules and the
overview see it.
- The host's health strip, the Hosts widget and the host glance grade / by
whichever is fuller and name inodes when they are ("Disk / 66% · inodes
100%", "Inodes on / ... past its critical threshold").
Hosts pick up the inode figures once their agent is updated to 1.13.0.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -269,7 +269,7 @@ async def agent_overview_by_host(
|
||||
stale_after = int(stale_after_seconds)
|
||||
latest = await latest_by_resource(
|
||||
db, "host_agent", list(host_names) + [h + ":/" for h in host_names],
|
||||
set(AGENT_GLANCE_METRICS) | {"disk_used_pct", "clock_off_secs"})
|
||||
set(AGENT_GLANCE_METRICS) | {"disk_used_pct", "inodes_used_pct", "clock_off_secs"})
|
||||
from steward.models.metrics import PluginMetricLatest
|
||||
cores: dict[str, int] = {}
|
||||
for res in (await db.execute(
|
||||
@@ -289,6 +289,8 @@ async def agent_overview_by_host(
|
||||
d = {m: r.value for m, r in own.items()}
|
||||
if root is not None:
|
||||
d["disk_root"] = root.value
|
||||
inodes = latest.get(name + ":/", {}).get("inodes_used_pct")
|
||||
d["inodes_root"] = inodes.value if inodes is not None else None
|
||||
ts = max((r.recorded_at for r in own.values()), default=None)
|
||||
d["last_at"] = ts
|
||||
d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
|
||||
@@ -314,6 +316,11 @@ HOST_NORMED = {"cpu": ("cpu_pct", "host_cpu_pct"), "mem": ("mem_used_pct", "host
|
||||
_KIND_OF = {metric: kind for kind, (metric, _f) in HOST_NORMED.items()}
|
||||
|
||||
|
||||
def fuller(space: float | None, inodes: float | None) -> float | None:
|
||||
"""How full a mount is: by space or by inodes, whichever is fuller (#5745)."""
|
||||
return max((v for v in (space, inodes) if v is not None), default=None)
|
||||
|
||||
|
||||
def host_normals(norms: dict, thresholds: dict) -> dict:
|
||||
"""{kind: (Norm, floor)} for worst_reading and anomalies.graded, from an
|
||||
agent glance's `norms`."""
|
||||
@@ -381,11 +388,13 @@ def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict
|
||||
worst, grades = None, {}
|
||||
if a and a.get("fresh"):
|
||||
grades = host_grades({"cpu": a.get("cpu_pct"), "mem": a.get("mem_used_pct"),
|
||||
"disk": a.get("disk_root"), "load": a.get("load_per_core")},
|
||||
"disk": fuller(a.get("disk_root"), a.get("inodes_root")),
|
||||
"load": a.get("load_per_core")},
|
||||
a.get("norms"), thresholds)
|
||||
worst = worst_reading((
|
||||
("CPU", a.get("cpu_pct"), "cpu"), ("Memory", a.get("mem_used_pct"), "mem"),
|
||||
("Disk /", a.get("disk_root"), "disk"), ("Load", a.get("load_per_core"), "load")),
|
||||
("Disk /", a.get("disk_root"), "disk"), ("Inodes on /", a.get("inodes_root"), "disk"),
|
||||
("Load", a.get("load_per_core"), "load")),
|
||||
thresholds, host_normals(a.get("norms"), thresholds))
|
||||
if st and st["state"] == "down":
|
||||
state = "down"
|
||||
|
||||
Reference in New Issue
Block a user