feat(host_agent): a disk out of inodes reads as full (#5745)

VDNT-Util01's / ran out of inodes (956 free of 60.8M) with 274G of space
left; every file create failed and Steward's page read healthy at 66%.

- Agent 1.13.0 reports each mount's inodes (statvfs; a filesystem reporting
  none, like btrfs, is left without).
- Ingest records inodes_used_pct per mount, and disk_used_pct_worst is the
  fullest mount by space or inodes, so existing disk-full rules and the
  overview see it.
- The host's health strip, the Hosts widget and the host glance grade / by
  whichever is fuller and name inodes when they are ("Disk / 66% · inodes
  100%", "Inodes on / ... past its critical threshold").

Hosts pick up the inode figures once their agent is updated to 1.13.0.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/mcp_tools.py

@@ -107,10 +107,17 @@ def task_error_text(err: dict | None) -> str:
     return f"{why}{code}{where}"
 
 
+# Services that started failing within this long of each other read as one
+# finding: they are usually one node's event (#5722).
+TOGETHER = timedelta(minutes=5)
+
+
 def service_findings(services: list[dict]) -> list[dict]:
-    """Swarm services that are failing: crash-looping, under-replicated, or that
-    lost a task to an error in the last 24h. Each names Swarm's last task error."""
-    out = []
+    """Swarm services failing now: crash-looping, or short of replicas, each
+    naming Swarm's last task error. One back to all its replicas is history,
+    kept on the Docker page, not a finding (#5722). Several short of replicas
+    since within TOGETHER of each other are one finding naming them all."""
+    out, short = [], []
     for s in services:
         replicas = f"{s['running']} of {s['desired']} replicas running"
         last = task_error_text(s.get("last_error"))
@@ -124,20 +131,40 @@ def service_findings(services: list[dict]) -> list[dict]:
                 f"last hour, {replicas}{tail}", since=since, ids=ids,
                 next="list_swarm_services()"))
         elif s["state"] != "healthy":
+            short.append((s, replicas, tail))
+    for group in _together(short):
+        if len(group) == 1:
+            s, replicas, tail = group[0]
             out.append(finding(
                 "critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
                 s["name"], f"swarm service {s['name']} has {replicas}{tail}",
-                since=since, ids=ids, next="list_swarm_services()"))
-        elif s.get("failures_24h"):
-            n = s["failures_24h"]
-            out.append(finding(
-                "warning", "swarm_task_failed", s["name"],
-                f"swarm service {s['name']} had {n} failed task{'s' if n != 1 else ''} in the last "
-                f"24h and is now {replicas}{tail}", since=since, ids=ids,
+                since=s.get("trouble_since"), ids={"service_name": s["name"]},
                 next="list_swarm_services()"))
+            continue
+        names = [s["name"] for s, _r, _t in group]
+        down = any(s["state"] == "down" for s, _r, _t in group)
+        out.append(finding(
+            "critical" if down else "warning", "swarm_services_short", f"{len(group)} swarm services",
+            f"{len(group)} swarm services started failing together and are short of replicas: "
+            + ", ".join(f"{s['name']} ({r})" for s, r, _t in group),
+            since=min(s["trouble_since"] for s, _r, _t in group),
+            ids={"service_names": names}, next="list_swarm_services()"))
     return out
 
 
+def _together(short: list[tuple]) -> list[list[tuple]]:
+    """Short services grouped by when they started failing: each group's
+    starts lie within TOGETHER of the one before. One without a start stands alone."""
+    timed = sorted((x for x in short if x[0].get("trouble_since")), key=lambda x: x[0]["trouble_since"])
+    groups: list[list[tuple]] = []
+    for x in timed:
+        if groups and x[0]["trouble_since"] - groups[-1][-1][0]["trouble_since"] <= TOGETHER:
+            groups[-1].append(x)
+        else:
+            groups.append([x])
+    return groups + [[x] for x in short if not x[0].get("trouble_since")]
+
+
 # ── loading ──────────────────────────────────────────────────────────────────
 
 async def _host_names(db) -> dict[str, str]:

plugins/docker/routes.py

@@ -245,7 +245,11 @@ async def widget():
     swarm_services = services_problems_first(build_swarm_services(
         all_containers, services, node_hostname, host_name,
         crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"])
-    failures = docker_svc.failures_24h(events, swarm_services, host_name)
+    # The dashboard shows what is wrong now: a service back to all its
+    # replicas has its failed tasks on the Docker page, not here (#5722).
+    still = {s["name"] for s in swarm_services if s["state"] != "healthy" or s.get("crash_looping")}
+    failures = [f for f in docker_svc.failures_24h(events, swarm_services, host_name)
+                if f["kind"] != "service" or f["name"] in still]
 
     standalone = [c for c in all_containers if not c.service_name]
     running = [c for c in standalone if c.status == "running"]

plugins/host_agent/__init__.py

@@ -53,9 +53,11 @@ def get_alert_metrics() -> dict[str, list[str]]:
     comes from the scheduler, not the agent: "agent offline" is > 180.
     clock_off_secs is how far the host's clock is from Steward's, either way
     (agent ≥ 1.12.0): "clock drifting" is > 30. load_pct is load_1m as a %
-    of the host's cores."""
+    of the host's cores. inodes_used_pct is per mount (agent ≥ 1.13.0), and
+    disk_used_pct_worst is the fullest mount by space or inodes."""
     return {"host_agent": [
         "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
-        "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m", "load_pct",
+        "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m",
+        "load_pct",
         "uptime_secs", "seconds_since_seen", "clock_off_secs",
     ]}

plugins/host_agent/agent.py

@@ -20,7 +20,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.12.0"
+AGENT_VERSION = "1.13.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -188,7 +188,18 @@ def collect_storage(mounts: list[str]) -> list[dict]:
             u = shutil.disk_usage(m)
         except (FileNotFoundError, PermissionError):
             continue
-        out.append({"mount": m, "total_bytes": u.total, "used_bytes": u.used})
+        row = {"mount": m, "total_bytes": u.total, "used_bytes": u.used}
+        # Inodes too (1.13.0): a filesystem can run out of them with space
+        # to spare, and every create then fails with ENOSPC. Some (btrfs)
+        # report none, and are left without.
+        try:
+            st = os.statvfs(m)
+        except OSError:
+            st = None
+        if st is not None and st.f_files:
+            row["inodes_total"] = st.f_files
+            row["inodes_used"] = st.f_files - st.f_ffree
+        out.append(row)
     return out
 
 

plugins/host_agent/routes.py

@@ -127,6 +127,9 @@ def _expand_sample_to_metrics(
     if sample.get("uptime_secs") is not None:
         row("uptime_secs", host_name, sample["uptime_secs"])
 
+    # The fullest disk is the fullest by space or by inodes (#5745): out of
+    # inodes, every file create fails with space to spare, so the disk rules
+    # and the overview, which read disk_used_pct_worst, see either.
     worst_pct = 0.0
     for disk in sample.get("storage") or []:
         total = disk.get("total_bytes", 0)
@@ -136,6 +139,10 @@ def _expand_sample_to_metrics(
         row("disk_used_pct", mount_res, pct)
         row("disk_used_bytes", mount_res, used)
         row("disk_total_bytes", mount_res, total)
+        if disk.get("inodes_total"):
+            inode_pct = 100.0 * disk.get("inodes_used", 0) / disk["inodes_total"]
+            row("inodes_used_pct", mount_res, inode_pct)
+            pct = max(pct, inode_pct)
         if pct > worst_pct:
             worst_pct = pct
     if sample.get("storage"):
@@ -750,6 +757,8 @@ async def host_vitals(host_id: str):
     # At-a-glance disk = the root filesystem (what people actually care about),
     # not the "worst" mount which is opaque. Worst stays on the full-metrics page.
     disk_root = (latest.get(host.name + ":/", {}) or {}).get("disk_used_pct")
+    # Out of inodes, "/" is full with space to spare (#5745): named when worse.
+    inodes_root = (latest.get(host.name + ":/", {}) or {}).get("inodes_used_pct")
     sparks = {k: sparkline_svg(v, height=24, fill_width=True) for k, v in series_raw.items()}
     # Normalize load by core count so it's comparable across hardware (load/core
     # as %). Cores derived from the per-core CPU sub-resources already collected.
@@ -767,12 +776,12 @@ async def host_vitals(host_id: str):
     cpu, mem = hostlvl.get("cpu_pct"), hostlvl.get("mem_used_pct")
     thresholds = current_app.config.get("THRESHOLDS", {})
     worst = worst_reading((("CPU", cpu, "cpu"), ("Memory", mem, "mem"), ("Disk /", disk_root, "disk"),
-                           ("Load per core", load_per_core, "load")),
+                           ("Inodes on /", inodes_root, "disk"), ("Load per core", load_per_core, "load")),
                           thresholds, host_svc.host_normals(norms, thresholds))
     # Each tile's grade; CPU, memory and load within the host's own normal
     # for the hour read healthy, and say what is usual (#5742).
-    grades = host_svc.host_grades({"cpu": cpu, "mem": mem, "disk": disk_root, "load": load_per_core},
-                                  norms, thresholds)
+    grades = host_svc.host_grades({"cpu": cpu, "mem": mem, "disk": host_svc.fuller(disk_root, inodes_root),
+                                   "load": load_per_core}, norms, thresholds)
     usual = [f"{label} {value:.0f}% is {grades[kind][1]}"
              for label, value, kind in (("CPU", cpu, "cpu"), ("Memory", mem, "mem"),
                                         ("load per core", load_per_core, "load"))
@@ -780,6 +789,7 @@ async def host_vitals(host_id: str):
     return await render_template(
         "_host_vitals.html",
         reg=reg, cpu=cpu, mem=mem, disk_root=disk_root, load_per_core=load_per_core,
+        inodes_root=inodes_root,
         sparks=sparks, psi=psi, reporting=reporting, stale=stale, worst=worst,
         grades=grades, usual=usual,
         clock=clock_words(reg.clock_offset_s if reg else None, current_app.config.get("THRESHOLDS", {})),

plugins/host_agent/templates/_host_vitals.html

@@ -25,7 +25,9 @@
   <div class="stat-strip host-health-tiles" style="margin-bottom:0;">
     {{ stat_tile("CPU", pct(cpu), state=(grades.cpu or [status_class(cpu, 'cpu')])[0], hint=sparks.cpu|safe) }}
     {{ stat_tile("Memory", pct(mem), state=(grades.mem or [status_class(mem, 'mem')])[0], hint=sparks.mem|safe) }}
-    {{ stat_tile("Disk /", pct(disk_root), state=(grades.disk or [status_class(disk_root, 'disk')])[0], hint=sparks.disk|safe) }}
+    {%- set inodes_worse = inodes_root is defined and inodes_root is not none and (disk_root is none or inodes_root > disk_root) %}
+    {{ stat_tile("Disk /", pct(disk_root) ~ (" · inodes " ~ pct(inodes_root) if inodes_worse else ""),
+                 state=(grades.disk or [status_class(disk_root, 'disk')])[0], hint=sparks.disk|safe) }}
     {{ stat_tile("Load per core", pct(load_per_core), state=(grades.load or [status_class(load_per_core, 'load')])[0], hint=sparks.load|safe) }}
   </div>
   {% if usual and not stale %}

steward/core/formatting.py

@@ -329,8 +329,9 @@ METRICS: dict[str, tuple[str, str]] = {
     "mem_used_pct": ("Memory used", "memory in use"),
     "mem_available_bytes": ("Memory available", "memory free for new work"),
     "swap_used_bytes": ("Swap used", "memory paged out to disk"),
-    "disk_used_pct_worst": ("Fullest disk", "the most-full mount on the host"),
+    "disk_used_pct_worst": ("Fullest disk", "the most-full mount on the host, by space or inodes"),
     "disk_used_pct": ("Disk used", "one mount (resource host:mount)"),
+    "inodes_used_pct": ("Inodes used", "one mount's file slots in use (resource host:mount); at 100% no file can be created"),
     "load_1m": ("Load (1 min)", "runnable processes, averaged over a minute"),
     "load_5m": ("Load (5 min)", "runnable processes, averaged over 5 minutes"),
     "load_15m": ("Load (15 min)", "runnable processes, averaged over 15 minutes"),

steward/hosts/service.py

@@ -269,7 +269,7 @@ async def agent_overview_by_host(
     stale_after = int(stale_after_seconds)
     latest = await latest_by_resource(
         db, "host_agent", list(host_names) + [h + ":/" for h in host_names],
-        set(AGENT_GLANCE_METRICS) | {"disk_used_pct", "clock_off_secs"})
+        set(AGENT_GLANCE_METRICS) | {"disk_used_pct", "inodes_used_pct", "clock_off_secs"})
     from steward.models.metrics import PluginMetricLatest
     cores: dict[str, int] = {}
     for res in (await db.execute(
@@ -289,6 +289,8 @@ async def agent_overview_by_host(
         d = {m: r.value for m, r in own.items()}
         if root is not None:
             d["disk_root"] = root.value
+        inodes = latest.get(name + ":/", {}).get("inodes_used_pct")
+        d["inodes_root"] = inodes.value if inodes is not None else None
         ts = max((r.recorded_at for r in own.values()), default=None)
         d["last_at"] = ts
         d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
@@ -314,6 +316,11 @@ HOST_NORMED = {"cpu": ("cpu_pct", "host_cpu_pct"), "mem": ("mem_used_pct", "host
 _KIND_OF = {metric: kind for kind, (metric, _f) in HOST_NORMED.items()}
 
 
+def fuller(space: float | None, inodes: float | None) -> float | None:
+    """How full a mount is: by space or by inodes, whichever is fuller (#5745)."""
+    return max((v for v in (space, inodes) if v is not None), default=None)
+
+
 def host_normals(norms: dict, thresholds: dict) -> dict:
     """{kind: (Norm, floor)} for worst_reading and anomalies.graded, from an
     agent glance's `norms`."""
@@ -381,11 +388,13 @@ def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict
         worst, grades = None, {}
         if a and a.get("fresh"):
             grades = host_grades({"cpu": a.get("cpu_pct"), "mem": a.get("mem_used_pct"),
-                                  "disk": a.get("disk_root"), "load": a.get("load_per_core")},
+                                  "disk": fuller(a.get("disk_root"), a.get("inodes_root")),
+                                  "load": a.get("load_per_core")},
                                  a.get("norms"), thresholds)
             worst = worst_reading((
                 ("CPU", a.get("cpu_pct"), "cpu"), ("Memory", a.get("mem_used_pct"), "mem"),
-                ("Disk /", a.get("disk_root"), "disk"), ("Load", a.get("load_per_core"), "load")),
+                ("Disk /", a.get("disk_root"), "disk"), ("Inodes on /", a.get("inodes_root"), "disk"),
+                ("Load", a.get("load_per_core"), "load")),
                 thresholds, host_normals(a.get("norms"), thresholds))
         if st and st["state"] == "down":
             state = "down"

steward/mcp/tools/overview.py

@@ -127,7 +127,7 @@ def alert_findings(active, silent: dict[str, list[str]] | None = None) -> list[d
 
 
 _AGENT_CHECKS = (("cpu_pct", "cpu", "CPU"), ("mem_used_pct", "mem", "memory"),
-                 ("disk_used_pct_worst", "disk", "fullest disk"))
+                 ("disk_used_pct_worst", "disk", "fullest disk (space or inodes)"))
 
 
 def agent_findings(hosts, agent: dict, thresholds: dict,

steward/templates/hosts/overview_widget.html

@@ -33,7 +33,8 @@
     </td>
     {% for v, kind in ((a.get('cpu_pct'), 'cpu'), (a.get('mem_used_pct'), 'mem'), (a.get('disk_root'), 'disk')) %}
     {% set g = (r.grades or {}).get(kind) or (none, none) %}
-    <td style="{{ grade_style(g[0]) }}"{% if g[1] %} title="{{ g[1]|capitalize }}"{% endif %}>{% if v is none %}—{% else %}{{ "%.0f"|format(v) }}%{% endif %}</td>
+    {% set inodes = a.get('inodes_root') if kind == 'disk' else none %}
+    <td style="{{ grade_style(g[0]) }}"{% if g[1] %} title="{{ g[1]|capitalize }}"{% elif inodes is not none and (v is none or inodes > v) %} title="Inodes {{ '%.0f'|format(inodes) }}% used: no file can be created once they run out"{% endif %}>{% if v is none %}—{% else %}{{ "%.0f"|format(v) }}%{% endif %}{% if inodes is not none and (v is none or inodes > v) and inodes >= 80 %} <span class="w-sub">inodes {{ "%.0f"|format(inodes) }}%</span>{% endif %}</td>
     {% endfor %}
     {% set g = (r.grades or {}).get('load') or (none, none) %}
     <td style="{{ grade_style(g[0]) }}"

tests/core/test_norm_grading.py

@@ -133,3 +133,12 @@ async def test_the_thresholds_page_offers_each_running_kinds_floor(app):
     assert 'name="norms.floor.host_cpu_pct"' in page and 'name="norms.floor.monitor_ms"' in page
     assert 'name="norms.floor.container_cpu_pct"' not in page         # docker isn't running
     assert 'name="norms.floor.router_4xx_pct"' not in page
+
+
+def test_a_host_out_of_inodes_reads_critical_and_says_inodes():
+    """#5745: space 66%, inodes 100% is a full disk, named as inodes."""
+    hosts = [NS(id="u", name="util01")]
+    agent = {"util01": {"disk_root": 66.0, "inodes_root": 100.0, "fresh": True}}
+    (row,) = host_glance(hosts, {}, agent, TH)
+    assert row["state"] == "crit" and row["worst"][0] == "Inodes on /"
+    assert row["grades"]["disk"][0] == "crit"

tests/plugins/docker/test_mcp_tools.py

@@ -96,7 +96,7 @@ def test_service_findings_name_the_crash_loop_and_swarms_error():
             {"name": "ok", "state": "healthy", "running": 1, "desired": 1, "crash_looping": False,
              "failures_1h": 0, "failures_24h": 0, "last_error": None}]
     got = {f["subject"]: f for f in t.service_findings(svcs)}
-    assert set(got) == {"db", "app", "web"}
+    assert set(got) == {"db", "app"}     # web is back to all its replicas: history (#5722)
     db = got["db"]
     assert db["kind"] == "swarm_service_crash_looping" and db["severity"] == "critical"
     assert "4 tasks failed in the last hour" in db["summary"]
@@ -104,9 +104,6 @@ def test_service_findings_name_the_crash_loop_and_swarms_error():
     # Down without a loop still names why: here Swarm could not even start it.
     assert got["app"]["kind"] == "swarm_service_down"
     assert got["app"]["summary"].endswith("last task error: No such image: app:dev")
-    # Recovered, but a task died in the last day: worth knowing, not critical.
-    assert got["web"]["kind"] == "swarm_task_failed" and got["web"]["severity"] == "warning"
-    assert "had 1 failed task in the last 24h" in got["web"]["summary"]
 
 
 def test_a_replaced_swarm_task_is_marked_and_judged_on_its_service():
@@ -178,3 +175,24 @@ def test_findings_carry_when_the_problem_began():
     svc = {"name": "db", "state": "down", "running": 0, "desired": 1, "trouble_since": began}
     (g,) = t.service_findings([svc])
     assert g["since"] == "2026-10-06T11:20:00Z"
+
+
+def test_services_that_failed_together_are_one_finding_and_recovered_ones_none():
+    """#5722: nine services lost tasks within three minutes on one node's bad
+    moment, and stayed on the overview for a day after they were back. Those
+    back to full replicas are history; those still short and started together
+    are one finding naming them, dated from the first."""
+    at = NOW - timedelta(minutes=30)
+    svcs = [{"name": "auth", "state": "degraded", "running": 1, "desired": 2, "trouble_since": at},
+            {"name": "wiki", "state": "down", "running": 0, "desired": 1,
+             "trouble_since": at + timedelta(minutes=2)},
+            {"name": "later", "state": "degraded", "running": 1, "desired": 2,
+             "trouble_since": at + timedelta(minutes=20)},
+            {"name": "back", "state": "healthy", "running": 1, "desired": 1, "failures_24h": 3,
+             "trouble_since": at}]
+    together, alone = t.service_findings(svcs)
+    assert together["kind"] == "swarm_services_short" and together["severity"] == "critical"
+    assert together["subject"] == "2 swarm services"
+    assert "auth (1 of 2 replicas running), wiki (0 of 1 replicas running)" in together["summary"]
+    assert together["since"] == "2026-10-06T11:30:00Z"
+    assert alone["subject"] == "later" and alone["kind"] == "swarm_service_degraded"

tests/plugins/host_agent/test_agent_collectors.py

@@ -78,10 +78,19 @@ def test_collect_storage():
         r.free = free
         return r
 
-    with patch.object(a.shutil, "disk_usage", side_effect=_fake_usage):
+    # Inodes (#5745): "/" has 100 with 1 free; /mnt/data (btrfs-like) reports none.
+    inodes = {"/": (100, 1), "/mnt/data": (0, 0)}
+
+    def _fake_statvfs(path):
+        files, ffree = inodes[path]
+        return type("S", (), {"f_files": files, "f_ffree": ffree})()
+
+    with patch.object(a.shutil, "disk_usage", side_effect=_fake_usage), \
+            patch.object(a.os, "statvfs", side_effect=_fake_statvfs):
         out = a.collect_storage(["/", "/mnt/data"])
     assert out == [
-        {"mount": "/",         "total_bytes": 1000, "used_bytes": 400},
+        {"mount": "/",         "total_bytes": 1000, "used_bytes": 400,
+         "inodes_total": 100, "inodes_used": 99},
         {"mount": "/mnt/data", "total_bytes": 2000, "used_bytes": 500},
     ]
 

tests/plugins/host_agent/test_routes_expand_metrics.py

@@ -96,3 +96,18 @@ def test_expand_empty_storage_emits_no_disk_worst():
     names = {r.metric_name for r in rows}
     assert "disk_used_pct_worst" not in names
     assert "cpu_pct" in names
+
+
+def test_a_mount_out_of_inodes_is_the_fullest_disk():
+    """#5745: VDNT-Util01's / was 66% full by space and 100% by inodes, every
+    file create failed, and Steward read it healthy. Inode use is recorded per
+    mount, and the fullest disk is the fullest by either."""
+    sample = _sample()
+    sample["storage"][0].update(inodes_total=60_850_176, inodes_used=60_849_220)
+    rows = _expand_sample_to_metrics(sample, "testhost", NOW)
+    (inodes,) = _by_metric(rows, "inodes_used_pct")
+    assert inodes.resource_name == "testhost:/" and inodes.value > 99.99
+    (worst,) = _by_metric(rows, "disk_used_pct_worst")
+    assert worst.value == inodes.value
+    # A filesystem reporting no inodes (btrfs) records none.
+    assert len(_by_metric(_expand_sample_to_metrics(_sample(), "testhost", NOW), "inodes_used_pct")) == 0