bvandeusenandClaude Opus 5.5
6c4ef368f9
The root helper gains one read-only action, `drives`: it finds the drives itself (smartctl --scan-open), reads each as JSON, and writes a summary the agent carries on every sample (agent 1.20.0). Steward judges the figures in drives.py (failing / worn / watch / ok / no SMART / unreadable), records them per drive and as host roll-ups for covering rules, keeps the newest report on the registration and lists it on the host page. install-deps.sh installs smartmontools with apt, dnf, yum, zypper, pacman or apk, run by install.sh and every agent playbook; one it can't install is reported, never fatal. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
106 lines
4.6 KiB
Python
106 lines
4.6 KiB
Python
"""Each drive's health, judged from what the root helper read with smartctl
|
|
(#5789). The helper reports figures, not verdicts, so how a drive is judged
|
|
can change here without an agent update.
|
|
|
|
A drive is:
|
|
- "failing" when its own self-assessment fails (SMART overall-health), an NVMe
|
|
drive raises a critical warning, or its spare blocks fall under threshold;
|
|
- "worn" when it has used WORN_PCT of its rated life;
|
|
- "watch" when it has bad sectors or media errors but otherwise passes: old
|
|
disks carry a few for years, so a count alone isn't a failure;
|
|
- "ok", or "unavailable" when it has no SMART to read (a virtual disk, a
|
|
bridge that hides it), or "unreadable" when smartctl couldn't read it.
|
|
|
|
Metrics, per drive on "<host>:drive:<device>": drive_failing (1/0),
|
|
drive_life_used_pct, drive_bad_sectors, drive_temp_c. Per host, for covering
|
|
rules: drives_failing (how many) and drive_life_used_max_pct.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
WORN_PCT = 90
|
|
DRIVE_METRICS = ("drive_failing", "drive_life_used_pct", "drive_bad_sectors", "drive_temp_c",
|
|
"drives_failing", "drive_life_used_max_pct")
|
|
# What a drive's own figures count as bad sectors or media errors, whichever
|
|
# its kind reports: NVMe media errors, ATA reallocated / pending / offline
|
|
# uncorrectable sectors, the SCSI grown defect list.
|
|
_BAD = ("media_errors", "reallocated", "pending", "offline_uncorrectable", "grown_defects")
|
|
_ORDER = {"failing": 0, "unreadable": 1, "worn": 2, "watch": 3, "ok": 4, "unavailable": 5}
|
|
|
|
|
|
def _num(v):
|
|
return v if isinstance(v, (int, float)) and not isinstance(v, bool) else None
|
|
|
|
|
|
def bad_sectors(d: dict) -> int | None:
|
|
counts = [_num(d.get(k)) for k in _BAD]
|
|
counts = [c for c in counts if c is not None]
|
|
return int(sum(counts)) if counts else None
|
|
|
|
|
|
def failing_reasons(d: dict) -> list[str]:
|
|
why = []
|
|
if d.get("passed") is False:
|
|
why.append("its SMART self-assessment failed")
|
|
if _num(d.get("critical_warning")):
|
|
why.append("critical warning %#x" % int(d["critical_warning"]))
|
|
spare, floor = _num(d.get("spare_pct")), _num(d.get("spare_threshold_pct"))
|
|
if spare is not None and floor is not None and spare < floor:
|
|
why.append(f"spare blocks {spare:.0f}% under the {floor:.0f}% threshold")
|
|
return why
|
|
|
|
|
|
def judge(d: dict) -> dict:
|
|
"""One drive as the host page shows it: its figures plus `state` and `why`."""
|
|
out = dict(d)
|
|
if d.get("error"):
|
|
out.update(state="unreadable", why=str(d["error"]))
|
|
elif d.get("available") is False:
|
|
out.update(state="unavailable", why="no SMART data (a virtual disk, or SMART is off)")
|
|
elif failing_reasons(d):
|
|
out.update(state="failing", why="; ".join(failing_reasons(d)))
|
|
elif (_num(d.get("life_used_pct")) or 0) >= WORN_PCT:
|
|
out.update(state="worn", why=f"{d['life_used_pct']:.0f}% of its rated life used")
|
|
elif bad_sectors(d):
|
|
out.update(state="watch", why=f"{bad_sectors(d)} bad sectors or media errors")
|
|
else:
|
|
out.update(state="ok", why="")
|
|
out["bad_sectors"] = bad_sectors(d)
|
|
return out
|
|
|
|
|
|
def drives_of(report) -> list[dict]:
|
|
"""The report's drives, judged, what needs attention first."""
|
|
drives = report.get("drives") if isinstance(report, dict) else None
|
|
judged = [judge(d) for d in drives or [] if isinstance(d, dict) and d.get("device")]
|
|
return sorted(judged, key=lambda d: (_ORDER[d["state"]], str(d["device"])))
|
|
|
|
|
|
def metric_rows(report, host_name: str) -> list[tuple[str, str, float]]:
|
|
"""(metric, resource, value) for every figure a drive has, and the host's
|
|
two roll-ups once any drive was read."""
|
|
rows: list[tuple[str, str, float]] = []
|
|
failing = 0
|
|
life_max: float | None = None
|
|
read = 0
|
|
for d in drives_of(report):
|
|
if d["state"] in ("unreadable", "unavailable"):
|
|
continue
|
|
read += 1
|
|
res = f"{host_name}:drive:{d['device']}"
|
|
is_failing = d["state"] == "failing"
|
|
failing += is_failing
|
|
rows.append(("drive_failing", res, 1.0 if is_failing else 0.0))
|
|
life = _num(d.get("life_used_pct"))
|
|
if life is not None:
|
|
rows.append(("drive_life_used_pct", res, float(life)))
|
|
life_max = life if life_max is None else max(life_max, life)
|
|
if d["bad_sectors"] is not None:
|
|
rows.append(("drive_bad_sectors", res, float(d["bad_sectors"])))
|
|
if _num(d.get("temp_c")) is not None:
|
|
rows.append(("drive_temp_c", res, float(d["temp_c"])))
|
|
if read:
|
|
rows.append(("drives_failing", host_name, float(failing)))
|
|
if life_max is not None:
|
|
rows.append(("drive_life_used_max_pct", host_name, float(life_max)))
|
|
return rows
|