bvandeusenandClaude Opus 5.5
e2055242f1
smartctl reports every disk behind a PERC/MegaRAID controller as /dev/bus/0 with type megaraid,N; the helper named a drive by its path alone, so all of VDNT-Docker02's and Docker03's disks read as one drive "0", each overwriting the last. The slot is now part of the name (bus0-megaraid3), and Steward gives a repeated name an ordinal so two drives never share a series. Agent 1.20.1. fix(ansible): Steward stopping mid-run logs no traceback (#5749) The run's done-callback called exception() on a cancelled task, which raises; it now logs a run cut short by shutdown at INFO. A finished run's in-memory state is dropped by a timer rather than a sleeping task, which shutdown found still pending. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
117 lines
5.0 KiB
Python
117 lines
5.0 KiB
Python
"""Each drive's health, judged from what the root helper read with smartctl
|
|
(#5789). The helper reports figures, not verdicts, so how a drive is judged
|
|
can change here without an agent update.
|
|
|
|
A drive is:
|
|
- "failing" when its own self-assessment fails (SMART overall-health), an NVMe
|
|
drive raises a critical warning, or its spare blocks fall under threshold;
|
|
- "worn" when it has used WORN_PCT of its rated life;
|
|
- "watch" when it has bad sectors or media errors but otherwise passes: old
|
|
disks carry a few for years, so a count alone isn't a failure;
|
|
- "ok", or "unavailable" when it has no SMART to read (a virtual disk, a
|
|
bridge that hides it), or "unreadable" when smartctl couldn't read it.
|
|
|
|
Metrics, per drive on "<host>:drive:<device>": drive_failing (1/0),
|
|
drive_life_used_pct, drive_bad_sectors, drive_temp_c. Per host, for covering
|
|
rules: drives_failing (how many) and drive_life_used_max_pct.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
WORN_PCT = 90
|
|
DRIVE_METRICS = ("drive_failing", "drive_life_used_pct", "drive_bad_sectors", "drive_temp_c",
|
|
"drives_failing", "drive_life_used_max_pct")
|
|
# What a drive's own figures count as bad sectors or media errors, whichever
|
|
# its kind reports: NVMe media errors, ATA reallocated / pending / offline
|
|
# uncorrectable sectors, the SCSI grown defect list.
|
|
_BAD = ("media_errors", "reallocated", "pending", "offline_uncorrectable", "grown_defects")
|
|
_ORDER = {"failing": 0, "unreadable": 1, "worn": 2, "watch": 3, "ok": 4, "unavailable": 5}
|
|
|
|
|
|
def _num(v):
|
|
return v if isinstance(v, (int, float)) and not isinstance(v, bool) else None
|
|
|
|
|
|
def bad_sectors(d: dict) -> int | None:
|
|
counts = [_num(d.get(k)) for k in _BAD]
|
|
counts = [c for c in counts if c is not None]
|
|
return int(sum(counts)) if counts else None
|
|
|
|
|
|
def failing_reasons(d: dict) -> list[str]:
|
|
why = []
|
|
if d.get("passed") is False:
|
|
why.append("its SMART self-assessment failed")
|
|
if _num(d.get("critical_warning")):
|
|
why.append("critical warning %#x" % int(d["critical_warning"]))
|
|
spare, floor = _num(d.get("spare_pct")), _num(d.get("spare_threshold_pct"))
|
|
if spare is not None and floor is not None and spare < floor:
|
|
why.append(f"spare blocks {spare:.0f}% under the {floor:.0f}% threshold")
|
|
return why
|
|
|
|
|
|
def judge(d: dict) -> dict:
|
|
"""One drive as the host page shows it: its figures plus `state` and `why`."""
|
|
out = dict(d)
|
|
if d.get("error"):
|
|
out.update(state="unreadable", why=str(d["error"]))
|
|
elif d.get("available") is False:
|
|
out.update(state="unavailable", why="no SMART data (a virtual disk, or SMART is off)")
|
|
elif failing_reasons(d):
|
|
out.update(state="failing", why="; ".join(failing_reasons(d)))
|
|
elif (_num(d.get("life_used_pct")) or 0) >= WORN_PCT:
|
|
out.update(state="worn", why=f"{d['life_used_pct']:.0f}% of its rated life used")
|
|
elif bad_sectors(d):
|
|
out.update(state="watch", why=f"{bad_sectors(d)} bad sectors or media errors")
|
|
else:
|
|
out.update(state="ok", why="")
|
|
out["bad_sectors"] = bad_sectors(d)
|
|
return out
|
|
|
|
|
|
def drives_of(report) -> list[dict]:
|
|
"""The report's drives, judged, what needs attention first. A name seen
|
|
twice gets an ordinal, so two drives never share one series: agent
|
|
1.20.0 named every disk behind a RAID controller "0" (#5806)."""
|
|
drives = report.get("drives") if isinstance(report, dict) else None
|
|
judged, seen = [], {}
|
|
for d in drives or []:
|
|
if not isinstance(d, dict) or not d.get("device"):
|
|
continue
|
|
d = judge(d)
|
|
name = str(d["device"])
|
|
seen[name] = seen.get(name, 0) + 1
|
|
if seen[name] > 1:
|
|
d["device"] = f"{name}#{seen[name]}"
|
|
judged.append(d)
|
|
return sorted(judged, key=lambda d: (_ORDER[d["state"]], str(d["device"])))
|
|
|
|
|
|
def metric_rows(report, host_name: str) -> list[tuple[str, str, float]]:
|
|
"""(metric, resource, value) for every figure a drive has, and the host's
|
|
two roll-ups once any drive was read."""
|
|
rows: list[tuple[str, str, float]] = []
|
|
failing = 0
|
|
life_max: float | None = None
|
|
read = 0
|
|
for d in drives_of(report):
|
|
if d["state"] in ("unreadable", "unavailable"):
|
|
continue
|
|
read += 1
|
|
res = f"{host_name}:drive:{d['device']}"
|
|
is_failing = d["state"] == "failing"
|
|
failing += is_failing
|
|
rows.append(("drive_failing", res, 1.0 if is_failing else 0.0))
|
|
life = _num(d.get("life_used_pct"))
|
|
if life is not None:
|
|
rows.append(("drive_life_used_pct", res, float(life)))
|
|
life_max = life if life_max is None else max(life_max, life)
|
|
if d["bad_sectors"] is not None:
|
|
rows.append(("drive_bad_sectors", res, float(d["bad_sectors"])))
|
|
if _num(d.get("temp_c")) is not None:
|
|
rows.append(("drive_temp_c", res, float(d["temp_c"])))
|
|
if read:
|
|
rows.append(("drives_failing", host_name, float(failing)))
|
|
if life_max is not None:
|
|
rows.append(("drive_life_used_max_pct", host_name, float(life_max)))
|
|
return rows
|