"""Each drive's health, judged from what the root helper read with smartctl (#5789). The helper reports figures, not verdicts, so how a drive is judged can change here without an agent update. A drive is: - "failing" when its own self-assessment fails (SMART overall-health), an NVMe drive raises a critical warning, or its spare blocks fall under threshold; - "worn" when it has used WORN_PCT of its rated life; - "watch" when it has bad sectors or media errors but otherwise passes: old disks carry a few for years, so a count alone isn't a failure; - "ok", or "unavailable" when it has no SMART to read (a virtual disk, a bridge that hides it), or "unreadable" when smartctl couldn't read it. Metrics, per drive on ":drive:": drive_failing (1/0), drive_life_used_pct, drive_bad_sectors, drive_temp_c. Per host, for covering rules: drives_failing (how many) and drive_life_used_max_pct. """ from __future__ import annotations WORN_PCT = 90 DRIVE_METRICS = ("drive_failing", "drive_life_used_pct", "drive_bad_sectors", "drive_temp_c", "drives_failing", "drive_life_used_max_pct") # What a drive's own figures count as bad sectors or media errors, whichever # its kind reports: NVMe media errors, ATA reallocated / pending / offline # uncorrectable sectors, the SCSI grown defect list. _BAD = ("media_errors", "reallocated", "pending", "offline_uncorrectable", "grown_defects") _ORDER = {"failing": 0, "unreadable": 1, "worn": 2, "watch": 3, "ok": 4, "unavailable": 5} def _num(v): return v if isinstance(v, (int, float)) and not isinstance(v, bool) else None def bad_sectors(d: dict) -> int | None: counts = [_num(d.get(k)) for k in _BAD] counts = [c for c in counts if c is not None] return int(sum(counts)) if counts else None def failing_reasons(d: dict) -> list[str]: why = [] if d.get("passed") is False: why.append("its SMART self-assessment failed") if _num(d.get("critical_warning")): why.append("critical warning %#x" % int(d["critical_warning"])) spare, floor = _num(d.get("spare_pct")), _num(d.get("spare_threshold_pct")) if spare is not None and floor is not None and spare < floor: why.append(f"spare blocks {spare:.0f}% under the {floor:.0f}% threshold") return why def judge(d: dict) -> dict: """One drive as the host page shows it: its figures plus `state` and `why`.""" out = dict(d) if d.get("error"): out.update(state="unreadable", why=str(d["error"])) elif d.get("available") is False: out.update(state="unavailable", why="no SMART data (a virtual disk, or SMART is off)") elif failing_reasons(d): out.update(state="failing", why="; ".join(failing_reasons(d))) elif (_num(d.get("life_used_pct")) or 0) >= WORN_PCT: out.update(state="worn", why=f"{d['life_used_pct']:.0f}% of its rated life used") elif bad_sectors(d): out.update(state="watch", why=f"{bad_sectors(d)} bad sectors or media errors") else: out.update(state="ok", why="") out["bad_sectors"] = bad_sectors(d) return out def drives_of(report) -> list[dict]: """The report's drives, judged, what needs attention first. A name seen twice gets an ordinal, so two drives never share one series: agent 1.20.0 named every disk behind a RAID controller "0" (#5806).""" drives = report.get("drives") if isinstance(report, dict) else None judged, seen = [], {} for d in drives or []: if not isinstance(d, dict) or not d.get("device"): continue d = judge(d) name = str(d["device"]) seen[name] = seen.get(name, 0) + 1 if seen[name] > 1: d["device"] = f"{name}#{seen[name]}" judged.append(d) return sorted(judged, key=lambda d: (_ORDER[d["state"]], str(d["device"]))) def metric_rows(report, host_name: str) -> list[tuple[str, str, float]]: """(metric, resource, value) for every figure a drive has, and the host's two roll-ups once any drive was read.""" rows: list[tuple[str, str, float]] = [] failing = 0 life_max: float | None = None read = 0 for d in drives_of(report): if d["state"] in ("unreadable", "unavailable"): continue read += 1 res = f"{host_name}:drive:{d['device']}" is_failing = d["state"] == "failing" failing += is_failing rows.append(("drive_failing", res, 1.0 if is_failing else 0.0)) life = _num(d.get("life_used_pct")) if life is not None: rows.append(("drive_life_used_pct", res, float(life))) life_max = life if life_max is None else max(life_max, life) if d["bad_sectors"] is not None: rows.append(("drive_bad_sectors", res, float(d["bad_sectors"]))) if _num(d.get("temp_c")) is not None: rows.append(("drive_temp_c", res, float(d["temp_c"]))) if read: rows.append(("drives_failing", host_name, float(failing))) if life_max is not None: rows.append(("drive_life_used_max_pct", host_name, float(life_max))) return rows