feat(host_agent): each drive's health, read with smartctl, alertable (#5789)

The root helper gains one read-only action, `drives`: it finds the drives
itself (smartctl --scan-open), reads each as JSON, and writes a summary the
agent carries on every sample (agent 1.20.0). Steward judges the figures in
drives.py (failing / worn / watch / ok / no SMART / unreadable), records them
per drive and as host roll-ups for covering rules, keeps the newest report on
the registration and lists it on the host page.

install-deps.sh installs smartmontools with apt, dnf, yum, zypper, pacman or
apk, run by install.sh and every agent playbook; one it can't install is
reported, never fatal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/files/install-deps.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+# Steward host agent: the packages the agent's helper uses, installed with
+# whichever package manager the host has (#5789). Served by Steward at
+# /plugins/host_agent/files/install-deps.sh and run, as root, by install.sh
+# and the host_agent playbooks: one copy of the list.
+#
+# smartmontools gives the helper smartctl, which reads each drive's health.
+# A package that can't be installed is reported, never fatal: the agent runs
+# without it and the host page says what is missing.
+#
+# Recommended / weak dependencies are skipped: on Debian, smartmontools
+# recommends a mail server.
+
+PACKAGES="smartmontools"
+
+have() { command -v "$1" >/dev/null 2>&1; }
+
+install_with() {
+    if have apt-get; then
+        export DEBIAN_FRONTEND=noninteractive
+        # A stale package list can't find the package: refresh once and retry.
+        apt-get install -y --no-install-recommends "$@" \
+            || { apt-get update && apt-get install -y --no-install-recommends "$@"; }
+    elif have dnf; then
+        dnf -y install --setopt=install_weak_deps=False "$@"
+    elif have yum; then
+        yum -y install "$@"
+    elif have zypper; then
+        zypper --non-interactive install --no-recommends "$@"
+    elif have pacman; then
+        pacman -S --needed --noconfirm "$@"
+    elif have apk; then
+        apk add --no-cache "$@"
+    else
+        echo "no package manager this script knows (apt, dnf, yum, zypper, pacman, apk)"
+        return 1
+    fi
+}
+
+missing=""
+have smartctl || missing="$missing smartmontools"
+[ -n "$missing" ] || { echo "dependencies present: $PACKAGES"; exit 0; }
+
+# shellcheck disable=SC2086  # one word per package
+if install_with $missing; then
+    echo "installed:$missing"
+else
+    echo "WARNING: could not install$missing; drive health will read \"smartctl not installed\" until it is"
+fi
+exit 0

plugins/host_agent/files/maint.py

@@ -10,6 +10,10 @@ the action, and writes the outcome to RESULTS (root's own directory, which the
 agent reads back and reports to Steward). A stolen agent token can therefore
 make its host update, reboot or drain, and nothing else.
 
+One action only reads: `drives` (#5789) asks smartctl for each disk's health,
+which needs root, and writes a summary to DRIVES, never a result: it is not a
+maintenance job.
+
 Stdlib only; served by Steward at /plugins/host_agent/files/maint.py.
 """
 from __future__ import annotations
@@ -25,7 +29,8 @@ from datetime import datetime, timezone
 
 REQUESTS = "/var/lib/steward-agent/requests"
 RESULTS = "/var/lib/steward-maint/results"
-ACTIONS = ("update", "reboot", "drain", "undrain")
+DRIVES = "/var/lib/steward-maint/drives.json"
+ACTIONS = ("update", "reboot", "drain", "undrain", "drives")
 REQUEST_MAX_BYTES = 4096
 RESULTS_KEPT_SECONDS = 7 * 86400
 UPDATE_DEADLINE_SECONDS = 3000
@@ -152,6 +157,130 @@ def do_node(availability: str, node: str | None) -> dict:
     return {"ok": ok, "node": node, "detail": out.strip()[-500:]}
 
 
+# ── drive health (#5789) ─────────────────────────────────────────────────────
+# smartctl ≥ 7.0 answers in JSON, the same for SATA, SAS and NVMe drives on any
+# distribution; an older one is reported, never parsed as text. The helper
+# finds the drives itself: the request names none.
+
+SMARTCTL_MIN = (7, 0)
+SMARTCTL_DEADLINE_SECONDS = 60
+DRIVES_MAX = 32
+# smartctl's exit status is a bit mask: 1 the command line, 2 the device
+# couldn't be opened. Higher bits describe the disk, and its JSON still holds.
+_SMARTCTL_UNREAD = 0b11
+# ATA attributes counted as bad sectors.
+_ATA_BAD = {5: "reallocated", 197: "pending", 198: "offline_uncorrectable"}
+
+
+def smartctl_version():
+    """(major, minor) of the installed smartctl, or None when there is none."""
+    if not shutil.which("smartctl"):
+        return None
+    ok, out = run(["smartctl", "--version"], deadline=30)
+    m = re.search(r"smartctl (\d+)\.(\d+)", out)
+    return (int(m.group(1)), int(m.group(2))) if m else (0, 0)
+
+
+def smartctl_json(args: list):
+    """(exit status, parsed JSON or None) of one smartctl call."""
+    try:
+        done = subprocess.run(["smartctl", "--json", *args], capture_output=True, text=True,
+                              timeout=SMARTCTL_DEADLINE_SECONDS,
+                              env=dict(os.environ, LANG="C", LC_ALL="C"))
+    except (subprocess.TimeoutExpired, OSError):
+        return None, None
+    try:
+        return done.returncode, json.loads(done.stdout)
+    except ValueError:
+        return done.returncode, None
+
+
+def _int(v):
+    return v if isinstance(v, int) and not isinstance(v, bool) else None
+
+
+def drive_summary(name: str, status, data) -> dict:
+    """The few figures Steward reads from one drive's `smartctl -a` JSON."""
+    out = {"device": os.path.basename(name)}
+    if not isinstance(data, dict) or status is None or status & _SMARTCTL_UNREAD:
+        messages = ((data.get("smartctl") or {}).get("messages") or []) if isinstance(data, dict) else []
+        text = "; ".join(str(m.get("string", "")) for m in messages if isinstance(m, dict))
+        out["error"] = (text or "smartctl couldn't read it")[:200]
+        return out
+    dev = data.get("device") or {}
+    out["protocol"] = str(dev.get("protocol") or "")[:16]
+    out["model"] = str(data.get("model_name") or data.get("scsi_model_name") or "")[:80]
+    cap = (data.get("user_capacity") or {}).get("bytes")
+    if _int(cap):
+        out["capacity_bytes"] = cap
+    support = data.get("smart_support") or {}
+    if support.get("available") is False or support.get("enabled") is False:
+        out["available"] = False             # a virtual disk, or SMART switched off
+        return out
+    passed = (data.get("smart_status") or {}).get("passed")
+    if isinstance(passed, bool):
+        out["passed"] = passed
+    temp = _int((data.get("temperature") or {}).get("current"))
+    if temp is not None:
+        out["temp_c"] = temp
+    hours = _int((data.get("power_on_time") or {}).get("hours"))
+    if hours is not None:
+        out["power_on_hours"] = hours
+    nvme = data.get("nvme_smart_health_information_log") or {}
+    for src, dst in (("critical_warning", "critical_warning"), ("percentage_used", "life_used_pct"),
+                     ("media_errors", "media_errors"), ("available_spare", "spare_pct"),
+                     ("available_spare_threshold", "spare_threshold_pct")):
+        if _int(nvme.get(src)) is not None:
+            out[dst] = nvme[src]
+    for attr in (data.get("ata_smart_attributes") or {}).get("table") or []:
+        key = _ATA_BAD.get(attr.get("id")) if isinstance(attr, dict) else None
+        raw = _int((attr.get("raw") or {}).get("value")) if key else None
+        if raw is not None:
+            out[key] = raw
+    used = _int((data.get("endurance_used") or {}).get("current_percent"))          # ATA SSDs
+    used = used if used is not None else _int(data.get("scsi_percentage_used_endurance_indicator"))
+    if used is not None and "life_used_pct" not in out:
+        out["life_used_pct"] = used
+    if _int(data.get("scsi_grown_defect_list")) is not None:
+        out["grown_defects"] = data["scsi_grown_defect_list"]
+    return out
+
+
+def do_drives() -> dict:
+    """Each drive's health, as {"checked_at", "smartctl", "drives"} or
+    {"checked_at", "error"} when smartctl is missing or too old."""
+    base = {"checked_at": now_iso()}
+    version = smartctl_version()
+    if version is None:
+        return {**base, "error": "smartctl not installed"}
+    base["smartctl"] = "%d.%d" % version
+    if version < SMARTCTL_MIN:
+        return {**base, "error": "smartctl %d.%d is too old: drive health needs 7.0 or newer" % version}
+    _, scan = smartctl_json(["--scan-open"])
+    devices = (scan or {}).get("devices") if isinstance(scan, dict) else None
+    if not isinstance(devices, list):
+        return {**base, "error": "smartctl couldn't list the drives"}
+    drives = []
+    for d in devices[:DRIVES_MAX]:
+        name, kind = (d.get("name"), d.get("type")) if isinstance(d, dict) else (None, None)
+        if not isinstance(name, str) or not name.startswith("/dev/"):
+            continue
+        args = ["-a", name] if not isinstance(kind, str) else ["-a", "-d", kind, name]
+        status, data = smartctl_json(args)
+        drives.append(drive_summary(name, status, data))
+    return {**base, "drives": drives}
+
+
+def write_drives(report: dict, path: str | None = None) -> None:
+    """DRIVES, written whole and readable by the agent."""
+    path = path or DRIVES
+    tmp = path + ".tmp"
+    with open(tmp, "w") as fh:
+        json.dump(report, fh)
+    os.chmod(tmp, 0o644)
+    os.replace(tmp, path)
+
+
 def write_result(result: dict) -> None:
     """RESULTS/<id>.json, written whole (a temp file renamed into place)."""
     os.makedirs(RESULTS, mode=0o755, exist_ok=True)
@@ -170,6 +299,9 @@ def already_handled(job_id: str) -> bool:
 
 def handle(req: dict) -> dict:
     base = {"id": req["id"], "action": req["action"], "started_at": now_iso()}
+    if req["action"] == "drives":               # a read, not a job: no result
+        write_drives(do_drives())
+        return {**base, "state": "done"}
     write_result({**base, "state": "running"})
     action = req["action"]
     if action == "update":

plugins/host_agent/files/steward-maint.service

@@ -1,5 +1,6 @@
 # Runs as root, and does only what maint.py's fixed list allows: update the
-# host's packages, reboot it, or drain / reactivate its swarm node (#5766).
+# host's packages, reboot it, drain / reactivate its swarm node (#5766), or
+# read its drives' health with smartctl (#5789).
 [Unit]
 Description=Steward maintenance helper