feat(host_agent): each drive's health, read with smartctl, alertable (#5789)

The root helper gains one read-only action, `drives`: it finds the drives
itself (smartctl --scan-open), reads each as JSON, and writes a summary the
agent carries on every sample (agent 1.20.0). Steward judges the figures in
drives.py (failing / worn / watch / ok / no SMART / unreadable), records them
per drive and as host roll-ups for covering rules, keeps the newest report on
the registration and lists it on the host page.

install-deps.sh installs smartmontools with apt, dnf, yum, zypper, pacman or
apk, run by install.sh and every agent playbook; one it can't install is
reported, never fatal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/__init__.py

@@ -86,10 +86,22 @@ def get_alert_metrics() -> dict[str, list[str]]:
     the whole host ("<host>") or one interface ("<host>:net:<iface>") or disk
     ("<host>:diskio:<device>").
 
+    Drive health (agent ≥ 1.20.0, #5789), read hourly with smartctl by the
+    root helper and judged in drives.py, per drive on "<host>:drive:<device>":
+    drive_failing is 1 when the drive fails its own self-assessment, raises
+    an NVMe critical warning or runs out of spare blocks; drive_life_used_pct
+    is how much of its rated life it has used; drive_bad_sectors counts its
+    media errors or reallocated / pending / uncorrectable sectors;
+    drive_temp_c is its own temperature. Per host, for covering rules:
+    drives_failing (how many) and drive_life_used_max_pct (the most worn):
+    "drive failing" is drives_failing > 0, "drive wearing out" is
+    drive_life_used_max_pct > 90.
+
     Recorded but not offered (NOT_ALERTABLE): disk_used_bytes and
     disk_total_bytes, where disk_used_pct is the figure to watch (and docker
     uses disk_total_bytes for something else), and mem_cached_bytes /
     mem_buffers_bytes, parts of what mem_available_bytes already counts."""
+    from .drives import DRIVE_METRICS
     return {"host_agent": [
         "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
         "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m",
@@ -98,6 +110,7 @@ def get_alert_metrics() -> dict[str, list[str]]:
         "agent_rss_bytes", "agent_cpu_pct",
         "updates_pending", "updates_security", "reboot_required", "maintenance_failed",
         "temp_c_max", "temp_c", "net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
+        *DRIVE_METRICS,
     ]}
 
 

plugins/host_agent/agent.py

@@ -22,7 +22,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.19.0"
+AGENT_VERSION = "1.20.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -1456,7 +1456,7 @@ def collect_reboot(path: str = REBOOT_REQUIRED):
 MAINT_REQUESTS = "/var/lib/steward-agent/requests"
 MAINT_RESULTS = "/var/lib/steward-maint/results"
 MAINT_PATH_UNIT = "/etc/systemd/system/steward-maint.path"
-MAINT_ACTIONS = ("update", "reboot", "drain", "undrain")
+MAINT_ACTIONS = ("update", "reboot", "drain", "undrain", "drives")
 MAINT_RESULT_MAX_BYTES = 8192
 # The helper keeps results a week; only this recent a one is reported, so a
 # job a reboot interrupted still reaches Steward from the agent's next start
@@ -1543,6 +1543,47 @@ def maintenance_results(seen: set, results_dir: str = MAINT_RESULTS,
     return events, keys
 
 
+# ── drive health (#5789) ─────────────────────────────────────────────────────
+# Reading SMART needs root, so the helper does it: hourly (and at start) the
+# agent leaves it a `drives` request, and the helper writes each drive's
+# figures to DRIVES_PATH. The last answer rides every sample, like updates.
+
+DRIVES_PATH = "/var/lib/steward-maint/drives.json"
+DRIVES_INTERVAL_SECONDS = 3600
+DRIVES_MAX_BYTES = 65536
+
+
+def refresh_drives(state: dict, now_mono: float | None = None,
+                   requests_dir: str = MAINT_REQUESTS, drives_path: str = DRIVES_PATH,
+                   unit: str = MAINT_PATH_UNIT) -> None:
+    """Ask the helper for a drive read when one is due, and keep its newest
+    answer in state["drives"]."""
+    now_mono = time.monotonic() if now_mono is None else now_mono
+    last = state.get("drives_asked_mono")
+    if last is None or now_mono - last >= DRIVES_INTERVAL_SECONDS:
+        state["drives_asked_mono"] = now_mono
+        if not maint_helper_installed(unit, requests_dir):
+            state["drives"] = {"error": "needs the maintenance helper (Update agent installs it)"}
+        else:
+            try:
+                request_maintenance(f"drives-{int(time.time())}", "drives", requests_dir=requests_dir)
+            except (ValueError, OSError) as exc:
+                _log("WARN", f"drive health not requested: {exc}")
+    try:
+        st = os.stat(drives_path)
+    except OSError:
+        return
+    if st.st_mtime == state.get("drives_mtime") or st.st_size > DRIVES_MAX_BYTES:
+        return
+    try:
+        with open(drives_path) as fh:
+            got = json.load(fh)
+    except (OSError, ValueError):
+        return                              # mid-write: the next sample reads it
+    if isinstance(got, dict):
+        state["drives"], state["drives_mtime"] = got, st.st_mtime
+
+
 def collect_self(state: dict) -> dict:
     """The agent's own footprint: resident memory and CPU % of one core since
     the last sample (#5764). A 1.13.0 agent once grew to 2.1 GB and spun for a
@@ -1687,6 +1728,14 @@ def build_sample(mounts: list[str], state: dict,
     if reboot is not None:
         sample["reboot"] = reboot
 
+    # Each drive's health, read hourly by the root helper (#5789).
+    try:
+        refresh_drives(state)
+    except Exception:
+        pass
+    if state.get("drives"):
+        sample["drives"] = state["drives"]
+
     # Last, so its CPU covers the whole of this sample's work.
     try:
         sample["agent"] = collect_self(state)

plugins/host_agent/drives.py

@@ -0,0 +1,105 @@
+"""Each drive's health, judged from what the root helper read with smartctl
+(#5789). The helper reports figures, not verdicts, so how a drive is judged
+can change here without an agent update.
+
+A drive is:
+- "failing" when its own self-assessment fails (SMART overall-health), an NVMe
+  drive raises a critical warning, or its spare blocks fall under threshold;
+- "worn" when it has used WORN_PCT of its rated life;
+- "watch" when it has bad sectors or media errors but otherwise passes: old
+  disks carry a few for years, so a count alone isn't a failure;
+- "ok", or "unavailable" when it has no SMART to read (a virtual disk, a
+  bridge that hides it), or "unreadable" when smartctl couldn't read it.
+
+Metrics, per drive on "<host>:drive:<device>": drive_failing (1/0),
+drive_life_used_pct, drive_bad_sectors, drive_temp_c. Per host, for covering
+rules: drives_failing (how many) and drive_life_used_max_pct.
+"""
+from __future__ import annotations
+
+WORN_PCT = 90
+DRIVE_METRICS = ("drive_failing", "drive_life_used_pct", "drive_bad_sectors", "drive_temp_c",
+                 "drives_failing", "drive_life_used_max_pct")
+# What a drive's own figures count as bad sectors or media errors, whichever
+# its kind reports: NVMe media errors, ATA reallocated / pending / offline
+# uncorrectable sectors, the SCSI grown defect list.
+_BAD = ("media_errors", "reallocated", "pending", "offline_uncorrectable", "grown_defects")
+_ORDER = {"failing": 0, "unreadable": 1, "worn": 2, "watch": 3, "ok": 4, "unavailable": 5}
+
+
+def _num(v):
+    return v if isinstance(v, (int, float)) and not isinstance(v, bool) else None
+
+
+def bad_sectors(d: dict) -> int | None:
+    counts = [_num(d.get(k)) for k in _BAD]
+    counts = [c for c in counts if c is not None]
+    return int(sum(counts)) if counts else None
+
+
+def failing_reasons(d: dict) -> list[str]:
+    why = []
+    if d.get("passed") is False:
+        why.append("its SMART self-assessment failed")
+    if _num(d.get("critical_warning")):
+        why.append("critical warning %#x" % int(d["critical_warning"]))
+    spare, floor = _num(d.get("spare_pct")), _num(d.get("spare_threshold_pct"))
+    if spare is not None and floor is not None and spare < floor:
+        why.append(f"spare blocks {spare:.0f}% under the {floor:.0f}% threshold")
+    return why
+
+
+def judge(d: dict) -> dict:
+    """One drive as the host page shows it: its figures plus `state` and `why`."""
+    out = dict(d)
+    if d.get("error"):
+        out.update(state="unreadable", why=str(d["error"]))
+    elif d.get("available") is False:
+        out.update(state="unavailable", why="no SMART data (a virtual disk, or SMART is off)")
+    elif failing_reasons(d):
+        out.update(state="failing", why="; ".join(failing_reasons(d)))
+    elif (_num(d.get("life_used_pct")) or 0) >= WORN_PCT:
+        out.update(state="worn", why=f"{d['life_used_pct']:.0f}% of its rated life used")
+    elif bad_sectors(d):
+        out.update(state="watch", why=f"{bad_sectors(d)} bad sectors or media errors")
+    else:
+        out.update(state="ok", why="")
+    out["bad_sectors"] = bad_sectors(d)
+    return out
+
+
+def drives_of(report) -> list[dict]:
+    """The report's drives, judged, what needs attention first."""
+    drives = report.get("drives") if isinstance(report, dict) else None
+    judged = [judge(d) for d in drives or [] if isinstance(d, dict) and d.get("device")]
+    return sorted(judged, key=lambda d: (_ORDER[d["state"]], str(d["device"])))
+
+
+def metric_rows(report, host_name: str) -> list[tuple[str, str, float]]:
+    """(metric, resource, value) for every figure a drive has, and the host's
+    two roll-ups once any drive was read."""
+    rows: list[tuple[str, str, float]] = []
+    failing = 0
+    life_max: float | None = None
+    read = 0
+    for d in drives_of(report):
+        if d["state"] in ("unreadable", "unavailable"):
+            continue
+        read += 1
+        res = f"{host_name}:drive:{d['device']}"
+        is_failing = d["state"] == "failing"
+        failing += is_failing
+        rows.append(("drive_failing", res, 1.0 if is_failing else 0.0))
+        life = _num(d.get("life_used_pct"))
+        if life is not None:
+            rows.append(("drive_life_used_pct", res, float(life)))
+            life_max = life if life_max is None else max(life_max, life)
+        if d["bad_sectors"] is not None:
+            rows.append(("drive_bad_sectors", res, float(d["bad_sectors"])))
+        if _num(d.get("temp_c")) is not None:
+            rows.append(("drive_temp_c", res, float(d["temp_c"])))
+    if read:
+        rows.append(("drives_failing", host_name, float(failing)))
+        if life_max is not None:
+            rows.append(("drive_life_used_max_pct", host_name, float(life_max)))
+    return rows

plugins/host_agent/files/install-deps.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+# Steward host agent: the packages the agent's helper uses, installed with
+# whichever package manager the host has (#5789). Served by Steward at
+# /plugins/host_agent/files/install-deps.sh and run, as root, by install.sh
+# and the host_agent playbooks: one copy of the list.
+#
+# smartmontools gives the helper smartctl, which reads each drive's health.
+# A package that can't be installed is reported, never fatal: the agent runs
+# without it and the host page says what is missing.
+#
+# Recommended / weak dependencies are skipped: on Debian, smartmontools
+# recommends a mail server.
+
+PACKAGES="smartmontools"
+
+have() { command -v "$1" >/dev/null 2>&1; }
+
+install_with() {
+    if have apt-get; then
+        export DEBIAN_FRONTEND=noninteractive
+        # A stale package list can't find the package: refresh once and retry.
+        apt-get install -y --no-install-recommends "$@" \
+            || { apt-get update && apt-get install -y --no-install-recommends "$@"; }
+    elif have dnf; then
+        dnf -y install --setopt=install_weak_deps=False "$@"
+    elif have yum; then
+        yum -y install "$@"
+    elif have zypper; then
+        zypper --non-interactive install --no-recommends "$@"
+    elif have pacman; then
+        pacman -S --needed --noconfirm "$@"
+    elif have apk; then
+        apk add --no-cache "$@"
+    else
+        echo "no package manager this script knows (apt, dnf, yum, zypper, pacman, apk)"
+        return 1
+    fi
+}
+
+missing=""
+have smartctl || missing="$missing smartmontools"
+[ -n "$missing" ] || { echo "dependencies present: $PACKAGES"; exit 0; }
+
+# shellcheck disable=SC2086  # one word per package
+if install_with $missing; then
+    echo "installed:$missing"
+else
+    echo "WARNING: could not install$missing; drive health will read \"smartctl not installed\" until it is"
+fi
+exit 0

plugins/host_agent/files/maint.py

@@ -10,6 +10,10 @@ the action, and writes the outcome to RESULTS (root's own directory, which the
 agent reads back and reports to Steward). A stolen agent token can therefore
 make its host update, reboot or drain, and nothing else.
 
+One action only reads: `drives` (#5789) asks smartctl for each disk's health,
+which needs root, and writes a summary to DRIVES, never a result: it is not a
+maintenance job.
+
 Stdlib only; served by Steward at /plugins/host_agent/files/maint.py.
 """
 from __future__ import annotations
@@ -25,7 +29,8 @@ from datetime import datetime, timezone
 
 REQUESTS = "/var/lib/steward-agent/requests"
 RESULTS = "/var/lib/steward-maint/results"
-ACTIONS = ("update", "reboot", "drain", "undrain")
+DRIVES = "/var/lib/steward-maint/drives.json"
+ACTIONS = ("update", "reboot", "drain", "undrain", "drives")
 REQUEST_MAX_BYTES = 4096
 RESULTS_KEPT_SECONDS = 7 * 86400
 UPDATE_DEADLINE_SECONDS = 3000
@@ -152,6 +157,130 @@ def do_node(availability: str, node: str | None) -> dict:
     return {"ok": ok, "node": node, "detail": out.strip()[-500:]}
 
 
+# ── drive health (#5789) ─────────────────────────────────────────────────────
+# smartctl ≥ 7.0 answers in JSON, the same for SATA, SAS and NVMe drives on any
+# distribution; an older one is reported, never parsed as text. The helper
+# finds the drives itself: the request names none.
+
+SMARTCTL_MIN = (7, 0)
+SMARTCTL_DEADLINE_SECONDS = 60
+DRIVES_MAX = 32
+# smartctl's exit status is a bit mask: 1 the command line, 2 the device
+# couldn't be opened. Higher bits describe the disk, and its JSON still holds.
+_SMARTCTL_UNREAD = 0b11
+# ATA attributes counted as bad sectors.
+_ATA_BAD = {5: "reallocated", 197: "pending", 198: "offline_uncorrectable"}
+
+
+def smartctl_version():
+    """(major, minor) of the installed smartctl, or None when there is none."""
+    if not shutil.which("smartctl"):
+        return None
+    ok, out = run(["smartctl", "--version"], deadline=30)
+    m = re.search(r"smartctl (\d+)\.(\d+)", out)
+    return (int(m.group(1)), int(m.group(2))) if m else (0, 0)
+
+
+def smartctl_json(args: list):
+    """(exit status, parsed JSON or None) of one smartctl call."""
+    try:
+        done = subprocess.run(["smartctl", "--json", *args], capture_output=True, text=True,
+                              timeout=SMARTCTL_DEADLINE_SECONDS,
+                              env=dict(os.environ, LANG="C", LC_ALL="C"))
+    except (subprocess.TimeoutExpired, OSError):
+        return None, None
+    try:
+        return done.returncode, json.loads(done.stdout)
+    except ValueError:
+        return done.returncode, None
+
+
+def _int(v):
+    return v if isinstance(v, int) and not isinstance(v, bool) else None
+
+
+def drive_summary(name: str, status, data) -> dict:
+    """The few figures Steward reads from one drive's `smartctl -a` JSON."""
+    out = {"device": os.path.basename(name)}
+    if not isinstance(data, dict) or status is None or status & _SMARTCTL_UNREAD:
+        messages = ((data.get("smartctl") or {}).get("messages") or []) if isinstance(data, dict) else []
+        text = "; ".join(str(m.get("string", "")) for m in messages if isinstance(m, dict))
+        out["error"] = (text or "smartctl couldn't read it")[:200]
+        return out
+    dev = data.get("device") or {}
+    out["protocol"] = str(dev.get("protocol") or "")[:16]
+    out["model"] = str(data.get("model_name") or data.get("scsi_model_name") or "")[:80]
+    cap = (data.get("user_capacity") or {}).get("bytes")
+    if _int(cap):
+        out["capacity_bytes"] = cap
+    support = data.get("smart_support") or {}
+    if support.get("available") is False or support.get("enabled") is False:
+        out["available"] = False             # a virtual disk, or SMART switched off
+        return out
+    passed = (data.get("smart_status") or {}).get("passed")
+    if isinstance(passed, bool):
+        out["passed"] = passed
+    temp = _int((data.get("temperature") or {}).get("current"))
+    if temp is not None:
+        out["temp_c"] = temp
+    hours = _int((data.get("power_on_time") or {}).get("hours"))
+    if hours is not None:
+        out["power_on_hours"] = hours
+    nvme = data.get("nvme_smart_health_information_log") or {}
+    for src, dst in (("critical_warning", "critical_warning"), ("percentage_used", "life_used_pct"),
+                     ("media_errors", "media_errors"), ("available_spare", "spare_pct"),
+                     ("available_spare_threshold", "spare_threshold_pct")):
+        if _int(nvme.get(src)) is not None:
+            out[dst] = nvme[src]
+    for attr in (data.get("ata_smart_attributes") or {}).get("table") or []:
+        key = _ATA_BAD.get(attr.get("id")) if isinstance(attr, dict) else None
+        raw = _int((attr.get("raw") or {}).get("value")) if key else None
+        if raw is not None:
+            out[key] = raw
+    used = _int((data.get("endurance_used") or {}).get("current_percent"))          # ATA SSDs
+    used = used if used is not None else _int(data.get("scsi_percentage_used_endurance_indicator"))
+    if used is not None and "life_used_pct" not in out:
+        out["life_used_pct"] = used
+    if _int(data.get("scsi_grown_defect_list")) is not None:
+        out["grown_defects"] = data["scsi_grown_defect_list"]
+    return out
+
+
+def do_drives() -> dict:
+    """Each drive's health, as {"checked_at", "smartctl", "drives"} or
+    {"checked_at", "error"} when smartctl is missing or too old."""
+    base = {"checked_at": now_iso()}
+    version = smartctl_version()
+    if version is None:
+        return {**base, "error": "smartctl not installed"}
+    base["smartctl"] = "%d.%d" % version
+    if version < SMARTCTL_MIN:
+        return {**base, "error": "smartctl %d.%d is too old: drive health needs 7.0 or newer" % version}
+    _, scan = smartctl_json(["--scan-open"])
+    devices = (scan or {}).get("devices") if isinstance(scan, dict) else None
+    if not isinstance(devices, list):
+        return {**base, "error": "smartctl couldn't list the drives"}
+    drives = []
+    for d in devices[:DRIVES_MAX]:
+        name, kind = (d.get("name"), d.get("type")) if isinstance(d, dict) else (None, None)
+        if not isinstance(name, str) or not name.startswith("/dev/"):
+            continue
+        args = ["-a", name] if not isinstance(kind, str) else ["-a", "-d", kind, name]
+        status, data = smartctl_json(args)
+        drives.append(drive_summary(name, status, data))
+    return {**base, "drives": drives}
+
+
+def write_drives(report: dict, path: str | None = None) -> None:
+    """DRIVES, written whole and readable by the agent."""
+    path = path or DRIVES
+    tmp = path + ".tmp"
+    with open(tmp, "w") as fh:
+        json.dump(report, fh)
+    os.chmod(tmp, 0o644)
+    os.replace(tmp, path)
+
+
 def write_result(result: dict) -> None:
     """RESULTS/<id>.json, written whole (a temp file renamed into place)."""
     os.makedirs(RESULTS, mode=0o755, exist_ok=True)
@@ -170,6 +299,9 @@ def already_handled(job_id: str) -> bool:
 
 def handle(req: dict) -> dict:
     base = {"id": req["id"], "action": req["action"], "started_at": now_iso()}
+    if req["action"] == "drives":               # a read, not a job: no result
+        write_drives(do_drives())
+        return {**base, "state": "done"}
     write_result({**base, "state": "running"})
     action = req["action"]
     if action == "update":

plugins/host_agent/files/steward-maint.service

@@ -1,5 +1,6 @@
 # Runs as root, and does only what maint.py's fixed list allows: update the
-# host's packages, reboot it, or drain / reactivate its swarm node (#5766).
+# host's packages, reboot it, drain / reactivate its swarm node (#5766), or
+# read its drives' health with smartctl (#5789).
 [Unit]
 Description=Steward maintenance helper
 

plugins/host_agent/migrations/versions/host_agent_009_drive_health.py

@@ -0,0 +1,24 @@
+# plugins/host_agent/migrations/versions/host_agent_009_drive_health.py
+"""host_agent: each host's newest drive-health report (#5789)
+
+Revision ID: host_agent_009_drive_health
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_009_drive_health"
+down_revision: Union[str, None] = "host_agent_008_maint_plans"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("host_agent_registrations", sa.Column("drive_health", sa.JSON(), nullable=True))
+    op.add_column("host_agent_registrations",
+                  sa.Column("drive_health_at", sa.DateTime(timezone=True), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("host_agent_registrations", "drive_health_at")
+    op.drop_column("host_agent_registrations", "drive_health")

plugins/host_agent/models.py

@@ -2,7 +2,7 @@
 from __future__ import annotations
 import uuid
 from datetime import datetime, timezone
-from sqlalchemy import Boolean, Column, String, DateTime, Float, ForeignKey, Integer, Text
+from sqlalchemy import JSON, Boolean, Column, String, DateTime, Float, ForeignKey, Integer, Text
 from steward.models.base import Base
 
 
@@ -51,6 +51,11 @@ class HostAgentRegistration(Base):
     # its request directory) that updates and reboots go through (#5766); None
     # until an agent that says reports.
     maint_helper = Column(Boolean, nullable=True)
+    # The host's newest drive-health report, as the root helper read it with
+    # smartctl (agent ≥ 1.20.0, #5789): {"checked_at", "smartctl", "drives"} or
+    # {"error"}. drives.py judges it; the figures are also metrics.
+    drive_health = Column(JSON, nullable=True)
+    drive_health_at = Column(DateTime(timezone=True), nullable=True)
     created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
     updated_at = Column(DateTime(timezone=True), nullable=False,
                         default=_utcnow, onupdate=_utcnow)

plugins/host_agent/routes.py

@@ -27,7 +27,7 @@ from steward.hosts import service as host_svc
 from steward.models.hosts import Host
 from steward.models.metrics import PluginMetric
 from .downtime import recent as recent_downtime, record_events
-from . import maintenance
+from . import drives as drive_health, maintenance
 from .models import HostAgentRegistration
 from .scheduler import arrivals
 # Query helpers live in a model-free module so integration tests can import them
@@ -193,6 +193,16 @@ def apply_updates(reg, sample: dict, now: datetime) -> bool:
                       reg.reboot_required_since, reg.reboot_packages)
 
 
+def apply_drives(reg, sample: dict, now: datetime) -> bool:
+    """Keep the registration's copy of the host's newest drive report (#5789),
+    which the host page lists: model, state and why, beside the figures."""
+    report = sample.get("drives")
+    if not isinstance(report, dict) or report == reg.drive_health:
+        return False
+    reg.drive_health, reg.drive_health_at = report, _ts_or(report.get("checked_at"), now)
+    return True
+
+
 def _expand_sample_to_metrics(
     sample: dict, host_name: str, recorded_at: datetime
 ) -> list[PluginMetric]:
@@ -331,6 +341,10 @@ def _expand_sample_to_metrics(
     if max_temp is not None:
         row("temp_c_max", host_name, max_temp)
 
+    # Each drive's health (#5789): the helper's hourly read rides every sample.
+    for metric, resource, value in drive_health.metric_rows(sample.get("drives"), host_name):
+        row(metric, resource, value)
+
     # Pressure stall information (PSI) — host-level gauges (mem/cpu/io some|full).
     for k, v in (sample.get("psi") or {}).items():
         if isinstance(v, (int, float)):
@@ -527,6 +541,8 @@ async def ingest():
                 changed = True
             if newest_sample is not None and apply_updates(reg, newest_sample, received_at):
                 changed = True
+            if newest_sample is not None and apply_drives(reg, newest_sample, received_at):
+                changed = True
             # Mirror the reported IP into Host.address only when it's blank.
             new_address = pick_host_address(host.address, host_ip)
             if new_address is not None:
@@ -685,6 +701,7 @@ async def agent_source():
 AGENT_FILES_DIR = Path(__file__).parent / "files"
 AGENT_FILES = {
     "maint.py": "text/x-python",
+    "install-deps.sh": "text/x-shellscript",
     "steward-agent.service": "text/plain",
     "steward-maint.service": "text/plain",
     "steward-maint.path": "text/plain",
@@ -842,6 +859,8 @@ def _split_host_metrics(host_name: str, latest: dict) -> tuple:
             disks_io[suffix[len("diskio:"):]] = metrics
         elif suffix.startswith("temp:"):
             temps[suffix[len("temp:"):]] = metrics.get("temp_c")
+        elif suffix.startswith("drive:"):
+            continue                        # listed from the drive report (#5789)
         else:
             mounts[suffix] = metrics
     cores.sort(key=lambda c: c[0])
@@ -882,12 +901,15 @@ async def host_detail_metrics(host_id: str):
         latest = await _latest_metrics_for_host(session, host.name)
     hostlvl, cores, nets, disks_io, temps, mounts = _split_host_metrics(host.name, latest)
     temps, unwired = split_temps(host.name, temps)
+    drive_report = reg.drive_health if reg else None
     ls = reg.last_seen_at if reg else None
     stale = ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()
     return await render_template(
         "_host_metrics.html",
         host=host, reg=reg, stale=stale, hostlvl=hostlvl,
         cores=cores, nets=nets, disks_io=disks_io, temps=temps, unwired_temps=unwired, mounts=mounts,
+        drives=drive_health.drives_of(drive_report), drives_error=(drive_report or {}).get("error"),
+        drives_at=reg.drive_health_at if reg else None,
     )
 
 

plugins/host_agent/templates/_host_metrics.html

@@ -119,6 +119,41 @@
 </div>
 {% endif %}
 
+{# ── Drive health (#5789): each drive as the root helper read it with
+   smartctl, hourly; what needs attention first (drives.py judges them). ──── #}
+{% set drive_badge = {"failing": ("crit", "Failing"), "unreadable": ("warn", "Unreadable"),
+                      "worn": ("warn", "Worn"), "watch": ("warn", "Watch"), "ok": ("ok", "OK"),
+                      "unavailable": ("inactive", "No SMART")} %}
+{% if drives or drives_error %}
+<div class="card">
+  <div class="section-title" style="margin-bottom:0.5rem;display:flex;justify-content:space-between;gap:0.75rem;">
+    <span>Drive health</span>
+    {% if drives_at %}<span style="font-weight:400;color:var(--text-dim);font-size:0.72rem;">read {{ drives_at|ago }}</span>{% endif %}
+  </div>
+  {% if drives_error %}
+  <p class="field-hint" style="margin:0;">Not read: {{ drives_error }}.</p>
+  {% endif %}
+  {% for d in drives %}
+  {% set tone, label = drive_badge[d.state] %}
+  <div style="display:flex;justify-content:space-between;align-items:baseline;gap:0.75rem;font-size:0.8rem;padding:0.2rem 0;">
+    <span style="min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="{{ d.model }}">
+      {{ status_badge(tone, label) }}
+      <span style="font-family:ui-monospace,monospace;">{{ d.device }}</span>
+      {% if d.model %}<span style="color:var(--text-dim);">{{ d.model }}{% if d.capacity_bytes %} · {{ d.capacity_bytes|bytes }}{% endif %}</span>{% endif %}
+      {% if d.why %}<span style="color:var(--text-muted);"> — {{ d.why }}</span>{% endif %}
+    </span>
+    <span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">
+      {%- set parts = [] -%}
+      {%- if d.get("life_used_pct") is not none %}{% set _ = parts.append("%d%% life used"|format(d.life_used_pct)) %}{% endif -%}
+      {%- if d.get("bad_sectors") is not none %}{% set _ = parts.append("%d bad sectors"|format(d.bad_sectors)) %}{% endif -%}
+      {%- if d.get("temp_c") is not none %}{% set _ = parts.append("%d°C"|format(d.temp_c)) %}{% endif -%}
+      {{ parts|join(" · ") if parts else "—" }}
+    </span>
+  </div>
+  {% endfor %}
+</div>
+{% endif %}
+
 {# ── Interfaces / disks (short panels, side by side at natural height) ─────── #}
 {% if nets or disks_io %}
 <div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(260px,1fr));gap:1rem;align-items:start;">

plugins/host_agent/templates/install.sh.j2

@@ -63,6 +63,12 @@ chmod 0640 "$CONF_FILE"
 curl -fsSL "${FILES_URL}/maint.py" -o "$AGENT_DIR/maint.py"
 chmod 0755 "$AGENT_DIR/maint.py"
 chown root:root "$AGENT_DIR/maint.py"
+# Its packages (smartctl for drive health, #5789), with whichever package
+# manager this host has; one that can't be installed is said, never fatal.
+curl -fsSL "${FILES_URL}/install-deps.sh" -o "$AGENT_DIR/install-deps.sh"
+chmod 0755 "$AGENT_DIR/install-deps.sh"
+chown root:root "$AGENT_DIR/install-deps.sh"
+"$AGENT_DIR/install-deps.sh" || true
 install -d -o root -g root -m 0755 /var/lib/steward-agent /var/lib/steward-maint /var/lib/steward-maint/results
 install -d -o "$AGENT_USER" -g "$AGENT_USER" -m 0700 /var/lib/steward-agent/requests
 for u in $UNITS; do