feat(host_agent): each drive's health, read with smartctl, alertable (#5789)

The root helper gains one read-only action, `drives`: it finds the drives
itself (smartctl --scan-open), reads each as JSON, and writes a summary the
agent carries on every sample (agent 1.20.0). Steward judges the figures in
drives.py (failing / worn / watch / ok / no SMART / unreadable), records them
per drive and as host roll-ups for covering rules, keeps the newest report on
the registration and lists it on the host page.

install-deps.sh installs smartmontools with apt, dnf, yum, zypper, pacman or
apk, run by install.sh and every agent playbook; one it can't install is
reported, never fatal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/service.py

@@ -385,17 +385,19 @@ async def resources_for(db, source_module: str) -> list[str]:
 # A host's parts, by the shape of the name after "<host>:" (#5475): the order
 # and heading each kind takes in the rule form's Part picker.
 PART_KINDS = (("mount", "Mounts"), ("core", "CPU cores"), ("disk", "Disks"),
-              ("net", "Interfaces"), ("temp", "Temperatures"), ("other", "Other"))
+              ("drive", "Drive health"), ("net", "Interfaces"), ("temp", "Temperatures"),
+              ("other", "Other"))
 
 
 def part_kind(part: str) -> tuple[str, str]:
     """(kind, label) for a host part: "/var" is a mount, "core3" a core,
-    "diskio:sda" a disk, "net:eth0" an interface, "temp:Core 0" a sensor."""
+    "diskio:sda" a disk, "drive:sda" its health (#5789), "net:eth0" an
+    interface, "temp:Core 0" a sensor."""
     if part.startswith("/"):
         return "mount", part
     if part.startswith("core") and part[4:].isdigit():
         return "core", f"core {part[4:]}"
-    for prefix, kind in (("diskio:", "disk"), ("net:", "net"), ("temp:", "temp")):
+    for prefix, kind in (("diskio:", "disk"), ("drive:", "drive"), ("net:", "net"), ("temp:", "temp")):
         if part.startswith(prefix):
             return kind, part[len(prefix):]
     return "other", part

steward/ansible/bundled/host_agent/install.yml

@@ -98,6 +98,27 @@
         mode: "0755"
         force: true
 
+    # The helper's packages (smartctl for drive health, #5789), with whichever
+    # package manager the host has. One that can't be installed is said, never
+    # fatal: the host page then reads "smartctl not installed".
+    - name: Download the dependency installer
+      ansible.builtin.get_url:
+        url: "{{ steward_url }}/plugins/host_agent/files/install-deps.sh"
+        dest: /usr/local/lib/steward-agent/install-deps.sh
+        owner: root
+        mode: "0755"
+        force: true
+
+    - name: Install the helper's packages
+      ansible.builtin.command: /usr/local/lib/steward-agent/install-deps.sh
+      register: helper_deps
+      changed_when: "'installed:' in helper_deps.stdout"
+
+    - name: Say what couldn't be installed
+      ansible.builtin.debug:
+        msg: "{{ helper_deps.stdout_lines | select('search', 'WARNING') | list }}"
+      when: "'WARNING' in helper_deps.stdout"
+
     - name: Install the agent and maintenance units
       ansible.builtin.get_url:
         url: "{{ steward_url }}/plugins/host_agent/files/{{ item }}"

steward/ansible/bundled/host_agent/provision.yml

@@ -154,6 +154,27 @@
         mode: "0755"
         force: true
 
+    # The helper's packages (smartctl for drive health, #5789), with whichever
+    # package manager the host has. One that can't be installed is said, never
+    # fatal: the host page then reads "smartctl not installed".
+    - name: Download the dependency installer
+      ansible.builtin.get_url:
+        url: "{{ steward_url }}/plugins/host_agent/files/install-deps.sh"
+        dest: /usr/local/lib/steward-agent/install-deps.sh
+        owner: root
+        mode: "0755"
+        force: true
+
+    - name: Install the helper's packages
+      ansible.builtin.command: /usr/local/lib/steward-agent/install-deps.sh
+      register: helper_deps
+      changed_when: "'installed:' in helper_deps.stdout"
+
+    - name: Say what couldn't be installed
+      ansible.builtin.debug:
+        msg: "{{ helper_deps.stdout_lines | select('search', 'WARNING') | list }}"
+      when: "'WARNING' in helper_deps.stdout"
+
     - name: Install the agent and maintenance units
       ansible.builtin.get_url:
         url: "{{ steward_url }}/plugins/host_agent/files/{{ item }}"

steward/ansible/bundled/host_agent/update.yml

@@ -1,5 +1,5 @@
 ---
-# description: Update the Steward host agent (refresh agent.py, its units and the maintenance helper, then restart) on hosts already running it.
+# description: Update the Steward host agent (refresh agent.py, its units, the maintenance helper and its packages, then restart) on hosts already running it.
 # steward:category: host-agent
 # Update an already-installed Steward host agent: refresh agent.py, its systemd
 # units and the maintenance helper (#5766), then restart.
@@ -66,6 +66,27 @@
         mode: "0755"
         force: true
 
+    # The helper's packages (smartctl for drive health, #5789), with whichever
+    # package manager the host has. One that can't be installed is said, never
+    # fatal: the host page then reads "smartctl not installed".
+    - name: Download the dependency installer
+      ansible.builtin.get_url:
+        url: "{{ steward_url }}/plugins/host_agent/files/install-deps.sh"
+        dest: /usr/local/lib/steward-agent/install-deps.sh
+        owner: root
+        mode: "0755"
+        force: true
+
+    - name: Install the helper's packages
+      ansible.builtin.command: /usr/local/lib/steward-agent/install-deps.sh
+      register: helper_deps
+      changed_when: "'installed:' in helper_deps.stdout"
+
+    - name: Say what couldn't be installed
+      ansible.builtin.debug:
+        msg: "{{ helper_deps.stdout_lines | select('search', 'WARNING') | list }}"
+      when: "'WARNING' in helper_deps.stdout"
+
     - name: Install the agent and maintenance units
       ansible.builtin.get_url:
         url: "{{ steward_url }}/plugins/host_agent/files/{{ item }}"

steward/core/formatting.py

@@ -352,6 +352,12 @@ METRICS: dict[str, tuple[str, str]] = {
     "net_tx_bps": ("Network out", "bytes per second sent, by the whole host or one interface"),
     "disk_read_bps": ("Disk reads", "bytes per second read, from all disks or one"),
     "disk_write_bps": ("Disk writes", "bytes per second written, to all disks or one"),
+    "drives_failing": ("Drives failing", "drives on the host failing their own health check, raising a critical warning or out of spare blocks"),
+    "drive_failing": ("Drive failing", "1 when this drive fails its own health check, raises a critical warning or is out of spare blocks"),
+    "drive_life_used_max_pct": ("Most worn drive", "the most life any of the host's SSDs has used of its rating"),
+    "drive_life_used_pct": ("Drive life used", "how much of its rated write life this drive has used"),
+    "drive_bad_sectors": ("Bad sectors", "media errors, or reallocated, pending and uncorrectable sectors, as the drive counts them"),
+    "drive_temp_c": ("Drive temperature", "the drive's own temperature sensor"),
 }
 
 # Where a rule's metric comes from, as the picker and the rules list say it.

steward/mcp/tools/hosts.py

@@ -40,6 +40,7 @@ def _agent_glance(a: dict | None) -> dict | None:
 _PART_GROUPS = (  # (prefix, group, the field that names one part)
     ("net:", "interfaces", "interface"),
     ("diskio:", "disk_io", "device"),
+    ("drive:", "drive_health", "device"),
     ("temp:", "temperatures", "sensor"),
 )
 # Sort each group by what makes a part worth a look first.
@@ -47,6 +48,7 @@ _PART_ORDER = {
     "mounts": lambda r: -(r.get("disk_used_pct") or 0),
     "interfaces": lambda r: -((r.get("net_rx_bps") or 0) + (r.get("net_tx_bps") or 0)),
     "disk_io": lambda r: -((r.get("disk_read_bps") or 0) + (r.get("disk_write_bps") or 0)),
+    "drive_health": lambda r: (-(r.get("drive_failing") or 0), -(r.get("drive_bad_sectors") or 0)),
     "temperatures": lambda r: -(r.get("temp_c") or 0),
     "other": lambda r: r["part"],
 }
@@ -59,7 +61,7 @@ def agent_parts(parts: dict[str, dict], host_name: str, limit: int) -> dict:
     metrics. Per-core CPU is summarised, not listed: the host's cpu_pct
     already says how busy it is, and 30+ cores would crowd out the rest.
     """
-    groups: dict[str, list] = {"mounts": [], "interfaces": [], "disk_io": [],
+    groups: dict[str, list] = {"mounts": [], "interfaces": [], "disk_io": [], "drive_health": [],
                                "temperatures": [], "other": []}
     cores = []
     for name, metrics in parts.items():