feat(host_agent): each drive's health, read with smartctl, alertable (#5789)
CI / lint (push) Successful in 3s
CI / integration (push) Successful in 1m24s
CI / unit (push) Successful in 1m32s
CI / publish (push) Successful in 1m10s

The root helper gains one read-only action, `drives`: it finds the drives
itself (smartctl --scan-open), reads each as JSON, and writes a summary the
agent carries on every sample (agent 1.20.0). Steward judges the figures in
drives.py (failing / worn / watch / ok / no SMART / unreadable), records them
per drive and as host roll-ups for covering rules, keeps the newest report on
the registration and lists it on the host page.

install-deps.sh installs smartmontools with apt, dnf, yum, zypper, pacman or
apk, run by install.sh and every agent playbook; one it can't install is
reported, never fatal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-15 09:17:50 -04:00
co-authored by Claude Opus 5.5
parent fdc5c07183
commit 6c4ef368f9
20 changed files with 848 additions and 11 deletions
+51 -2
View File
@@ -22,7 +22,7 @@ from collections import deque
from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
from datetime import datetime, timezone
AGENT_VERSION = "1.19.0"
AGENT_VERSION = "1.20.0"
# Default path to the local Docker Engine socket. Overridable via the
# `docker_socket` config key; collection is silently skipped if it's absent or
@@ -1456,7 +1456,7 @@ def collect_reboot(path: str = REBOOT_REQUIRED):
MAINT_REQUESTS = "/var/lib/steward-agent/requests"
MAINT_RESULTS = "/var/lib/steward-maint/results"
MAINT_PATH_UNIT = "/etc/systemd/system/steward-maint.path"
MAINT_ACTIONS = ("update", "reboot", "drain", "undrain")
MAINT_ACTIONS = ("update", "reboot", "drain", "undrain", "drives")
MAINT_RESULT_MAX_BYTES = 8192
# The helper keeps results a week; only this recent a one is reported, so a
# job a reboot interrupted still reaches Steward from the agent's next start
@@ -1543,6 +1543,47 @@ def maintenance_results(seen: set, results_dir: str = MAINT_RESULTS,
return events, keys
# ── drive health (#5789) ─────────────────────────────────────────────────────
# Reading SMART needs root, so the helper does it: hourly (and at start) the
# agent leaves it a `drives` request, and the helper writes each drive's
# figures to DRIVES_PATH. The last answer rides every sample, like updates.
DRIVES_PATH = "/var/lib/steward-maint/drives.json"
DRIVES_INTERVAL_SECONDS = 3600
DRIVES_MAX_BYTES = 65536
def refresh_drives(state: dict, now_mono: float | None = None,
requests_dir: str = MAINT_REQUESTS, drives_path: str = DRIVES_PATH,
unit: str = MAINT_PATH_UNIT) -> None:
"""Ask the helper for a drive read when one is due, and keep its newest
answer in state["drives"]."""
now_mono = time.monotonic() if now_mono is None else now_mono
last = state.get("drives_asked_mono")
if last is None or now_mono - last >= DRIVES_INTERVAL_SECONDS:
state["drives_asked_mono"] = now_mono
if not maint_helper_installed(unit, requests_dir):
state["drives"] = {"error": "needs the maintenance helper (Update agent installs it)"}
else:
try:
request_maintenance(f"drives-{int(time.time())}", "drives", requests_dir=requests_dir)
except (ValueError, OSError) as exc:
_log("WARN", f"drive health not requested: {exc}")
try:
st = os.stat(drives_path)
except OSError:
return
if st.st_mtime == state.get("drives_mtime") or st.st_size > DRIVES_MAX_BYTES:
return
try:
with open(drives_path) as fh:
got = json.load(fh)
except (OSError, ValueError):
return # mid-write: the next sample reads it
if isinstance(got, dict):
state["drives"], state["drives_mtime"] = got, st.st_mtime
def collect_self(state: dict) -> dict:
"""The agent's own footprint: resident memory and CPU % of one core since
the last sample (#5764). A 1.13.0 agent once grew to 2.1 GB and spun for a
@@ -1687,6 +1728,14 @@ def build_sample(mounts: list[str], state: dict,
if reboot is not None:
sample["reboot"] = reboot
# Each drive's health, read hourly by the root helper (#5789).
try:
refresh_drives(state)
except Exception:
pass
if state.get("drives"):
sample["drives"] = state["drives"]
# Last, so its CPU covers the whole of this sample's work.
try:
sample["agent"] = collect_self(state)