feat(host_agent): each drive's health, read with smartctl, alertable (#5789)
The root helper gains one read-only action, `drives`: it finds the drives itself (smartctl --scan-open), reads each as JSON, and writes a summary the agent carries on every sample (agent 1.20.0). Steward judges the figures in drives.py (failing / worn / watch / ok / no SMART / unreadable), records them per drive and as host roll-ups for covering rules, keeps the newest report on the registration and lists it on the host page. install-deps.sh installs smartmontools with apt, dnf, yum, zypper, pacman or apk, run by install.sh and every agent playbook; one it can't install is reported, never fatal. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -22,7 +22,7 @@ from collections import deque
|
||||
from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
|
||||
from datetime import datetime, timezone
|
||||
|
||||
AGENT_VERSION = "1.19.0"
|
||||
AGENT_VERSION = "1.20.0"
|
||||
|
||||
# Default path to the local Docker Engine socket. Overridable via the
|
||||
# `docker_socket` config key; collection is silently skipped if it's absent or
|
||||
@@ -1456,7 +1456,7 @@ def collect_reboot(path: str = REBOOT_REQUIRED):
|
||||
MAINT_REQUESTS = "/var/lib/steward-agent/requests"
|
||||
MAINT_RESULTS = "/var/lib/steward-maint/results"
|
||||
MAINT_PATH_UNIT = "/etc/systemd/system/steward-maint.path"
|
||||
MAINT_ACTIONS = ("update", "reboot", "drain", "undrain")
|
||||
MAINT_ACTIONS = ("update", "reboot", "drain", "undrain", "drives")
|
||||
MAINT_RESULT_MAX_BYTES = 8192
|
||||
# The helper keeps results a week; only this recent a one is reported, so a
|
||||
# job a reboot interrupted still reaches Steward from the agent's next start
|
||||
@@ -1543,6 +1543,47 @@ def maintenance_results(seen: set, results_dir: str = MAINT_RESULTS,
|
||||
return events, keys
|
||||
|
||||
|
||||
# ── drive health (#5789) ─────────────────────────────────────────────────────
|
||||
# Reading SMART needs root, so the helper does it: hourly (and at start) the
|
||||
# agent leaves it a `drives` request, and the helper writes each drive's
|
||||
# figures to DRIVES_PATH. The last answer rides every sample, like updates.
|
||||
|
||||
DRIVES_PATH = "/var/lib/steward-maint/drives.json"
|
||||
DRIVES_INTERVAL_SECONDS = 3600
|
||||
DRIVES_MAX_BYTES = 65536
|
||||
|
||||
|
||||
def refresh_drives(state: dict, now_mono: float | None = None,
|
||||
requests_dir: str = MAINT_REQUESTS, drives_path: str = DRIVES_PATH,
|
||||
unit: str = MAINT_PATH_UNIT) -> None:
|
||||
"""Ask the helper for a drive read when one is due, and keep its newest
|
||||
answer in state["drives"]."""
|
||||
now_mono = time.monotonic() if now_mono is None else now_mono
|
||||
last = state.get("drives_asked_mono")
|
||||
if last is None or now_mono - last >= DRIVES_INTERVAL_SECONDS:
|
||||
state["drives_asked_mono"] = now_mono
|
||||
if not maint_helper_installed(unit, requests_dir):
|
||||
state["drives"] = {"error": "needs the maintenance helper (Update agent installs it)"}
|
||||
else:
|
||||
try:
|
||||
request_maintenance(f"drives-{int(time.time())}", "drives", requests_dir=requests_dir)
|
||||
except (ValueError, OSError) as exc:
|
||||
_log("WARN", f"drive health not requested: {exc}")
|
||||
try:
|
||||
st = os.stat(drives_path)
|
||||
except OSError:
|
||||
return
|
||||
if st.st_mtime == state.get("drives_mtime") or st.st_size > DRIVES_MAX_BYTES:
|
||||
return
|
||||
try:
|
||||
with open(drives_path) as fh:
|
||||
got = json.load(fh)
|
||||
except (OSError, ValueError):
|
||||
return # mid-write: the next sample reads it
|
||||
if isinstance(got, dict):
|
||||
state["drives"], state["drives_mtime"] = got, st.st_mtime
|
||||
|
||||
|
||||
def collect_self(state: dict) -> dict:
|
||||
"""The agent's own footprint: resident memory and CPU % of one core since
|
||||
the last sample (#5764). A 1.13.0 agent once grew to 2.1 GB and spun for a
|
||||
@@ -1687,6 +1728,14 @@ def build_sample(mounts: list[str], state: dict,
|
||||
if reboot is not None:
|
||||
sample["reboot"] = reboot
|
||||
|
||||
# Each drive's health, read hourly by the root helper (#5789).
|
||||
try:
|
||||
refresh_drives(state)
|
||||
except Exception:
|
||||
pass
|
||||
if state.get("drives"):
|
||||
sample["drives"] = state["drives"]
|
||||
|
||||
# Last, so its CPU covers the whole of this sample's work.
|
||||
try:
|
||||
sample["agent"] = collect_self(state)
|
||||
|
||||
Reference in New Issue
Block a user