diff --git a/docs/plugins/host-agent-design.md b/docs/plugins/host-agent-design.md index 760c180..7bb324e 100644 --- a/docs/plugins/host-agent-design.md +++ b/docs/plugins/host-agent-design.md @@ -266,6 +266,29 @@ The figures are recorded at most once every 30 s, whichever manager sent them, s A maintenance job on a swarm node opens a second window over `docker` / `swarm` beside its host's (`maintenance._swarm_window`). An alert the drain or reboot causes then waits for the job, and one still firing when the window ends is sent then. +### Drive health (1.20.0, #5789) + +Reading SMART needs root, so the root helper does it, as the one action on its fixed list that only reads: `drives`. Hourly, and at start, the agent leaves a `drives` request. The helper runs `smartctl --scan-open` to find the drives itself (the request names none), reads each with `smartctl --json -a -d `, and writes a summary to `/var/lib/steward-maint/drives.json` (0644). It writes no job result, because a drive read is not a maintenance job. The agent's newest copy rides every sample, as updates do. + +smartctl ≥ 7.0 answers in JSON, the same on every distribution and for SATA, SAS and NVMe drives. A missing or older smartctl is reported (`smartctl not installed`, `… too old`) and its text output is never parsed. The helper reports figures only: + +- `passed` and `temp_c` for every drive; +- `critical_warning`, `life_used_pct`, `media_errors` and spare % against threshold for NVMe; +- reallocated / pending / offline-uncorrectable sectors for ATA, and life used where the drive reports it; +- grown defects for SCSI. + +A drive with no SMART (a virtual disk, a bridge that hides it) reads `available: false`, and one smartctl couldn't open carries `error`. + +Steward judges the figures in `drives.py`, so the judgment can change without an agent update: + +- **failing**: the drive's self-assessment fails, an NVMe critical warning is raised, or spare blocks fall under threshold; +- **worn**: at least 90% of its rated life is used; +- **watch**: it has bad sectors or media errors but otherwise passes. Old disks carry a few for years, so a count alone isn't a failure. + +The figures become metrics on `:drive:`: `drive_failing`, `drive_life_used_pct`, `drive_bad_sectors` and `drive_temp_c`. The host gets `drives_failing` and `drive_life_used_max_pct` for covering rules; they are recorded only when at least one drive was read, so a VM's virtual disks don't claim "0 failing". The registration keeps the newest report (`drive_health`), and the host page lists it under **Drive health**, what needs attention first. + +**Installing smartctl on any common distribution.** `files/install-deps.sh` installs smartmontools with whichever package manager the host has: apt (without recommends, which would pull in a mail server), dnf, yum, zypper, pacman or apk. install.sh and the install / provision / update playbooks all fetch and run that one copy. A package that can't be installed is reported and never fails the install. + ### What happened to the host (1.14.0, #5756) A post may carry `events`, said once each, with the next post that gets through: diff --git a/plugins/host_agent/__init__.py b/plugins/host_agent/__init__.py index 3140582..9e3cc7a 100644 --- a/plugins/host_agent/__init__.py +++ b/plugins/host_agent/__init__.py @@ -86,10 +86,22 @@ def get_alert_metrics() -> dict[str, list[str]]: the whole host ("") or one interface (":net:") or disk (":diskio:"). + Drive health (agent ≥ 1.20.0, #5789), read hourly with smartctl by the + root helper and judged in drives.py, per drive on ":drive:": + drive_failing is 1 when the drive fails its own self-assessment, raises + an NVMe critical warning or runs out of spare blocks; drive_life_used_pct + is how much of its rated life it has used; drive_bad_sectors counts its + media errors or reallocated / pending / uncorrectable sectors; + drive_temp_c is its own temperature. Per host, for covering rules: + drives_failing (how many) and drive_life_used_max_pct (the most worn): + "drive failing" is drives_failing > 0, "drive wearing out" is + drive_life_used_max_pct > 90. + Recorded but not offered (NOT_ALERTABLE): disk_used_bytes and disk_total_bytes, where disk_used_pct is the figure to watch (and docker uses disk_total_bytes for something else), and mem_cached_bytes / mem_buffers_bytes, parts of what mem_available_bytes already counts.""" + from .drives import DRIVE_METRICS return {"host_agent": [ "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes", "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m", @@ -98,6 +110,7 @@ def get_alert_metrics() -> dict[str, list[str]]: "agent_rss_bytes", "agent_cpu_pct", "updates_pending", "updates_security", "reboot_required", "maintenance_failed", "temp_c_max", "temp_c", "net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps", + *DRIVE_METRICS, ]} diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py index 9349c6f..559ec4e 100644 --- a/plugins/host_agent/agent.py +++ b/plugins/host_agent/agent.py @@ -22,7 +22,7 @@ from collections import deque from concurrent.futures import ThreadPoolExecutor, wait as wait_futures from datetime import datetime, timezone -AGENT_VERSION = "1.19.0" +AGENT_VERSION = "1.20.0" # Default path to the local Docker Engine socket. Overridable via the # `docker_socket` config key; collection is silently skipped if it's absent or @@ -1456,7 +1456,7 @@ def collect_reboot(path: str = REBOOT_REQUIRED): MAINT_REQUESTS = "/var/lib/steward-agent/requests" MAINT_RESULTS = "/var/lib/steward-maint/results" MAINT_PATH_UNIT = "/etc/systemd/system/steward-maint.path" -MAINT_ACTIONS = ("update", "reboot", "drain", "undrain") +MAINT_ACTIONS = ("update", "reboot", "drain", "undrain", "drives") MAINT_RESULT_MAX_BYTES = 8192 # The helper keeps results a week; only this recent a one is reported, so a # job a reboot interrupted still reaches Steward from the agent's next start @@ -1543,6 +1543,47 @@ def maintenance_results(seen: set, results_dir: str = MAINT_RESULTS, return events, keys +# ── drive health (#5789) ───────────────────────────────────────────────────── +# Reading SMART needs root, so the helper does it: hourly (and at start) the +# agent leaves it a `drives` request, and the helper writes each drive's +# figures to DRIVES_PATH. The last answer rides every sample, like updates. + +DRIVES_PATH = "/var/lib/steward-maint/drives.json" +DRIVES_INTERVAL_SECONDS = 3600 +DRIVES_MAX_BYTES = 65536 + + +def refresh_drives(state: dict, now_mono: float | None = None, + requests_dir: str = MAINT_REQUESTS, drives_path: str = DRIVES_PATH, + unit: str = MAINT_PATH_UNIT) -> None: + """Ask the helper for a drive read when one is due, and keep its newest + answer in state["drives"].""" + now_mono = time.monotonic() if now_mono is None else now_mono + last = state.get("drives_asked_mono") + if last is None or now_mono - last >= DRIVES_INTERVAL_SECONDS: + state["drives_asked_mono"] = now_mono + if not maint_helper_installed(unit, requests_dir): + state["drives"] = {"error": "needs the maintenance helper (Update agent installs it)"} + else: + try: + request_maintenance(f"drives-{int(time.time())}", "drives", requests_dir=requests_dir) + except (ValueError, OSError) as exc: + _log("WARN", f"drive health not requested: {exc}") + try: + st = os.stat(drives_path) + except OSError: + return + if st.st_mtime == state.get("drives_mtime") or st.st_size > DRIVES_MAX_BYTES: + return + try: + with open(drives_path) as fh: + got = json.load(fh) + except (OSError, ValueError): + return # mid-write: the next sample reads it + if isinstance(got, dict): + state["drives"], state["drives_mtime"] = got, st.st_mtime + + def collect_self(state: dict) -> dict: """The agent's own footprint: resident memory and CPU % of one core since the last sample (#5764). A 1.13.0 agent once grew to 2.1 GB and spun for a @@ -1687,6 +1728,14 @@ def build_sample(mounts: list[str], state: dict, if reboot is not None: sample["reboot"] = reboot + # Each drive's health, read hourly by the root helper (#5789). + try: + refresh_drives(state) + except Exception: + pass + if state.get("drives"): + sample["drives"] = state["drives"] + # Last, so its CPU covers the whole of this sample's work. try: sample["agent"] = collect_self(state) diff --git a/plugins/host_agent/drives.py b/plugins/host_agent/drives.py new file mode 100644 index 0000000..521fbc4 --- /dev/null +++ b/plugins/host_agent/drives.py @@ -0,0 +1,105 @@ +"""Each drive's health, judged from what the root helper read with smartctl +(#5789). The helper reports figures, not verdicts, so how a drive is judged +can change here without an agent update. + +A drive is: +- "failing" when its own self-assessment fails (SMART overall-health), an NVMe + drive raises a critical warning, or its spare blocks fall under threshold; +- "worn" when it has used WORN_PCT of its rated life; +- "watch" when it has bad sectors or media errors but otherwise passes: old + disks carry a few for years, so a count alone isn't a failure; +- "ok", or "unavailable" when it has no SMART to read (a virtual disk, a + bridge that hides it), or "unreadable" when smartctl couldn't read it. + +Metrics, per drive on ":drive:": drive_failing (1/0), +drive_life_used_pct, drive_bad_sectors, drive_temp_c. Per host, for covering +rules: drives_failing (how many) and drive_life_used_max_pct. +""" +from __future__ import annotations + +WORN_PCT = 90 +DRIVE_METRICS = ("drive_failing", "drive_life_used_pct", "drive_bad_sectors", "drive_temp_c", + "drives_failing", "drive_life_used_max_pct") +# What a drive's own figures count as bad sectors or media errors, whichever +# its kind reports: NVMe media errors, ATA reallocated / pending / offline +# uncorrectable sectors, the SCSI grown defect list. +_BAD = ("media_errors", "reallocated", "pending", "offline_uncorrectable", "grown_defects") +_ORDER = {"failing": 0, "unreadable": 1, "worn": 2, "watch": 3, "ok": 4, "unavailable": 5} + + +def _num(v): + return v if isinstance(v, (int, float)) and not isinstance(v, bool) else None + + +def bad_sectors(d: dict) -> int | None: + counts = [_num(d.get(k)) for k in _BAD] + counts = [c for c in counts if c is not None] + return int(sum(counts)) if counts else None + + +def failing_reasons(d: dict) -> list[str]: + why = [] + if d.get("passed") is False: + why.append("its SMART self-assessment failed") + if _num(d.get("critical_warning")): + why.append("critical warning %#x" % int(d["critical_warning"])) + spare, floor = _num(d.get("spare_pct")), _num(d.get("spare_threshold_pct")) + if spare is not None and floor is not None and spare < floor: + why.append(f"spare blocks {spare:.0f}% under the {floor:.0f}% threshold") + return why + + +def judge(d: dict) -> dict: + """One drive as the host page shows it: its figures plus `state` and `why`.""" + out = dict(d) + if d.get("error"): + out.update(state="unreadable", why=str(d["error"])) + elif d.get("available") is False: + out.update(state="unavailable", why="no SMART data (a virtual disk, or SMART is off)") + elif failing_reasons(d): + out.update(state="failing", why="; ".join(failing_reasons(d))) + elif (_num(d.get("life_used_pct")) or 0) >= WORN_PCT: + out.update(state="worn", why=f"{d['life_used_pct']:.0f}% of its rated life used") + elif bad_sectors(d): + out.update(state="watch", why=f"{bad_sectors(d)} bad sectors or media errors") + else: + out.update(state="ok", why="") + out["bad_sectors"] = bad_sectors(d) + return out + + +def drives_of(report) -> list[dict]: + """The report's drives, judged, what needs attention first.""" + drives = report.get("drives") if isinstance(report, dict) else None + judged = [judge(d) for d in drives or [] if isinstance(d, dict) and d.get("device")] + return sorted(judged, key=lambda d: (_ORDER[d["state"]], str(d["device"]))) + + +def metric_rows(report, host_name: str) -> list[tuple[str, str, float]]: + """(metric, resource, value) for every figure a drive has, and the host's + two roll-ups once any drive was read.""" + rows: list[tuple[str, str, float]] = [] + failing = 0 + life_max: float | None = None + read = 0 + for d in drives_of(report): + if d["state"] in ("unreadable", "unavailable"): + continue + read += 1 + res = f"{host_name}:drive:{d['device']}" + is_failing = d["state"] == "failing" + failing += is_failing + rows.append(("drive_failing", res, 1.0 if is_failing else 0.0)) + life = _num(d.get("life_used_pct")) + if life is not None: + rows.append(("drive_life_used_pct", res, float(life))) + life_max = life if life_max is None else max(life_max, life) + if d["bad_sectors"] is not None: + rows.append(("drive_bad_sectors", res, float(d["bad_sectors"]))) + if _num(d.get("temp_c")) is not None: + rows.append(("drive_temp_c", res, float(d["temp_c"]))) + if read: + rows.append(("drives_failing", host_name, float(failing))) + if life_max is not None: + rows.append(("drive_life_used_max_pct", host_name, float(life_max))) + return rows diff --git a/plugins/host_agent/files/install-deps.sh b/plugins/host_agent/files/install-deps.sh new file mode 100644 index 0000000..3ad626a --- /dev/null +++ b/plugins/host_agent/files/install-deps.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# Steward host agent: the packages the agent's helper uses, installed with +# whichever package manager the host has (#5789). Served by Steward at +# /plugins/host_agent/files/install-deps.sh and run, as root, by install.sh +# and the host_agent playbooks: one copy of the list. +# +# smartmontools gives the helper smartctl, which reads each drive's health. +# A package that can't be installed is reported, never fatal: the agent runs +# without it and the host page says what is missing. +# +# Recommended / weak dependencies are skipped: on Debian, smartmontools +# recommends a mail server. + +PACKAGES="smartmontools" + +have() { command -v "$1" >/dev/null 2>&1; } + +install_with() { + if have apt-get; then + export DEBIAN_FRONTEND=noninteractive + # A stale package list can't find the package: refresh once and retry. + apt-get install -y --no-install-recommends "$@" \ + || { apt-get update && apt-get install -y --no-install-recommends "$@"; } + elif have dnf; then + dnf -y install --setopt=install_weak_deps=False "$@" + elif have yum; then + yum -y install "$@" + elif have zypper; then + zypper --non-interactive install --no-recommends "$@" + elif have pacman; then + pacman -S --needed --noconfirm "$@" + elif have apk; then + apk add --no-cache "$@" + else + echo "no package manager this script knows (apt, dnf, yum, zypper, pacman, apk)" + return 1 + fi +} + +missing="" +have smartctl || missing="$missing smartmontools" +[ -n "$missing" ] || { echo "dependencies present: $PACKAGES"; exit 0; } + +# shellcheck disable=SC2086 # one word per package +if install_with $missing; then + echo "installed:$missing" +else + echo "WARNING: could not install$missing; drive health will read \"smartctl not installed\" until it is" +fi +exit 0 diff --git a/plugins/host_agent/files/maint.py b/plugins/host_agent/files/maint.py index 792ca70..07a1e11 100644 --- a/plugins/host_agent/files/maint.py +++ b/plugins/host_agent/files/maint.py @@ -10,6 +10,10 @@ the action, and writes the outcome to RESULTS (root's own directory, which the agent reads back and reports to Steward). A stolen agent token can therefore make its host update, reboot or drain, and nothing else. +One action only reads: `drives` (#5789) asks smartctl for each disk's health, +which needs root, and writes a summary to DRIVES, never a result: it is not a +maintenance job. + Stdlib only; served by Steward at /plugins/host_agent/files/maint.py. """ from __future__ import annotations @@ -25,7 +29,8 @@ from datetime import datetime, timezone REQUESTS = "/var/lib/steward-agent/requests" RESULTS = "/var/lib/steward-maint/results" -ACTIONS = ("update", "reboot", "drain", "undrain") +DRIVES = "/var/lib/steward-maint/drives.json" +ACTIONS = ("update", "reboot", "drain", "undrain", "drives") REQUEST_MAX_BYTES = 4096 RESULTS_KEPT_SECONDS = 7 * 86400 UPDATE_DEADLINE_SECONDS = 3000 @@ -152,6 +157,130 @@ def do_node(availability: str, node: str | None) -> dict: return {"ok": ok, "node": node, "detail": out.strip()[-500:]} +# ── drive health (#5789) ───────────────────────────────────────────────────── +# smartctl ≥ 7.0 answers in JSON, the same for SATA, SAS and NVMe drives on any +# distribution; an older one is reported, never parsed as text. The helper +# finds the drives itself: the request names none. + +SMARTCTL_MIN = (7, 0) +SMARTCTL_DEADLINE_SECONDS = 60 +DRIVES_MAX = 32 +# smartctl's exit status is a bit mask: 1 the command line, 2 the device +# couldn't be opened. Higher bits describe the disk, and its JSON still holds. +_SMARTCTL_UNREAD = 0b11 +# ATA attributes counted as bad sectors. +_ATA_BAD = {5: "reallocated", 197: "pending", 198: "offline_uncorrectable"} + + +def smartctl_version(): + """(major, minor) of the installed smartctl, or None when there is none.""" + if not shutil.which("smartctl"): + return None + ok, out = run(["smartctl", "--version"], deadline=30) + m = re.search(r"smartctl (\d+)\.(\d+)", out) + return (int(m.group(1)), int(m.group(2))) if m else (0, 0) + + +def smartctl_json(args: list): + """(exit status, parsed JSON or None) of one smartctl call.""" + try: + done = subprocess.run(["smartctl", "--json", *args], capture_output=True, text=True, + timeout=SMARTCTL_DEADLINE_SECONDS, + env=dict(os.environ, LANG="C", LC_ALL="C")) + except (subprocess.TimeoutExpired, OSError): + return None, None + try: + return done.returncode, json.loads(done.stdout) + except ValueError: + return done.returncode, None + + +def _int(v): + return v if isinstance(v, int) and not isinstance(v, bool) else None + + +def drive_summary(name: str, status, data) -> dict: + """The few figures Steward reads from one drive's `smartctl -a` JSON.""" + out = {"device": os.path.basename(name)} + if not isinstance(data, dict) or status is None or status & _SMARTCTL_UNREAD: + messages = ((data.get("smartctl") or {}).get("messages") or []) if isinstance(data, dict) else [] + text = "; ".join(str(m.get("string", "")) for m in messages if isinstance(m, dict)) + out["error"] = (text or "smartctl couldn't read it")[:200] + return out + dev = data.get("device") or {} + out["protocol"] = str(dev.get("protocol") or "")[:16] + out["model"] = str(data.get("model_name") or data.get("scsi_model_name") or "")[:80] + cap = (data.get("user_capacity") or {}).get("bytes") + if _int(cap): + out["capacity_bytes"] = cap + support = data.get("smart_support") or {} + if support.get("available") is False or support.get("enabled") is False: + out["available"] = False # a virtual disk, or SMART switched off + return out + passed = (data.get("smart_status") or {}).get("passed") + if isinstance(passed, bool): + out["passed"] = passed + temp = _int((data.get("temperature") or {}).get("current")) + if temp is not None: + out["temp_c"] = temp + hours = _int((data.get("power_on_time") or {}).get("hours")) + if hours is not None: + out["power_on_hours"] = hours + nvme = data.get("nvme_smart_health_information_log") or {} + for src, dst in (("critical_warning", "critical_warning"), ("percentage_used", "life_used_pct"), + ("media_errors", "media_errors"), ("available_spare", "spare_pct"), + ("available_spare_threshold", "spare_threshold_pct")): + if _int(nvme.get(src)) is not None: + out[dst] = nvme[src] + for attr in (data.get("ata_smart_attributes") or {}).get("table") or []: + key = _ATA_BAD.get(attr.get("id")) if isinstance(attr, dict) else None + raw = _int((attr.get("raw") or {}).get("value")) if key else None + if raw is not None: + out[key] = raw + used = _int((data.get("endurance_used") or {}).get("current_percent")) # ATA SSDs + used = used if used is not None else _int(data.get("scsi_percentage_used_endurance_indicator")) + if used is not None and "life_used_pct" not in out: + out["life_used_pct"] = used + if _int(data.get("scsi_grown_defect_list")) is not None: + out["grown_defects"] = data["scsi_grown_defect_list"] + return out + + +def do_drives() -> dict: + """Each drive's health, as {"checked_at", "smartctl", "drives"} or + {"checked_at", "error"} when smartctl is missing or too old.""" + base = {"checked_at": now_iso()} + version = smartctl_version() + if version is None: + return {**base, "error": "smartctl not installed"} + base["smartctl"] = "%d.%d" % version + if version < SMARTCTL_MIN: + return {**base, "error": "smartctl %d.%d is too old: drive health needs 7.0 or newer" % version} + _, scan = smartctl_json(["--scan-open"]) + devices = (scan or {}).get("devices") if isinstance(scan, dict) else None + if not isinstance(devices, list): + return {**base, "error": "smartctl couldn't list the drives"} + drives = [] + for d in devices[:DRIVES_MAX]: + name, kind = (d.get("name"), d.get("type")) if isinstance(d, dict) else (None, None) + if not isinstance(name, str) or not name.startswith("/dev/"): + continue + args = ["-a", name] if not isinstance(kind, str) else ["-a", "-d", kind, name] + status, data = smartctl_json(args) + drives.append(drive_summary(name, status, data)) + return {**base, "drives": drives} + + +def write_drives(report: dict, path: str | None = None) -> None: + """DRIVES, written whole and readable by the agent.""" + path = path or DRIVES + tmp = path + ".tmp" + with open(tmp, "w") as fh: + json.dump(report, fh) + os.chmod(tmp, 0o644) + os.replace(tmp, path) + + def write_result(result: dict) -> None: """RESULTS/.json, written whole (a temp file renamed into place).""" os.makedirs(RESULTS, mode=0o755, exist_ok=True) @@ -170,6 +299,9 @@ def already_handled(job_id: str) -> bool: def handle(req: dict) -> dict: base = {"id": req["id"], "action": req["action"], "started_at": now_iso()} + if req["action"] == "drives": # a read, not a job: no result + write_drives(do_drives()) + return {**base, "state": "done"} write_result({**base, "state": "running"}) action = req["action"] if action == "update": diff --git a/plugins/host_agent/files/steward-maint.service b/plugins/host_agent/files/steward-maint.service index 5eb8ea8..43feb08 100644 --- a/plugins/host_agent/files/steward-maint.service +++ b/plugins/host_agent/files/steward-maint.service @@ -1,5 +1,6 @@ # Runs as root, and does only what maint.py's fixed list allows: update the -# host's packages, reboot it, or drain / reactivate its swarm node (#5766). +# host's packages, reboot it, drain / reactivate its swarm node (#5766), or +# read its drives' health with smartctl (#5789). [Unit] Description=Steward maintenance helper diff --git a/plugins/host_agent/migrations/versions/host_agent_009_drive_health.py b/plugins/host_agent/migrations/versions/host_agent_009_drive_health.py new file mode 100644 index 0000000..25d12e7 --- /dev/null +++ b/plugins/host_agent/migrations/versions/host_agent_009_drive_health.py @@ -0,0 +1,24 @@ +# plugins/host_agent/migrations/versions/host_agent_009_drive_health.py +"""host_agent: each host's newest drive-health report (#5789) + +Revision ID: host_agent_009_drive_health +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "host_agent_009_drive_health" +down_revision: Union[str, None] = "host_agent_008_maint_plans" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("host_agent_registrations", sa.Column("drive_health", sa.JSON(), nullable=True)) + op.add_column("host_agent_registrations", + sa.Column("drive_health_at", sa.DateTime(timezone=True), nullable=True)) + + +def downgrade() -> None: + op.drop_column("host_agent_registrations", "drive_health_at") + op.drop_column("host_agent_registrations", "drive_health") diff --git a/plugins/host_agent/models.py b/plugins/host_agent/models.py index f7cd6b8..d932cb7 100644 --- a/plugins/host_agent/models.py +++ b/plugins/host_agent/models.py @@ -2,7 +2,7 @@ from __future__ import annotations import uuid from datetime import datetime, timezone -from sqlalchemy import Boolean, Column, String, DateTime, Float, ForeignKey, Integer, Text +from sqlalchemy import JSON, Boolean, Column, String, DateTime, Float, ForeignKey, Integer, Text from steward.models.base import Base @@ -51,6 +51,11 @@ class HostAgentRegistration(Base): # its request directory) that updates and reboots go through (#5766); None # until an agent that says reports. maint_helper = Column(Boolean, nullable=True) + # The host's newest drive-health report, as the root helper read it with + # smartctl (agent ≥ 1.20.0, #5789): {"checked_at", "smartctl", "drives"} or + # {"error"}. drives.py judges it; the figures are also metrics. + drive_health = Column(JSON, nullable=True) + drive_health_at = Column(DateTime(timezone=True), nullable=True) created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow) updated_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow, onupdate=_utcnow) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index eaba7c7..813b62a 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -27,7 +27,7 @@ from steward.hosts import service as host_svc from steward.models.hosts import Host from steward.models.metrics import PluginMetric from .downtime import recent as recent_downtime, record_events -from . import maintenance +from . import drives as drive_health, maintenance from .models import HostAgentRegistration from .scheduler import arrivals # Query helpers live in a model-free module so integration tests can import them @@ -193,6 +193,16 @@ def apply_updates(reg, sample: dict, now: datetime) -> bool: reg.reboot_required_since, reg.reboot_packages) +def apply_drives(reg, sample: dict, now: datetime) -> bool: + """Keep the registration's copy of the host's newest drive report (#5789), + which the host page lists: model, state and why, beside the figures.""" + report = sample.get("drives") + if not isinstance(report, dict) or report == reg.drive_health: + return False + reg.drive_health, reg.drive_health_at = report, _ts_or(report.get("checked_at"), now) + return True + + def _expand_sample_to_metrics( sample: dict, host_name: str, recorded_at: datetime ) -> list[PluginMetric]: @@ -331,6 +341,10 @@ def _expand_sample_to_metrics( if max_temp is not None: row("temp_c_max", host_name, max_temp) + # Each drive's health (#5789): the helper's hourly read rides every sample. + for metric, resource, value in drive_health.metric_rows(sample.get("drives"), host_name): + row(metric, resource, value) + # Pressure stall information (PSI) — host-level gauges (mem/cpu/io some|full). for k, v in (sample.get("psi") or {}).items(): if isinstance(v, (int, float)): @@ -527,6 +541,8 @@ async def ingest(): changed = True if newest_sample is not None and apply_updates(reg, newest_sample, received_at): changed = True + if newest_sample is not None and apply_drives(reg, newest_sample, received_at): + changed = True # Mirror the reported IP into Host.address only when it's blank. new_address = pick_host_address(host.address, host_ip) if new_address is not None: @@ -685,6 +701,7 @@ async def agent_source(): AGENT_FILES_DIR = Path(__file__).parent / "files" AGENT_FILES = { "maint.py": "text/x-python", + "install-deps.sh": "text/x-shellscript", "steward-agent.service": "text/plain", "steward-maint.service": "text/plain", "steward-maint.path": "text/plain", @@ -842,6 +859,8 @@ def _split_host_metrics(host_name: str, latest: dict) -> tuple: disks_io[suffix[len("diskio:"):]] = metrics elif suffix.startswith("temp:"): temps[suffix[len("temp:"):]] = metrics.get("temp_c") + elif suffix.startswith("drive:"): + continue # listed from the drive report (#5789) else: mounts[suffix] = metrics cores.sort(key=lambda c: c[0]) @@ -882,12 +901,15 @@ async def host_detail_metrics(host_id: str): latest = await _latest_metrics_for_host(session, host.name) hostlvl, cores, nets, disks_io, temps, mounts = _split_host_metrics(host.name, latest) temps, unwired = split_temps(host.name, temps) + drive_report = reg.drive_health if reg else None ls = reg.last_seen_at if reg else None stale = ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds() return await render_template( "_host_metrics.html", host=host, reg=reg, stale=stale, hostlvl=hostlvl, cores=cores, nets=nets, disks_io=disks_io, temps=temps, unwired_temps=unwired, mounts=mounts, + drives=drive_health.drives_of(drive_report), drives_error=(drive_report or {}).get("error"), + drives_at=reg.drive_health_at if reg else None, ) diff --git a/plugins/host_agent/templates/_host_metrics.html b/plugins/host_agent/templates/_host_metrics.html index 625e71d..a6d44d8 100644 --- a/plugins/host_agent/templates/_host_metrics.html +++ b/plugins/host_agent/templates/_host_metrics.html @@ -119,6 +119,41 @@ {% endif %} +{# ── Drive health (#5789): each drive as the root helper read it with + smartctl, hourly; what needs attention first (drives.py judges them). ──── #} +{% set drive_badge = {"failing": ("crit", "Failing"), "unreadable": ("warn", "Unreadable"), + "worn": ("warn", "Worn"), "watch": ("warn", "Watch"), "ok": ("ok", "OK"), + "unavailable": ("inactive", "No SMART")} %} +{% if drives or drives_error %} +
+
+ Drive health + {% if drives_at %}read {{ drives_at|ago }}{% endif %} +
+ {% if drives_error %} +

Not read: {{ drives_error }}.

+ {% endif %} + {% for d in drives %} + {% set tone, label = drive_badge[d.state] %} +
+ + {{ status_badge(tone, label) }} + {{ d.device }} + {% if d.model %}{{ d.model }}{% if d.capacity_bytes %} · {{ d.capacity_bytes|bytes }}{% endif %}{% endif %} + {% if d.why %} — {{ d.why }}{% endif %} + + + {%- set parts = [] -%} + {%- if d.get("life_used_pct") is not none %}{% set _ = parts.append("%d%% life used"|format(d.life_used_pct)) %}{% endif -%} + {%- if d.get("bad_sectors") is not none %}{% set _ = parts.append("%d bad sectors"|format(d.bad_sectors)) %}{% endif -%} + {%- if d.get("temp_c") is not none %}{% set _ = parts.append("%d°C"|format(d.temp_c)) %}{% endif -%} + {{ parts|join(" · ") if parts else "—" }} + +
+ {% endfor %} +
+{% endif %} + {# ── Interfaces / disks (short panels, side by side at natural height) ─────── #} {% if nets or disks_io %}
diff --git a/plugins/host_agent/templates/install.sh.j2 b/plugins/host_agent/templates/install.sh.j2 index 3ad1d65..48e4965 100644 --- a/plugins/host_agent/templates/install.sh.j2 +++ b/plugins/host_agent/templates/install.sh.j2 @@ -63,6 +63,12 @@ chmod 0640 "$CONF_FILE" curl -fsSL "${FILES_URL}/maint.py" -o "$AGENT_DIR/maint.py" chmod 0755 "$AGENT_DIR/maint.py" chown root:root "$AGENT_DIR/maint.py" +# Its packages (smartctl for drive health, #5789), with whichever package +# manager this host has; one that can't be installed is said, never fatal. +curl -fsSL "${FILES_URL}/install-deps.sh" -o "$AGENT_DIR/install-deps.sh" +chmod 0755 "$AGENT_DIR/install-deps.sh" +chown root:root "$AGENT_DIR/install-deps.sh" +"$AGENT_DIR/install-deps.sh" || true install -d -o root -g root -m 0755 /var/lib/steward-agent /var/lib/steward-maint /var/lib/steward-maint/results install -d -o "$AGENT_USER" -g "$AGENT_USER" -m 0700 /var/lib/steward-agent/requests for u in $UNITS; do diff --git a/steward/alerts/service.py b/steward/alerts/service.py index 2f28237..3349912 100644 --- a/steward/alerts/service.py +++ b/steward/alerts/service.py @@ -385,17 +385,19 @@ async def resources_for(db, source_module: str) -> list[str]: # A host's parts, by the shape of the name after ":" (#5475): the order # and heading each kind takes in the rule form's Part picker. PART_KINDS = (("mount", "Mounts"), ("core", "CPU cores"), ("disk", "Disks"), - ("net", "Interfaces"), ("temp", "Temperatures"), ("other", "Other")) + ("drive", "Drive health"), ("net", "Interfaces"), ("temp", "Temperatures"), + ("other", "Other")) def part_kind(part: str) -> tuple[str, str]: """(kind, label) for a host part: "/var" is a mount, "core3" a core, - "diskio:sda" a disk, "net:eth0" an interface, "temp:Core 0" a sensor.""" + "diskio:sda" a disk, "drive:sda" its health (#5789), "net:eth0" an + interface, "temp:Core 0" a sensor.""" if part.startswith("/"): return "mount", part if part.startswith("core") and part[4:].isdigit(): return "core", f"core {part[4:]}" - for prefix, kind in (("diskio:", "disk"), ("net:", "net"), ("temp:", "temp")): + for prefix, kind in (("diskio:", "disk"), ("drive:", "drive"), ("net:", "net"), ("temp:", "temp")): if part.startswith(prefix): return kind, part[len(prefix):] return "other", part diff --git a/steward/ansible/bundled/host_agent/install.yml b/steward/ansible/bundled/host_agent/install.yml index 589ad7f..0b8deb7 100644 --- a/steward/ansible/bundled/host_agent/install.yml +++ b/steward/ansible/bundled/host_agent/install.yml @@ -98,6 +98,27 @@ mode: "0755" force: true + # The helper's packages (smartctl for drive health, #5789), with whichever + # package manager the host has. One that can't be installed is said, never + # fatal: the host page then reads "smartctl not installed". + - name: Download the dependency installer + ansible.builtin.get_url: + url: "{{ steward_url }}/plugins/host_agent/files/install-deps.sh" + dest: /usr/local/lib/steward-agent/install-deps.sh + owner: root + mode: "0755" + force: true + + - name: Install the helper's packages + ansible.builtin.command: /usr/local/lib/steward-agent/install-deps.sh + register: helper_deps + changed_when: "'installed:' in helper_deps.stdout" + + - name: Say what couldn't be installed + ansible.builtin.debug: + msg: "{{ helper_deps.stdout_lines | select('search', 'WARNING') | list }}" + when: "'WARNING' in helper_deps.stdout" + - name: Install the agent and maintenance units ansible.builtin.get_url: url: "{{ steward_url }}/plugins/host_agent/files/{{ item }}" diff --git a/steward/ansible/bundled/host_agent/provision.yml b/steward/ansible/bundled/host_agent/provision.yml index a9cfd26..2f21745 100644 --- a/steward/ansible/bundled/host_agent/provision.yml +++ b/steward/ansible/bundled/host_agent/provision.yml @@ -154,6 +154,27 @@ mode: "0755" force: true + # The helper's packages (smartctl for drive health, #5789), with whichever + # package manager the host has. One that can't be installed is said, never + # fatal: the host page then reads "smartctl not installed". + - name: Download the dependency installer + ansible.builtin.get_url: + url: "{{ steward_url }}/plugins/host_agent/files/install-deps.sh" + dest: /usr/local/lib/steward-agent/install-deps.sh + owner: root + mode: "0755" + force: true + + - name: Install the helper's packages + ansible.builtin.command: /usr/local/lib/steward-agent/install-deps.sh + register: helper_deps + changed_when: "'installed:' in helper_deps.stdout" + + - name: Say what couldn't be installed + ansible.builtin.debug: + msg: "{{ helper_deps.stdout_lines | select('search', 'WARNING') | list }}" + when: "'WARNING' in helper_deps.stdout" + - name: Install the agent and maintenance units ansible.builtin.get_url: url: "{{ steward_url }}/plugins/host_agent/files/{{ item }}" diff --git a/steward/ansible/bundled/host_agent/update.yml b/steward/ansible/bundled/host_agent/update.yml index bcc2573..c2bbc65 100644 --- a/steward/ansible/bundled/host_agent/update.yml +++ b/steward/ansible/bundled/host_agent/update.yml @@ -1,5 +1,5 @@ --- -# description: Update the Steward host agent (refresh agent.py, its units and the maintenance helper, then restart) on hosts already running it. +# description: Update the Steward host agent (refresh agent.py, its units, the maintenance helper and its packages, then restart) on hosts already running it. # steward:category: host-agent # Update an already-installed Steward host agent: refresh agent.py, its systemd # units and the maintenance helper (#5766), then restart. @@ -66,6 +66,27 @@ mode: "0755" force: true + # The helper's packages (smartctl for drive health, #5789), with whichever + # package manager the host has. One that can't be installed is said, never + # fatal: the host page then reads "smartctl not installed". + - name: Download the dependency installer + ansible.builtin.get_url: + url: "{{ steward_url }}/plugins/host_agent/files/install-deps.sh" + dest: /usr/local/lib/steward-agent/install-deps.sh + owner: root + mode: "0755" + force: true + + - name: Install the helper's packages + ansible.builtin.command: /usr/local/lib/steward-agent/install-deps.sh + register: helper_deps + changed_when: "'installed:' in helper_deps.stdout" + + - name: Say what couldn't be installed + ansible.builtin.debug: + msg: "{{ helper_deps.stdout_lines | select('search', 'WARNING') | list }}" + when: "'WARNING' in helper_deps.stdout" + - name: Install the agent and maintenance units ansible.builtin.get_url: url: "{{ steward_url }}/plugins/host_agent/files/{{ item }}" diff --git a/steward/core/formatting.py b/steward/core/formatting.py index 493fe24..b26ab98 100644 --- a/steward/core/formatting.py +++ b/steward/core/formatting.py @@ -352,6 +352,12 @@ METRICS: dict[str, tuple[str, str]] = { "net_tx_bps": ("Network out", "bytes per second sent, by the whole host or one interface"), "disk_read_bps": ("Disk reads", "bytes per second read, from all disks or one"), "disk_write_bps": ("Disk writes", "bytes per second written, to all disks or one"), + "drives_failing": ("Drives failing", "drives on the host failing their own health check, raising a critical warning or out of spare blocks"), + "drive_failing": ("Drive failing", "1 when this drive fails its own health check, raises a critical warning or is out of spare blocks"), + "drive_life_used_max_pct": ("Most worn drive", "the most life any of the host's SSDs has used of its rating"), + "drive_life_used_pct": ("Drive life used", "how much of its rated write life this drive has used"), + "drive_bad_sectors": ("Bad sectors", "media errors, or reallocated, pending and uncorrectable sectors, as the drive counts them"), + "drive_temp_c": ("Drive temperature", "the drive's own temperature sensor"), } # Where a rule's metric comes from, as the picker and the rules list say it. diff --git a/steward/mcp/tools/hosts.py b/steward/mcp/tools/hosts.py index d3bd5ff..9632b4a 100644 --- a/steward/mcp/tools/hosts.py +++ b/steward/mcp/tools/hosts.py @@ -40,6 +40,7 @@ def _agent_glance(a: dict | None) -> dict | None: _PART_GROUPS = ( # (prefix, group, the field that names one part) ("net:", "interfaces", "interface"), ("diskio:", "disk_io", "device"), + ("drive:", "drive_health", "device"), ("temp:", "temperatures", "sensor"), ) # Sort each group by what makes a part worth a look first. @@ -47,6 +48,7 @@ _PART_ORDER = { "mounts": lambda r: -(r.get("disk_used_pct") or 0), "interfaces": lambda r: -((r.get("net_rx_bps") or 0) + (r.get("net_tx_bps") or 0)), "disk_io": lambda r: -((r.get("disk_read_bps") or 0) + (r.get("disk_write_bps") or 0)), + "drive_health": lambda r: (-(r.get("drive_failing") or 0), -(r.get("drive_bad_sectors") or 0)), "temperatures": lambda r: -(r.get("temp_c") or 0), "other": lambda r: r["part"], } @@ -59,7 +61,7 @@ def agent_parts(parts: dict[str, dict], host_name: str, limit: int) -> dict: metrics. Per-core CPU is summarised, not listed: the host's cpu_pct already says how busy it is, and 30+ cores would crowd out the rest. """ - groups: dict[str, list] = {"mounts": [], "interfaces": [], "disk_io": [], + groups: dict[str, list] = {"mounts": [], "interfaces": [], "disk_io": [], "drive_health": [], "temperatures": [], "other": []} cores = [] for name, metrics in parts.items(): diff --git a/tests/mcp/test_tools.py b/tests/mcp/test_tools.py index 8194e61..77a4777 100644 --- a/tests/mcp/test_tools.py +++ b/tests/mcp/test_tools.py @@ -267,6 +267,8 @@ def test_get_host_groups_parts_by_kind_and_summarises_cores(): "net:br-1": {"net_rx_bps": 900.0, "net_tx_bps": 0.0}, "net:eth1": {"net_rx_bps": 0.0, "net_tx_bps": 0.0}, "diskio:sda": {"disk_read_bps": 1.0, "disk_write_bps": 2.0}, + "drive:sda": {"drive_failing": 0.0, "drive_bad_sectors": 0.0}, + "drive:nvme0": {"drive_failing": 1.0, "drive_life_used_pct": 12.0}, "temp:Package id 0": {"temp_c": 61.0}, }) got = agent_parts(parts, "box", limit=2) @@ -280,6 +282,7 @@ def test_get_host_groups_parts_by_kind_and_summarises_cores(): assert nets["total"] == 3 and nets["truncated"] is True assert "parts_limit" in nets["more"] and 'resource="box:"' in nets["more"] assert got["disk_io"]["items"][0]["device"] == "sda" + assert [d["device"] for d in got["drive_health"]["items"]] == ["nvme0", "sda"] # failing first assert got["temperatures"]["items"][0] == {"sensor": "Package id 0", "part": "temp:Package id 0", "temp_c": 61.0} assert "other" not in got diff --git a/tests/plugins/host_agent/test_drive_health.py b/tests/plugins/host_agent/test_drive_health.py new file mode 100644 index 0000000..7c31d22 --- /dev/null +++ b/tests/plugins/host_agent/test_drive_health.py @@ -0,0 +1,296 @@ +# tests/plugins/host_agent/test_drive_health.py +"""#5789: each drive's health, read with smartctl by the root helper, judged by +Steward, recorded as metrics, listed on the host page. + +The helper's half reads smartctl's JSON, which is the same on every +distribution and for SATA, SAS and NVMe drives; the fixtures are trimmed from +real `smartctl --json -a` output (7.2/7.3). +""" +import importlib.util +import json +import os +import subprocess +from datetime import datetime, timezone +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from plugins.host_agent import agent as a +from plugins.host_agent import drives as dh +from tests._templates import render_plugin_template + +ROOT = Path(__file__).resolve().parents[3] +FILES = ROOT / "plugins" / "host_agent" / "files" + +NVME = {"smartctl": {"version": [7, 2], "exit_status": 0}, + "device": {"name": "/dev/nvme0", "type": "nvme", "protocol": "NVMe"}, + "model_name": "Samsung SSD 980 PRO 1TB", "user_capacity": {"bytes": 1000204886016}, + "smart_status": {"passed": True, "nvme": {"value": 0}}, + "temperature": {"current": 41}, "power_on_time": {"hours": 9120}, + "nvme_smart_health_information_log": {"critical_warning": 0, "temperature": 41, + "available_spare": 100, "available_spare_threshold": 10, + "percentage_used": 7, "media_errors": 0}} +ATA = {"smartctl": {"version": [7, 3], "exit_status": 0}, + "device": {"name": "/dev/sda", "type": "sat", "protocol": "ATA"}, + "model_name": "WDC WD40EFRX-68N32N0", "user_capacity": {"bytes": 4000787030016}, + "smart_support": {"available": True, "enabled": True}, + "smart_status": {"passed": True}, "temperature": {"current": 34}, + "ata_smart_attributes": {"table": [ + {"id": 5, "name": "Reallocated_Sector_Ct", "raw": {"value": 8}}, + {"id": 9, "name": "Power_On_Hours", "raw": {"value": 30211}}, + {"id": 197, "name": "Current_Pending_Sector", "raw": {"value": 0}}, + {"id": 198, "name": "Offline_Uncorrectable", "raw": {"value": 0}}]}} +VIRTUAL = {"smartctl": {"version": [7, 2], "exit_status": 4}, + "device": {"name": "/dev/sdb", "type": "scsi", "protocol": "SCSI"}, + "scsi_model_name": "QEMU HARDDISK", "smart_support": {"available": False}} +GONE = {"smartctl": {"version": [7, 2], "exit_status": 2, + "messages": [{"string": "Smartctl open device: /dev/sdc failed: No such device", + "severity": "error"}]}} + + +@pytest.fixture +def maint(): + spec = importlib.util.spec_from_file_location("steward_maint", FILES / "maint.py") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +# ── the helper reads ───────────────────────────────────────────────────────── + +def test_an_nvme_drive_s_figures_are_read(maint): + got = maint.drive_summary("/dev/nvme0", 0, NVME) + assert got == {"device": "nvme0", "protocol": "NVMe", "model": "Samsung SSD 980 PRO 1TB", + "capacity_bytes": 1000204886016, "passed": True, "temp_c": 41, + "power_on_hours": 9120, "critical_warning": 0, "life_used_pct": 7, + "media_errors": 0, "spare_pct": 100, "spare_threshold_pct": 10} + + +def test_a_sata_drive_s_bad_sectors_are_read_from_its_attributes(maint): + got = maint.drive_summary("/dev/sda", 0, ATA) + assert (got["reallocated"], got["pending"], got["offline_uncorrectable"]) == (8, 0, 0) + assert got["passed"] is True and got["temp_c"] == 34 and "life_used_pct" not in got + + +def test_a_drive_with_no_smart_and_one_that_can_t_be_opened_say_so(maint): + assert maint.drive_summary("/dev/sdb", 4, VIRTUAL)["available"] is False + got = maint.drive_summary("/dev/sdc", 2, GONE) + assert got == {"device": "sdc", "error": "Smartctl open device: /dev/sdc failed: No such device"} + assert maint.drive_summary("/dev/sdd", None, None)["error"] == "smartctl couldn't read it" + + +def test_a_failing_disk_s_figures_still_count(maint): + """Exit bit 3 says the disk is failing; its JSON is the point, not an error.""" + failing = {**ATA, "smart_status": {"passed": False}} + assert maint.drive_summary("/dev/sda", 8, failing)["passed"] is False + + +def test_no_smartctl_or_an_old_one_is_reported_not_parsed(maint, monkeypatch): + monkeypatch.setattr(maint, "smartctl_version", lambda: None) + assert maint.do_drives()["error"] == "smartctl not installed" + monkeypatch.setattr(maint, "smartctl_version", lambda: (6, 6)) + got = maint.do_drives() + assert got["smartctl"] == "6.6" and "too old" in got["error"] and "drives" not in got + + +def test_the_helper_finds_the_drives_itself(maint, monkeypatch): + calls = [] + + def fake(args): + calls.append(args) + if args == ["--scan-open"]: + return 0, {"devices": [{"name": "/dev/sda", "type": "sat"}, + {"name": "/dev/nvme0", "type": "nvme"}, + {"name": "not-a-device"}]} + return 0, {"/dev/sda": ATA, "/dev/nvme0": NVME}[args[-1]] + monkeypatch.setattr(maint, "smartctl_version", lambda: (7, 2)) + monkeypatch.setattr(maint, "smartctl_json", fake) + got = maint.do_drives() + assert [d["device"] for d in got["drives"]] == ["sda", "nvme0"] and got["smartctl"] == "7.2" + assert calls[1:] == [["-a", "-d", "sat", "/dev/sda"], ["-a", "-d", "nvme", "/dev/nvme0"]] + + +def test_a_drives_request_writes_the_report_and_no_job_result(maint, tmp_path, monkeypatch): + monkeypatch.setattr(maint, "DRIVES", str(tmp_path / "drives.json")) + monkeypatch.setattr(maint, "RESULTS", str(tmp_path / "results")) + monkeypatch.setattr(maint, "do_drives", lambda: {"checked_at": "t", "drives": []}) + req = {"id": "drives-1", "action": "drives"} + path = tmp_path / "r.json" + path.write_text(json.dumps(req)) + assert maint.read_request(str(path)) == req # on the helper's list + assert maint.handle(req)["state"] == "done" + assert json.loads((tmp_path / "drives.json").read_text()) == {"checked_at": "t", "drives": []} + assert oct(os.stat(tmp_path / "drives.json").st_mode & 0o777) == "0o644" + assert not (tmp_path / "results").exists() # not a maintenance job + + +# ── the agent asks hourly and carries the answer ───────────────────────────── + +def _helper(tmp_path): + unit, requests = tmp_path / "steward-maint.path", tmp_path / "requests" + unit.write_text("[Path]\n") + requests.mkdir() + return dict(unit=str(unit), requests_dir=str(requests), drives_path=str(tmp_path / "drives.json")) + + +def test_the_agent_asks_at_start_then_hourly(tmp_path): + paths, state = _helper(tmp_path), {} + a.refresh_drives(state, now_mono=100.0, **paths) + assert [json.loads((tmp_path / "requests" / n).read_text())["action"] + for n in os.listdir(tmp_path / "requests")] == ["drives"] + for n in os.listdir(tmp_path / "requests"): + os.unlink(tmp_path / "requests" / n) + a.refresh_drives(state, now_mono=100.0 + 3599, **paths) + assert os.listdir(tmp_path / "requests") == [] + a.refresh_drives(state, now_mono=100.0 + 3600, **paths) + assert len(os.listdir(tmp_path / "requests")) == 1 + + +def test_the_agent_carries_the_newest_report(tmp_path): + paths, state = _helper(tmp_path), {} + a.refresh_drives(state, now_mono=0.0, **paths) + assert "drives" not in state # nothing read yet + (tmp_path / "drives.json").write_text(json.dumps({"checked_at": "t1", "drives": []})) + a.refresh_drives(state, now_mono=1.0, **paths) + assert state["drives"]["checked_at"] == "t1" + (tmp_path / "drives.json").write_text("{half") # mid-write + os.utime(tmp_path / "drives.json", (5, 5)) + a.refresh_drives(state, now_mono=2.0, **paths) + assert state["drives"]["checked_at"] == "t1" + + +def test_without_the_helper_the_agent_says_what_drive_health_needs(tmp_path): + state = {} + a.refresh_drives(state, now_mono=0.0, unit=str(tmp_path / "none.path"), + requests_dir=str(tmp_path / "none"), drives_path=str(tmp_path / "d.json")) + assert "maintenance helper" in state["drives"]["error"] + + +def test_the_agent_and_the_helper_agree_on_the_report_s_place(maint): + assert a.DRIVES_PATH == maint.DRIVES and "drives" in a.MAINT_ACTIONS + + +# ── Steward judges ─────────────────────────────────────────────────────────── + +@pytest.mark.parametrize("drive, state", [ + ({"device": "sda", "passed": True, "reallocated": 0}, "ok"), + ({"device": "sda", "passed": False}, "failing"), + ({"device": "nvme0", "passed": True, "critical_warning": 4}, "failing"), + ({"device": "nvme0", "passed": True, "spare_pct": 5, "spare_threshold_pct": 10}, "failing"), + ({"device": "nvme0", "passed": True, "life_used_pct": 93}, "worn"), + ({"device": "sda", "passed": True, "reallocated": 8}, "watch"), + ({"device": "sdb", "available": False}, "unavailable"), + ({"device": "sdc", "error": "No such device"}, "unreadable"), +]) +def test_each_drive_is_judged(drive, state): + assert dh.judge(drive)["state"] == state + + +def test_what_needs_attention_is_listed_first(): + report = {"drives": [{"device": "sda", "passed": True}, {"device": "sdb", "available": False}, + {"device": "nvme0", "passed": False}, {"device": "sdc", "error": "x"}]} + assert [d["device"] for d in dh.drives_of(report)] == ["nvme0", "sdc", "sda", "sdb"] + assert dh.drives_of({"error": "smartctl not installed"}) == [] == dh.drives_of(None) + + +def test_each_read_drive_is_a_set_of_metrics_and_the_host_rolls_them_up(): + report = {"drives": [{"device": "nvme0", "passed": True, "life_used_pct": 7, "media_errors": 0, + "temp_c": 41}, + {"device": "sda", "passed": False, "reallocated": 8, "pending": 2}, + {"device": "sdb", "available": False}]} + rows = {(m, r): v for m, r, v in dh.metric_rows(report, "box")} + assert rows == { + ("drive_failing", "box:drive:sda"): 1.0, ("drive_bad_sectors", "box:drive:sda"): 10.0, + ("drive_failing", "box:drive:nvme0"): 0.0, ("drive_life_used_pct", "box:drive:nvme0"): 7.0, + ("drive_bad_sectors", "box:drive:nvme0"): 0.0, ("drive_temp_c", "box:drive:nvme0"): 41.0, + ("drives_failing", "box"): 1.0, ("drive_life_used_max_pct", "box"): 7.0} + + +def test_a_host_with_no_drive_read_records_no_roll_up(): + """A VM's virtual disks, or smartctl missing: 0 failing would claim a look + that never happened.""" + assert dh.metric_rows({"drives": [{"device": "vda", "available": False}]}, "box") == [] + assert dh.metric_rows({"error": "smartctl not installed"}, "box") == [] + + +def test_every_drive_metric_is_offered_to_rules_with_a_plain_label(): + from plugins.host_agent import get_alert_metrics + from steward.core.formatting import METRICS, metric_unit + assert set(dh.DRIVE_METRICS) <= set(get_alert_metrics()["host_agent"]) + assert all(m in METRICS for m in dh.DRIVE_METRICS) + assert metric_unit("drive_temp_c") == "°C" and metric_unit("drive_life_used_max_pct") == "%" + + +def test_ingest_records_the_drives_and_keeps_the_report_once(): + from plugins.host_agent.routes import _expand_sample_to_metrics, apply_drives + now = datetime(2026, 10, 15, tzinfo=timezone.utc) + report = {"checked_at": "2026-10-15T00:00:00+00:00", "smartctl": "7.2", + "drives": [{"device": "sda", "passed": True}]} + rows = _expand_sample_to_metrics({"drives": report}, "box", now) + assert {(r.metric_name, r.resource_name) for r in rows} == { + ("drive_failing", "box:drive:sda"), ("drives_failing", "box")} + reg = SimpleNamespace(drive_health=None, drive_health_at=None) + assert apply_drives(reg, {"drives": report}, now) is True + assert reg.drive_health == report and reg.drive_health_at == now + assert apply_drives(reg, {"drives": dict(report)}, now) is False # unchanged: no write + assert apply_drives(reg, {}, now) is False + + +# ── installed on any common distribution ───────────────────────────────────── + +def test_the_dependency_installer_knows_the_common_package_managers(): + script = (FILES / "install-deps.sh").read_text() + for manager in ("apt-get", "dnf", "yum", "zypper", "pacman", "apk"): + assert f"have {manager}" in script, manager + assert "--no-install-recommends" in script # no mail server pulled in on Debian + subprocess.run(["sh", "-n", str(FILES / "install-deps.sh")], check=True) + + +def test_the_dependency_installer_never_fails_the_install(tmp_path): + """No package manager it knows (PATH holds none): it says so and exits 0.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + done = subprocess.run(["/bin/sh", str(FILES / "install-deps.sh")], capture_output=True, text=True, + env={"PATH": str(bin_dir)}) + assert done.returncode == 0 + assert "WARNING: could not install smartmontools" in done.stdout + + +def test_install_sh_and_every_agent_playbook_run_it(): + install = (ROOT / "plugins" / "host_agent" / "templates" / "install.sh.j2").read_text() + assert '"$AGENT_DIR/install-deps.sh" || true' in install + for name in ("install", "provision", "update"): + book = (ROOT / "steward" / "ansible" / "bundled" / "host_agent" / f"{name}.yml").read_text() + assert "files/install-deps.sh" in book and "command: /usr/local/lib/steward-agent/install-deps.sh" in book, name + + +# ── the host page ──────────────────────────────────────────────────────────── + +async def test_the_host_page_lists_each_drive_failing_first(app): + report = {"drives": [{"device": "nvme0", "passed": True, "life_used_pct": 7, "temp_c": 41, + "model": "Samsung SSD 980 PRO 1TB"}, + {"device": "sda", "passed": False, "reallocated": 8, "model": "WDC WD40EFRX"}]} + async with app.test_request_context("/plugins/host_agent/h1/metrics"): + page = await render_plugin_template( + app, "host_agent", "_host_metrics.html", host=SimpleNamespace(address="10.0.0.2"), stale=False, + reg=SimpleNamespace(distro=None, kernel=None, arch=None, agent_version=None, + last_seen_at=datetime.now(timezone.utc)), + hostlvl={"cpu_pct": 5.0}, cores=[], nets={}, disks_io={}, temps={}, mounts={}, + drives=dh.drives_of(report), drives_error=None, drives_at=datetime.now(timezone.utc)) + page = " ".join(page.split()) + assert "Drive health" in page + assert page.index('badge-crit">Failing') < page.index('badge-ok">OK') + assert "its SMART self-assessment failed" in page and "8 bad sectors" in page + assert "7% life used · 0 bad sectors" not in page and "7% life used · 41°C" in page + + +async def test_the_host_page_says_why_drives_weren_t_read(app): + async with app.test_request_context("/plugins/host_agent/h1/metrics"): + page = await render_plugin_template( + app, "host_agent", "_host_metrics.html", host=SimpleNamespace(address="10.0.0.2"), stale=False, + reg=SimpleNamespace(distro=None, kernel=None, arch=None, agent_version=None, + last_seen_at=datetime.now(timezone.utc)), + hostlvl={"cpu_pct": 5.0}, cores=[], nets={}, disks_io={}, temps={}, mounts={}, + drives=[], drives_error="smartctl not installed", drives_at=None) + assert "Not read: smartctl not installed." in page