bvandeusenandClaude Opus 5.5
715b58a09f
Each weekly run drains Docker02 then Docker03, and the swarm never moves a task back by itself, so every run ended with nearly every service on one node. On 2026-10-25 the operator had that spread out by hand. - Agent 1.26.0 reports each swarm service's pinned digest, and every repo@sha256 its host holds (from /system/df, every 10 min). - Once a run's last host is back in service, Steward picks services on the fullest node to move to the emptiest until they are within one task. It takes services with several replicas on one node first. A service moves only when the emptier node already holds its exact image. One the plan keeps in place, one below its replicas, and one with no pinned digest stay put, and the run says why. - A swarm manager's helper carries it out through a new action, rebalance, with the service names (validated, at most 48). It force-updates each in turn with the spec unchanged and no registry lookup, skips any a constraint places, and gives each 5 min to settle. - The run ends done with what moved, or failed if a move didn't settle or the manager didn't answer in 60 min. A docker-scoped window covers the restarts, and the step goes on the timeline. - Plans gain "Spread swarm services back out" (on by default) and "Keep in place", on the create form and in an "After the run" form on each plan. list_maintenance_plans shows both and the latest run's spread. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
645 lines
28 KiB
Python
645 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
"""Steward maintenance helper (#5766): runs as root, started by steward-maint.path
|
|
when the host agent leaves a request, and does only what ACTIONS lists.
|
|
|
|
The agent (an unprivileged user under NoNewPrivileges) can't update or reboot
|
|
its host itself. It writes `<id>.json` = {"id", "action", ["node"]} into
|
|
REQUESTS, the one directory it may write. This helper reads each request
|
|
without following links, refuses anything outside the list, deletes it, does
|
|
the action, and writes the outcome to RESULTS (root's own directory, which the
|
|
agent reads back and reports to Steward). A stolen agent token can therefore
|
|
make its host update, reboot or drain, or on a swarm manager restart named
|
|
services where the swarm puts them, and nothing else.
|
|
|
|
One action only reads: `drives` (#5789) asks smartctl for each disk's health,
|
|
which needs root, and writes a summary to DRIVES, never a result: it is not a
|
|
maintenance job. `check` (#6098) changes nothing but the package lists: it
|
|
says what an update would upgrade, whether Docker is among it, and which
|
|
containers run outside the swarm.
|
|
|
|
`rebalance` spreads swarm services back out after a maintenance run left them
|
|
on one node. Steward names the services, having checked that the emptier node
|
|
holds each one's image; the helper force-updates them one at a time, spec
|
|
unchanged and no image pulled, so the swarm places each on its emptiest node.
|
|
A service a constraint or a preference places is left alone.
|
|
|
|
Stdlib only; served by Steward at /plugins/host_agent/files/maint.py.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone
|
|
|
|
REQUESTS = "/var/lib/steward-agent/requests"
|
|
RESULTS = "/var/lib/steward-maint/results"
|
|
DRIVES = "/var/lib/steward-maint/drives.json"
|
|
ACTIONS = ("check", "update", "reboot", "drain", "undrain", "drives", "rebalance")
|
|
REQUEST_MAX_BYTES = 4096
|
|
RESULTS_KEPT_SECONDS = 7 * 86400
|
|
UPDATE_DEADLINE_SECONDS = 3000
|
|
COMMAND_DEADLINE_SECONDS = 600
|
|
OUTPUT_KEPT_CHARS = 2000
|
|
# A listing of what an update would change is read whole, not just its tail.
|
|
LISTING_KEPT_CHARS = 1_000_000
|
|
_ID = re.compile(r"^[A-Za-z0-9-]{1,64}$")
|
|
_NODE = re.compile(r"^[a-z0-9]{25}$")
|
|
# A swarm service's name as Docker allows it, never a flag.
|
|
_SERVICE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,62}$")
|
|
SERVICES_MAX = 48
|
|
MOVE_DEADLINE_SECONDS = 300
|
|
# A rebalance starts no move after this long.
|
|
REBALANCE_SECONDS = 2400
|
|
|
|
|
|
def now_iso() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def read_request(path: str):
|
|
"""The request at `path` as a dict, or None when it is not a small regular
|
|
file holding a valid request. Never follows a link the agent planted."""
|
|
try:
|
|
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
except OSError:
|
|
return None
|
|
try:
|
|
st = os.fstat(fd)
|
|
if not (st.st_mode & 0o170000 == 0o100000) or st.st_size > REQUEST_MAX_BYTES:
|
|
return None
|
|
raw = os.read(fd, REQUEST_MAX_BYTES + 1)
|
|
finally:
|
|
os.close(fd)
|
|
try:
|
|
req = json.loads(raw.decode("utf-8"))
|
|
except (UnicodeDecodeError, ValueError):
|
|
return None
|
|
return req if valid(req) else None
|
|
|
|
|
|
def valid(req) -> bool:
|
|
"""A request names a known action, has a plain id, a node id only in the
|
|
swarm's own form, and, for a rebalance alone, a list of service names."""
|
|
if not isinstance(req, dict) or req.get("action") not in ACTIONS:
|
|
return False
|
|
if not isinstance(req.get("id"), str) or not _ID.match(req["id"]):
|
|
return False
|
|
services = req.get("services")
|
|
if req["action"] == "rebalance":
|
|
if not (isinstance(services, list) and 0 < len(services) <= SERVICES_MAX
|
|
and all(isinstance(s, str) and _SERVICE.match(s) for s in services)):
|
|
return False
|
|
elif services is not None:
|
|
return False
|
|
node = req.get("node")
|
|
return node is None or (isinstance(node, str) and bool(_NODE.match(node)))
|
|
|
|
|
|
def run(cmd: list, deadline: int = COMMAND_DEADLINE_SECONDS, env: dict | None = None,
|
|
ok_codes: tuple = (0,), keep: int = OUTPUT_KEPT_CHARS):
|
|
"""(ok, output tail) of a command, bounded by `deadline`. `ok_codes` are
|
|
the exit codes that mean it worked (zypper says more than 0); `keep` is
|
|
how much of the output's end is kept."""
|
|
try:
|
|
done = subprocess.run(cmd, capture_output=True, text=True, timeout=deadline,
|
|
env=dict(os.environ, LANG="C", LC_ALL="C", **(env or {})))
|
|
except subprocess.TimeoutExpired:
|
|
return False, f"{cmd[0]} didn't finish in {deadline} s"
|
|
except OSError as exc:
|
|
return False, str(exc)
|
|
out = (done.stdout or "") + (done.stderr or "")
|
|
return done.returncode in ok_codes, out[-keep:]
|
|
|
|
|
|
REBOOT_MARKER = "/var/run/reboot-required" # Debian and Ubuntu
|
|
KERNEL_MODULES = "/usr/lib/modules"
|
|
# zypper's own exit codes for "done, and": 102 a reboot is needed, 103 zypper
|
|
# updated itself and would run again (its next run picks up the rest).
|
|
ZYPPER_OK = (0, 102, 103)
|
|
|
|
|
|
def reboot_required() -> bool | None:
|
|
"""Whether the host needs a reboot after an update, asked the way its
|
|
package manager says it; None when it can't say."""
|
|
if os.path.exists(REBOOT_MARKER):
|
|
return True
|
|
if shutil.which("apt-get"):
|
|
return False
|
|
try:
|
|
if shutil.which("needs-restarting"): # dnf, yum
|
|
return subprocess.run(["needs-restarting", "-r"], capture_output=True,
|
|
timeout=60).returncode == 1
|
|
if shutil.which("zypper"):
|
|
code = subprocess.run(["zypper", "needs-rebooting"], capture_output=True,
|
|
timeout=60).returncode
|
|
return {0: False, 102: True}.get(code)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
return None
|
|
if shutil.which("pacman"):
|
|
# An upgraded kernel removes the running one's modules.
|
|
return not os.path.isdir(os.path.join(KERNEL_MODULES, os.uname().release))
|
|
return None
|
|
|
|
|
|
def upgraded_count(output: str) -> int | None:
|
|
"""Packages an update changed, from the manager's summary line."""
|
|
m = re.search(r"(\d+) upgraded, (\d+) newly installed", output) # apt
|
|
if m:
|
|
return int(m.group(1)) + int(m.group(2))
|
|
m = re.search(r"(\d+) packages? to upgrade(?:, (\d+) new)?", output) # zypper
|
|
if m:
|
|
return int(m.group(1)) + int(m.group(2) or 0)
|
|
m = re.search(r"^Packages \((\d+)\)", output, re.MULTILINE) # pacman
|
|
if m:
|
|
return int(m.group(1))
|
|
counts = re.findall(r"^\s*(?:Upgrade|Install)\s+(\d+) Packages?", output, re.MULTILINE) # dnf, yum
|
|
return sum(int(c) for c in counts) if counts else None
|
|
|
|
|
|
# The same as the agent's: this helper runs as root on its own and imports
|
|
# nothing of the agent's, so it keeps its own copy.
|
|
def _os_release_id(path: str = "/etc/os-release") -> str:
|
|
try:
|
|
with open(path) as fh:
|
|
for line in fh:
|
|
if line.startswith("ID="):
|
|
return line.split("=", 1)[1].strip().strip('"')
|
|
except OSError:
|
|
pass
|
|
return ""
|
|
|
|
|
|
def update_steps(manager: str) -> list:
|
|
"""The commands an update runs on a host with `manager`, in order: each
|
|
(what it is, command, deadline, env, ok exit codes). The last one is the upgrade
|
|
whose output is counted. Existing config files are kept, and needrestart
|
|
only lists, leaving service restarts to a reboot. A package's own install
|
|
script can still restart its service: Docker's does, which is why the job
|
|
checks first and drains a swarm node before such an update (#6098)."""
|
|
long = UPDATE_DEADLINE_SECONDS
|
|
if manager == "apt":
|
|
env = {"DEBIAN_FRONTEND": "noninteractive", "NEEDRESTART_MODE": "l"}
|
|
return [("apt-get update", ["apt-get", "update"], COMMAND_DEADLINE_SECONDS, env, (0,)),
|
|
("apt-get dist-upgrade", ["apt-get", "-y", "-o", "Dpkg::Options::=--force-confdef",
|
|
"-o", "Dpkg::Options::=--force-confold", "dist-upgrade"],
|
|
long, env, (0,))]
|
|
if manager == "dnf":
|
|
return [("dnf upgrade", ["dnf", "-y", "upgrade", "--refresh"], long, None, (0,))]
|
|
if manager == "yum":
|
|
return [("yum update", ["yum", "-y", "update"], long, None, (0,))]
|
|
if manager == "zypper":
|
|
# Tumbleweed is a rolling release: it upgrades with dup, not update.
|
|
upgrade = "dup" if _os_release_id() == "opensuse-tumbleweed" else "update"
|
|
return [("zypper refresh", ["zypper", "--non-interactive", "refresh"],
|
|
COMMAND_DEADLINE_SECONDS, None, ZYPPER_OK),
|
|
("zypper " + upgrade, ["zypper", "--non-interactive", upgrade, "--auto-agree-with-licenses"],
|
|
long, None, ZYPPER_OK)]
|
|
if manager == "pacman":
|
|
# Packages from the AUR aren't updated: no AUR helper runs as root.
|
|
return [("pacman -Syu", ["pacman", "-Syu", "--noconfirm"], long, None, (0,))]
|
|
return []
|
|
|
|
|
|
# Which manager a host has; dnf before yum, since dnf hosts often keep a yum
|
|
# that is dnf underneath.
|
|
UPDATE_MANAGERS = (("apt", "apt-get"), ("dnf", "dnf"), ("yum", "yum"), ("zypper", "zypper"),
|
|
("pacman", "pacman"))
|
|
|
|
|
|
def host_manager() -> str | None:
|
|
for manager, binary in UPDATE_MANAGERS:
|
|
if shutil.which(binary):
|
|
return manager
|
|
return None
|
|
|
|
|
|
def do_update() -> dict:
|
|
"""Upgrade every package with whichever package manager the host has."""
|
|
manager = host_manager()
|
|
if manager is None:
|
|
return {"ok": False, "detail": "no package manager this helper knows "
|
|
"(apt, dnf, yum, zypper, pacman) on this host"}
|
|
out = ""
|
|
for label, cmd, deadline, env, ok_codes in update_steps(manager):
|
|
ok, out = run(cmd, deadline=deadline, env=env, ok_codes=ok_codes)
|
|
if not ok:
|
|
return {"ok": False, "detail": f"{label} failed: " + out}
|
|
changed = upgraded_count(out)
|
|
if manager == "apt":
|
|
run(["apt-get", "-y", "autoremove"], env={"DEBIAN_FRONTEND": "noninteractive"})
|
|
return {"ok": True, "packages": changed, "reboot_required": reboot_required(),
|
|
"detail": out[-500:]}
|
|
|
|
|
|
# ── what an update would change, and what runs outside the swarm (#6098) ─────
|
|
# Docker's own packages restart the daemon from their install scripts, whatever
|
|
# needrestart is told, and every container on the host stops with it. So the
|
|
# job asks first: a swarm node is drained before an update that upgrades
|
|
# Docker, not after it, and the containers running outside the swarm are
|
|
# noted, to see each of them running again once the job is over.
|
|
|
|
DOCKER_PACKAGES = re.compile(
|
|
r"^(docker|docker-ce|docker\.io|docker-engine|moby-engine|containerd|containerd\.io)$")
|
|
CONTAINERS_MAX = 40
|
|
CONTAINER_NAME_MAX = 64
|
|
|
|
|
|
def pending_steps(manager: str) -> list:
|
|
"""The commands that list what an update would upgrade, changing nothing
|
|
but the package lists: each (what it is, command, ok exit codes). The
|
|
last one's output names the packages."""
|
|
if manager == "apt":
|
|
return [("apt-get update", ["apt-get", "update"], (0,)),
|
|
("apt-get -s dist-upgrade", ["apt-get", "-s", "dist-upgrade"], (0,))]
|
|
if manager == "dnf":
|
|
# check-update exits 100 when there is something to update.
|
|
return [("dnf check-update", ["dnf", "-q", "check-update", "--refresh"], (0, 100))]
|
|
if manager == "yum":
|
|
return [("yum check-update", ["yum", "-q", "check-update"], (0, 100))]
|
|
if manager == "zypper":
|
|
if _os_release_id() == "opensuse-tumbleweed":
|
|
listing = ("zypper dup --dry-run", ["zypper", "--non-interactive", "dup", "--dry-run",
|
|
"--auto-agree-with-licenses"])
|
|
else:
|
|
listing = ("zypper list-updates", ["zypper", "--non-interactive", "list-updates"])
|
|
return [("zypper refresh", ["zypper", "--non-interactive", "refresh"], ZYPPER_OK),
|
|
(*listing, ZYPPER_OK)]
|
|
if manager == "pacman":
|
|
# -Sy refreshes the lists only; the update's -Syu follows it. -Qu exits
|
|
# 1 when there is nothing to upgrade.
|
|
return [("pacman -Sy", ["pacman", "-Sy"], (0,)), ("pacman -Qu", ["pacman", "-Qu"], (0, 1))]
|
|
return []
|
|
|
|
|
|
def pending_names(manager: str, output: str) -> list:
|
|
"""The packages a listing from pending_steps says would be upgraded."""
|
|
if manager == "apt":
|
|
return re.findall(r"^Inst (\S+)", output, re.MULTILINE)
|
|
if manager in ("dnf", "yum"):
|
|
# "docker-ce.x86_64 3:27.1.1-1.el9 docker-ce-stable"
|
|
return [parts[0].rsplit(".", 1)[0] for parts in (line.split() for line in output.splitlines())
|
|
if len(parts) == 3 and "." in parts[0]]
|
|
if manager == "zypper":
|
|
names, column, listing = [], None, False
|
|
for line in output.splitlines():
|
|
cells = [c.strip() for c in line.split("|")]
|
|
if len(cells) > 2 and column is None and "Name" in cells:
|
|
column = cells.index("Name") # list-updates' table
|
|
elif column is not None and len(cells) > column and cells[0] == "v":
|
|
names.append(cells[column])
|
|
elif re.search(r"going to be (upgraded|installed)", line):
|
|
listing = True # dup --dry-run's lists
|
|
elif listing and line.startswith(" ") and line.strip():
|
|
names.extend(line.split())
|
|
else:
|
|
listing = False
|
|
return names
|
|
if manager == "pacman":
|
|
return [line.split()[0] for line in output.splitlines() if line.strip()]
|
|
return []
|
|
|
|
|
|
def standalone_containers() -> list:
|
|
"""The running containers that aren't swarm tasks, each with its restart
|
|
policy, by name; at most CONTAINERS_MAX, so the result stays small."""
|
|
if not shutil.which("docker"):
|
|
return []
|
|
ok, out = run(["docker", "ps", "-q", "--no-trunc"], deadline=30, keep=LISTING_KEPT_CHARS)
|
|
ids = out.split() if ok else []
|
|
if not ids:
|
|
return []
|
|
fmt = '{{.Name}}|{{.HostConfig.RestartPolicy.Name}}|{{index .Config.Labels "com.docker.swarm.task.id"}}'
|
|
ok, out = run(["docker", "inspect", "--format", fmt, *ids], deadline=60, keep=LISTING_KEPT_CHARS)
|
|
if not ok:
|
|
return []
|
|
found = []
|
|
for line in out.splitlines():
|
|
parts = line.split("|")
|
|
if len(parts) != 3 or parts[2].strip():
|
|
continue # a swarm task: the drain moves it
|
|
found.append({"name": parts[0].lstrip("/")[:CONTAINER_NAME_MAX],
|
|
"restart": parts[1].strip() or "no"})
|
|
return sorted(found, key=lambda c: c["name"])[:CONTAINERS_MAX]
|
|
|
|
|
|
def do_check() -> dict:
|
|
"""What an update would upgrade, whether Docker is among it, and the
|
|
containers running outside the swarm. Only the package lists change."""
|
|
manager = host_manager()
|
|
if manager is None:
|
|
return {"ok": False, "detail": "no package manager this helper knows "
|
|
"(apt, dnf, yum, zypper, pacman) on this host"}
|
|
env = {"DEBIAN_FRONTEND": "noninteractive"} if manager == "apt" else None
|
|
out = ""
|
|
for label, cmd, ok_codes in pending_steps(manager):
|
|
ok, out = run(cmd, env=env, ok_codes=ok_codes, keep=LISTING_KEPT_CHARS)
|
|
if not ok:
|
|
return {"ok": False, "detail": f"{label} failed: " + out[-500:]}
|
|
names = pending_names(manager, out)
|
|
docker = sorted({n for n in names if DOCKER_PACKAGES.match(n)})
|
|
return {"ok": True, "pending": len(names), "docker_update": bool(docker),
|
|
"docker_packages": docker[:10], "containers": standalone_containers()}
|
|
|
|
|
|
def own_node() -> str | None:
|
|
ok, out = run(["docker", "info", "--format", "{{.Swarm.NodeID}}"], deadline=30)
|
|
node = out.strip() if ok else ""
|
|
return node if _NODE.match(node) else None
|
|
|
|
|
|
def do_node(availability: str, node: str | None) -> dict:
|
|
"""Set a swarm node's availability (drain before a reboot, active after).
|
|
Only a manager can; a worker's helper says so and Steward asks a manager."""
|
|
if not shutil.which("docker"):
|
|
return {"ok": False, "detail": "no docker on this host"}
|
|
node = node or own_node()
|
|
if node is None:
|
|
return {"ok": False, "detail": "this host is not in a swarm"}
|
|
ok, out = run(["docker", "node", "update", "--availability", availability, node], deadline=60)
|
|
return {"ok": ok, "node": node, "detail": out.strip()[-500:]}
|
|
|
|
|
|
def placed_by_rule(name: str) -> bool | None:
|
|
"""Whether a constraint or a preference decides where the service runs
|
|
(a force-update would leave it where it is); None when it can't be read."""
|
|
ok, out = run(["docker", "service", "inspect", "--format",
|
|
"{{json .Spec.TaskTemplate.Placement}}", name], deadline=30)
|
|
if not ok:
|
|
return None
|
|
try:
|
|
placement = json.loads(out.strip() or "null") or {}
|
|
except ValueError:
|
|
return None
|
|
return bool(placement.get("Constraints") or placement.get("Preferences"))
|
|
|
|
|
|
def do_rebalance(services: list, clock=time.monotonic) -> dict:
|
|
"""Force-update each named service in turn, waiting for it to settle, so
|
|
the swarm starts it again on its emptiest node. Fails when a move didn't
|
|
settle; what was left and why is in `left`."""
|
|
if not shutil.which("docker"):
|
|
return {"ok": False, "detail": "no docker on this host"}
|
|
began = clock()
|
|
moved, left, failed = [], [], []
|
|
for name in services:
|
|
if clock() - began > REBALANCE_SECONDS:
|
|
left.append({"service": name, "reason": "out of time"})
|
|
continue
|
|
rule = placed_by_rule(name)
|
|
if rule is None:
|
|
left.append({"service": name, "reason": "not found on this manager"})
|
|
continue
|
|
if rule:
|
|
left.append({"service": name, "reason": "placed by a constraint"})
|
|
continue
|
|
ok, out = run(["docker", "service", "update", "--force", "--detach=false", "--quiet", name],
|
|
deadline=MOVE_DEADLINE_SECONDS)
|
|
if ok:
|
|
moved.append(name)
|
|
else:
|
|
failed.append(name)
|
|
left.append({"service": name, "reason": f"didn't settle: {out.strip()[-120:]}"})
|
|
detail = f"moved {len(moved)} of {len(services)}"
|
|
if failed:
|
|
detail += "; didn't settle: " + ", ".join(failed)
|
|
return {"ok": not failed, "moved": moved, "left": left, "detail": detail}
|
|
|
|
|
|
# ── drive health (#5789) ─────────────────────────────────────────────────────
|
|
# smartctl ≥ 7.0 answers in JSON, the same for SATA, SAS and NVMe drives on any
|
|
# distribution; an older one is reported, never parsed as text. The helper
|
|
# finds the drives itself: the request names none.
|
|
|
|
SMARTCTL_MIN = (7, 0)
|
|
SMARTCTL_DEADLINE_SECONDS = 60
|
|
DRIVES_MAX = 32
|
|
# smartctl's exit status is a bit mask: 1 the command line, 2 the device
|
|
# couldn't be opened. Higher bits describe the disk, and its JSON still holds.
|
|
_SMARTCTL_UNREAD = 0b11
|
|
# ATA attributes counted as bad sectors.
|
|
_ATA_BAD = {5: "reallocated", 197: "pending", 198: "offline_uncorrectable"}
|
|
|
|
|
|
def smartctl_version():
|
|
"""(major, minor) of the installed smartctl, or None when there is none."""
|
|
if not shutil.which("smartctl"):
|
|
return None
|
|
ok, out = run(["smartctl", "--version"], deadline=30)
|
|
m = re.search(r"smartctl (\d+)\.(\d+)", out)
|
|
return (int(m.group(1)), int(m.group(2))) if m else (0, 0)
|
|
|
|
|
|
def smartctl_json(args: list):
|
|
"""(exit status, parsed JSON or None) of one smartctl call."""
|
|
try:
|
|
done = subprocess.run(["smartctl", "--json", *args], capture_output=True, text=True,
|
|
timeout=SMARTCTL_DEADLINE_SECONDS,
|
|
env=dict(os.environ, LANG="C", LC_ALL="C"))
|
|
except (subprocess.TimeoutExpired, OSError):
|
|
return None, None
|
|
try:
|
|
return done.returncode, json.loads(done.stdout)
|
|
except ValueError:
|
|
return done.returncode, None
|
|
|
|
|
|
def _int(v):
|
|
return v if isinstance(v, int) and not isinstance(v, bool) else None
|
|
|
|
|
|
def drive_name(name: str, kind) -> str:
|
|
"""A drive's name, unique on its host. Disks behind a RAID controller
|
|
share one path (/dev/bus/0) and differ only by the slot in their type
|
|
(megaraid,3; cciss,0), so the slot is part of the name (#5806)."""
|
|
base = os.path.basename(name)
|
|
if name.startswith("/dev/bus/"):
|
|
base = "bus" + base
|
|
if isinstance(kind, str) and "," in kind:
|
|
base += "-" + re.sub(r"[^A-Za-z0-9]", "", kind.split("+")[-1])
|
|
return base
|
|
|
|
|
|
def drive_summary(name: str, status, data, kind=None) -> dict:
|
|
"""The few figures Steward reads from one drive's `smartctl -a` JSON."""
|
|
out = {"device": drive_name(name, kind)}
|
|
if not isinstance(data, dict) or status is None or status & _SMARTCTL_UNREAD:
|
|
messages = ((data.get("smartctl") or {}).get("messages") or []) if isinstance(data, dict) else []
|
|
text = "; ".join(str(m.get("string", "")) for m in messages if isinstance(m, dict))
|
|
out["error"] = (text or "smartctl couldn't read it")[:200]
|
|
return out
|
|
dev = data.get("device") or {}
|
|
out["protocol"] = str(dev.get("protocol") or "")[:16]
|
|
out["model"] = str(data.get("model_name") or data.get("scsi_model_name") or "")[:80]
|
|
cap = (data.get("user_capacity") or {}).get("bytes")
|
|
if _int(cap):
|
|
out["capacity_bytes"] = cap
|
|
support = data.get("smart_support") or {}
|
|
if support.get("available") is False or support.get("enabled") is False:
|
|
out["available"] = False # a virtual disk, or SMART switched off
|
|
return out
|
|
passed = (data.get("smart_status") or {}).get("passed")
|
|
if isinstance(passed, bool):
|
|
out["passed"] = passed
|
|
temp = _int((data.get("temperature") or {}).get("current"))
|
|
if temp is not None:
|
|
out["temp_c"] = temp
|
|
hours = _int((data.get("power_on_time") or {}).get("hours"))
|
|
if hours is not None:
|
|
out["power_on_hours"] = hours
|
|
nvme = data.get("nvme_smart_health_information_log") or {}
|
|
for src, dst in (("critical_warning", "critical_warning"), ("percentage_used", "life_used_pct"),
|
|
("media_errors", "media_errors"), ("available_spare", "spare_pct"),
|
|
("available_spare_threshold", "spare_threshold_pct")):
|
|
if _int(nvme.get(src)) is not None:
|
|
out[dst] = nvme[src]
|
|
for attr in (data.get("ata_smart_attributes") or {}).get("table") or []:
|
|
key = _ATA_BAD.get(attr.get("id")) if isinstance(attr, dict) else None
|
|
raw = _int((attr.get("raw") or {}).get("value")) if key else None
|
|
if raw is not None:
|
|
out[key] = raw
|
|
used = _int((data.get("endurance_used") or {}).get("current_percent")) # ATA SSDs
|
|
used = used if used is not None else _int(data.get("scsi_percentage_used_endurance_indicator"))
|
|
if used is not None and "life_used_pct" not in out:
|
|
out["life_used_pct"] = used
|
|
if _int(data.get("scsi_grown_defect_list")) is not None:
|
|
out["grown_defects"] = data["scsi_grown_defect_list"]
|
|
return out
|
|
|
|
|
|
def do_drives() -> dict:
|
|
"""Each drive's health, as {"checked_at", "smartctl", "drives"} or
|
|
{"checked_at", "error"} when smartctl is missing or too old."""
|
|
base = {"checked_at": now_iso()}
|
|
version = smartctl_version()
|
|
if version is None:
|
|
return {**base, "error": "smartctl not installed"}
|
|
base["smartctl"] = "%d.%d" % version
|
|
if version < SMARTCTL_MIN:
|
|
return {**base, "error": "smartctl %d.%d is too old: drive health needs 7.0 or newer" % version}
|
|
_, scan = smartctl_json(["--scan-open"])
|
|
devices = (scan or {}).get("devices") if isinstance(scan, dict) else None
|
|
if not isinstance(devices, list):
|
|
return {**base, "error": "smartctl couldn't list the drives"}
|
|
drives = []
|
|
for d in devices[:DRIVES_MAX]:
|
|
name, kind = (d.get("name"), d.get("type")) if isinstance(d, dict) else (None, None)
|
|
if not isinstance(name, str) or not name.startswith("/dev/"):
|
|
continue
|
|
args = ["-a", name] if not isinstance(kind, str) else ["-a", "-d", kind, name]
|
|
status, data = smartctl_json(args)
|
|
drives.append(drive_summary(name, status, data, kind))
|
|
return {**base, "drives": drives}
|
|
|
|
|
|
def write_drives(report: dict, path: str | None = None) -> None:
|
|
"""DRIVES, written whole and readable by the agent."""
|
|
path = path or DRIVES
|
|
tmp = path + ".tmp"
|
|
with open(tmp, "w") as fh:
|
|
json.dump(report, fh)
|
|
os.chmod(tmp, 0o644)
|
|
os.replace(tmp, path)
|
|
|
|
|
|
def write_result(result: dict) -> None:
|
|
"""RESULTS/<id>.json, written whole (a temp file renamed into place)."""
|
|
os.makedirs(RESULTS, mode=0o755, exist_ok=True)
|
|
final = os.path.join(RESULTS, result["id"] + ".json")
|
|
tmp = final + ".tmp"
|
|
with open(tmp, "w") as fh:
|
|
json.dump(result, fh)
|
|
os.chmod(tmp, 0o644)
|
|
os.replace(tmp, final)
|
|
|
|
|
|
def already_handled(job_id: str) -> bool:
|
|
"""A request the agent sent again, for a job already run, is not run twice."""
|
|
return os.path.exists(os.path.join(RESULTS, job_id + ".json"))
|
|
|
|
|
|
def handle(req: dict) -> dict:
|
|
base = {"id": req["id"], "action": req["action"], "started_at": now_iso()}
|
|
if req["action"] == "drives": # a read, not a job: no result
|
|
write_drives(do_drives())
|
|
return {**base, "state": "done"}
|
|
write_result({**base, "state": "running"})
|
|
action = req["action"]
|
|
if action == "check":
|
|
outcome = do_check()
|
|
elif action == "update":
|
|
outcome = do_update()
|
|
elif action in ("drain", "undrain"):
|
|
outcome = do_node("drain" if action == "drain" else "active", req.get("node"))
|
|
elif action == "rebalance":
|
|
outcome = do_rebalance(req["services"])
|
|
else: # reboot
|
|
result = {**base, "state": "rebooting", "ok": True, "finished_at": now_iso()}
|
|
write_result(result)
|
|
os.sync()
|
|
run(["systemctl", "reboot"], deadline=60)
|
|
return result
|
|
result = {**base, **outcome, "state": "done" if outcome.get("ok") else "failed",
|
|
"finished_at": now_iso()}
|
|
write_result(result)
|
|
return result
|
|
|
|
|
|
def prune_results(now: float | None = None) -> None:
|
|
now = time.time() if now is None else now
|
|
try:
|
|
names = os.listdir(RESULTS)
|
|
except OSError:
|
|
return
|
|
for name in names:
|
|
path = os.path.join(RESULTS, name)
|
|
try:
|
|
if now - os.lstat(path).st_mtime > RESULTS_KEPT_SECONDS:
|
|
os.unlink(path)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def pending_requests() -> list:
|
|
"""Request files, oldest first."""
|
|
try:
|
|
names = [n for n in os.listdir(REQUESTS) if n.endswith(".json")]
|
|
except OSError:
|
|
return []
|
|
paths = [os.path.join(REQUESTS, n) for n in names]
|
|
|
|
def age(p):
|
|
try:
|
|
return os.lstat(p).st_mtime
|
|
except OSError:
|
|
return 0.0
|
|
return sorted(paths, key=age)
|
|
|
|
|
|
def main() -> int:
|
|
prune_results()
|
|
for path in pending_requests():
|
|
req = read_request(path)
|
|
try:
|
|
os.unlink(path) # whatever it held: never re-read in a loop
|
|
except OSError:
|
|
pass
|
|
if req is None:
|
|
sys.stderr.write(f"[steward-maint] ignored {os.path.basename(path)}: not a valid request\n")
|
|
continue
|
|
if already_handled(req["id"]):
|
|
continue
|
|
result = handle(req)
|
|
sys.stderr.write(f"[steward-maint] {req['action']} {req['id']}: {result['state']}\n")
|
|
if result["state"] == "rebooting":
|
|
break # the rest waits for the next boot
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|