feat(host_agent): a root helper the agent can start, for updates, reboots and drains only (#5766)
CI / lint (push) Successful in 2s
CI / integration (push) Successful in 1m22s
CI / unit (push) Successful in 1m24s
CI / publish (push) Successful in 1m12s

#5766 "A root helper the agent can start, for a fixed set of maintenance
actions and nothing else", milestone 514.

The agent runs unprivileged under NoNewPrivileges, so it can't update or
reboot its host. It now leaves a request in /var/lib/steward-agent/requests,
the one path its unit lets it write; steward-maint.path starts maint.py as
root, which reads it without following links, accepts only update, reboot,
drain or undrain, deletes it, and writes the outcome to
/var/lib/steward-maint/results. The agent (1.17.0) reports each new result
state once as a `maintenance` event, re-checks pending updates when an update
finishes, and says in its metadata whether the helper is installed.

The units and the helper are served from one copy at
/plugins/host_agent/files/<name> (a fixed list), and install.sh plus the
install, provision and update playbooks all fetch them, replacing three
inline copies of the unit. Update agent installs the helper on existing
hosts; the Agent panel says under Manage agent whether it is there.

Steward handing out jobs is #5767; the cadence is #5768.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-14 18:34:28 -04:00
co-authored by Claude Opus 5.5
parent 210a35c956
commit 6a68bd75d9
17 changed files with 851 additions and 175 deletions
+87 -2
View File
@@ -22,7 +22,7 @@ from collections import deque
from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
from datetime import datetime, timezone
AGENT_VERSION = "1.16.0"
AGENT_VERSION = "1.17.0"
# Default path to the local Docker Engine socket. Overridable via the
# `docker_socket` config key; collection is silently skipped if it's absent or
@@ -291,6 +291,7 @@ def collect_metadata() -> dict:
"distro": distro,
"arch": u.machine,
"host_ip": detect_primary_ip(),
"maint_helper": maint_helper_installed(),
}
@@ -1441,6 +1442,82 @@ def collect_reboot(path: str = REBOOT_REQUIRED):
"packages": packages}
# ── maintenance helper (#5766) ───────────────────────────────────────────────
# The agent can't update or reboot its host: it runs unprivileged under
# NoNewPrivileges. It leaves a request in MAINT_REQUESTS, the one directory its
# unit lets it write; steward-maint.path starts the root helper (maint.py),
# which does only update / reboot / drain / undrain and writes each outcome to
# MAINT_RESULTS. The agent reads those back and reports each new state once.
MAINT_REQUESTS = "/var/lib/steward-agent/requests"
MAINT_RESULTS = "/var/lib/steward-maint/results"
MAINT_PATH_UNIT = "/etc/systemd/system/steward-maint.path"
MAINT_ACTIONS = ("update", "reboot", "drain", "undrain")
MAINT_RESULT_MAX_BYTES = 8192
# The helper keeps results a week; only this recent a one is reported, so a
# job a reboot interrupted still reaches Steward from the agent's next start
# and old ones don't repeat on every start.
MAINT_REPORT_WINDOW_SECONDS = 86400
_MAINT_ID = re.compile(r"^[A-Za-z0-9-]{1,64}$")
_MAINT_FIELDS = ("action", "state", "ok", "packages", "reboot_required", "node",
"detail", "started_at", "finished_at")
def maint_helper_installed(unit: str = MAINT_PATH_UNIT,
requests_dir: str = MAINT_REQUESTS) -> bool:
"""The helper's path unit is installed and the agent may leave it a request."""
return os.path.exists(unit) and os.access(requests_dir, os.W_OK)
def request_maintenance(job_id: str, action: str, node: str | None = None,
requests_dir: str = MAINT_REQUESTS) -> None:
"""Leave the helper a request, whole: written to a dot-file the path unit
doesn't match, then renamed to `<id>.json`."""
if action not in MAINT_ACTIONS or not _MAINT_ID.match(job_id or ""):
raise ValueError(f"not a maintenance request: {action!r} {job_id!r}")
req = {"id": job_id, "action": action}
if node:
req["node"] = node
tmp = os.path.join(requests_dir, f".{job_id}.tmp")
with open(tmp, "w") as fh:
json.dump(req, fh)
os.replace(tmp, os.path.join(requests_dir, f"{job_id}.json"))
def maintenance_results(seen: set, results_dir: str = MAINT_RESULTS,
now: float | None = None) -> tuple:
"""(events, keys): a `maintenance` event for each result whose (id, state)
isn't in `seen`. The caller adds `keys` to `seen` once Steward has them."""
now = time.time() if now is None else now
try:
names = sorted(n for n in os.listdir(results_dir) if n.endswith(".json"))
except OSError:
return [], []
events, keys = [], []
for name in names:
path = os.path.join(results_dir, name)
try:
st = os.stat(path)
if st.st_size > MAINT_RESULT_MAX_BYTES:
continue
with open(path) as fh:
res = json.load(fh)
except (OSError, ValueError):
continue # mid-write or gone: next interval
if not isinstance(res, dict) or not _MAINT_ID.match(str(res.get("id", ""))):
continue
key = (res["id"], res.get("state"))
if key in seen:
continue
if now - st.st_mtime > MAINT_REPORT_WINDOW_SECONDS:
continue # long finished: not news to anyone
event = {"kind": "maintenance", "id": res["id"]}
event.update({k: res[k] for k in _MAINT_FIELDS if k in res})
events.append(event)
keys.append(key)
return events, keys
def collect_self(state: dict) -> dict:
"""The agent's own footprint: resident memory and CPU % of one core since
the last sample (#5764). A 1.13.0 agent once grew to 2.1 GB and spun for a
@@ -1764,6 +1841,7 @@ def main_loop(conf_path: str) -> int:
lost_since = None
# Carries previous net/disk counters + monotonic ts for rate computation.
rate_state: dict = {}
maint_seen: set = set()
_log("INFO", f"steward-host-agent {AGENT_VERSION} starting "
f"(url={cfg['url']}, interval={cfg['interval_seconds']}s)")
@@ -1787,11 +1865,13 @@ def main_loop(conf_path: str) -> int:
docker_log_exclude=docker_log_exclude)
buffered = buffer.drain()
send, held = split_backlog(buffered)
maint_events, maint_keys = maintenance_results(maint_seen)
payload = build_payload(
samples=send + [sample],
hostname=hostname,
metadata=metadata,
events=events + ([lost_contact_event(lost_since)] if lost_since else []),
events=events + ([lost_contact_event(lost_since)] if lost_since else [])
+ maint_events,
)
ok, status = post_payload(cfg["url"], cfg["token"], payload)
@@ -1799,6 +1879,11 @@ def main_loop(conf_path: str) -> int:
if lost_since:
_log("INFO", f"reached Steward again (unreachable since {lost_since})")
events, lost_since = [], None
maint_seen.update(maint_keys)
# An update just finished: what's left waiting is known now, not in an hour.
if any(e.get("action") == "update" and e.get("state") in ("done", "failed")
for e in maint_events):
rate_state["updates_due"] = True
for s in held:
buffer.push(s)
if send:
+243
View File
@@ -0,0 +1,243 @@
#!/usr/bin/env python3
"""Steward maintenance helper (#5766): runs as root, started by steward-maint.path
when the host agent leaves a request, and does only what ACTIONS lists.
The agent (an unprivileged user under NoNewPrivileges) can't update or reboot
its host itself. It writes `<id>.json` = {"id", "action", ["node"]} into
REQUESTS, the one directory it may write. This helper reads each request
without following links, refuses anything outside the list, deletes it, does
the action, and writes the outcome to RESULTS (root's own directory, which the
agent reads back and reports to Steward). A stolen agent token can therefore
make its host update, reboot or drain, and nothing else.
Stdlib only; served by Steward at /plugins/host_agent/files/maint.py.
"""
from __future__ import annotations
import json
import os
import re
import shutil
import subprocess
import sys
import time
from datetime import datetime, timezone
REQUESTS = "/var/lib/steward-agent/requests"
RESULTS = "/var/lib/steward-maint/results"
ACTIONS = ("update", "reboot", "drain", "undrain")
REQUEST_MAX_BYTES = 4096
RESULTS_KEPT_SECONDS = 7 * 86400
UPDATE_DEADLINE_SECONDS = 3000
COMMAND_DEADLINE_SECONDS = 600
OUTPUT_KEPT_CHARS = 2000
_ID = re.compile(r"^[A-Za-z0-9-]{1,64}$")
_NODE = re.compile(r"^[a-z0-9]{25}$")
def now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
def read_request(path: str):
"""The request at `path` as a dict, or None when it is not a small regular
file holding a valid request. Never follows a link the agent planted."""
try:
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
except OSError:
return None
try:
st = os.fstat(fd)
if not (st.st_mode & 0o170000 == 0o100000) or st.st_size > REQUEST_MAX_BYTES:
return None
raw = os.read(fd, REQUEST_MAX_BYTES + 1)
finally:
os.close(fd)
try:
req = json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, ValueError):
return None
return req if valid(req) else None
def valid(req) -> bool:
"""A request names a known action, has a plain id, and a node id only in
the swarm's own form."""
if not isinstance(req, dict) or req.get("action") not in ACTIONS:
return False
if not isinstance(req.get("id"), str) or not _ID.match(req["id"]):
return False
node = req.get("node")
return node is None or (isinstance(node, str) and bool(_NODE.match(node)))
def run(cmd: list, deadline: int = COMMAND_DEADLINE_SECONDS, env: dict | None = None):
"""(ok, output tail) of a command, bounded by `deadline`."""
try:
done = subprocess.run(cmd, capture_output=True, text=True, timeout=deadline,
env=dict(os.environ, LANG="C", LC_ALL="C", **(env or {})))
except subprocess.TimeoutExpired:
return False, f"{cmd[0]} didn't finish in {deadline} s"
except OSError as exc:
return False, str(exc)
out = (done.stdout or "") + (done.stderr or "")
return done.returncode == 0, out[-OUTPUT_KEPT_CHARS:]
def reboot_required() -> bool | None:
if os.path.exists("/var/run/reboot-required"):
return True
if shutil.which("apt-get"):
return False
if shutil.which("needs-restarting"):
try:
return subprocess.run(["needs-restarting", "-r"], capture_output=True,
timeout=60).returncode == 1
except (OSError, subprocess.TimeoutExpired):
return None
return None
def upgraded_count(output: str) -> int | None:
"""Packages an apt or dnf run changed, from its summary line."""
m = re.search(r"(\d+) upgraded, (\d+) newly installed", output) # apt
if m:
return int(m.group(1)) + int(m.group(2))
counts = re.findall(r"^\s*(?:Upgrade|Install)\s+(\d+) Packages?", output, re.MULTILINE) # dnf
return sum(int(c) for c in counts) if counts else None
def do_update() -> dict:
"""Upgrade every package, keeping existing config files and leaving
service restarts to a reboot: needrestart only lists, so the Docker daemon
isn't restarted under its containers mid-upgrade."""
if shutil.which("apt-get"):
env = {"DEBIAN_FRONTEND": "noninteractive", "NEEDRESTART_MODE": "l"}
ok, out = run(["apt-get", "update"], env=env)
if not ok:
return {"ok": False, "detail": "apt-get update failed: " + out}
ok, out = run(["apt-get", "-y", "-o", "Dpkg::Options::=--force-confdef",
"-o", "Dpkg::Options::=--force-confold", "dist-upgrade"],
deadline=UPDATE_DEADLINE_SECONDS, env=env)
if not ok:
return {"ok": False, "detail": "apt-get dist-upgrade failed: " + out}
changed = upgraded_count(out)
run(["apt-get", "-y", "autoremove"], env=env)
elif shutil.which("dnf"):
ok, out = run(["dnf", "-y", "upgrade", "--refresh"], deadline=UPDATE_DEADLINE_SECONDS)
if not ok:
return {"ok": False, "detail": "dnf upgrade failed: " + out}
changed = upgraded_count(out)
else:
return {"ok": False, "detail": "no apt-get or dnf on this host"}
return {"ok": True, "packages": changed, "reboot_required": reboot_required(),
"detail": out[-500:]}
def own_node() -> str | None:
ok, out = run(["docker", "info", "--format", "{{.Swarm.NodeID}}"], deadline=30)
node = out.strip() if ok else ""
return node if _NODE.match(node) else None
def do_node(availability: str, node: str | None) -> dict:
"""Set a swarm node's availability (drain before a reboot, active after).
Only a manager can; a worker's helper says so and Steward asks a manager."""
if not shutil.which("docker"):
return {"ok": False, "detail": "no docker on this host"}
node = node or own_node()
if node is None:
return {"ok": False, "detail": "this host is not in a swarm"}
ok, out = run(["docker", "node", "update", "--availability", availability, node], deadline=60)
return {"ok": ok, "node": node, "detail": out.strip()[-500:]}
def write_result(result: dict) -> None:
"""RESULTS/<id>.json, written whole (a temp file renamed into place)."""
os.makedirs(RESULTS, mode=0o755, exist_ok=True)
final = os.path.join(RESULTS, result["id"] + ".json")
tmp = final + ".tmp"
with open(tmp, "w") as fh:
json.dump(result, fh)
os.chmod(tmp, 0o644)
os.replace(tmp, final)
def already_handled(job_id: str) -> bool:
"""A request the agent sent again, for a job already run, is not run twice."""
return os.path.exists(os.path.join(RESULTS, job_id + ".json"))
def handle(req: dict) -> dict:
base = {"id": req["id"], "action": req["action"], "started_at": now_iso()}
write_result({**base, "state": "running"})
action = req["action"]
if action == "update":
outcome = do_update()
elif action in ("drain", "undrain"):
outcome = do_node("drain" if action == "drain" else "active", req.get("node"))
else: # reboot
result = {**base, "state": "rebooting", "ok": True, "finished_at": now_iso()}
write_result(result)
os.sync()
run(["systemctl", "reboot"], deadline=60)
return result
result = {**base, **outcome, "state": "done" if outcome.get("ok") else "failed",
"finished_at": now_iso()}
write_result(result)
return result
def prune_results(now: float | None = None) -> None:
now = time.time() if now is None else now
try:
names = os.listdir(RESULTS)
except OSError:
return
for name in names:
path = os.path.join(RESULTS, name)
try:
if now - os.lstat(path).st_mtime > RESULTS_KEPT_SECONDS:
os.unlink(path)
except OSError:
pass
def pending_requests() -> list:
"""Request files, oldest first."""
try:
names = [n for n in os.listdir(REQUESTS) if n.endswith(".json")]
except OSError:
return []
paths = [os.path.join(REQUESTS, n) for n in names]
def age(p):
try:
return os.lstat(p).st_mtime
except OSError:
return 0.0
return sorted(paths, key=age)
def main() -> int:
prune_results()
for path in pending_requests():
req = read_request(path)
try:
os.unlink(path) # whatever it held: never re-read in a loop
except OSError:
pass
if req is None:
sys.stderr.write(f"[steward-maint] ignored {os.path.basename(path)}: not a valid request\n")
continue
if already_handled(req["id"]):
continue
result = handle(req)
sys.stderr.write(f"[steward-maint] {req['action']} {req['id']}: {result['state']}\n")
if result["state"] == "rebooting":
break # the rest waits for the next boot
return 0
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,26 @@
# Served by Steward at /plugins/host_agent/files/steward-agent.service and
# installed by install.sh and the host_agent playbooks: one copy of the unit.
[Unit]
Description=Steward host agent
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=steward-agent
Environment=STEWARD_AGENT_CONFIG=/etc/steward-agent.conf
ExecStart=/usr/bin/env python3 /usr/local/lib/steward-agent/agent.py
Restart=on-failure
RestartSec=5
# Stopping waits for the agent's last post, bounded at 10 s.
TimeoutStopSec=30
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=true
PrivateTmp=true
ReadOnlyPaths=/proc /sys
# The one place the agent writes: requests to the maintenance helper (#5766).
ReadWritePaths=-/var/lib/steward-agent/requests
[Install]
WantedBy=multi-user.target
@@ -0,0 +1,10 @@
# Starts the maintenance helper when the host agent leaves a request (#5766).
[Unit]
Description=Steward maintenance requests from the host agent
[Path]
PathExistsGlob=/var/lib/steward-agent/requests/*.json
Unit=steward-maint.service
[Install]
WantedBy=paths.target
@@ -0,0 +1,10 @@
# Runs as root, and does only what maint.py's fixed list allows: update the
# host's packages, reboot it, or drain / reactivate its swarm node (#5766).
[Unit]
Description=Steward maintenance helper
[Service]
Type=oneshot
ExecStart=/usr/bin/env python3 /usr/local/lib/steward-agent/maint.py
# A full upgrade on a slow mirror; past this systemd stops it.
TimeoutStartSec=3600
@@ -0,0 +1,21 @@
# plugins/host_agent/migrations/versions/host_agent_006_maint_helper.py
"""host_agent: whether a host has the maintenance helper (#5766)
Revision ID: host_agent_006_maint_helper
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "host_agent_006_maint_helper"
down_revision: Union[str, None] = "host_agent_005_updates"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("host_agent_registrations", sa.Column("maint_helper", sa.Boolean(), nullable=True))
def downgrade() -> None:
op.drop_column("host_agent_registrations", "maint_helper")
+4
View File
@@ -47,6 +47,10 @@ class HostAgentRegistration(Base):
reboot_required = Column(Boolean, nullable=True)
reboot_required_since = Column(DateTime(timezone=True), nullable=True)
reboot_packages = Column(Text, nullable=True)
# Whether the host has the root maintenance helper (steward-maint.path and
# its request directory) that updates and reboots go through (#5766); None
# until an agent that says reports.
maint_helper = Column(Boolean, nullable=True)
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
updated_at = Column(DateTime(timezone=True), nullable=False,
default=_utcnow, onupdate=_utcnow)
+27
View File
@@ -475,6 +475,10 @@ async def ingest():
if field in md and getattr(reg, field) != md[field]:
setattr(reg, field, md[field])
changed = True
# Whether the maintenance helper is installed (agent ≥ 1.17.0, #5766).
if isinstance(md.get("maint_helper"), bool) and reg.maint_helper != md["maint_helper"]:
reg.maint_helper = md["maint_helper"]
changed = True
host_ip = md.get("host_ip")
if host_ip and reg.host_ip != host_ip:
reg.host_ip = host_ip
@@ -631,6 +635,29 @@ async def agent_source():
return Response(body, mimetype="text/x-python")
# What a host installs besides agent.py: the units and the maintenance helper
# (#5766), served from one copy so install.sh and every playbook agree.
AGENT_FILES_DIR = Path(__file__).parent / "files"
AGENT_FILES = {
"maint.py": "text/x-python",
"steward-agent.service": "text/plain",
"steward-maint.service": "text/plain",
"steward-maint.path": "text/plain",
}
@host_agent_bp.get("/files/<name>")
@public_route("fetched by the installer and the agent playbooks; holds no secrets")
async def agent_file(name: str):
if name not in AGENT_FILES:
return _error(404, "not_found")
try:
body = (AGENT_FILES_DIR / name).read_text(encoding="utf-8")
except OSError:
return _error(500, "agent_missing")
return Response(body, mimetype=AGENT_FILES[name])
_FLEET_ORDER = {"stale": 0, "pending": 1, "up": 2}
+24 -28
View File
@@ -11,7 +11,9 @@ AGENT_VERSION="{{ agent_version }}"
AGENT_USER="steward-agent"
AGENT_DIR="/usr/local/lib/steward-agent"
CONF_FILE="/etc/steward-agent.conf"
UNIT_FILE="/etc/systemd/system/steward-agent.service"
UNIT_DIR="/etc/systemd/system"
UNITS="steward-agent.service steward-maint.service steward-maint.path"
FILES_URL="${STEWARD_URL}/plugins/host_agent/files"
# ── preflight ────────────────────────────────────────────────────────────────
[ "$(id -u)" = "0" ] || { echo "must run as root (use sudo)"; exit 1; }
@@ -20,9 +22,10 @@ command -v python3 >/dev/null 2>&1 || { echo "python3 not found — install py
# Handle --uninstall
if [ "${1:-}" = "--uninstall" ]; then
systemctl disable --now steward-agent.service 2>/dev/null || true
rm -f "$UNIT_FILE" "$CONF_FILE"
rm -rf "$AGENT_DIR"
systemctl disable --now steward-maint.path steward-agent.service 2>/dev/null || true
for u in $UNITS; do rm -f "$UNIT_DIR/$u"; done
rm -f "$CONF_FILE"
rm -rf "$AGENT_DIR" /var/lib/steward-agent /var/lib/steward-maint
systemctl daemon-reload
# keep the system user — removing it risks orphaned files elsewhere
echo "uninstalled"
@@ -36,7 +39,7 @@ fi
# ── drop the agent file ──────────────────────────────────────────────────────
mkdir -p "$AGENT_DIR"
curl -sSL "${STEWARD_URL}/plugins/host_agent/agent.py" -o "$AGENT_DIR/agent.py"
curl -fsSL "${STEWARD_URL}/plugins/host_agent/agent.py" -o "$AGENT_DIR/agent.py"
chmod 0755 "$AGENT_DIR/agent.py"
chown root:root "$AGENT_DIR/agent.py"
@@ -53,31 +56,24 @@ EOF
chown "root:$AGENT_USER" "$CONF_FILE"
chmod 0640 "$CONF_FILE"
# ── write systemd unit ───────────────────────────────────────────────────────
cat > "$UNIT_FILE" <<EOF
[Unit]
Description=Steward host agent
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=$AGENT_USER
ExecStart=/usr/bin/python3 $AGENT_DIR/agent.py
Restart=on-failure
RestartSec=10
NoNewPrivileges=yes
ProtectSystem=strict
ProtectHome=yes
PrivateTmp=yes
ReadOnlyPaths=/proc /sys
[Install]
WantedBy=multi-user.target
EOF
# ── the maintenance helper and the units (#5766) ─────────────────────────────
# One copy, served by Steward, the same the playbooks install. The agent may
# write only its request directory; the helper (root) and its results don't
# belong to it.
curl -fsSL "${FILES_URL}/maint.py" -o "$AGENT_DIR/maint.py"
chmod 0755 "$AGENT_DIR/maint.py"
chown root:root "$AGENT_DIR/maint.py"
install -d -o root -g root -m 0755 /var/lib/steward-agent /var/lib/steward-maint /var/lib/steward-maint/results
install -d -o "$AGENT_USER" -g "$AGENT_USER" -m 0700 /var/lib/steward-agent/requests
for u in $UNITS; do
curl -fsSL "${FILES_URL}/$u" -o "$UNIT_DIR/$u"
chmod 0644 "$UNIT_DIR/$u"
done
systemctl daemon-reload
systemctl enable --now steward-agent.service
systemctl enable --now steward-maint.path
systemctl enable steward-agent.service
systemctl restart steward-agent.service
echo
echo "Steward host agent $AGENT_VERSION installed and running."
+5
View File
@@ -78,6 +78,11 @@
Link an Ansible target (below) to update the agent from here.
</p>
{% endif %}
{# The root helper that carries out scheduled updates and reboots (#5766). #}
<p style="font-size:0.8rem;margin:0.6rem 0 0;color:var(--text-dim);">
{% if reg.maint_helper %}Maintenance helper installed: Steward can update and reboot this host.
{% else %}No maintenance helper yet: Update agent installs it, so Steward can update and reboot this host.{% endif %}
</p>
{% if ansible_available and target and managed_key_set %}
<details style="margin-top:0.7rem;">