feat(host_agent): a root helper the agent can start, for updates, reboots and drains only (#5766)

#5766 "A root helper the agent can start, for a fixed set of maintenance
actions and nothing else", milestone 514.

The agent runs unprivileged under NoNewPrivileges, so it can't update or
reboot its host. It now leaves a request in /var/lib/steward-agent/requests,
the one path its unit lets it write; steward-maint.path starts maint.py as
root, which reads it without following links, accepts only update, reboot,
drain or undrain, deletes it, and writes the outcome to
/var/lib/steward-maint/results. The agent (1.17.0) reports each new result
state once as a `maintenance` event, re-checks pending updates when an update
finishes, and says in its metadata whether the helper is installed.

The units and the helper are served from one copy at
/plugins/host_agent/files/<name> (a fixed list), and install.sh plus the
install, provision and update playbooks all fetch them, replacing three
inline copies of the unit. Update agent installs the helper on existing
hosts; the Agent panel says under Manage agent whether it is there.

Steward handing out jobs is #5767; the cadence is #5768.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/agent.py

@@ -22,7 +22,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.16.0"
+AGENT_VERSION = "1.17.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -291,6 +291,7 @@ def collect_metadata() -> dict:
         "distro": distro,
         "arch": u.machine,
         "host_ip": detect_primary_ip(),
+        "maint_helper": maint_helper_installed(),
     }
 
 
@@ -1441,6 +1442,82 @@ def collect_reboot(path: str = REBOOT_REQUIRED):
             "packages": packages}
 
 
+# ── maintenance helper (#5766) ───────────────────────────────────────────────
+# The agent can't update or reboot its host: it runs unprivileged under
+# NoNewPrivileges. It leaves a request in MAINT_REQUESTS, the one directory its
+# unit lets it write; steward-maint.path starts the root helper (maint.py),
+# which does only update / reboot / drain / undrain and writes each outcome to
+# MAINT_RESULTS. The agent reads those back and reports each new state once.
+
+MAINT_REQUESTS = "/var/lib/steward-agent/requests"
+MAINT_RESULTS = "/var/lib/steward-maint/results"
+MAINT_PATH_UNIT = "/etc/systemd/system/steward-maint.path"
+MAINT_ACTIONS = ("update", "reboot", "drain", "undrain")
+MAINT_RESULT_MAX_BYTES = 8192
+# The helper keeps results a week; only this recent a one is reported, so a
+# job a reboot interrupted still reaches Steward from the agent's next start
+# and old ones don't repeat on every start.
+MAINT_REPORT_WINDOW_SECONDS = 86400
+_MAINT_ID = re.compile(r"^[A-Za-z0-9-]{1,64}$")
+_MAINT_FIELDS = ("action", "state", "ok", "packages", "reboot_required", "node",
+                 "detail", "started_at", "finished_at")
+
+
+def maint_helper_installed(unit: str = MAINT_PATH_UNIT,
+                           requests_dir: str = MAINT_REQUESTS) -> bool:
+    """The helper's path unit is installed and the agent may leave it a request."""
+    return os.path.exists(unit) and os.access(requests_dir, os.W_OK)
+
+
+def request_maintenance(job_id: str, action: str, node: str | None = None,
+                        requests_dir: str = MAINT_REQUESTS) -> None:
+    """Leave the helper a request, whole: written to a dot-file the path unit
+    doesn't match, then renamed to `<id>.json`."""
+    if action not in MAINT_ACTIONS or not _MAINT_ID.match(job_id or ""):
+        raise ValueError(f"not a maintenance request: {action!r} {job_id!r}")
+    req = {"id": job_id, "action": action}
+    if node:
+        req["node"] = node
+    tmp = os.path.join(requests_dir, f".{job_id}.tmp")
+    with open(tmp, "w") as fh:
+        json.dump(req, fh)
+    os.replace(tmp, os.path.join(requests_dir, f"{job_id}.json"))
+
+
+def maintenance_results(seen: set, results_dir: str = MAINT_RESULTS,
+                        now: float | None = None) -> tuple:
+    """(events, keys): a `maintenance` event for each result whose (id, state)
+    isn't in `seen`. The caller adds `keys` to `seen` once Steward has them."""
+    now = time.time() if now is None else now
+    try:
+        names = sorted(n for n in os.listdir(results_dir) if n.endswith(".json"))
+    except OSError:
+        return [], []
+    events, keys = [], []
+    for name in names:
+        path = os.path.join(results_dir, name)
+        try:
+            st = os.stat(path)
+            if st.st_size > MAINT_RESULT_MAX_BYTES:
+                continue
+            with open(path) as fh:
+                res = json.load(fh)
+        except (OSError, ValueError):
+            continue                        # mid-write or gone: next interval
+        if not isinstance(res, dict) or not _MAINT_ID.match(str(res.get("id", ""))):
+            continue
+        key = (res["id"], res.get("state"))
+        if key in seen:
+            continue
+        if now - st.st_mtime > MAINT_REPORT_WINDOW_SECONDS:
+            continue                        # long finished: not news to anyone
+        event = {"kind": "maintenance", "id": res["id"]}
+        event.update({k: res[k] for k in _MAINT_FIELDS if k in res})
+        events.append(event)
+        keys.append(key)
+    return events, keys
+
+
 def collect_self(state: dict) -> dict:
     """The agent's own footprint: resident memory and CPU % of one core since
     the last sample (#5764). A 1.13.0 agent once grew to 2.1 GB and spun for a
@@ -1764,6 +1841,7 @@ def main_loop(conf_path: str) -> int:
     lost_since = None
     # Carries previous net/disk counters + monotonic ts for rate computation.
     rate_state: dict = {}
+    maint_seen: set = set()
 
     _log("INFO", f"steward-host-agent {AGENT_VERSION} starting "
                  f"(url={cfg['url']}, interval={cfg['interval_seconds']}s)")
@@ -1787,11 +1865,13 @@ def main_loop(conf_path: str) -> int:
                               docker_log_exclude=docker_log_exclude)
         buffered = buffer.drain()
         send, held = split_backlog(buffered)
+        maint_events, maint_keys = maintenance_results(maint_seen)
         payload = build_payload(
             samples=send + [sample],
             hostname=hostname,
             metadata=metadata,
-            events=events + ([lost_contact_event(lost_since)] if lost_since else []),
+            events=events + ([lost_contact_event(lost_since)] if lost_since else [])
+            + maint_events,
         )
         ok, status = post_payload(cfg["url"], cfg["token"], payload)
 
@@ -1799,6 +1879,11 @@ def main_loop(conf_path: str) -> int:
             if lost_since:
                 _log("INFO", f"reached Steward again (unreachable since {lost_since})")
             events, lost_since = [], None
+            maint_seen.update(maint_keys)
+            # An update just finished: what's left waiting is known now, not in an hour.
+            if any(e.get("action") == "update" and e.get("state") in ("done", "failed")
+                   for e in maint_events):
+                rate_state["updates_due"] = True
             for s in held:
                 buffer.push(s)
             if send: