feat(host_agent): a host says what updates it has waiting and whether it needs a reboot (#5765)

Step 1 of milestone 514 "Updates and reboots on a cadence, carried out by
each host's agent…": #5765 "The agent reports pending updates and whether
a reboot is required".

- Agent 1.16.0: an hourly background check (apt-check, apt list, or dnf -C
  with updateinfo and needs-restarting), each command bounded at 120 s, no
  root and no network. Debian's reboot-required marker and its packages
  are read every sample.
- Ingest records updates_pending, updates_security and reboot_required for
  rules, and keeps the newest on the registration (host_agent_005_updates).
- The Agent card leads with "Reboot required since …", then the updates
  waiting or "Up to date", with when the check ran. list_agents carries
  them too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/__init__.py

@@ -67,11 +67,15 @@ def get_alert_metrics() -> dict[str, list[str]]:
     mounts_stalled counts the mounts that didn't answer the agent in time
     (agent ≥ 1.14.0): "an NFS share stopped" is > 0. agent_rss_bytes and
     agent_cpu_pct are the agent's own memory and CPU (agent ≥ 1.15.0): a
-    healthy agent sits in tens of MB and a few %."""
+    healthy agent sits in tens of MB and a few %. updates_pending and
+    updates_security are what the package manager has waiting, and
+    reboot_required is 1 when the host needs a reboot (agent ≥ 1.16.0):
+    "security updates waiting" is updates_security > 0."""
     return {"host_agent": [
         "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
         "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m",
         "load_pct",
         "uptime_secs", "seconds_since_seen", "clock_off_secs", "mounts_stalled",
         "agent_rss_bytes", "agent_cpu_pct",
+        "updates_pending", "updates_security", "reboot_required",
     ]}

plugins/host_agent/agent.py

@@ -22,7 +22,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.15.0"
+AGENT_VERSION = "1.16.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -1303,6 +1303,144 @@ class RingBuffer:
         return len(self._dq)
 
 
+# ─── pending updates and reboots (#5765) ─────────────────────────────────────
+#
+# What the package manager has waiting, read once an hour in a background
+# thread so a slow apt never holds up a sample, and whether the host needs a
+# reboot, read every sample (a file test). Nothing here needs root or the
+# network: apt-check and `apt list` read the lists apt-daily already fetched,
+# and dnf runs from its cache (-C).
+
+UPDATES_INTERVAL_SECONDS = 3600.0
+UPDATES_DEADLINE_SECONDS = 120.0
+REBOOT_REQUIRED = "/var/run/reboot-required"
+REBOOT_PACKAGES_MAX = 20
+
+
+def _run(cmd: list, deadline: float = UPDATES_DEADLINE_SECONDS):
+    """(returncode, stdout, stderr) of `cmd`, or None when it isn't installed.
+    Raises subprocess.TimeoutExpired past the deadline (the child is killed)."""
+    if shutil.which(cmd[0]) is None and not os.path.exists(cmd[0]):
+        return None
+    env = dict(os.environ, LANG="C", LC_ALL="C")
+    done = subprocess.run(cmd, capture_output=True, text=True, timeout=deadline, env=env)
+    return done.returncode, done.stdout, done.stderr
+
+
+def parse_apt_check(stderr: str):
+    """apt-check writes "<pending>;<security>" to stderr; (pending, security) or None."""
+    m = re.search(r"(\d+);(\d+)", stderr or "")
+    return (int(m.group(1)), int(m.group(2))) if m else None
+
+
+def parse_apt_list(stdout: str) -> tuple:
+    """(pending, security) from `apt list --upgradable`: one line per package,
+    "name/suite[,suite…] version arch [upgradable from: …]"; a security update
+    comes from a "-security" suite."""
+    pending = security = 0
+    for line in (stdout or "").splitlines():
+        if "upgradable from" not in line:
+            continue
+        pending += 1
+        suites = line.split("/", 1)[1].split(" ", 1)[0] if "/" in line else ""
+        if "-security" in suites:
+            security += 1
+    return pending, security
+
+
+def parse_dnf_check_update(stdout: str) -> int:
+    """Packages `dnf check-update` lists, up to its "Obsoleting Packages" part."""
+    count = 0
+    for line in (stdout or "").splitlines():
+        if line.startswith("Obsoleting"):
+            break
+        if line.strip() and not line.startswith((" ", "Last metadata")):
+            count += 1
+    return count
+
+
+def collect_updates() -> dict:
+    """{"manager", "pending", "security", "checked_at"} (security None when the
+    manager can't say), {"error"} when the check failed, or {} on a host with
+    no package manager we know. dnf hosts also say whether a reboot is
+    required, from needs-restarting."""
+    at = _now_iso()
+    try:
+        got = _run(["/usr/lib/update-notifier/apt-check"])
+        if got is not None:
+            counts = parse_apt_check(got[2])
+            if counts:
+                return {"manager": "apt", "pending": counts[0], "security": counts[1],
+                        "checked_at": at}
+        got = _run(["apt", "list", "--upgradable"])
+        if got is not None and got[0] == 0:
+            pending, security = parse_apt_list(got[1])
+            return {"manager": "apt", "pending": pending, "security": security, "checked_at": at}
+        got = _run(["dnf", "-q", "-C", "check-update"])
+        if got is not None:
+            if got[0] not in (0, 100):        # 100: updates are waiting
+                return {"manager": "dnf", "error": (got[2] or "dnf check-update failed").strip()[:200],
+                        "checked_at": at}
+            out = {"manager": "dnf", "pending": parse_dnf_check_update(got[1]),
+                   "security": None, "checked_at": at}
+            sec = _run(["dnf", "-q", "-C", "updateinfo", "list", "--security"])
+            if sec is not None and sec[0] == 0:
+                out["security"] = len({ln.split()[-1] for ln in sec[1].splitlines() if ln.strip()})
+            restart = _run(["needs-restarting", "-r"])
+            if restart is not None and restart[0] in (0, 1):
+                out["reboot_required"] = restart[0] == 1
+            return out
+    except subprocess.TimeoutExpired:
+        return {"error": "the package manager didn't answer in %.0f s" % UPDATES_DEADLINE_SECONDS,
+                "checked_at": at}
+    except OSError as exc:
+        return {"error": (str(exc) or type(exc).__name__)[:200], "checked_at": at}
+    return {}
+
+
+def refresh_updates(state: dict, now_mono: float | None = None) -> None:
+    """Start an update check in the background when one is due (hourly, and at
+    start). Its answer lands in state["updates"] for the samples that follow;
+    one check runs at a time. state["updates_due"] = True asks for one now."""
+    now_mono = time.monotonic() if now_mono is None else now_mono
+    if state.get("updates_running"):
+        return
+    last = state.get("updates_mono")
+    if not state.pop("updates_due", False) and last is not None \
+            and now_mono - last < UPDATES_INTERVAL_SECONDS:
+        return
+    state["updates_running"], state["updates_mono"] = True, now_mono
+
+    def work():
+        try:
+            state["updates"] = collect_updates()
+        except Exception as exc:              # never take the thread down silently
+            state["updates"] = {"error": (str(exc) or type(exc).__name__)[:200],
+                                "checked_at": _now_iso()}
+        finally:
+            state["updates_running"] = False
+
+    threading.Thread(target=work, name="updates", daemon=True).start()
+
+
+def collect_reboot(path: str = REBOOT_REQUIRED):
+    """{"required", "since", "packages"} from Debian's reboot-required marker,
+    or None where there is no such marker system (no apt) to ask."""
+    try:
+        st = os.stat(path)
+    except FileNotFoundError:
+        return {"required": False} if shutil.which("apt-get") else None
+    packages: list = []
+    try:
+        with open(path + ".pkgs") as fh:
+            packages = sorted({ln.strip() for ln in fh if ln.strip()})[:REBOOT_PACKAGES_MAX]
+    except OSError:
+        pass
+    return {"required": True,
+            "since": datetime.fromtimestamp(st.st_mtime, timezone.utc).isoformat(),
+            "packages": packages}
+
+
 def collect_self(state: dict) -> dict:
     """The agent's own footprint: resident memory and CPU % of one core since
     the last sample (#5764). A 1.13.0 agent once grew to 2.1 GB and spun for a
@@ -1436,6 +1574,17 @@ def build_sample(mounts: list[str], state: dict,
         if logs:
             sample["docker_logs"] = logs
 
+    # Pending updates: the last hourly answer rides every sample (#5765).
+    try:
+        refresh_updates(state)
+        reboot = collect_reboot()
+    except Exception:
+        reboot = None
+    if state.get("updates"):
+        sample["updates"] = state["updates"]
+    if reboot is not None:
+        sample["reboot"] = reboot
+
     # Last, so its CPU covers the whole of this sample's work.
     try:
         sample["agent"] = collect_self(state)

plugins/host_agent/mcp_tools.py

@@ -33,6 +33,19 @@ async def _agents(db):
     )).all()
 
 
+def _updates(reg) -> dict | None:
+    """A registration's pending updates and reboot, as list_agents says them (#5765)."""
+    if reg.updates_checked_at is None and reg.reboot_required is None:
+        return None
+    return {
+        "manager": reg.updates_manager, "pending": reg.updates_pending,
+        "security": reg.updates_security, "checked_at": iso(reg.updates_checked_at),
+        "error": reg.updates_error, "reboot_required": reg.reboot_required,
+        "reboot_required_since": iso(reg.reboot_required_since),
+        "reboot_packages": reg.reboot_packages.split(", ") if reg.reboot_packages else [],
+    }
+
+
 async def list_agents() -> dict:
     """Every host agent Steward has registered, with its version and whether it is reporting.
 
@@ -41,6 +54,11 @@ async def list_agents() -> dict:
     (registered, never checked in, usually an install that is still running
     or that failed). Each row carries the agent version, kernel, distro, arch
     and the IP the agent reports. get_host shows an agent's latest metrics.
+
+    `updates` is what the host's package manager has waiting, from the
+    agent's hourly check (agent ≥ 1.16.0): manager, pending, security (null
+    when the manager can't say), checked_at, error, and reboot_required with
+    since when and the packages that asked for it. null before the first check.
     """
     now = now_utc()
     stale_after = host_svc.agent_stale_after_seconds(mcp_app().config)
@@ -54,6 +72,7 @@ async def list_agents() -> dict:
             "agent_version": reg.agent_version, "kernel": reg.kernel, "distro": reg.distro,
             "arch": reg.arch, "reported_ip": reg.host_ip,
             "registered_at": iso(reg.created_at),
+            "updates": _updates(reg),
         }
         for reg, h in rows
     ]

plugins/host_agent/migrations/versions/host_agent_005_updates.py

@@ -0,0 +1,34 @@
+# plugins/host_agent/migrations/versions/host_agent_005_updates.py
+"""host_agent: a host's pending updates and whether it needs a reboot (#5765)
+
+Revision ID: host_agent_005_updates
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_005_updates"
+down_revision: Union[str, None] = "host_agent_004_downtime"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+_COLUMNS = (
+    ("updates_manager", sa.String(8)),
+    ("updates_pending", sa.Integer()),
+    ("updates_security", sa.Integer()),
+    ("updates_checked_at", sa.DateTime(timezone=True)),
+    ("updates_error", sa.String(200)),
+    ("reboot_required", sa.Boolean()),
+    ("reboot_required_since", sa.DateTime(timezone=True)),
+    ("reboot_packages", sa.Text()),
+)
+
+
+def upgrade() -> None:
+    for name, kind in _COLUMNS:
+        op.add_column("host_agent_registrations", sa.Column(name, kind, nullable=True))
+
+
+def downgrade() -> None:
+    for name, _kind in reversed(_COLUMNS):
+        op.drop_column("host_agent_registrations", name)

plugins/host_agent/models.py

@@ -2,7 +2,7 @@
 from __future__ import annotations
 import uuid
 from datetime import datetime, timezone
-from sqlalchemy import Column, String, DateTime, Float, ForeignKey
+from sqlalchemy import Boolean, Column, String, DateTime, Float, ForeignKey, Integer, Text
 from steward.models.base import Base
 
 
@@ -34,6 +34,19 @@ class HostAgentRegistration(Base):
     # and when that was. None until an agent that stamps its posts reports.
     clock_offset_s = Column(Float, nullable=True)
     clock_checked_at = Column(DateTime(timezone=True), nullable=True)
+    # What the package manager has waiting, from the agent's hourly check
+    # (agent ≥ 1.16.0, #5765). updates_security is None when the manager
+    # can't say; updates_error is why the last check failed.
+    updates_manager = Column(String(8), nullable=True)
+    updates_pending = Column(Integer, nullable=True)
+    updates_security = Column(Integer, nullable=True)
+    updates_checked_at = Column(DateTime(timezone=True), nullable=True)
+    updates_error = Column(String(200), nullable=True)
+    # Whether the host needs a reboot (None: it can't say), since when, and
+    # the packages that asked for it, comma-separated.
+    reboot_required = Column(Boolean, nullable=True)
+    reboot_required_since = Column(DateTime(timezone=True), nullable=True)
+    reboot_packages = Column(Text, nullable=True)
     created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
     updated_at = Column(DateTime(timezone=True), nullable=False,
                         default=_utcnow, onupdate=_utcnow)

plugins/host_agent/routes.py

@@ -111,6 +111,62 @@ def counts_as_hottest(label: str, celsius: float) -> bool:
     return not (_BOARD_INPUT.match(label) and celsius >= BOARD_INPUT_CEILING_C)
 
 
+def _ts_or(value, fallback: datetime) -> datetime:
+    """An agent-sent ISO time, or `fallback` when it isn't one."""
+    try:
+        got = _parse_ts(value)
+    except (AttributeError, TypeError, ValueError):
+        return fallback
+    return got if got.tzinfo else got.replace(tzinfo=timezone.utc)
+
+
+def reboot_required(sample: dict):
+    """Whether the sample says its host needs a reboot: from the reboot-required
+    marker (apt hosts) or the hourly update check (dnf's needs-restarting);
+    None when it can't say."""
+    reboot = sample.get("reboot") if isinstance(sample.get("reboot"), dict) else {}
+    if isinstance(reboot.get("required"), bool):
+        return reboot["required"]
+    upd = sample.get("updates") if isinstance(sample.get("updates"), dict) else {}
+    return upd["reboot_required"] if isinstance(upd.get("reboot_required"), bool) else None
+
+
+def apply_updates(reg, sample: dict, now: datetime) -> bool:
+    """Keep the registration's view of the host's updates and reboot current
+    from its newest sample (#5765); True when anything changed."""
+    before = (reg.updates_manager, reg.updates_pending, reg.updates_security,
+              reg.updates_checked_at, reg.updates_error, reg.reboot_required,
+              reg.reboot_required_since, reg.reboot_packages)
+    upd = sample.get("updates") if isinstance(sample.get("updates"), dict) else None
+    if upd:
+        reg.updates_checked_at = _ts_or(upd.get("checked_at"), now)
+        if upd.get("manager") in ("apt", "dnf"):
+            reg.updates_manager = upd["manager"]
+        if upd.get("error"):
+            reg.updates_error = str(upd["error"])[:200]
+        elif isinstance(upd.get("pending"), int):
+            reg.updates_pending = upd["pending"]
+            reg.updates_security = upd["security"] if isinstance(upd.get("security"), int) else None
+            reg.updates_error = None
+    required = reboot_required(sample)
+    if required is not None:
+        reg.reboot_required = required
+        if not required:
+            reg.reboot_required_since, reg.reboot_packages = None, None
+        else:
+            reboot = sample.get("reboot") if isinstance(sample.get("reboot"), dict) else {}
+            if reboot.get("since") is not None:
+                reg.reboot_required_since = _ts_or(reboot["since"], now)
+            elif reg.reboot_required_since is None:
+                reg.reboot_required_since = now
+            packages = reboot.get("packages")
+            if isinstance(packages, list):
+                reg.reboot_packages = ", ".join(str(p) for p in packages[:20])[:2000] or None
+    return before != (reg.updates_manager, reg.updates_pending, reg.updates_security,
+                      reg.updates_checked_at, reg.updates_error, reg.reboot_required,
+                      reg.reboot_required_since, reg.reboot_packages)
+
+
 def _expand_sample_to_metrics(
     sample: dict, host_name: str, recorded_at: datetime
 ) -> list[PluginMetric]:
@@ -149,6 +205,17 @@ def _expand_sample_to_metrics(
     if sample.get("uptime_secs") is not None:
         row("uptime_secs", host_name, sample["uptime_secs"])
 
+    # What the host has waiting (#5765): the agent's hourly answer rides
+    # every sample, so a rule on it reads like any other host metric.
+    upd = sample.get("updates") if isinstance(sample.get("updates"), dict) else {}
+    if isinstance(upd.get("pending"), int):
+        row("updates_pending", host_name, upd["pending"])
+        if isinstance(upd.get("security"), int):
+            row("updates_security", host_name, upd["security"])
+    required = reboot_required(sample)
+    if required is not None:
+        row("reboot_required", host_name, 1 if required else 0)
+
     # The agent's own footprint (#5764), so a runaway agent shows before it
     # goes silent.
     me = sample.get("agent") or {}
@@ -312,6 +379,7 @@ async def ingest():
             latest_disk_ts: datetime | None = None
             # Whether Docker answered, from the newest sample that says (agent ≥ 1.9.0).
             docker_status: dict | None = None
+            newest_sample: dict | None = None
             for sample in samples:
                 try:
                     recorded_at = _parse_ts(sample["ts"])
@@ -346,7 +414,7 @@ async def ingest():
                     latest_disk, latest_disk_ts = disk, recorded_at
                 accepted += 1
                 if latest_ts is None or recorded_at > latest_ts:
-                    latest_ts = recorded_at
+                    latest_ts, newest_sample = recorded_at, sample
 
             if accepted == 0 and not events:
                 await session.rollback()
@@ -411,6 +479,8 @@ async def ingest():
             if host_ip and reg.host_ip != host_ip:
                 reg.host_ip = host_ip
                 changed = True
+            if newest_sample is not None and apply_updates(reg, newest_sample, received_at):
+                changed = True
             # Mirror the reported IP into Host.address only when it's blank.
             new_address = pick_host_address(host.address, host_ip)
             if new_address is not None:

plugins/host_agent/templates/panel.html

@@ -19,6 +19,27 @@
      "Manage agent" (#5278). #}
   <a href="/plugins/host_agent/{{ host.id }}/" class="btn btn-sm btn-ghost">Full metrics →</a>
 
+  {# What the host has waiting, from the agent's hourly check (#5765): a
+     needed reboot first, then the updates. #}
+  {% if reg.updates_checked_at or reg.reboot_required %}
+  <div style="margin-top:0.8rem;font-size:0.82rem;display:flex;flex-direction:column;gap:0.2rem;">
+    {% if reg.reboot_required %}
+    <div class="tone-warn" title="{{ reg.reboot_packages or '' }}">
+      Reboot required{% if reg.reboot_required_since %} since {{ reg.reboot_required_since|ago }}{% endif %}{% if reg.reboot_packages %}
+      <span style="color:var(--text-muted);">· {{ reg.reboot_packages|truncate(80) }}</span>{% endif %}
+    </div>
+    {% endif %}
+    {% if reg.updates_checked_at %}
+    <div>
+      {% if reg.updates_error %}<span class="tone-warn">Update check failed: {{ reg.updates_error }}</span>
+      {% elif reg.updates_pending %}<span class="{{ 'tone-warn' if reg.updates_security else '' }}">{{ reg.updates_pending }} update{{ 's' if reg.updates_pending != 1 }} waiting{% if reg.updates_security %} ({{ reg.updates_security }} security){% endif %}</span>
+      {% else %}Up to date{% endif %}
+      <span style="color:var(--text-muted);">· checked {{ reg.updates_checked_at|ago }}</span>
+    </div>
+    {% endif %}
+  </div>
+  {% endif %}
+
   {# What the agent said happened while it was away, last 7 days (#5756). #}
   {% if downtime %}
   <div style="margin-top:0.8rem;">