From 210a35c956c203f12ceed5a00b361a01e1d89181 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 14 Oct 2026 18:23:44 -0400 Subject: [PATCH] feat(host_agent): a host says what updates it has waiting and whether it needs a reboot (#5765) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 1 of milestone 514 "Updates and reboots on a cadence, carried out by each host's agent…": #5765 "The agent reports pending updates and whether a reboot is required". - Agent 1.16.0: an hourly background check (apt-check, apt list, or dnf -C with updateinfo and needs-restarting), each command bounded at 120 s, no root and no network. Debian's reboot-required marker and its packages are read every sample. - Ingest records updates_pending, updates_security and reboot_required for rules, and keeps the newest on the registration (host_agent_005_updates). - The Agent card leads with "Reboot required since …", then the updates waiting or "Up to date", with when the check ran. list_agents carries them too. Co-Authored-By: Claude Opus 5.5 --- docs/plugins/host-agent-design.md | 4 + plugins/host_agent/__init__.py | 6 +- plugins/host_agent/agent.py | 151 +++++++++++++++++- plugins/host_agent/mcp_tools.py | 19 +++ .../versions/host_agent_005_updates.py | 34 ++++ plugins/host_agent/models.py | 15 +- plugins/host_agent/routes.py | 72 ++++++++- plugins/host_agent/templates/panel.html | 21 +++ steward/core/formatting.py | 3 + tests/core/test_host_page.py | 25 +++ .../plugins/host_agent/test_agent_updates.py | 92 +++++++++++ .../plugins/host_agent/test_expand_metrics.py | 34 ++++ 12 files changed, 472 insertions(+), 4 deletions(-) create mode 100644 plugins/host_agent/migrations/versions/host_agent_005_updates.py create mode 100644 tests/plugins/host_agent/test_agent_updates.py diff --git a/docs/plugins/host-agent-design.md b/docs/plugins/host-agent-design.md index b4cbfa7..6e32fd5 100644 --- a/docs/plugins/host-agent-design.md +++ b/docs/plugins/host-agent-design.md @@ -124,6 +124,10 @@ Each mount is read in a daemon thread under one 5 s deadline. A hard-mounted NFS One Docker answer must arrive within 15 s and fit in 32 MiB, and one log answer within 5 s and 4 MiB. Past either limit the read is cut off, not waited on. A per-read timeout alone never ends a body that keeps trickling in. Container logs are read since each container's cursor, but only its newest 2,000 lines, all containers within 10 s. A container that writes more than that gets a `_steward` note saying older lines were skipped. A container whose answer is too big skips ahead to now. Before this, a 1.13.0 agent re-read a growing backlog every interval and went silent for a day at 2.1 GB. Each sample also carries the agent's own `agent` footprint (`rss_bytes`, `cpu_pct`), which Steward records as `agent_rss_bytes` and `agent_cpu_pct`. +### What the host has waiting (1.16.0, #5765) + +Once an hour, and when the agent starts, a background thread asks the package manager what it would upgrade, so a slow apt never holds up a sample. On apt hosts it uses `/usr/lib/update-notifier/apt-check` (pending;security), or `apt list --upgradable` when that's missing. On dnf hosts it uses `dnf -C check-update`, `dnf -C updateinfo list --security`, and `needs-restarting -r` for whether to reboot. None of this needs root or the network: the commands read lists the system's own timers already fetched. Each command has 120 s. The last answer rides every sample as `updates`. Every sample also reads Debian's `/var/run/reboot-required` (and `.pkgs`) as `reboot`. Steward records `updates_pending`, `updates_security` and `reboot_required` (1/0) for rules, and keeps the newest on the registration. The host page's Agent card leads with "Reboot required since …", then "N updates waiting (M security)" or "Up to date", with when the check ran. `state["updates_due"] = True` asks for a check on the next sample; a maintenance job uses it once it has updated the host. + ### What happened to the host (1.14.0, #5756) A post may carry `events`, said once each, with the next post that gets through: diff --git a/plugins/host_agent/__init__.py b/plugins/host_agent/__init__.py index e05fc90..56565e4 100644 --- a/plugins/host_agent/__init__.py +++ b/plugins/host_agent/__init__.py @@ -67,11 +67,15 @@ def get_alert_metrics() -> dict[str, list[str]]: mounts_stalled counts the mounts that didn't answer the agent in time (agent ≥ 1.14.0): "an NFS share stopped" is > 0. agent_rss_bytes and agent_cpu_pct are the agent's own memory and CPU (agent ≥ 1.15.0): a - healthy agent sits in tens of MB and a few %.""" + healthy agent sits in tens of MB and a few %. updates_pending and + updates_security are what the package manager has waiting, and + reboot_required is 1 when the host needs a reboot (agent ≥ 1.16.0): + "security updates waiting" is updates_security > 0.""" return {"host_agent": [ "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes", "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m", "load_pct", "uptime_secs", "seconds_since_seen", "clock_off_secs", "mounts_stalled", "agent_rss_bytes", "agent_cpu_pct", + "updates_pending", "updates_security", "reboot_required", ]} diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py index 141862e..59cdd5e 100644 --- a/plugins/host_agent/agent.py +++ b/plugins/host_agent/agent.py @@ -22,7 +22,7 @@ from collections import deque from concurrent.futures import ThreadPoolExecutor, wait as wait_futures from datetime import datetime, timezone -AGENT_VERSION = "1.15.0" +AGENT_VERSION = "1.16.0" # Default path to the local Docker Engine socket. Overridable via the # `docker_socket` config key; collection is silently skipped if it's absent or @@ -1303,6 +1303,144 @@ class RingBuffer: return len(self._dq) +# ─── pending updates and reboots (#5765) ───────────────────────────────────── +# +# What the package manager has waiting, read once an hour in a background +# thread so a slow apt never holds up a sample, and whether the host needs a +# reboot, read every sample (a file test). Nothing here needs root or the +# network: apt-check and `apt list` read the lists apt-daily already fetched, +# and dnf runs from its cache (-C). + +UPDATES_INTERVAL_SECONDS = 3600.0 +UPDATES_DEADLINE_SECONDS = 120.0 +REBOOT_REQUIRED = "/var/run/reboot-required" +REBOOT_PACKAGES_MAX = 20 + + +def _run(cmd: list, deadline: float = UPDATES_DEADLINE_SECONDS): + """(returncode, stdout, stderr) of `cmd`, or None when it isn't installed. + Raises subprocess.TimeoutExpired past the deadline (the child is killed).""" + if shutil.which(cmd[0]) is None and not os.path.exists(cmd[0]): + return None + env = dict(os.environ, LANG="C", LC_ALL="C") + done = subprocess.run(cmd, capture_output=True, text=True, timeout=deadline, env=env) + return done.returncode, done.stdout, done.stderr + + +def parse_apt_check(stderr: str): + """apt-check writes ";" to stderr; (pending, security) or None.""" + m = re.search(r"(\d+);(\d+)", stderr or "") + return (int(m.group(1)), int(m.group(2))) if m else None + + +def parse_apt_list(stdout: str) -> tuple: + """(pending, security) from `apt list --upgradable`: one line per package, + "name/suite[,suite…] version arch [upgradable from: …]"; a security update + comes from a "-security" suite.""" + pending = security = 0 + for line in (stdout or "").splitlines(): + if "upgradable from" not in line: + continue + pending += 1 + suites = line.split("/", 1)[1].split(" ", 1)[0] if "/" in line else "" + if "-security" in suites: + security += 1 + return pending, security + + +def parse_dnf_check_update(stdout: str) -> int: + """Packages `dnf check-update` lists, up to its "Obsoleting Packages" part.""" + count = 0 + for line in (stdout or "").splitlines(): + if line.startswith("Obsoleting"): + break + if line.strip() and not line.startswith((" ", "Last metadata")): + count += 1 + return count + + +def collect_updates() -> dict: + """{"manager", "pending", "security", "checked_at"} (security None when the + manager can't say), {"error"} when the check failed, or {} on a host with + no package manager we know. dnf hosts also say whether a reboot is + required, from needs-restarting.""" + at = _now_iso() + try: + got = _run(["/usr/lib/update-notifier/apt-check"]) + if got is not None: + counts = parse_apt_check(got[2]) + if counts: + return {"manager": "apt", "pending": counts[0], "security": counts[1], + "checked_at": at} + got = _run(["apt", "list", "--upgradable"]) + if got is not None and got[0] == 0: + pending, security = parse_apt_list(got[1]) + return {"manager": "apt", "pending": pending, "security": security, "checked_at": at} + got = _run(["dnf", "-q", "-C", "check-update"]) + if got is not None: + if got[0] not in (0, 100): # 100: updates are waiting + return {"manager": "dnf", "error": (got[2] or "dnf check-update failed").strip()[:200], + "checked_at": at} + out = {"manager": "dnf", "pending": parse_dnf_check_update(got[1]), + "security": None, "checked_at": at} + sec = _run(["dnf", "-q", "-C", "updateinfo", "list", "--security"]) + if sec is not None and sec[0] == 0: + out["security"] = len({ln.split()[-1] for ln in sec[1].splitlines() if ln.strip()}) + restart = _run(["needs-restarting", "-r"]) + if restart is not None and restart[0] in (0, 1): + out["reboot_required"] = restart[0] == 1 + return out + except subprocess.TimeoutExpired: + return {"error": "the package manager didn't answer in %.0f s" % UPDATES_DEADLINE_SECONDS, + "checked_at": at} + except OSError as exc: + return {"error": (str(exc) or type(exc).__name__)[:200], "checked_at": at} + return {} + + +def refresh_updates(state: dict, now_mono: float | None = None) -> None: + """Start an update check in the background when one is due (hourly, and at + start). Its answer lands in state["updates"] for the samples that follow; + one check runs at a time. state["updates_due"] = True asks for one now.""" + now_mono = time.monotonic() if now_mono is None else now_mono + if state.get("updates_running"): + return + last = state.get("updates_mono") + if not state.pop("updates_due", False) and last is not None \ + and now_mono - last < UPDATES_INTERVAL_SECONDS: + return + state["updates_running"], state["updates_mono"] = True, now_mono + + def work(): + try: + state["updates"] = collect_updates() + except Exception as exc: # never take the thread down silently + state["updates"] = {"error": (str(exc) or type(exc).__name__)[:200], + "checked_at": _now_iso()} + finally: + state["updates_running"] = False + + threading.Thread(target=work, name="updates", daemon=True).start() + + +def collect_reboot(path: str = REBOOT_REQUIRED): + """{"required", "since", "packages"} from Debian's reboot-required marker, + or None where there is no such marker system (no apt) to ask.""" + try: + st = os.stat(path) + except FileNotFoundError: + return {"required": False} if shutil.which("apt-get") else None + packages: list = [] + try: + with open(path + ".pkgs") as fh: + packages = sorted({ln.strip() for ln in fh if ln.strip()})[:REBOOT_PACKAGES_MAX] + except OSError: + pass + return {"required": True, + "since": datetime.fromtimestamp(st.st_mtime, timezone.utc).isoformat(), + "packages": packages} + + def collect_self(state: dict) -> dict: """The agent's own footprint: resident memory and CPU % of one core since the last sample (#5764). A 1.13.0 agent once grew to 2.1 GB and spun for a @@ -1436,6 +1574,17 @@ def build_sample(mounts: list[str], state: dict, if logs: sample["docker_logs"] = logs + # Pending updates: the last hourly answer rides every sample (#5765). + try: + refresh_updates(state) + reboot = collect_reboot() + except Exception: + reboot = None + if state.get("updates"): + sample["updates"] = state["updates"] + if reboot is not None: + sample["reboot"] = reboot + # Last, so its CPU covers the whole of this sample's work. try: sample["agent"] = collect_self(state) diff --git a/plugins/host_agent/mcp_tools.py b/plugins/host_agent/mcp_tools.py index 77d8c98..4346918 100644 --- a/plugins/host_agent/mcp_tools.py +++ b/plugins/host_agent/mcp_tools.py @@ -33,6 +33,19 @@ async def _agents(db): )).all() +def _updates(reg) -> dict | None: + """A registration's pending updates and reboot, as list_agents says them (#5765).""" + if reg.updates_checked_at is None and reg.reboot_required is None: + return None + return { + "manager": reg.updates_manager, "pending": reg.updates_pending, + "security": reg.updates_security, "checked_at": iso(reg.updates_checked_at), + "error": reg.updates_error, "reboot_required": reg.reboot_required, + "reboot_required_since": iso(reg.reboot_required_since), + "reboot_packages": reg.reboot_packages.split(", ") if reg.reboot_packages else [], + } + + async def list_agents() -> dict: """Every host agent Steward has registered, with its version and whether it is reporting. @@ -41,6 +54,11 @@ async def list_agents() -> dict: (registered, never checked in, usually an install that is still running or that failed). Each row carries the agent version, kernel, distro, arch and the IP the agent reports. get_host shows an agent's latest metrics. + + `updates` is what the host's package manager has waiting, from the + agent's hourly check (agent ≥ 1.16.0): manager, pending, security (null + when the manager can't say), checked_at, error, and reboot_required with + since when and the packages that asked for it. null before the first check. """ now = now_utc() stale_after = host_svc.agent_stale_after_seconds(mcp_app().config) @@ -54,6 +72,7 @@ async def list_agents() -> dict: "agent_version": reg.agent_version, "kernel": reg.kernel, "distro": reg.distro, "arch": reg.arch, "reported_ip": reg.host_ip, "registered_at": iso(reg.created_at), + "updates": _updates(reg), } for reg, h in rows ] diff --git a/plugins/host_agent/migrations/versions/host_agent_005_updates.py b/plugins/host_agent/migrations/versions/host_agent_005_updates.py new file mode 100644 index 0000000..94b7027 --- /dev/null +++ b/plugins/host_agent/migrations/versions/host_agent_005_updates.py @@ -0,0 +1,34 @@ +# plugins/host_agent/migrations/versions/host_agent_005_updates.py +"""host_agent: a host's pending updates and whether it needs a reboot (#5765) + +Revision ID: host_agent_005_updates +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "host_agent_005_updates" +down_revision: Union[str, None] = "host_agent_004_downtime" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_COLUMNS = ( + ("updates_manager", sa.String(8)), + ("updates_pending", sa.Integer()), + ("updates_security", sa.Integer()), + ("updates_checked_at", sa.DateTime(timezone=True)), + ("updates_error", sa.String(200)), + ("reboot_required", sa.Boolean()), + ("reboot_required_since", sa.DateTime(timezone=True)), + ("reboot_packages", sa.Text()), +) + + +def upgrade() -> None: + for name, kind in _COLUMNS: + op.add_column("host_agent_registrations", sa.Column(name, kind, nullable=True)) + + +def downgrade() -> None: + for name, _kind in reversed(_COLUMNS): + op.drop_column("host_agent_registrations", name) diff --git a/plugins/host_agent/models.py b/plugins/host_agent/models.py index 8612e8c..2099bc9 100644 --- a/plugins/host_agent/models.py +++ b/plugins/host_agent/models.py @@ -2,7 +2,7 @@ from __future__ import annotations import uuid from datetime import datetime, timezone -from sqlalchemy import Column, String, DateTime, Float, ForeignKey +from sqlalchemy import Boolean, Column, String, DateTime, Float, ForeignKey, Integer, Text from steward.models.base import Base @@ -34,6 +34,19 @@ class HostAgentRegistration(Base): # and when that was. None until an agent that stamps its posts reports. clock_offset_s = Column(Float, nullable=True) clock_checked_at = Column(DateTime(timezone=True), nullable=True) + # What the package manager has waiting, from the agent's hourly check + # (agent ≥ 1.16.0, #5765). updates_security is None when the manager + # can't say; updates_error is why the last check failed. + updates_manager = Column(String(8), nullable=True) + updates_pending = Column(Integer, nullable=True) + updates_security = Column(Integer, nullable=True) + updates_checked_at = Column(DateTime(timezone=True), nullable=True) + updates_error = Column(String(200), nullable=True) + # Whether the host needs a reboot (None: it can't say), since when, and + # the packages that asked for it, comma-separated. + reboot_required = Column(Boolean, nullable=True) + reboot_required_since = Column(DateTime(timezone=True), nullable=True) + reboot_packages = Column(Text, nullable=True) created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow) updated_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow, onupdate=_utcnow) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 4ddb7be..ff168e9 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -111,6 +111,62 @@ def counts_as_hottest(label: str, celsius: float) -> bool: return not (_BOARD_INPUT.match(label) and celsius >= BOARD_INPUT_CEILING_C) +def _ts_or(value, fallback: datetime) -> datetime: + """An agent-sent ISO time, or `fallback` when it isn't one.""" + try: + got = _parse_ts(value) + except (AttributeError, TypeError, ValueError): + return fallback + return got if got.tzinfo else got.replace(tzinfo=timezone.utc) + + +def reboot_required(sample: dict): + """Whether the sample says its host needs a reboot: from the reboot-required + marker (apt hosts) or the hourly update check (dnf's needs-restarting); + None when it can't say.""" + reboot = sample.get("reboot") if isinstance(sample.get("reboot"), dict) else {} + if isinstance(reboot.get("required"), bool): + return reboot["required"] + upd = sample.get("updates") if isinstance(sample.get("updates"), dict) else {} + return upd["reboot_required"] if isinstance(upd.get("reboot_required"), bool) else None + + +def apply_updates(reg, sample: dict, now: datetime) -> bool: + """Keep the registration's view of the host's updates and reboot current + from its newest sample (#5765); True when anything changed.""" + before = (reg.updates_manager, reg.updates_pending, reg.updates_security, + reg.updates_checked_at, reg.updates_error, reg.reboot_required, + reg.reboot_required_since, reg.reboot_packages) + upd = sample.get("updates") if isinstance(sample.get("updates"), dict) else None + if upd: + reg.updates_checked_at = _ts_or(upd.get("checked_at"), now) + if upd.get("manager") in ("apt", "dnf"): + reg.updates_manager = upd["manager"] + if upd.get("error"): + reg.updates_error = str(upd["error"])[:200] + elif isinstance(upd.get("pending"), int): + reg.updates_pending = upd["pending"] + reg.updates_security = upd["security"] if isinstance(upd.get("security"), int) else None + reg.updates_error = None + required = reboot_required(sample) + if required is not None: + reg.reboot_required = required + if not required: + reg.reboot_required_since, reg.reboot_packages = None, None + else: + reboot = sample.get("reboot") if isinstance(sample.get("reboot"), dict) else {} + if reboot.get("since") is not None: + reg.reboot_required_since = _ts_or(reboot["since"], now) + elif reg.reboot_required_since is None: + reg.reboot_required_since = now + packages = reboot.get("packages") + if isinstance(packages, list): + reg.reboot_packages = ", ".join(str(p) for p in packages[:20])[:2000] or None + return before != (reg.updates_manager, reg.updates_pending, reg.updates_security, + reg.updates_checked_at, reg.updates_error, reg.reboot_required, + reg.reboot_required_since, reg.reboot_packages) + + def _expand_sample_to_metrics( sample: dict, host_name: str, recorded_at: datetime ) -> list[PluginMetric]: @@ -149,6 +205,17 @@ def _expand_sample_to_metrics( if sample.get("uptime_secs") is not None: row("uptime_secs", host_name, sample["uptime_secs"]) + # What the host has waiting (#5765): the agent's hourly answer rides + # every sample, so a rule on it reads like any other host metric. + upd = sample.get("updates") if isinstance(sample.get("updates"), dict) else {} + if isinstance(upd.get("pending"), int): + row("updates_pending", host_name, upd["pending"]) + if isinstance(upd.get("security"), int): + row("updates_security", host_name, upd["security"]) + required = reboot_required(sample) + if required is not None: + row("reboot_required", host_name, 1 if required else 0) + # The agent's own footprint (#5764), so a runaway agent shows before it # goes silent. me = sample.get("agent") or {} @@ -312,6 +379,7 @@ async def ingest(): latest_disk_ts: datetime | None = None # Whether Docker answered, from the newest sample that says (agent ≥ 1.9.0). docker_status: dict | None = None + newest_sample: dict | None = None for sample in samples: try: recorded_at = _parse_ts(sample["ts"]) @@ -346,7 +414,7 @@ async def ingest(): latest_disk, latest_disk_ts = disk, recorded_at accepted += 1 if latest_ts is None or recorded_at > latest_ts: - latest_ts = recorded_at + latest_ts, newest_sample = recorded_at, sample if accepted == 0 and not events: await session.rollback() @@ -411,6 +479,8 @@ async def ingest(): if host_ip and reg.host_ip != host_ip: reg.host_ip = host_ip changed = True + if newest_sample is not None and apply_updates(reg, newest_sample, received_at): + changed = True # Mirror the reported IP into Host.address only when it's blank. new_address = pick_host_address(host.address, host_ip) if new_address is not None: diff --git a/plugins/host_agent/templates/panel.html b/plugins/host_agent/templates/panel.html index 07f0cf2..5b51d29 100644 --- a/plugins/host_agent/templates/panel.html +++ b/plugins/host_agent/templates/panel.html @@ -19,6 +19,27 @@ "Manage agent" (#5278). #} Full metrics → + {# What the host has waiting, from the agent's hourly check (#5765): a + needed reboot first, then the updates. #} + {% if reg.updates_checked_at or reg.reboot_required %} +
+ {% if reg.reboot_required %} +
+ Reboot required{% if reg.reboot_required_since %} since {{ reg.reboot_required_since|ago }}{% endif %}{% if reg.reboot_packages %} + · {{ reg.reboot_packages|truncate(80) }}{% endif %} +
+ {% endif %} + {% if reg.updates_checked_at %} +
+ {% if reg.updates_error %}Update check failed: {{ reg.updates_error }} + {% elif reg.updates_pending %}{{ reg.updates_pending }} update{{ 's' if reg.updates_pending != 1 }} waiting{% if reg.updates_security %} ({{ reg.updates_security }} security){% endif %} + {% else %}Up to date{% endif %} + · checked {{ reg.updates_checked_at|ago }} +
+ {% endif %} +
+ {% endif %} + {# What the agent said happened while it was away, last 7 days (#5756). #} {% if downtime %}
diff --git a/steward/core/formatting.py b/steward/core/formatting.py index 361e52f..9c1b4a9 100644 --- a/steward/core/formatting.py +++ b/steward/core/formatting.py @@ -342,6 +342,9 @@ METRICS: dict[str, tuple[str, str]] = { "mounts_stalled": ("Mounts not answering", "mounts the agent couldn't read in 5 s, e.g. an NFS share whose server stopped"), "agent_rss_bytes": ("Agent memory", "the host agent's own resident memory; tens of MB is normal"), "agent_cpu_pct": ("Agent CPU", "the host agent's own CPU, % of one core; a few % is normal"), + "updates_pending": ("Updates waiting", "packages the host's package manager would upgrade, checked hourly"), + "updates_security": ("Security updates waiting", "of the updates waiting, those from a security source"), + "reboot_required": ("Reboot required", "1 when the host needs a reboot to finish an update, 0 when not"), } # Where a rule's metric comes from, as the picker and the rules list say it. diff --git a/tests/core/test_host_page.py b/tests/core/test_host_page.py index 01cf463..83e25ba 100644 --- a/tests/core/test_host_page.py +++ b/tests/core/test_host_page.py @@ -235,3 +235,28 @@ async def test_the_agent_card_says_what_happened_while_the_host_was_away(app): assert page.index("Rebooted") < page.index("Lost contact") assert "going on, 5m" in page and "1h" in page assert "Downtime, last 7 days" not in quiet + + +async def test_the_agent_card_leads_with_a_needed_reboot_then_the_updates(app): + """#5765: what the host has waiting, from the agent's hourly check.""" + from datetime import datetime, timedelta, timezone + from tests._templates import render_plugin_template + now = datetime.now(timezone.utc) + host = SimpleNamespace(id="h1", name="box", address="10.0.0.5") + waiting = SimpleNamespace( + agent_version="1.16.0", last_seen_at=now, updates_manager="apt", updates_pending=12, + updates_security=3, updates_checked_at=now - timedelta(minutes=20), updates_error=None, + reboot_required=True, reboot_required_since=now - timedelta(hours=5), + reboot_packages="libc6, linux-image-6.8.0-146-generic") + current = SimpleNamespace(**{**vars(waiting), "updates_pending": 0, "updates_security": 0, + "reboot_required": False, "reboot_required_since": None, + "reboot_packages": None}) + kw = dict(host=host, reporting=True, stale=False, target=None, ansible_available=False, + managed_key_set=False, downtime=[]) + async with app.test_request_context("/plugins/host_agent/panel/h1"): + session["user_role"] = "viewer" + page = await render_plugin_template(app, "host_agent", "panel.html", reg=waiting, **kw) + quiet = await render_plugin_template(app, "host_agent", "panel.html", reg=current, **kw) + assert page.index("Reboot required") < page.index("12 updates waiting (3 security)") + assert "libc6" in page + assert "Up to date" in quiet and "Reboot required" not in quiet diff --git a/tests/plugins/host_agent/test_agent_updates.py b/tests/plugins/host_agent/test_agent_updates.py new file mode 100644 index 0000000..231da8d --- /dev/null +++ b/tests/plugins/host_agent/test_agent_updates.py @@ -0,0 +1,92 @@ +# tests/plugins/host_agent/test_agent_updates.py +"""#5765: the agent says what the package manager has waiting and whether the +host needs a reboot, without root, without the network, and without a slow +apt ever holding up a sample.""" +import os +import subprocess +import threading +import time + +from plugins.host_agent import agent as a + + +def test_apt_check_counts_are_read_from_its_stderr(): + assert a.parse_apt_check("12;3") == (12, 3) + assert a.parse_apt_check("") is None + + +def test_apt_list_counts_security_updates_by_their_suite(): + out = ("Listing...\n" + "libc6/noble-updates,noble-security 2.39-0ubuntu8.4 amd64 [upgradable from: 2.39-0ubuntu8.3]\n" + "curl/noble-updates 8.5.0-2ubuntu10.5 amd64 [upgradable from: 8.5.0-2ubuntu10.4]\n") + assert a.parse_apt_list(out) == (2, 1) + + +def test_dnf_check_update_counts_packages_and_stops_at_obsoletes(): + out = ("\nkernel.x86_64 5.14.0-503 baseos\nopenssl.x86_64 3.2.2-6 baseos\n" + "Obsoleting Packages\ngrub2.x86_64 2.06 baseos\n") + assert a.parse_dnf_check_update(out) == 2 + + +def test_apt_hosts_are_checked_with_apt_check(monkeypatch): + calls = [] + + def fake_run(cmd, deadline=a.UPDATES_DEADLINE_SECONDS): + calls.append(cmd[0]) + return (0, "", "7;2") if cmd[0].endswith("apt-check") else None + + monkeypatch.setattr(a, "_run", fake_run) + got = a.collect_updates() + assert (got["manager"], got["pending"], got["security"]) == ("apt", 7, 2) + assert calls == ["/usr/lib/update-notifier/apt-check"] + + +def test_a_package_manager_that_hangs_is_an_error_not_a_stall(monkeypatch): + def hang(cmd, deadline=a.UPDATES_DEADLINE_SECONDS): + raise subprocess.TimeoutExpired(cmd, deadline) + + monkeypatch.setattr(a, "_run", hang) + assert "didn't answer" in a.collect_updates()["error"] + + +def test_the_check_runs_in_the_background_once_an_hour(monkeypatch): + started, release = threading.Event(), threading.Event() + + def slow(): + started.set() + release.wait(5) + return {"manager": "apt", "pending": 1, "security": 0, "checked_at": "x"} + + monkeypatch.setattr(a, "collect_updates", slow) + state: dict = {} + a.refresh_updates(state, now_mono=1000.0) + assert started.wait(2) + a.refresh_updates(state, now_mono=1001.0) # still running: not started twice + release.set() + for _ in range(50): + if not state.get("updates_running"): + break + time.sleep(0.02) + assert state["updates"]["pending"] == 1 + + monkeypatch.setattr(a, "collect_updates", lambda: {"pending": 2}) + a.refresh_updates(state, now_mono=1000.0 + 600) # inside the hour: no new check + assert state["updates"]["pending"] == 1 + state["updates_due"] = True # asked for now: checked now + a.refresh_updates(state, now_mono=1000.0 + 601) + for _ in range(50): + if state["updates"].get("pending") == 2: + break + time.sleep(0.02) + assert state["updates"]["pending"] == 2 + + +def test_a_reboot_is_required_when_debian_says_so(tmp_path): + marker = tmp_path / "reboot-required" + marker.write_text("*** System restart required ***\n") + (tmp_path / "reboot-required.pkgs").write_text("linux-image-6.8.0-146-generic\nlibc6\nlibc6\n") + os.utime(marker, (1792000000, 1792000000)) + got = a.collect_reboot(str(marker)) + assert got["required"] is True + assert got["packages"] == ["libc6", "linux-image-6.8.0-146-generic"] + assert got["since"].startswith("2026-10-14") diff --git a/tests/plugins/host_agent/test_expand_metrics.py b/tests/plugins/host_agent/test_expand_metrics.py index 7341577..6f48fd4 100644 --- a/tests/plugins/host_agent/test_expand_metrics.py +++ b/tests/plugins/host_agent/test_expand_metrics.py @@ -122,3 +122,37 @@ def test_the_agent_s_own_footprint_is_recorded(): idx = _index(_expand_sample_to_metrics(sample, "d3", datetime.now(timezone.utc))) assert idx[("agent_rss_bytes", "d3")] == 18_000_000 assert idx[("agent_cpu_pct", "d3")] == 1.5 + + +def test_pending_updates_and_a_needed_reboot_are_metrics(): + """#5765: rules can watch what a host has waiting.""" + sample = {"ts": "2026-10-14T22:00:00+00:00", + "updates": {"manager": "apt", "pending": 12, "security": 3, "checked_at": "2026-10-14T21:30:00+00:00"}, + "reboot": {"required": True, "since": "2026-10-14T06:00:00+00:00", "packages": ["libc6"]}} + idx = _index(_expand_sample_to_metrics(sample, "d3", datetime.now(timezone.utc))) + assert idx[("updates_pending", "d3")] == 12 + assert idx[("updates_security", "d3")] == 3 + assert idx[("reboot_required", "d3")] == 1 + + +def test_the_registration_keeps_the_host_s_updates_and_reboot(): + from types import SimpleNamespace + from plugins.host_agent.routes import apply_updates + now = datetime(2026, 10, 14, 22, tzinfo=timezone.utc) + reg = SimpleNamespace(updates_manager=None, updates_pending=None, updates_security=None, + updates_checked_at=None, updates_error=None, reboot_required=None, + reboot_required_since=None, reboot_packages=None) + sample = {"updates": {"manager": "apt", "pending": 4, "security": 1, + "checked_at": "2026-10-14T21:30:00+00:00"}, + "reboot": {"required": True, "since": "2026-10-14T06:00:00Z", + "packages": ["libc6", "linux-image-6.8.0-146-generic"]}} + assert apply_updates(reg, sample, now) is True + assert (reg.updates_pending, reg.updates_security, reg.reboot_required) == (4, 1, True) + assert reg.reboot_required_since.hour == 6 + assert reg.reboot_packages == "libc6, linux-image-6.8.0-146-generic" + assert apply_updates(reg, sample, now) is False # nothing new + # A failed check keeps the last counts and says why; the reboot clears. + apply_updates(reg, {"updates": {"error": "apt is locked", "checked_at": "2026-10-14T22:30:00Z"}, + "reboot": {"required": False}}, now) + assert reg.updates_pending == 4 and reg.updates_error == "apt is locked" + assert reg.reboot_required is False and reg.reboot_required_since is None