From 4cce3f0d12380bb04191867bb07b3be969692b28 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 30 Oct 2026 19:04:34 -0400 Subject: [PATCH] feat(host_agent): the heat alert says which sensor set the temperature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Host running hot" read the host's hottest sensor and didn't name it, so the operator couldn't tell what hit 100 °C on VDNT-Util01. The alert email now names the sensor and its reading, through the same names_behind a count uses. Co-Authored-By: Claude Opus 5.5 --- plugins/host_agent/__init__.py | 4 ++++ plugins/host_agent/routes.py | 21 +++++++++++++++++++ steward/core/alerts.py | 7 ++++--- .../plugins/host_agent/test_expand_metrics.py | 7 +++++++ 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/plugins/host_agent/__init__.py b/plugins/host_agent/__init__.py index d64a3ee..b497d70 100644 --- a/plugins/host_agent/__init__.py +++ b/plugins/host_agent/__init__.py @@ -25,6 +25,10 @@ def setup(app: "Quart") -> None: from steward.hosts.service import agent_stale_after_seconds agent_heartbeat("host_agent", "seconds_since_seen", lambda: agent_stale_after_seconds(app.config)) agent_fed("host_agent", lambda resource: resource.split(":", 1)[0]) + # temp_c_max stands for one of many sensors: its alert says which. + from steward.core.alerts import names_behind + from .routes import name_hottest_sensor + names_behind("host_agent", "temp_c_max", name_hottest_sensor) # A reboot the agent announced is silence with a reason, for a while (#5756). from datetime import datetime, timezone from steward.core.alerts import agents_away diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index c8e2c5f..bf32cc6 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -122,6 +122,27 @@ def is_thermometer(host_name: str, label: str, celsius: float) -> bool: return label not in sensors.unwired(host_name) and counts_as_hottest(label, celsius) +def hottest_sensor(host_name: str, temps: dict[str, float]) -> list[str]: + """Which sensor sets a host's temperature, as an alert on it says: + ["Package id 0 at 86°C"], or [] with no thermometer among `temps`.""" + from steward.core.formatting import metric_value + real = {label: c for label, c in temps.items() if is_thermometer(host_name, label, c)} + if not real: + return [] + label = max(real, key=real.get) + return [f"{label} at {metric_value('temp_c', real[label])}"] + + +async def name_hottest_sensor(session, resource: str) -> list[str]: + """The sensor behind temp_c_max on host `resource` right now.""" + from steward.models.metrics import PluginMetricLatest + prefix = f"{resource}:temp:" + rows = (await session.execute(select(PluginMetricLatest.resource_name, PluginMetricLatest.value).where( + PluginMetricLatest.source_module == "host_agent", PluginMetricLatest.metric_name == "temp_c", + PluginMetricLatest.resource_name.startswith(prefix, autoescape=True)))).all() + return hottest_sensor(resource, {name[len(prefix):]: value for name, value in rows}) + + def split_temps(host_name: str, temps: dict) -> tuple[dict, dict]: """(real, unwired): a host's sensors, hottest first, apart from the ones is_thermometer leaves out. Unwired ones map to (reading, why); they read diff --git a/steward/core/alerts.py b/steward/core/alerts.py index d236ef4..acb092d 100644 --- a/steward/core/alerts.py +++ b/steward/core/alerts.py @@ -81,9 +81,10 @@ def agents_away(hosts_away: Callable[[AsyncSession], Awaitable[set[str]]]) -> No def names_behind(source_module: str, metric_name: str, names: Callable[[AsyncSession, str], Awaitable[list[str]]]) -> None: - """A figure that counts things: `await names(session, resource)` names - the things it counts right now ("web_app (0 of 1 running)"), so the - alert about it can say which. The plugin recording the figure registers it.""" + """A figure that counts things, or stands for one of several: `await + names(session, resource)` names them right now ("web_app (0 of 1 running)", + "Package id 0 at 86°C"), so the alert about it can say which. The plugin + recording the figure registers it.""" _NAMED[(source_module, metric_name)] = names diff --git a/tests/plugins/host_agent/test_expand_metrics.py b/tests/plugins/host_agent/test_expand_metrics.py index 87d2e11..3d5a08d 100644 --- a/tests/plugins/host_agent/test_expand_metrics.py +++ b/tests/plugins/host_agent/test_expand_metrics.py @@ -107,6 +107,13 @@ def test_stalled_mounts_are_counted_and_have_no_figures(): assert ("disk_used_pct", "d3:/nfs/media") not in idx +def test_the_heat_alert_names_the_sensor_that_set_it(): + from plugins.host_agent.routes import hottest_sensor + temps = {"Package id 0": 86.0, "Composite": 40.0, "AUXTIN0": 110.0, "PCH_CHIP_CPU_MAX_TEMP": 0.0} + assert hottest_sensor("box", temps) == ["Package id 0 at 86°C"] + assert hottest_sensor("box", {"AUXTIN0": 110.0}) == [] + + def test_a_wired_board_input_still_counts(): """#5759: below 100 °C a board input is a real reading.""" from plugins.host_agent.routes import counts_as_hottest