feat(host_agent): the heat alert says which sensor set the temperature

"Host running hot" read the host's hottest sensor and didn't name it, so the
operator couldn't tell what hit 100 °C on VDNT-Util01. The alert email now
names the sensor and its reading, through the same names_behind a count uses.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-30 19:04:34 -04:00
co-authored by Claude Opus 5.5
parent e51d969c43
commit 4cce3f0d12
4 changed files with 36 additions and 3 deletions
+4
View File
@@ -25,6 +25,10 @@ def setup(app: "Quart") -> None:
from steward.hosts.service import agent_stale_after_seconds
agent_heartbeat("host_agent", "seconds_since_seen", lambda: agent_stale_after_seconds(app.config))
agent_fed("host_agent", lambda resource: resource.split(":", 1)[0])
# temp_c_max stands for one of many sensors: its alert says which.
from steward.core.alerts import names_behind
from .routes import name_hottest_sensor
names_behind("host_agent", "temp_c_max", name_hottest_sensor)
# A reboot the agent announced is silence with a reason, for a while (#5756).
from datetime import datetime, timezone
from steward.core.alerts import agents_away
+21
View File
@@ -122,6 +122,27 @@ def is_thermometer(host_name: str, label: str, celsius: float) -> bool:
return label not in sensors.unwired(host_name) and counts_as_hottest(label, celsius)
def hottest_sensor(host_name: str, temps: dict[str, float]) -> list[str]:
"""Which sensor sets a host's temperature, as an alert on it says:
["Package id 0 at 86°C"], or [] with no thermometer among `temps`."""
from steward.core.formatting import metric_value
real = {label: c for label, c in temps.items() if is_thermometer(host_name, label, c)}
if not real:
return []
label = max(real, key=real.get)
return [f"{label} at {metric_value('temp_c', real[label])}"]
async def name_hottest_sensor(session, resource: str) -> list[str]:
"""The sensor behind temp_c_max on host `resource` right now."""
from steward.models.metrics import PluginMetricLatest
prefix = f"{resource}:temp:"
rows = (await session.execute(select(PluginMetricLatest.resource_name, PluginMetricLatest.value).where(
PluginMetricLatest.source_module == "host_agent", PluginMetricLatest.metric_name == "temp_c",
PluginMetricLatest.resource_name.startswith(prefix, autoescape=True)))).all()
return hottest_sensor(resource, {name[len(prefix):]: value for name, value in rows})
def split_temps(host_name: str, temps: dict) -> tuple[dict, dict]:
"""(real, unwired): a host's sensors, hottest first, apart from the ones
is_thermometer leaves out. Unwired ones map to (reading, why); they read
+4 -3
View File
@@ -81,9 +81,10 @@ def agents_away(hosts_away: Callable[[AsyncSession], Awaitable[set[str]]]) -> No
def names_behind(source_module: str, metric_name: str,
names: Callable[[AsyncSession, str], Awaitable[list[str]]]) -> None:
"""A figure that counts things: `await names(session, resource)` names
the things it counts right now ("web_app (0 of 1 running)"), so the
alert about it can say which. The plugin recording the figure registers it."""
"""A figure that counts things, or stands for one of several: `await
names(session, resource)` names them right now ("web_app (0 of 1 running)",
"Package id 0 at 86°C"), so the alert about it can say which. The plugin
recording the figure registers it."""
_NAMED[(source_module, metric_name)] = names
@@ -107,6 +107,13 @@ def test_stalled_mounts_are_counted_and_have_no_figures():
assert ("disk_used_pct", "d3:/nfs/media") not in idx
def test_the_heat_alert_names_the_sensor_that_set_it():
from plugins.host_agent.routes import hottest_sensor
temps = {"Package id 0": 86.0, "Composite": 40.0, "AUXTIN0": 110.0, "PCH_CHIP_CPU_MAX_TEMP": 0.0}
assert hottest_sensor("box", temps) == ["Package id 0 at 86°C"]
assert hottest_sensor("box", {"AUXTIN0": 110.0}) == []
def test_a_wired_board_input_still_counts():
"""#5759: below 100 °C a board input is a real reading."""
from plugins.host_agent.routes import counts_as_hottest