feat(host_agent): the heat alert says which sensor set the temperature
"Host running hot" read the host's hottest sensor and didn't name it, so the operator couldn't tell what hit 100 °C on VDNT-Util01. The alert email now names the sensor and its reading, through the same names_behind a count uses. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -25,6 +25,10 @@ def setup(app: "Quart") -> None:
|
||||
from steward.hosts.service import agent_stale_after_seconds
|
||||
agent_heartbeat("host_agent", "seconds_since_seen", lambda: agent_stale_after_seconds(app.config))
|
||||
agent_fed("host_agent", lambda resource: resource.split(":", 1)[0])
|
||||
# temp_c_max stands for one of many sensors: its alert says which.
|
||||
from steward.core.alerts import names_behind
|
||||
from .routes import name_hottest_sensor
|
||||
names_behind("host_agent", "temp_c_max", name_hottest_sensor)
|
||||
# A reboot the agent announced is silence with a reason, for a while (#5756).
|
||||
from datetime import datetime, timezone
|
||||
from steward.core.alerts import agents_away
|
||||
|
||||
@@ -122,6 +122,27 @@ def is_thermometer(host_name: str, label: str, celsius: float) -> bool:
|
||||
return label not in sensors.unwired(host_name) and counts_as_hottest(label, celsius)
|
||||
|
||||
|
||||
def hottest_sensor(host_name: str, temps: dict[str, float]) -> list[str]:
|
||||
"""Which sensor sets a host's temperature, as an alert on it says:
|
||||
["Package id 0 at 86°C"], or [] with no thermometer among `temps`."""
|
||||
from steward.core.formatting import metric_value
|
||||
real = {label: c for label, c in temps.items() if is_thermometer(host_name, label, c)}
|
||||
if not real:
|
||||
return []
|
||||
label = max(real, key=real.get)
|
||||
return [f"{label} at {metric_value('temp_c', real[label])}"]
|
||||
|
||||
|
||||
async def name_hottest_sensor(session, resource: str) -> list[str]:
|
||||
"""The sensor behind temp_c_max on host `resource` right now."""
|
||||
from steward.models.metrics import PluginMetricLatest
|
||||
prefix = f"{resource}:temp:"
|
||||
rows = (await session.execute(select(PluginMetricLatest.resource_name, PluginMetricLatest.value).where(
|
||||
PluginMetricLatest.source_module == "host_agent", PluginMetricLatest.metric_name == "temp_c",
|
||||
PluginMetricLatest.resource_name.startswith(prefix, autoescape=True)))).all()
|
||||
return hottest_sensor(resource, {name[len(prefix):]: value for name, value in rows})
|
||||
|
||||
|
||||
def split_temps(host_name: str, temps: dict) -> tuple[dict, dict]:
|
||||
"""(real, unwired): a host's sensors, hottest first, apart from the ones
|
||||
is_thermometer leaves out. Unwired ones map to (reading, why); they read
|
||||
|
||||
@@ -81,9 +81,10 @@ def agents_away(hosts_away: Callable[[AsyncSession], Awaitable[set[str]]]) -> No
|
||||
|
||||
def names_behind(source_module: str, metric_name: str,
|
||||
names: Callable[[AsyncSession, str], Awaitable[list[str]]]) -> None:
|
||||
"""A figure that counts things: `await names(session, resource)` names
|
||||
the things it counts right now ("web_app (0 of 1 running)"), so the
|
||||
alert about it can say which. The plugin recording the figure registers it."""
|
||||
"""A figure that counts things, or stands for one of several: `await
|
||||
names(session, resource)` names them right now ("web_app (0 of 1 running)",
|
||||
"Package id 0 at 86°C"), so the alert about it can say which. The plugin
|
||||
recording the figure registers it."""
|
||||
_NAMED[(source_module, metric_name)] = names
|
||||
|
||||
|
||||
|
||||
@@ -107,6 +107,13 @@ def test_stalled_mounts_are_counted_and_have_no_figures():
|
||||
assert ("disk_used_pct", "d3:/nfs/media") not in idx
|
||||
|
||||
|
||||
def test_the_heat_alert_names_the_sensor_that_set_it():
|
||||
from plugins.host_agent.routes import hottest_sensor
|
||||
temps = {"Package id 0": 86.0, "Composite": 40.0, "AUXTIN0": 110.0, "PCH_CHIP_CPU_MAX_TEMP": 0.0}
|
||||
assert hottest_sensor("box", temps) == ["Package id 0 at 86°C"]
|
||||
assert hottest_sensor("box", {"AUXTIN0": 110.0}) == []
|
||||
|
||||
|
||||
def test_a_wired_board_input_still_counts():
|
||||
"""#5759: below 100 °C a board input is a real reading."""
|
||||
from plugins.host_agent.routes import counts_as_hottest
|
||||
|
||||
Reference in New Issue
Block a user