feat(host_agent): the heat alert says which sensor set the temperature
"Host running hot" read the host's hottest sensor and didn't name it, so the operator couldn't tell what hit 100 °C on VDNT-Util01. The alert email now names the sensor and its reading, through the same names_behind a count uses. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -25,6 +25,10 @@ def setup(app: "Quart") -> None:
|
||||
from steward.hosts.service import agent_stale_after_seconds
|
||||
agent_heartbeat("host_agent", "seconds_since_seen", lambda: agent_stale_after_seconds(app.config))
|
||||
agent_fed("host_agent", lambda resource: resource.split(":", 1)[0])
|
||||
# temp_c_max stands for one of many sensors: its alert says which.
|
||||
from steward.core.alerts import names_behind
|
||||
from .routes import name_hottest_sensor
|
||||
names_behind("host_agent", "temp_c_max", name_hottest_sensor)
|
||||
# A reboot the agent announced is silence with a reason, for a while (#5756).
|
||||
from datetime import datetime, timezone
|
||||
from steward.core.alerts import agents_away
|
||||
|
||||
@@ -122,6 +122,27 @@ def is_thermometer(host_name: str, label: str, celsius: float) -> bool:
|
||||
return label not in sensors.unwired(host_name) and counts_as_hottest(label, celsius)
|
||||
|
||||
|
||||
def hottest_sensor(host_name: str, temps: dict[str, float]) -> list[str]:
|
||||
"""Which sensor sets a host's temperature, as an alert on it says:
|
||||
["Package id 0 at 86°C"], or [] with no thermometer among `temps`."""
|
||||
from steward.core.formatting import metric_value
|
||||
real = {label: c for label, c in temps.items() if is_thermometer(host_name, label, c)}
|
||||
if not real:
|
||||
return []
|
||||
label = max(real, key=real.get)
|
||||
return [f"{label} at {metric_value('temp_c', real[label])}"]
|
||||
|
||||
|
||||
async def name_hottest_sensor(session, resource: str) -> list[str]:
|
||||
"""The sensor behind temp_c_max on host `resource` right now."""
|
||||
from steward.models.metrics import PluginMetricLatest
|
||||
prefix = f"{resource}:temp:"
|
||||
rows = (await session.execute(select(PluginMetricLatest.resource_name, PluginMetricLatest.value).where(
|
||||
PluginMetricLatest.source_module == "host_agent", PluginMetricLatest.metric_name == "temp_c",
|
||||
PluginMetricLatest.resource_name.startswith(prefix, autoescape=True)))).all()
|
||||
return hottest_sensor(resource, {name[len(prefix):]: value for name, value in rows})
|
||||
|
||||
|
||||
def split_temps(host_name: str, temps: dict) -> tuple[dict, dict]:
|
||||
"""(real, unwired): a host's sensors, hottest first, apart from the ones
|
||||
is_thermometer leaves out. Unwired ones map to (reading, why); they read
|
||||
|
||||
Reference in New Issue
Block a user