feat(alerts): the host's temperature and throughput can drive alert rules (#5787)
CI / lint (push) Successful in 2s
CI / integration (push) Successful in 1m23s
CI / unit (push) Successful in 1m30s
CI / publish (push) Successful in 1m23s

temp_c_max was recorded on every ingest, but host_agent didn't offer it
to rules, so "host running hot" couldn't be watched. The catalog guard
didn't see the row(...) helper ingest builds its rows with. Now offered:
temp_c_max, temp_c per sensor, and network and disk throughput per host,
interface and disk. A new guard holds every literal row(...) name to the
catalog, except NOT_ALERTABLE, where each exclusion is given a reason.
temp_c_max reads in °C.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-14 22:20:11 -04:00
co-authored by Claude Opus 5.5
parent 9a11f5788f
commit 26a3ec3339
5 changed files with 57 additions and 3 deletions
+4
View File
@@ -178,4 +178,8 @@ def test_metric_names_and_units_read_in_words():
assert metric_label("if_in_octets") == "if in octets"
assert metric_unit("cert_expiry_days") == "days" and metric_unit("seconds_since_seen") == "seconds"
assert metric_unit("is_up") == "" and metric_unit("request_rate") == "req/s"
# The host's hottest sensor reads in °C though its name ends in _max (#5787).
from steward.core.formatting import metric_value
assert metric_unit("temp_c_max") == "°C" and metric_value("temp_c_max", 71.4) == "71°C"
assert metric_label("temp_c_max") == "Hottest sensor"
assert source_label("host_agent") == "Host agent" and source_label("my_plugin") == "my plugin"
+24
View File
@@ -102,3 +102,27 @@ def test_every_recorded_metric_is_in_the_catalog():
# offers whatever it has recorded, so any name it writes is offerable.
missing = sorted(p for p in pairs if cat.get(p[0]) != set() and p[1] not in cat.get(p[0], set()))
assert not missing, f"recorded but not in the alert catalog: {missing}"
def host_agent_rows(source: str) -> set[str]:
"""Literal metric names host_agent's ingest writes through the `row(...)`
helper inside _expand_sample_to_metrics: it builds PluginMetric rows
itself, so the batch.add walk above never sees them."""
names: set[str] = set()
for func in ast.walk(ast.parse(source)):
if isinstance(func, ast.FunctionDef) and func.name == "_expand_sample_to_metrics":
for call in ast.walk(func):
if (isinstance(call, ast.Call) and isinstance(call.func, ast.Name)
and call.func.id == "row" and call.args and _const(call.args[0])):
names.add(call.args[0].value)
return names
def test_every_metric_host_agent_ingest_writes_is_in_the_catalog():
"""temp_c_max was recorded for weeks and no rule could watch it (#5787)."""
names = host_agent_rows((_REPO / "plugins/host_agent/routes.py").read_text())
assert {"cpu_pct", "temp_c_max", "mounts_stalled"} <= names and len(names) >= 20, names
from plugins.host_agent import NOT_ALERTABLE
missing = sorted(names - catalog()["host_agent"] - NOT_ALERTABLE)
assert not missing, f"host_agent records these but no rule can watch them: {missing}"
assert NOT_ALERTABLE <= names, "NOT_ALERTABLE names something host_agent no longer records"