feat(alerts): the host's temperature and throughput can drive alert rules (#5787)

temp_c_max was recorded on every ingest, but host_agent didn't offer it
to rules, so "host running hot" couldn't be watched. The catalog guard
didn't see the row(...) helper ingest builds its rows with. Now offered:
temp_c_max, temp_c per sensor, and network and disk throughput per host,
interface and disk. A new guard holds every literal row(...) name to the
catalog, except NOT_ALERTABLE, where each exclusion is given a reason.
temp_c_max reads in °C.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/plugins/host-agent-design.md

@@ -271,6 +271,8 @@ A board's spare thermistor inputs (a Nuvoton's SYSTIN, AUXTIN*) are often wired
 
 The verdicts are kept in memory and rebuilt at startup, so ingest and the host page read them without a query. The host page lists the sensors left out under "Not counted", each with its reason. A host with no CPU reading, or too little history, falls back to the value rule alone.
 
+`temp_c_max` can drive an alert rule (#5787), e.g. "Host running hot": all hosts, over 85 °C for 10 consecutive checks. So can each sensor's `temp_c`, and the per-interface and per-disk throughput. `tests/core/test_metric_catalog.py` holds every metric ingest writes to the alert catalog, except the few `NOT_ALERTABLE` names in `plugins/host_agent/__init__.py`.
+
 ### Logging
 
 To stderr only (systemd captures to journal). No file logging, no log rotation. Levels:

plugins/host_agent/__init__.py

@@ -78,7 +78,18 @@ def get_alert_metrics() -> dict[str, list[str]]:
     reboot_required is 1 when the host needs a reboot (agent ≥ 1.16.0):
     "security updates waiting" is updates_security > 0. maintenance_failed
     comes from the scheduler (#5768): 1 when the host's latest maintenance job
-    failed or timed out, 0 once one succeeds: "maintenance failed" is > 0."""
+    failed or timed out, 0 once one succeeds: "maintenance failed" is > 0.
+    temp_c_max is the host's hottest real thermometer, in °C (#5787); sensors
+    with nothing wired to them are left out (#5759, #5783): "running hot" is
+    > 85. temp_c is one sensor, resource "<host>:temp:<label>". net_rx_bps /
+    net_tx_bps and disk_read_bps / disk_write_bps are bytes per second, for
+    the whole host ("<host>") or one interface ("<host>:net:<iface>") or disk
+    ("<host>:diskio:<device>").
+
+    Recorded but not offered (NOT_ALERTABLE): disk_used_bytes and
+    disk_total_bytes, where disk_used_pct is the figure to watch (and docker
+    uses disk_total_bytes for something else), and mem_cached_bytes /
+    mem_buffers_bytes, parts of what mem_available_bytes already counts."""
     return {"host_agent": [
         "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
         "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m",
@@ -86,4 +97,11 @@ def get_alert_metrics() -> dict[str, list[str]]:
         "uptime_secs", "seconds_since_seen", "clock_off_secs", "mounts_stalled",
         "agent_rss_bytes", "agent_cpu_pct",
         "updates_pending", "updates_security", "reboot_required", "maintenance_failed",
+        "temp_c_max", "temp_c", "net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
     ]}
+
+
+# Metrics host_agent records that no rule is offered, each for the reason
+# get_alert_metrics gives; tests/core/test_metric_catalog.py holds every other
+# recorded name to the catalog.
+NOT_ALERTABLE = frozenset({"disk_used_bytes", "disk_total_bytes", "mem_cached_bytes", "mem_buffers_bytes"})

steward/core/formatting.py

@@ -279,7 +279,7 @@ def metric_value(name: str, value: Any) -> str:
     if n.endswith("_secs") or n.startswith("seconds_") or (
             n.endswith("_seconds") and not n.endswith("_timestamp_seconds")):
         return human_duration(value)
-    if n.endswith("_c"):
+    if n.endswith("_c") or n.startswith("temp_c"):     # temp_c_max too
         return f"{value:.0f}°C"
     if n.endswith("_days"):
         return f"{value:.0f} days"
@@ -346,6 +346,12 @@ METRICS: dict[str, tuple[str, str]] = {
     "updates_security": ("Security updates waiting", "of the updates waiting, those from a security source"),
     "reboot_required": ("Reboot required", "1 when the host needs a reboot to finish an update, 0 when not"),
     "maintenance_failed": ("Maintenance failed", "1 when the host's latest scheduled or manual update failed, timed out or didn't come back into service"),
+    "temp_c_max": ("Hottest sensor", "the host's hottest real thermometer; inputs with nothing wired to them are left out"),
+    "temp_c": ("Sensor temperature", "one temperature sensor (resource host:temp:label)"),
+    "net_rx_bps": ("Network in", "bytes per second received, by the whole host or one interface"),
+    "net_tx_bps": ("Network out", "bytes per second sent, by the whole host or one interface"),
+    "disk_read_bps": ("Disk reads", "bytes per second read, from all disks or one"),
+    "disk_write_bps": ("Disk writes", "bytes per second written, to all disks or one"),
 }
 
 # Where a rule's metric comes from, as the picker and the rules list say it.
@@ -418,7 +424,7 @@ def metric_unit(name: str) -> str:
         return "bytes/s"
     if n.endswith("_secs") or n.startswith("seconds_"):
         return "seconds"
-    if n.endswith("_c"):
+    if n.endswith("_c") or n.startswith("temp_c"):     # temp_c_max too
         return "°C"
     if n.endswith("_days"):
         return "days"

tests/core/test_formatting.py

@@ -178,4 +178,8 @@ def test_metric_names_and_units_read_in_words():
     assert metric_label("if_in_octets") == "if in octets"
     assert metric_unit("cert_expiry_days") == "days" and metric_unit("seconds_since_seen") == "seconds"
     assert metric_unit("is_up") == "" and metric_unit("request_rate") == "req/s"
+    # The host's hottest sensor reads in °C though its name ends in _max (#5787).
+    from steward.core.formatting import metric_value
+    assert metric_unit("temp_c_max") == "°C" and metric_value("temp_c_max", 71.4) == "71°C"
+    assert metric_label("temp_c_max") == "Hottest sensor"
     assert source_label("host_agent") == "Host agent" and source_label("my_plugin") == "my plugin"

tests/core/test_metric_catalog.py

@@ -102,3 +102,27 @@ def test_every_recorded_metric_is_in_the_catalog():
     # offers whatever it has recorded, so any name it writes is offerable.
     missing = sorted(p for p in pairs if cat.get(p[0]) != set() and p[1] not in cat.get(p[0], set()))
     assert not missing, f"recorded but not in the alert catalog: {missing}"
+
+
+def host_agent_rows(source: str) -> set[str]:
+    """Literal metric names host_agent's ingest writes through the `row(...)`
+    helper inside _expand_sample_to_metrics: it builds PluginMetric rows
+    itself, so the batch.add walk above never sees them."""
+    names: set[str] = set()
+    for func in ast.walk(ast.parse(source)):
+        if isinstance(func, ast.FunctionDef) and func.name == "_expand_sample_to_metrics":
+            for call in ast.walk(func):
+                if (isinstance(call, ast.Call) and isinstance(call.func, ast.Name)
+                        and call.func.id == "row" and call.args and _const(call.args[0])):
+                    names.add(call.args[0].value)
+    return names
+
+
+def test_every_metric_host_agent_ingest_writes_is_in_the_catalog():
+    """temp_c_max was recorded for weeks and no rule could watch it (#5787)."""
+    names = host_agent_rows((_REPO / "plugins/host_agent/routes.py").read_text())
+    assert {"cpu_pct", "temp_c_max", "mounts_stalled"} <= names and len(names) >= 20, names
+    from plugins.host_agent import NOT_ALERTABLE
+    missing = sorted(names - catalog()["host_agent"] - NOT_ALERTABLE)
+    assert not missing, f"host_agent records these but no rule can watch them: {missing}"
+    assert NOT_ALERTABLE <= names, "NOT_ALERTABLE names something host_agent no longer records"