feat(host_agent): a temperature sensor is judged by how it moves against the CPU (#5783)

A board's spare thermistor inputs report whatever an open input reads, and
the value rule from #5759 only catches the impossible ones. Once an hour,
sensors.refresh judges each sensor against the host's CPU (Package id N,
Tctl/Tdie/TccdN, PECI Agent N) over a week of hourly averages. A sensor is
ruled out if it falls as the CPU warms (r <= -0.5) or holds within 0.5 °C
while the CPU moved 10 °C or more. A judgment needs at least 72 shared hours.

The host temperature leaves judged sensors out at ingest. The host page
lists them under "Not counted", with each reason given once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/__init__.py

@@ -43,8 +43,8 @@ def get_audit_entities() -> dict:
 
 
 def get_scheduled_tasks() -> list:
-    from .scheduler import make_maintenance_task, make_task
-    return [make_task(_app), make_maintenance_task(_app)]
+    from .scheduler import make_maintenance_task, make_sensor_task, make_task
+    return [make_task(_app), make_maintenance_task(_app), make_sensor_task(_app)]
 
 
 def get_blueprint():

plugins/host_agent/routes.py

@@ -113,15 +113,28 @@ def counts_as_hottest(label: str, celsius: float) -> bool:
     return not (_BOARD_INPUT.match(label) and celsius >= BOARD_INPUT_CEILING_C)
 
 
-def split_temps(temps: dict) -> tuple[dict, dict]:
+def is_thermometer(host_name: str, label: str, celsius: float) -> bool:
+    """Whether a sensor's reading may count as the host's temperature: not one
+    judged unwired from how it moves against the CPU (sensors.py, #5783),
+    and a reading counts_as_hottest allows."""
+    from . import sensors
+    return label not in sensors.unwired(host_name) and counts_as_hottest(label, celsius)
+
+
+def split_temps(host_name: str, temps: dict) -> tuple[dict, dict]:
     """(real, unwired): a host's sensors, hottest first, apart from the ones
-    counts_as_hottest leaves out (a board input with nothing wired to it, a
-    reading at or below 0 °C). Those read a constant at any load, so the
-    page lists them apart instead of colouring them as the host's heat."""
-    def by_heat(d: dict) -> dict:
-        return dict(sorted(d.items(), key=lambda kv: -(kv[1] if kv[1] is not None else -999)))
-    real = {k: c for k, c in temps.items() if c is None or counts_as_hottest(k, c)}
-    return by_heat(real), by_heat({k: c for k, c in temps.items() if k not in real})
+    is_thermometer leaves out. Unwired ones map to (reading, why); they read
+    a constant or run against the CPU, so the page lists them apart instead
+    of colouring them as the host's heat."""
+    from . import sensors
+    judged = sensors.unwired(host_name)
+
+    def by_heat(d: dict, value=lambda v: v) -> dict:
+        return dict(sorted(d.items(), key=lambda kv: -(value(kv[1]) if value(kv[1]) is not None else -999)))
+    real = {k: c for k, c in temps.items() if c is None or is_thermometer(host_name, k, c)}
+    unwired = {k: (c, judged.get(k, "reads a value no wired sensor gives"))
+               for k, c in temps.items() if k not in real}
+    return by_heat(real), by_heat(unwired, lambda v: v[0])
 
 
 def _ts_or(value, fallback: datetime) -> datetime:
@@ -305,14 +318,15 @@ def _expand_sample_to_metrics(
         row("disk_write_bps", host_name, dw_total)
 
     # Temperatures — per sensor plus the host max for at-a-glance. The max
-    # counts only readings that can be real (#5759).
+    # counts only readings that can be real (#5759), from sensors that behave
+    # like thermometers (#5783).
     max_temp: float | None = None
     for t in sample.get("temps") or []:
         c = t.get("celsius")
         if c is None:
             continue
         row("temp_c", f"{host_name}:temp:{t['label']}", c)
-        if counts_as_hottest(str(t["label"]), c) and (max_temp is None or c > max_temp):
+        if is_thermometer(host_name, str(t["label"]), c) and (max_temp is None or c > max_temp):
             max_temp = c
     if max_temp is not None:
         row("temp_c_max", host_name, max_temp)
@@ -867,7 +881,7 @@ async def host_detail_metrics(host_id: str):
             HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
         latest = await _latest_metrics_for_host(session, host.name)
     hostlvl, cores, nets, disks_io, temps, mounts = _split_host_metrics(host.name, latest)
-    temps, unwired = split_temps(temps)
+    temps, unwired = split_temps(host.name, temps)
     ls = reg.last_seen_at if reg else None
     stale = ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()
     return await render_template(

plugins/host_agent/scheduler.py

@@ -121,6 +121,29 @@ def make_task(app) -> ScheduledTask:
     )
 
 
+def make_sensor_task(app) -> ScheduledTask:
+    """Re-judges which temperature sensors are thermometers (#5783), hourly
+    and at startup, from the hourly averages: verdicts live in memory, so a
+    restart rebuilds them before the first hour is out."""
+    async def _judge():
+        from . import sensors
+        try:
+            async with app.db_sessionmaker() as session:
+                found = await sensors.refresh(session, datetime.now(timezone.utc))
+            for host, labels in found.items():
+                logger.info("host_agent: %s sensors not counted as temperature: %s",
+                            host, ", ".join(sorted(labels)))
+        except Exception:
+            logger.exception("host_agent: judging temperature sensors failed")
+
+    return ScheduledTask(
+        name="host_agent_sensors",
+        coro_factory=_judge,
+        interval_seconds=3600,
+        run_on_startup=True,
+    )
+
+
 def make_maintenance_task(app) -> ScheduledTask:
     """Gives up on maintenance jobs past their deadline (#5767), starts due
     maintenance plans and moves their runs on one host at a time (#5768), and

plugins/host_agent/sensors.py

@@ -0,0 +1,123 @@
+"""Which of a host's temperature sensors are thermometers (#5783).
+
+A board's spare thermistor inputs often have nothing wired to them, and the
+kernel reports them anyway. No flag says so; their behaviour does. A
+thermometer in a host warms as its CPU warms. An open input reads the chip's
+own reference, which drifts the other way as the chip warms, or holds still:
+VDNT-Util01's SYSTIN ran at r = −0.81 against its CPU package over a week,
+while CPUTIN beside it ran at +0.83 (#5782).
+
+So each sensor is judged against the host's CPU over its hourly averages, by
+a scheduled task, and the verdicts are kept here for ingest and the pages to
+read. Until a host has enough history, or when its CPU never moved enough to
+tell, nothing is judged and routes.counts_as_hottest's value rule decides.
+"""
+from __future__ import annotations
+
+import logging
+import re
+from datetime import datetime, timedelta
+from math import sqrt
+
+logger = logging.getLogger(__name__)
+
+# Hours of history judged, the shared hours a verdict needs, and how far the
+# CPU must have moved in them for "it didn't follow" to mean anything.
+WINDOW = timedelta(days=7)
+MIN_HOURS = 72
+REFERENCE_SWING_C = 10.0
+# Falling as the CPU warms, this strongly, is no thermometer's behaviour.
+# A wired board or drive sensor may barely follow the CPU (r near 0), so
+# only a clear negative counts, not a weak one.
+FALLS_R = -0.5
+# Held within this over days while the CPU moved: a constant, not a reading.
+FLAT_C = 0.5
+
+# The host's CPU, in order of preference: Intel coretemp's package, AMD
+# k10temp's control/die/CCD readings, then the PECI agent.
+_REFERENCES = (re.compile(r"^Package id \d+$", re.I), re.compile(r"^(Tctl|Tdie)$", re.I),
+               re.compile(r"^Tccd\d+$", re.I), re.compile(r"^PECI Agent \d+$", re.I))
+
+# {host name: {sensor label: why it isn't a thermometer}}, from the last refresh.
+_unwired: dict[str, dict[str, str]] = {}
+
+
+def unwired(host_name: str) -> dict[str, str]:
+    """The sensors judged not to be thermometers on a host, with why."""
+    return _unwired.get(host_name, {})
+
+
+def reference_label(labels) -> str | None:
+    """The sensor that reads the host's CPU, or None when it has none."""
+    labels = sorted(labels)
+    for pattern in _REFERENCES:
+        found = [label for label in labels if pattern.match(label)]
+        if found:
+            return found[0]
+    return None
+
+
+def pearson(xs: list[float], ys: list[float]) -> float | None:
+    """Pearson's r, or None when either series doesn't move."""
+    n = len(xs)
+    mx, my = sum(xs) / n, sum(ys) / n
+    sxx = sum((x - mx) ** 2 for x in xs)
+    syy = sum((y - my) ** 2 for y in ys)
+    if sxx == 0 or syy == 0:
+        return None
+    return sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / sqrt(sxx * syy)
+
+
+def judge(reference: dict[datetime, float], sensor: dict[datetime, float]) -> str | None:
+    """Why `sensor` isn't a thermometer, judged against the host's CPU over
+    the hours both have; None when it behaves like one, or when there isn't
+    enough to tell (too few hours, or a CPU that never moved)."""
+    hours = sorted(set(reference) & set(sensor))
+    if len(hours) < MIN_HOURS:
+        return None
+    cpu = [reference[h] for h in hours]
+    got = [sensor[h] for h in hours]
+    if max(cpu) - min(cpu) < REFERENCE_SWING_C:
+        return None
+    if max(got) - min(got) < FLAT_C:
+        return f"holds at {sum(got) / len(got):.0f} °C while the CPU moved {min(cpu):.0f}–{max(cpu):.0f} °C"
+    r = pearson(cpu, got)
+    if r is not None and r <= FALLS_R:
+        return f"falls as the CPU warms (r {r:.2f} over {len(hours)} h)"
+    return None
+
+
+def judge_hosts(rows) -> dict[str, dict[str, str]]:
+    """{host: {label: why}} from hourly rows of (resource, bucket, value_avg),
+    where resource is "<host>:temp:<label>"."""
+    series: dict[str, dict[str, dict[datetime, float]]] = {}
+    for resource, bucket, value in rows:
+        host, sep, label = resource.partition(":temp:")
+        if sep and value is not None:
+            series.setdefault(host, {}).setdefault(label, {})[bucket] = value
+    out: dict[str, dict[str, str]] = {}
+    for host, sensors in series.items():
+        ref = reference_label(sensors)
+        if ref is None:
+            continue
+        for label, points in sensors.items():
+            if label == ref:
+                continue
+            why = judge(sensors[ref], points)
+            if why:
+                out.setdefault(host, {})[label] = why
+    return out
+
+
+async def refresh(session, now: datetime) -> dict[str, dict[str, str]]:
+    """Re-judge every host's sensors from the last WINDOW of hourly averages,
+    and keep the verdicts for ingest and the pages."""
+    global _unwired
+    from sqlalchemy import select
+    from steward.models.metrics import PluginMetricHourly as H
+    rows = (await session.execute(
+        select(H.resource_name, H.bucket, H.value_avg)
+        .where(H.source_module == "host_agent", H.metric_name == "temp_c",
+               H.resource_name.like("%:temp:%"), H.bucket >= now - WINDOW))).all()
+    _unwired = judge_hosts(rows)
+    return _unwired

plugins/host_agent/templates/_host_metrics.html

@@ -160,10 +160,12 @@
     {% endfor %}
   </div>
   {% if unwired_temps %}
-  {# Board inputs with nothing wired to them, and readings at or below 0 °C:
-     constant at any load, left out of the host's temperature (#5759). #}
-  <p class="field-hint" style="margin:0.5rem 0 0;">Not counted, reading a constant with nothing wired to them:
-    {% for label, c in unwired_temps.items() %}{{ label }} {{ "%.0f"|format(c) }}°C{{ ", " if not loop.last }}{% endfor %}.</p>
+  {# Sensors that aren't thermometers: they run against the CPU or hold a
+     constant (#5783), or read a value no wired sensor gives (#5759). Left
+     out of the host's temperature. Each reason is said once, after its
+     sensors, hottest first within it. #}
+  <p class="field-hint" style="margin:0.5rem 0 0;">Not counted, likely nothing wired to them:
+    {% for why, group in unwired_temps.items()|groupby("1.1") %}{% for label, v in group %}{{ label }} {{ "%.0f"|format(v[0]) }}°C{{ ", " if not loop.last }}{% endfor %} ({{ why }}){{ "; " if not loop.last }}{% endfor %}.</p>
   {% endif %}
 </div>
 {% endif %}