feat(host_agent): a temperature sensor is judged by how it moves against the CPU (#5783)

A board's spare thermistor inputs report whatever an open input reads, and
the value rule from #5759 only catches the impossible ones. Once an hour,
sensors.refresh judges each sensor against the host's CPU (Package id N,
Tctl/Tdie/TccdN, PECI Agent N) over a week of hourly averages. A sensor is
ruled out if it falls as the CPU warms (r <= -0.5) or holds within 0.5 °C
while the CPU moved 10 °C or more. A judgment needs at least 72 shared hours.

The host temperature leaves judged sensors out at ingest. The host page
lists them under "Not counted", with each reason given once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/plugins/host-agent-design.md

@@ -262,6 +262,15 @@ A post may carry `events`, said once each, with the next post that gets through:
 
 Steward turns them into the host's downtime (`host_downtime`): **reboot** (a stop, then a boot after it), **agent_stop** (the host stayed up), **unclean** (a boot after the agent was last heard that nobody announced: power loss or a crash) and **lost_contact**. An announced stop explains the silence for `reboot_grace_seconds` (Settings → Plugins → host_agent, 10 minutes): `seconds_since_seen` reads 0 until then, and the host's other rules go quiet with it, as when it is offline. Past the grace, agent offline fires. The events live in memory, so a span the agent saw is lost if it restarts mid-outage; the boot time still reads the restart as unclean.
 
+### Which temperature sensors are thermometers (#5782, #5783)
+
+A board's spare thermistor inputs (a Nuvoton's SYSTIN, AUXTIN*) are often wired to nothing, and the kernel reports them anyway, at whatever the open input reads. The host's temperature (`temp_c_max`) leaves a sensor out when:
+
+- **its value is impossible** (at or below 0 °C, or at or above 125 °C; `counts_as_hottest`), or
+- **it doesn't behave like a thermometer.** Once an hour, `sensors.refresh` judges each sensor against the host's CPU (Intel `Package id N`, AMD `Tctl`/`Tdie`/`TccdN`, else `PECI Agent N`), using the last 7 days of hourly averages. It needs at least 72 shared hours and a CPU that moved at least 10 °C. A sensor is ruled out when it **falls as the CPU warms** (Pearson r ≤ −0.5, the shape of an open input reading the chip's own reference) or **holds within 0.5 °C** while the CPU moved. A wired board or drive sensor that barely follows the CPU (r near 0) stays in.
+
+The verdicts are kept in memory and rebuilt at startup, so ingest and the host page read them without a query. The host page lists the sensors left out under "Not counted", each with its reason. A host with no CPU reading, or too little history, falls back to the value rule alone.
+
 ### Logging
 
 To stderr only (systemd captures to journal). No file logging, no log rotation. Levels:

plugins/host_agent/__init__.py

@@ -43,8 +43,8 @@ def get_audit_entities() -> dict:
 
 
 def get_scheduled_tasks() -> list:
-    from .scheduler import make_maintenance_task, make_task
+    from .scheduler import make_maintenance_task, make_sensor_task, make_task
-    return [make_task(_app), make_maintenance_task(_app)]
+    return [make_task(_app), make_maintenance_task(_app), make_sensor_task(_app)]
 
 
 def get_blueprint():

plugins/host_agent/routes.py

@@ -113,15 +113,28 @@ def counts_as_hottest(label: str, celsius: float) -> bool:
     return not (_BOARD_INPUT.match(label) and celsius >= BOARD_INPUT_CEILING_C)
 
 
-def split_temps(temps: dict) -> tuple[dict, dict]:
+def is_thermometer(host_name: str, label: str, celsius: float) -> bool:
+    """Whether a sensor's reading may count as the host's temperature: not one
+    judged unwired from how it moves against the CPU (sensors.py, #5783),
+    and a reading counts_as_hottest allows."""
+    from . import sensors
+    return label not in sensors.unwired(host_name) and counts_as_hottest(label, celsius)
+
+
+def split_temps(host_name: str, temps: dict) -> tuple[dict, dict]:
     """(real, unwired): a host's sensors, hottest first, apart from the ones
-    counts_as_hottest leaves out (a board input with nothing wired to it, a
+    is_thermometer leaves out. Unwired ones map to (reading, why); they read
-    reading at or below 0 °C). Those read a constant at any load, so the
+    a constant or run against the CPU, so the page lists them apart instead
-    page lists them apart instead of colouring them as the host's heat."""
+    of colouring them as the host's heat."""
-    def by_heat(d: dict) -> dict:
+    from . import sensors
-        return dict(sorted(d.items(), key=lambda kv: -(kv[1] if kv[1] is not None else -999)))
+    judged = sensors.unwired(host_name)
-    real = {k: c for k, c in temps.items() if c is None or counts_as_hottest(k, c)}
+
-    return by_heat(real), by_heat({k: c for k, c in temps.items() if k not in real})
+    def by_heat(d: dict, value=lambda v: v) -> dict:
+        return dict(sorted(d.items(), key=lambda kv: -(value(kv[1]) if value(kv[1]) is not None else -999)))
+    real = {k: c for k, c in temps.items() if c is None or is_thermometer(host_name, k, c)}
+    unwired = {k: (c, judged.get(k, "reads a value no wired sensor gives"))
+               for k, c in temps.items() if k not in real}
+    return by_heat(real), by_heat(unwired, lambda v: v[0])
 
 
 def _ts_or(value, fallback: datetime) -> datetime:
@@ -305,14 +318,15 @@ def _expand_sample_to_metrics(
         row("disk_write_bps", host_name, dw_total)
 
     # Temperatures — per sensor plus the host max for at-a-glance. The max
-    # counts only readings that can be real (#5759).
+    # counts only readings that can be real (#5759), from sensors that behave
+    # like thermometers (#5783).
     max_temp: float | None = None
     for t in sample.get("temps") or []:
         c = t.get("celsius")
         if c is None:
             continue
         row("temp_c", f"{host_name}:temp:{t['label']}", c)
-        if counts_as_hottest(str(t["label"]), c) and (max_temp is None or c > max_temp):
+        if is_thermometer(host_name, str(t["label"]), c) and (max_temp is None or c > max_temp):
             max_temp = c
     if max_temp is not None:
         row("temp_c_max", host_name, max_temp)
@@ -867,7 +881,7 @@ async def host_detail_metrics(host_id: str):
             HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
         latest = await _latest_metrics_for_host(session, host.name)
     hostlvl, cores, nets, disks_io, temps, mounts = _split_host_metrics(host.name, latest)
-    temps, unwired = split_temps(temps)
+    temps, unwired = split_temps(host.name, temps)
     ls = reg.last_seen_at if reg else None
     stale = ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()
     return await render_template(

plugins/host_agent/scheduler.py

@@ -121,6 +121,29 @@ def make_task(app) -> ScheduledTask:
     )
 
 
+def make_sensor_task(app) -> ScheduledTask:
+    """Re-judges which temperature sensors are thermometers (#5783), hourly
+    and at startup, from the hourly averages: verdicts live in memory, so a
+    restart rebuilds them before the first hour is out."""
+    async def _judge():
+        from . import sensors
+        try:
+            async with app.db_sessionmaker() as session:
+                found = await sensors.refresh(session, datetime.now(timezone.utc))
+            for host, labels in found.items():
+                logger.info("host_agent: %s sensors not counted as temperature: %s",
+                            host, ", ".join(sorted(labels)))
+        except Exception:
+            logger.exception("host_agent: judging temperature sensors failed")
+
+    return ScheduledTask(
+        name="host_agent_sensors",
+        coro_factory=_judge,
+        interval_seconds=3600,
+        run_on_startup=True,
+    )
+
+
 def make_maintenance_task(app) -> ScheduledTask:
     """Gives up on maintenance jobs past their deadline (#5767), starts due
     maintenance plans and moves their runs on one host at a time (#5768), and

plugins/host_agent/sensors.py

@@ -0,0 +1,123 @@
+"""Which of a host's temperature sensors are thermometers (#5783).
+
+A board's spare thermistor inputs often have nothing wired to them, and the
+kernel reports them anyway. No flag says so; their behaviour does. A
+thermometer in a host warms as its CPU warms. An open input reads the chip's
+own reference, which drifts the other way as the chip warms, or holds still:
+VDNT-Util01's SYSTIN ran at r = −0.81 against its CPU package over a week,
+while CPUTIN beside it ran at +0.83 (#5782).
+
+So each sensor is judged against the host's CPU over its hourly averages, by
+a scheduled task, and the verdicts are kept here for ingest and the pages to
+read. Until a host has enough history, or when its CPU never moved enough to
+tell, nothing is judged and routes.counts_as_hottest's value rule decides.
+"""
+from __future__ import annotations
+
+import logging
+import re
+from datetime import datetime, timedelta
+from math import sqrt
+
+logger = logging.getLogger(__name__)
+
+# Hours of history judged, the shared hours a verdict needs, and how far the
+# CPU must have moved in them for "it didn't follow" to mean anything.
+WINDOW = timedelta(days=7)
+MIN_HOURS = 72
+REFERENCE_SWING_C = 10.0
+# Falling as the CPU warms, this strongly, is no thermometer's behaviour.
+# A wired board or drive sensor may barely follow the CPU (r near 0), so
+# only a clear negative counts, not a weak one.
+FALLS_R = -0.5
+# Held within this over days while the CPU moved: a constant, not a reading.
+FLAT_C = 0.5
+
+# The host's CPU, in order of preference: Intel coretemp's package, AMD
+# k10temp's control/die/CCD readings, then the PECI agent.
+_REFERENCES = (re.compile(r"^Package id \d+$", re.I), re.compile(r"^(Tctl|Tdie)$", re.I),
+               re.compile(r"^Tccd\d+$", re.I), re.compile(r"^PECI Agent \d+$", re.I))
+
+# {host name: {sensor label: why it isn't a thermometer}}, from the last refresh.
+_unwired: dict[str, dict[str, str]] = {}
+
+
+def unwired(host_name: str) -> dict[str, str]:
+    """The sensors judged not to be thermometers on a host, with why."""
+    return _unwired.get(host_name, {})
+
+
+def reference_label(labels) -> str | None:
+    """The sensor that reads the host's CPU, or None when it has none."""
+    labels = sorted(labels)
+    for pattern in _REFERENCES:
+        found = [label for label in labels if pattern.match(label)]
+        if found:
+            return found[0]
+    return None
+
+
+def pearson(xs: list[float], ys: list[float]) -> float | None:
+    """Pearson's r, or None when either series doesn't move."""
+    n = len(xs)
+    mx, my = sum(xs) / n, sum(ys) / n
+    sxx = sum((x - mx) ** 2 for x in xs)
+    syy = sum((y - my) ** 2 for y in ys)
+    if sxx == 0 or syy == 0:
+        return None
+    return sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / sqrt(sxx * syy)
+
+
+def judge(reference: dict[datetime, float], sensor: dict[datetime, float]) -> str | None:
+    """Why `sensor` isn't a thermometer, judged against the host's CPU over
+    the hours both have; None when it behaves like one, or when there isn't
+    enough to tell (too few hours, or a CPU that never moved)."""
+    hours = sorted(set(reference) & set(sensor))
+    if len(hours) < MIN_HOURS:
+        return None
+    cpu = [reference[h] for h in hours]
+    got = [sensor[h] for h in hours]
+    if max(cpu) - min(cpu) < REFERENCE_SWING_C:
+        return None
+    if max(got) - min(got) < FLAT_C:
+        return f"holds at {sum(got) / len(got):.0f} °C while the CPU moved {min(cpu):.0f}–{max(cpu):.0f} °C"
+    r = pearson(cpu, got)
+    if r is not None and r <= FALLS_R:
+        return f"falls as the CPU warms (r {r:.2f} over {len(hours)} h)"
+    return None
+
+
+def judge_hosts(rows) -> dict[str, dict[str, str]]:
+    """{host: {label: why}} from hourly rows of (resource, bucket, value_avg),
+    where resource is "<host>:temp:<label>"."""
+    series: dict[str, dict[str, dict[datetime, float]]] = {}
+    for resource, bucket, value in rows:
+        host, sep, label = resource.partition(":temp:")
+        if sep and value is not None:
+            series.setdefault(host, {}).setdefault(label, {})[bucket] = value
+    out: dict[str, dict[str, str]] = {}
+    for host, sensors in series.items():
+        ref = reference_label(sensors)
+        if ref is None:
+            continue
+        for label, points in sensors.items():
+            if label == ref:
+                continue
+            why = judge(sensors[ref], points)
+            if why:
+                out.setdefault(host, {})[label] = why
+    return out
+
+
+async def refresh(session, now: datetime) -> dict[str, dict[str, str]]:
+    """Re-judge every host's sensors from the last WINDOW of hourly averages,
+    and keep the verdicts for ingest and the pages."""
+    global _unwired
+    from sqlalchemy import select
+    from steward.models.metrics import PluginMetricHourly as H
+    rows = (await session.execute(
+        select(H.resource_name, H.bucket, H.value_avg)
+        .where(H.source_module == "host_agent", H.metric_name == "temp_c",
+               H.resource_name.like("%:temp:%"), H.bucket >= now - WINDOW))).all()
+    _unwired = judge_hosts(rows)
+    return _unwired

plugins/host_agent/templates/_host_metrics.html

@@ -160,10 +160,12 @@
     {% endfor %}
   </div>
   {% if unwired_temps %}
-  {# Board inputs with nothing wired to them, and readings at or below 0 °C:
+  {# Sensors that aren't thermometers: they run against the CPU or hold a
-     constant at any load, left out of the host's temperature (#5759). #}
+     constant (#5783), or read a value no wired sensor gives (#5759). Left
-  <p class="field-hint" style="margin:0.5rem 0 0;">Not counted, reading a constant with nothing wired to them:
+     out of the host's temperature. Each reason is said once, after its
-    {% for label, c in unwired_temps.items() %}{{ label }} {{ "%.0f"|format(c) }}°C{{ ", " if not loop.last }}{% endfor %}.</p>
+     sensors, hottest first within it. #}
+  <p class="field-hint" style="margin:0.5rem 0 0;">Not counted, likely nothing wired to them:
+    {% for why, group in unwired_temps.items()|groupby("1.1") %}{% for label, v in group %}{{ label }} {{ "%.0f"|format(v[0]) }}°C{{ ", " if not loop.last }}{% endfor %} ({{ why }}){{ "; " if not loop.last }}{% endfor %}.</p>
   {% endif %}
 </div>
 {% endif %}

tests/plugins/host_agent/test_expand_metrics.py

@@ -162,7 +162,8 @@ def test_the_page_lists_unwired_sensors_apart_hottest_first():
     """VDNT-Util01's sensor list read SYSTIN 115 °C in red at the top, though
     the host's temperature already left it out (#5759)."""
     from plugins.host_agent.routes import split_temps
-    real, unwired = split_temps({"SYSTIN": 115.0, "AUXTIN0": 110.0, "Package id 0": 52.0,
+    real, unwired = split_temps("h", {"SYSTIN": 115.0, "AUXTIN0": 110.0, "Package id 0": 52.0,
-                                 "Composite": 48.9, "PCH_CHIP_TEMP": 0.0, "CPUTIN": 47.0, "gone": None})
+                                      "Composite": 48.9, "PCH_CHIP_TEMP": 0.0, "CPUTIN": 47.0, "gone": None})
     assert list(real) == ["Package id 0", "Composite", "CPUTIN", "gone"]
     assert list(unwired) == ["SYSTIN", "AUXTIN0", "PCH_CHIP_TEMP"]
+    assert unwired["SYSTIN"] == (115.0, "reads a value no wired sensor gives")

tests/plugins/host_agent/test_host_health.py

@@ -77,3 +77,21 @@ async def test_the_metrics_page_grades_by_the_same_cutoffs(app):
     assert 'class="stat-val tone-warn" style="font-size:1.6rem;">72°C' in page
     assert '<span class="tone-crit" style="font-variant-numeric:tabular-nums;">90°C' in page
     assert not hex_colours(page), hex_colours(page)
+
+
+async def test_sensors_not_counted_are_listed_once_per_reason(app):
+    """Util01's five open inputs read as one list with its reason, not five
+    repeats of it; a sensor judged against the CPU carries its own (#5783)."""
+    held = "reads a value no wired sensor gives"
+    falls = "falls as the CPU warms (r -0.81 over 168 h)"
+    async with app.test_request_context("/plugins/host_agent/h1/metrics"):
+        page = await render_plugin_template(
+            app, "host_agent", "_host_metrics.html", host=SimpleNamespace(address="10.0.0.2"), stale=False,
+            reg=SimpleNamespace(distro=None, kernel=None, arch=None, agent_version=None,
+                                last_seen_at=datetime.now(timezone.utc)),
+            hostlvl={"temp_c_max": 52.0}, cores=[], nets={}, disks_io={}, temps={"Package id 0": 52.0},
+            unwired_temps={"SYSTIN": (115.0, held), "AUXTIN0": (110.0, held), "temp3": (62.0, falls)},
+            mounts={})
+    page = " ".join(page.split())
+    assert (f"Not counted, likely nothing wired to them: temp3 62°C ({falls}); "
+            f"SYSTIN 115°C, AUXTIN0 110°C ({held}).") in page

tests/plugins/host_agent/test_sensors.py

@@ -0,0 +1,74 @@
+"""Which temperature sensors are thermometers, judged from how each moves
+against the host's CPU (#5783). VDNT-Util01's SYSTIN ran at r = −0.81
+against its CPU package over a week; CPUTIN beside it at +0.83 (#5782)."""
+
+import math
+from datetime import datetime, timedelta, timezone
+
+from plugins.host_agent import sensors as s
+
+T0 = datetime(2026, 10, 8, tzinfo=timezone.utc)
+
+
+def _hours(n, f):
+    """{hour: f(i)} for n hourly buckets."""
+    return {T0 + timedelta(hours=i): f(i) for i in range(n)}
+
+
+def cpu(i):                                  # a CPU that swings 50-80 °C through the day
+    return 65 + 15 * math.sin(i / 24 * 2 * math.pi)
+
+
+def test_a_sensor_that_falls_as_the_cpu_warms_is_not_a_thermometer():
+    ref = _hours(96, cpu)
+    systin = _hours(96, lambda i: 114 - 0.1 * (cpu(i) - 65))       # Util01's shape
+    why = s.judge(ref, systin)
+    assert why.startswith("falls as the CPU warms (r -1.00 over 96 h")
+
+
+def test_a_sensor_that_holds_still_is_not_a_thermometer():
+    why = s.judge(_hours(96, cpu), _hours(96, lambda i: 110.0 + (0.1 if i % 2 else 0.0)))
+    assert why == "holds at 110 °C while the CPU moved 50–80 °C"
+
+
+def test_a_wired_sensor_is_left_alone_even_when_it_barely_follows_the_cpu():
+    ref = _hours(96, cpu)
+    assert s.judge(ref, _hours(96, lambda i: 40 + 0.3 * (cpu(i) - 65))) is None     # a board sensor
+    assert s.judge(ref, _hours(96, lambda i: 45 + 3 * math.sin(i))) is None          # a drive, its own rhythm
+
+
+def test_nothing_is_judged_without_enough_history_or_a_cpu_that_moved():
+    falls = lambda i: 114 - 0.1 * (cpu(i) - 65)                                      # noqa: E731
+    assert s.judge(_hours(48, cpu), _hours(48, falls)) is None                      # 48 h < MIN_HOURS
+    idle = _hours(96, lambda i: 50 + (i % 3))                                       # moved 2 °C
+    assert s.judge(idle, _hours(96, lambda i: 110 - (i % 3))) is None
+
+
+def test_the_cpu_reference_is_found_by_its_driver_names():
+    assert s.reference_label(["SYSTIN", "Core 0", "Package id 1", "Package id 0"]) == "Package id 0"
+    assert s.reference_label(["Tccd1", "Tctl", "Composite"]) == "Tctl"
+    assert s.reference_label(["PECI Agent 0", "SYSTIN"]) == "PECI Agent 0"
+    assert s.reference_label(["Composite", "temp1"]) is None
+
+
+def test_hosts_are_judged_from_hourly_rows_and_a_host_without_a_cpu_reading_is_skipped():
+    rows = []
+    for i in range(96):
+        h = T0 + timedelta(hours=i)
+        rows += [("util:temp:Package id 0", h, cpu(i)),
+                 ("util:temp:SYSTIN", h, 114 - 0.1 * (cpu(i) - 65)),
+                 ("util:temp:CPUTIN", h, 40 + 0.3 * (cpu(i) - 65)),
+                 ("nas:temp:temp1", h, 110 - 0.1 * (cpu(i) - 65)),
+                 ("util", h, 1.0)]                                                   # not a sensor row
+    got = s.judge_hosts(rows)
+    assert list(got) == ["util"] and list(got["util"]) == ["SYSTIN"]
+
+
+def test_a_judged_sensor_is_left_out_of_the_host_temperature_and_listed_with_why(monkeypatch):
+    from plugins.host_agent.routes import is_thermometer, split_temps
+    monkeypatch.setattr(s, "_unwired", {"util": {"temp3": "falls as the CPU warms (r -0.81 over 168 h)"}})
+    assert not is_thermometer("util", "temp3", 62.0)       # a plausible value, still not a thermometer
+    assert is_thermometer("other", "temp3", 62.0)
+    real, unwired = split_temps("util", {"temp3": 62.0, "Package id 0": 55.0})
+    assert list(real) == ["Package id 0"]
+    assert unwired == {"temp3": (62.0, "falls as the CPU warms (r -0.81 over 168 h)")}