feat(host_agent): a temperature sensor is judged by how it moves against the CPU (#5783)

A board's spare thermistor inputs report whatever an open input reads, and
the value rule from #5759 only catches the impossible ones. Once an hour,
sensors.refresh judges each sensor against the host's CPU (Package id N,
Tctl/Tdie/TccdN, PECI Agent N) over a week of hourly averages. A sensor is
ruled out if it falls as the CPU warms (r <= -0.5) or holds within 0.5 °C
while the CPU moved 10 °C or more. A judgment needs at least 72 shared hours.

The host temperature leaves judged sensors out at ingest. The host page
lists them under "Not counted", with each reason given once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/sensors.py

@@ -0,0 +1,123 @@
+"""Which of a host's temperature sensors are thermometers (#5783).
+
+A board's spare thermistor inputs often have nothing wired to them, and the
+kernel reports them anyway. No flag says so; their behaviour does. A
+thermometer in a host warms as its CPU warms. An open input reads the chip's
+own reference, which drifts the other way as the chip warms, or holds still:
+VDNT-Util01's SYSTIN ran at r = −0.81 against its CPU package over a week,
+while CPUTIN beside it ran at +0.83 (#5782).
+
+So each sensor is judged against the host's CPU over its hourly averages, by
+a scheduled task, and the verdicts are kept here for ingest and the pages to
+read. Until a host has enough history, or when its CPU never moved enough to
+tell, nothing is judged and routes.counts_as_hottest's value rule decides.
+"""
+from __future__ import annotations
+
+import logging
+import re
+from datetime import datetime, timedelta
+from math import sqrt
+
+logger = logging.getLogger(__name__)
+
+# Hours of history judged, the shared hours a verdict needs, and how far the
+# CPU must have moved in them for "it didn't follow" to mean anything.
+WINDOW = timedelta(days=7)
+MIN_HOURS = 72
+REFERENCE_SWING_C = 10.0
+# Falling as the CPU warms, this strongly, is no thermometer's behaviour.
+# A wired board or drive sensor may barely follow the CPU (r near 0), so
+# only a clear negative counts, not a weak one.
+FALLS_R = -0.5
+# Held within this over days while the CPU moved: a constant, not a reading.
+FLAT_C = 0.5
+
+# The host's CPU, in order of preference: Intel coretemp's package, AMD
+# k10temp's control/die/CCD readings, then the PECI agent.
+_REFERENCES = (re.compile(r"^Package id \d+$", re.I), re.compile(r"^(Tctl|Tdie)$", re.I),
+               re.compile(r"^Tccd\d+$", re.I), re.compile(r"^PECI Agent \d+$", re.I))
+
+# {host name: {sensor label: why it isn't a thermometer}}, from the last refresh.
+_unwired: dict[str, dict[str, str]] = {}
+
+
+def unwired(host_name: str) -> dict[str, str]:
+    """The sensors judged not to be thermometers on a host, with why."""
+    return _unwired.get(host_name, {})
+
+
+def reference_label(labels) -> str | None:
+    """The sensor that reads the host's CPU, or None when it has none."""
+    labels = sorted(labels)
+    for pattern in _REFERENCES:
+        found = [label for label in labels if pattern.match(label)]
+        if found:
+            return found[0]
+    return None
+
+
+def pearson(xs: list[float], ys: list[float]) -> float | None:
+    """Pearson's r, or None when either series doesn't move."""
+    n = len(xs)
+    mx, my = sum(xs) / n, sum(ys) / n
+    sxx = sum((x - mx) ** 2 for x in xs)
+    syy = sum((y - my) ** 2 for y in ys)
+    if sxx == 0 or syy == 0:
+        return None
+    return sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / sqrt(sxx * syy)
+
+
+def judge(reference: dict[datetime, float], sensor: dict[datetime, float]) -> str | None:
+    """Why `sensor` isn't a thermometer, judged against the host's CPU over
+    the hours both have; None when it behaves like one, or when there isn't
+    enough to tell (too few hours, or a CPU that never moved)."""
+    hours = sorted(set(reference) & set(sensor))
+    if len(hours) < MIN_HOURS:
+        return None
+    cpu = [reference[h] for h in hours]
+    got = [sensor[h] for h in hours]
+    if max(cpu) - min(cpu) < REFERENCE_SWING_C:
+        return None
+    if max(got) - min(got) < FLAT_C:
+        return f"holds at {sum(got) / len(got):.0f} °C while the CPU moved {min(cpu):.0f}–{max(cpu):.0f} °C"
+    r = pearson(cpu, got)
+    if r is not None and r <= FALLS_R:
+        return f"falls as the CPU warms (r {r:.2f} over {len(hours)} h)"
+    return None
+
+
+def judge_hosts(rows) -> dict[str, dict[str, str]]:
+    """{host: {label: why}} from hourly rows of (resource, bucket, value_avg),
+    where resource is "<host>:temp:<label>"."""
+    series: dict[str, dict[str, dict[datetime, float]]] = {}
+    for resource, bucket, value in rows:
+        host, sep, label = resource.partition(":temp:")
+        if sep and value is not None:
+            series.setdefault(host, {}).setdefault(label, {})[bucket] = value
+    out: dict[str, dict[str, str]] = {}
+    for host, sensors in series.items():
+        ref = reference_label(sensors)
+        if ref is None:
+            continue
+        for label, points in sensors.items():
+            if label == ref:
+                continue
+            why = judge(sensors[ref], points)
+            if why:
+                out.setdefault(host, {})[label] = why
+    return out
+
+
+async def refresh(session, now: datetime) -> dict[str, dict[str, str]]:
+    """Re-judge every host's sensors from the last WINDOW of hourly averages,
+    and keep the verdicts for ingest and the pages."""
+    global _unwired
+    from sqlalchemy import select
+    from steward.models.metrics import PluginMetricHourly as H
+    rows = (await session.execute(
+        select(H.resource_name, H.bucket, H.value_avg)
+        .where(H.source_module == "host_agent", H.metric_name == "temp_c",
+               H.resource_name.like("%:temp:%"), H.bucket >= now - WINDOW))).all()
+    _unwired = judge_hosts(rows)
+    return _unwired