diff --git a/docs/plugins/host-agent-design.md b/docs/plugins/host-agent-design.md index d6f9688..c30387f 100644 --- a/docs/plugins/host-agent-design.md +++ b/docs/plugins/host-agent-design.md @@ -262,6 +262,15 @@ A post may carry `events`, said once each, with the next post that gets through: Steward turns them into the host's downtime (`host_downtime`): **reboot** (a stop, then a boot after it), **agent_stop** (the host stayed up), **unclean** (a boot after the agent was last heard that nobody announced: power loss or a crash) and **lost_contact**. An announced stop explains the silence for `reboot_grace_seconds` (Settings → Plugins → host_agent, 10 minutes): `seconds_since_seen` reads 0 until then, and the host's other rules go quiet with it, as when it is offline. Past the grace, agent offline fires. The events live in memory, so a span the agent saw is lost if it restarts mid-outage; the boot time still reads the restart as unclean. +### Which temperature sensors are thermometers (#5782, #5783) + +A board's spare thermistor inputs (a Nuvoton's SYSTIN, AUXTIN*) are often wired to nothing, and the kernel reports them anyway, at whatever the open input reads. The host's temperature (`temp_c_max`) leaves a sensor out when: + +- **its value is impossible** (at or below 0 °C, or at or above 125 °C; `counts_as_hottest`), or +- **it doesn't behave like a thermometer.** Once an hour, `sensors.refresh` judges each sensor against the host's CPU (Intel `Package id N`, AMD `Tctl`/`Tdie`/`TccdN`, else `PECI Agent N`), using the last 7 days of hourly averages. It needs at least 72 shared hours and a CPU that moved at least 10 °C. A sensor is ruled out when it **falls as the CPU warms** (Pearson r ≤ −0.5, the shape of an open input reading the chip's own reference) or **holds within 0.5 °C** while the CPU moved. A wired board or drive sensor that barely follows the CPU (r near 0) stays in. + +The verdicts are kept in memory and rebuilt at startup, so ingest and the host page read them without a query. The host page lists the sensors left out under "Not counted", each with its reason. A host with no CPU reading, or too little history, falls back to the value rule alone. + ### Logging To stderr only (systemd captures to journal). No file logging, no log rotation. Levels: diff --git a/plugins/host_agent/__init__.py b/plugins/host_agent/__init__.py index 5d83b0b..b8a7ba5 100644 --- a/plugins/host_agent/__init__.py +++ b/plugins/host_agent/__init__.py @@ -43,8 +43,8 @@ def get_audit_entities() -> dict: def get_scheduled_tasks() -> list: - from .scheduler import make_maintenance_task, make_task - return [make_task(_app), make_maintenance_task(_app)] + from .scheduler import make_maintenance_task, make_sensor_task, make_task + return [make_task(_app), make_maintenance_task(_app), make_sensor_task(_app)] def get_blueprint(): diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index a845f0a..eaba7c7 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -113,15 +113,28 @@ def counts_as_hottest(label: str, celsius: float) -> bool: return not (_BOARD_INPUT.match(label) and celsius >= BOARD_INPUT_CEILING_C) -def split_temps(temps: dict) -> tuple[dict, dict]: +def is_thermometer(host_name: str, label: str, celsius: float) -> bool: + """Whether a sensor's reading may count as the host's temperature: not one + judged unwired from how it moves against the CPU (sensors.py, #5783), + and a reading counts_as_hottest allows.""" + from . import sensors + return label not in sensors.unwired(host_name) and counts_as_hottest(label, celsius) + + +def split_temps(host_name: str, temps: dict) -> tuple[dict, dict]: """(real, unwired): a host's sensors, hottest first, apart from the ones - counts_as_hottest leaves out (a board input with nothing wired to it, a - reading at or below 0 °C). Those read a constant at any load, so the - page lists them apart instead of colouring them as the host's heat.""" - def by_heat(d: dict) -> dict: - return dict(sorted(d.items(), key=lambda kv: -(kv[1] if kv[1] is not None else -999))) - real = {k: c for k, c in temps.items() if c is None or counts_as_hottest(k, c)} - return by_heat(real), by_heat({k: c for k, c in temps.items() if k not in real}) + is_thermometer leaves out. Unwired ones map to (reading, why); they read + a constant or run against the CPU, so the page lists them apart instead + of colouring them as the host's heat.""" + from . import sensors + judged = sensors.unwired(host_name) + + def by_heat(d: dict, value=lambda v: v) -> dict: + return dict(sorted(d.items(), key=lambda kv: -(value(kv[1]) if value(kv[1]) is not None else -999))) + real = {k: c for k, c in temps.items() if c is None or is_thermometer(host_name, k, c)} + unwired = {k: (c, judged.get(k, "reads a value no wired sensor gives")) + for k, c in temps.items() if k not in real} + return by_heat(real), by_heat(unwired, lambda v: v[0]) def _ts_or(value, fallback: datetime) -> datetime: @@ -305,14 +318,15 @@ def _expand_sample_to_metrics( row("disk_write_bps", host_name, dw_total) # Temperatures — per sensor plus the host max for at-a-glance. The max - # counts only readings that can be real (#5759). + # counts only readings that can be real (#5759), from sensors that behave + # like thermometers (#5783). max_temp: float | None = None for t in sample.get("temps") or []: c = t.get("celsius") if c is None: continue row("temp_c", f"{host_name}:temp:{t['label']}", c) - if counts_as_hottest(str(t["label"]), c) and (max_temp is None or c > max_temp): + if is_thermometer(host_name, str(t["label"]), c) and (max_temp is None or c > max_temp): max_temp = c if max_temp is not None: row("temp_c_max", host_name, max_temp) @@ -867,7 +881,7 @@ async def host_detail_metrics(host_id: str): HostAgentRegistration.host_id == host_id))).scalar_one_or_none() latest = await _latest_metrics_for_host(session, host.name) hostlvl, cores, nets, disks_io, temps, mounts = _split_host_metrics(host.name, latest) - temps, unwired = split_temps(temps) + temps, unwired = split_temps(host.name, temps) ls = reg.last_seen_at if reg else None stale = ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds() return await render_template( diff --git a/plugins/host_agent/scheduler.py b/plugins/host_agent/scheduler.py index 1f9b8e7..96ca90c 100644 --- a/plugins/host_agent/scheduler.py +++ b/plugins/host_agent/scheduler.py @@ -121,6 +121,29 @@ def make_task(app) -> ScheduledTask: ) +def make_sensor_task(app) -> ScheduledTask: + """Re-judges which temperature sensors are thermometers (#5783), hourly + and at startup, from the hourly averages: verdicts live in memory, so a + restart rebuilds them before the first hour is out.""" + async def _judge(): + from . import sensors + try: + async with app.db_sessionmaker() as session: + found = await sensors.refresh(session, datetime.now(timezone.utc)) + for host, labels in found.items(): + logger.info("host_agent: %s sensors not counted as temperature: %s", + host, ", ".join(sorted(labels))) + except Exception: + logger.exception("host_agent: judging temperature sensors failed") + + return ScheduledTask( + name="host_agent_sensors", + coro_factory=_judge, + interval_seconds=3600, + run_on_startup=True, + ) + + def make_maintenance_task(app) -> ScheduledTask: """Gives up on maintenance jobs past their deadline (#5767), starts due maintenance plans and moves their runs on one host at a time (#5768), and diff --git a/plugins/host_agent/sensors.py b/plugins/host_agent/sensors.py new file mode 100644 index 0000000..8579d4c --- /dev/null +++ b/plugins/host_agent/sensors.py @@ -0,0 +1,123 @@ +"""Which of a host's temperature sensors are thermometers (#5783). + +A board's spare thermistor inputs often have nothing wired to them, and the +kernel reports them anyway. No flag says so; their behaviour does. A +thermometer in a host warms as its CPU warms. An open input reads the chip's +own reference, which drifts the other way as the chip warms, or holds still: +VDNT-Util01's SYSTIN ran at r = −0.81 against its CPU package over a week, +while CPUTIN beside it ran at +0.83 (#5782). + +So each sensor is judged against the host's CPU over its hourly averages, by +a scheduled task, and the verdicts are kept here for ingest and the pages to +read. Until a host has enough history, or when its CPU never moved enough to +tell, nothing is judged and routes.counts_as_hottest's value rule decides. +""" +from __future__ import annotations + +import logging +import re +from datetime import datetime, timedelta +from math import sqrt + +logger = logging.getLogger(__name__) + +# Hours of history judged, the shared hours a verdict needs, and how far the +# CPU must have moved in them for "it didn't follow" to mean anything. +WINDOW = timedelta(days=7) +MIN_HOURS = 72 +REFERENCE_SWING_C = 10.0 +# Falling as the CPU warms, this strongly, is no thermometer's behaviour. +# A wired board or drive sensor may barely follow the CPU (r near 0), so +# only a clear negative counts, not a weak one. +FALLS_R = -0.5 +# Held within this over days while the CPU moved: a constant, not a reading. +FLAT_C = 0.5 + +# The host's CPU, in order of preference: Intel coretemp's package, AMD +# k10temp's control/die/CCD readings, then the PECI agent. +_REFERENCES = (re.compile(r"^Package id \d+$", re.I), re.compile(r"^(Tctl|Tdie)$", re.I), + re.compile(r"^Tccd\d+$", re.I), re.compile(r"^PECI Agent \d+$", re.I)) + +# {host name: {sensor label: why it isn't a thermometer}}, from the last refresh. +_unwired: dict[str, dict[str, str]] = {} + + +def unwired(host_name: str) -> dict[str, str]: + """The sensors judged not to be thermometers on a host, with why.""" + return _unwired.get(host_name, {}) + + +def reference_label(labels) -> str | None: + """The sensor that reads the host's CPU, or None when it has none.""" + labels = sorted(labels) + for pattern in _REFERENCES: + found = [label for label in labels if pattern.match(label)] + if found: + return found[0] + return None + + +def pearson(xs: list[float], ys: list[float]) -> float | None: + """Pearson's r, or None when either series doesn't move.""" + n = len(xs) + mx, my = sum(xs) / n, sum(ys) / n + sxx = sum((x - mx) ** 2 for x in xs) + syy = sum((y - my) ** 2 for y in ys) + if sxx == 0 or syy == 0: + return None + return sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / sqrt(sxx * syy) + + +def judge(reference: dict[datetime, float], sensor: dict[datetime, float]) -> str | None: + """Why `sensor` isn't a thermometer, judged against the host's CPU over + the hours both have; None when it behaves like one, or when there isn't + enough to tell (too few hours, or a CPU that never moved).""" + hours = sorted(set(reference) & set(sensor)) + if len(hours) < MIN_HOURS: + return None + cpu = [reference[h] for h in hours] + got = [sensor[h] for h in hours] + if max(cpu) - min(cpu) < REFERENCE_SWING_C: + return None + if max(got) - min(got) < FLAT_C: + return f"holds at {sum(got) / len(got):.0f} °C while the CPU moved {min(cpu):.0f}–{max(cpu):.0f} °C" + r = pearson(cpu, got) + if r is not None and r <= FALLS_R: + return f"falls as the CPU warms (r {r:.2f} over {len(hours)} h)" + return None + + +def judge_hosts(rows) -> dict[str, dict[str, str]]: + """{host: {label: why}} from hourly rows of (resource, bucket, value_avg), + where resource is ":temp: