fix: error-pages graded healthy, unwired sensors listed apart, Maintenance failed reads 0 (#5780, #5781, #5782)
- #5780 "The error-pages service is graded on other routers' errors". The Traefik scrape reads which services an errors middleware sends to, and grades them healthy: their 4xx and 5xx are the codes of the errors they cover for, already graded on the routers that failed. - #5781 "The Maintenance failed rules sit at no data until a host has had a maintenance job". Every host with an agent reads 0 until a job fails. - #5782 "VDNT-Util01's host page lists SYSTIN at 115 °C in red". The host page lists sensors that counts_as_hottest leaves out (unwired board inputs, 0 °C or below) on one muted line, not as heat. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -285,12 +285,17 @@ async def tick(session, now: datetime) -> list[str]:
|
||||
|
||||
|
||||
async def failed_hosts(session, now: datetime) -> dict[str, float]:
|
||||
"""{host name: 1 or 0}: whether each host's latest finished maintenance job
|
||||
failed or timed out, for the maintenance_failed figure an alert rule
|
||||
watches. A later job that succeeds puts it back to 0."""
|
||||
"""{host name: 1 or 0} for every host with an agent: whether its latest
|
||||
finished maintenance job failed or timed out, for the maintenance_failed
|
||||
figure an alert rule watches. A host with no job yet reads 0, so a rule
|
||||
on it reports rather than sitting at "no data". A later job that
|
||||
succeeds puts it back to 0."""
|
||||
from sqlalchemy import select
|
||||
from steward.models.hosts import Host
|
||||
from .models import HostMaintenanceJob
|
||||
from .models import HostAgentRegistration, HostMaintenanceJob
|
||||
agents = (await session.execute(
|
||||
select(Host.name).join(HostAgentRegistration, HostAgentRegistration.host_id == Host.id))).scalars()
|
||||
out = {name: 0.0 for name in agents}
|
||||
rows = (await session.execute(
|
||||
select(Host.name, HostMaintenanceJob.state, HostMaintenanceJob.finished_at)
|
||||
.join(Host, Host.id == HostMaintenanceJob.host_id)
|
||||
@@ -300,4 +305,5 @@ async def failed_hosts(session, now: datetime) -> dict[str, float]:
|
||||
for name, state, finished in rows:
|
||||
if name not in latest or finished > latest[name][1]:
|
||||
latest[name] = (state, finished)
|
||||
return {name: 0.0 if state == "done" else 1.0 for name, (state, _) in latest.items()}
|
||||
out.update({name: 0.0 if state == "done" else 1.0 for name, (state, _) in latest.items()})
|
||||
return out
|
||||
|
||||
@@ -113,6 +113,17 @@ def counts_as_hottest(label: str, celsius: float) -> bool:
|
||||
return not (_BOARD_INPUT.match(label) and celsius >= BOARD_INPUT_CEILING_C)
|
||||
|
||||
|
||||
def split_temps(temps: dict) -> tuple[dict, dict]:
|
||||
"""(real, unwired): a host's sensors, hottest first, apart from the ones
|
||||
counts_as_hottest leaves out (a board input with nothing wired to it, a
|
||||
reading at or below 0 °C). Those read a constant at any load, so the
|
||||
page lists them apart instead of colouring them as the host's heat."""
|
||||
def by_heat(d: dict) -> dict:
|
||||
return dict(sorted(d.items(), key=lambda kv: -(kv[1] if kv[1] is not None else -999)))
|
||||
real = {k: c for k, c in temps.items() if c is None or counts_as_hottest(k, c)}
|
||||
return by_heat(real), by_heat({k: c for k, c in temps.items() if k not in real})
|
||||
|
||||
|
||||
def _ts_or(value, fallback: datetime) -> datetime:
|
||||
"""An agent-sent ISO time, or `fallback` when it isn't one."""
|
||||
try:
|
||||
@@ -856,12 +867,13 @@ async def host_detail_metrics(host_id: str):
|
||||
HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
|
||||
latest = await _latest_metrics_for_host(session, host.name)
|
||||
hostlvl, cores, nets, disks_io, temps, mounts = _split_host_metrics(host.name, latest)
|
||||
temps, unwired = split_temps(temps)
|
||||
ls = reg.last_seen_at if reg else None
|
||||
stale = ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()
|
||||
return await render_template(
|
||||
"_host_metrics.html",
|
||||
host=host, reg=reg, stale=stale, hostlvl=hostlvl,
|
||||
cores=cores, nets=nets, disks_io=disks_io, temps=temps, mounts=mounts,
|
||||
cores=cores, nets=nets, disks_io=disks_io, temps=temps, unwired_temps=unwired, mounts=mounts,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -159,6 +159,12 @@
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% if unwired_temps %}
|
||||
{# Board inputs with nothing wired to them, and readings at or below 0 °C:
|
||||
constant at any load, left out of the host's temperature (#5759). #}
|
||||
<p class="field-hint" style="margin:0.5rem 0 0;">Not counted, reading a constant with nothing wired to them:
|
||||
{% for label, c in unwired_temps.items() %}{{ label }} {{ "%.0f"|format(c) }}°C{{ ", " if not loop.last }}{% endfor %}.</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user