fix: error-pages graded healthy, unwired sensors listed apart, Maintenance failed reads 0 (#5780, #5781, #5782)

- #5780 "The error-pages service is graded on other routers' errors". The
  Traefik scrape reads which services an errors middleware sends to, and
  grades them healthy: their 4xx and 5xx are the codes of the errors they
  cover for, already graded on the routers that failed.
- #5781 "The Maintenance failed rules sit at no data until a host has had
  a maintenance job". Every host with an agent reads 0 until a job fails.
- #5782 "VDNT-Util01's host page lists SYSTIN at 115 °C in red". The host
  page lists sensors that counts_as_hottest leaves out (unwired board
  inputs, 0 °C or below) on one muted line, not as heat.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/cadence.py

@@ -285,12 +285,17 @@ async def tick(session, now: datetime) -> list[str]:
 
 
 async def failed_hosts(session, now: datetime) -> dict[str, float]:
-    """{host name: 1 or 0}: whether each host's latest finished maintenance job
+    """{host name: 1 or 0} for every host with an agent: whether its latest
-    failed or timed out, for the maintenance_failed figure an alert rule
+    finished maintenance job failed or timed out, for the maintenance_failed
-    watches. A later job that succeeds puts it back to 0."""
+    figure an alert rule watches. A host with no job yet reads 0, so a rule
+    on it reports rather than sitting at "no data". A later job that
+    succeeds puts it back to 0."""
     from sqlalchemy import select
     from steward.models.hosts import Host
-    from .models import HostMaintenanceJob
+    from .models import HostAgentRegistration, HostMaintenanceJob
+    agents = (await session.execute(
+        select(Host.name).join(HostAgentRegistration, HostAgentRegistration.host_id == Host.id))).scalars()
+    out = {name: 0.0 for name in agents}
     rows = (await session.execute(
         select(Host.name, HostMaintenanceJob.state, HostMaintenanceJob.finished_at)
         .join(Host, Host.id == HostMaintenanceJob.host_id)
@@ -300,4 +305,5 @@ async def failed_hosts(session, now: datetime) -> dict[str, float]:
     for name, state, finished in rows:
         if name not in latest or finished > latest[name][1]:
             latest[name] = (state, finished)
-    return {name: 0.0 if state == "done" else 1.0 for name, (state, _) in latest.items()}
+    out.update({name: 0.0 if state == "done" else 1.0 for name, (state, _) in latest.items()})
+    return out

plugins/host_agent/routes.py

@@ -113,6 +113,17 @@ def counts_as_hottest(label: str, celsius: float) -> bool:
     return not (_BOARD_INPUT.match(label) and celsius >= BOARD_INPUT_CEILING_C)
 
 
+def split_temps(temps: dict) -> tuple[dict, dict]:
+    """(real, unwired): a host's sensors, hottest first, apart from the ones
+    counts_as_hottest leaves out (a board input with nothing wired to it, a
+    reading at or below 0 °C). Those read a constant at any load, so the
+    page lists them apart instead of colouring them as the host's heat."""
+    def by_heat(d: dict) -> dict:
+        return dict(sorted(d.items(), key=lambda kv: -(kv[1] if kv[1] is not None else -999)))
+    real = {k: c for k, c in temps.items() if c is None or counts_as_hottest(k, c)}
+    return by_heat(real), by_heat({k: c for k, c in temps.items() if k not in real})
+
+
 def _ts_or(value, fallback: datetime) -> datetime:
     """An agent-sent ISO time, or `fallback` when it isn't one."""
     try:
@@ -856,12 +867,13 @@ async def host_detail_metrics(host_id: str):
             HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
         latest = await _latest_metrics_for_host(session, host.name)
     hostlvl, cores, nets, disks_io, temps, mounts = _split_host_metrics(host.name, latest)
+    temps, unwired = split_temps(temps)
     ls = reg.last_seen_at if reg else None
     stale = ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()
     return await render_template(
         "_host_metrics.html",
         host=host, reg=reg, stale=stale, hostlvl=hostlvl,
-        cores=cores, nets=nets, disks_io=disks_io, temps=temps, mounts=mounts,
+        cores=cores, nets=nets, disks_io=disks_io, temps=temps, unwired_temps=unwired, mounts=mounts,
     )
 
 

plugins/host_agent/templates/_host_metrics.html

@@ -159,6 +159,12 @@
     </div>
     {% endfor %}
   </div>
+  {% if unwired_temps %}
+  {# Board inputs with nothing wired to them, and readings at or below 0 °C:
+     constant at any load, left out of the host's temperature (#5759). #}
+  <p class="field-hint" style="margin:0.5rem 0 0;">Not counted, reading a constant with nothing wired to them:
+    {% for label, c in unwired_temps.items() %}{{ label }} {{ "%.0f"|format(c) }}°C{{ ", " if not loop.last }}{% endfor %}.</p>
+  {% endif %}
 </div>
 {% endif %}
 

plugins/traefik/routes.py

@@ -18,7 +18,7 @@ from steward.core.settings import floor_of
 from steward.models.metrics import PluginMetricLatest
 from . import meaning
 from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
-from .scheduler import router_domains
+from .scheduler import error_page_services, router_domains
 from .scraper import cert_names, covers
 
 traefik_bp = Blueprint("traefik", __name__, template_folder="templates")
@@ -175,12 +175,21 @@ async def status_codes(db, names: list[str], since: datetime) -> dict[str, list[
     return out
 
 
-def grade_routers(routers: list[dict], window: dict, thresholds: dict, norms: dict | None = None) -> None:
+ERROR_PAGES_WHY = "serves other routers' error pages: its 4xx and 5xx are theirs, graded on them"
+
+
+def grade_routers(routers: list[dict], window: dict, thresholds: dict, norms: dict | None = None,
+                  error_pages: frozenset[str] = frozenset()) -> None:
     """Set each router's state and why from its recent window, or from its
     latest scrape when the window holds nothing, against its normals
-    (router_norms') where it has them."""
+    (router_norms') where it has them. A service Traefik's errors middleware
+    sends error pages to (`error_pages`) is healthy whatever its codes: it
+    answers with the code of the error it covers for (#5780)."""
     norms = norms or {}
     for r in routers:
+        if r["name"] in error_pages:
+            r["state"], r["why"], r["kind"], r["normed"] = "ok", ERROR_PAGES_WHY, None, False
+            continue
         reading, own = window.get(r["name"]), norms.get(r["name"])
         r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS, own) if reading
                                 else router_state(r["latest"], thresholds, norms=own))
@@ -365,7 +374,7 @@ async def rows():
         history = await router_history(db, names, since, bucket_seconds(since))
         phases.mark("history")
         grade_routers(current, await recent_window(db, names, now), thresholds,
-                      await router_norms(db, names, now))
+                      await router_norms(db, names, now), error_page_services())
         phases.mark("window")
         # The routers' metric anomalies, last 30 days (#5741).
         anomalies = await anom.views(db, await anom.list_anomalies(
@@ -479,7 +488,7 @@ async def widget():
     served, _old = current_certs(certs, scrape)
     domains = router_domains()
     by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
-    grade_routers(current, window, thresholds, norms)
+    grade_routers(current, window, thresholds, norms, error_page_services())
     for r in current:
         r["cert"] = by_router.get(r["name"])
         if r["cert"]:

plugins/traefik/scheduler.py

@@ -26,6 +26,16 @@ def router_domains() -> dict[str, list[str]] | None:
     return _router_domains
 
 
+# The services Traefik's errors middlewares send error pages to (#5780), as
+# last read from its API. Kept through an API blip: that config rarely moves,
+# and dropping it would grade the error pages as failing for that while.
+_error_page_services: frozenset[str] = frozenset()
+
+
+def error_page_services() -> frozenset[str]:
+    return _error_page_services
+
+
 # Access log state
 _geoip_db = None
 _access_log_last_tick: "datetime | None" = None
@@ -48,7 +58,7 @@ def make_scrape_task(app: "Quart") -> ScheduledTask:
 
 async def _do_scrape(app: "Quart") -> None:
     """Fetch Traefik metrics, write to DB, and emit plugin_metrics."""
-    global _prev_metrics, _prev_time, _router_domains
+    global _prev_metrics, _prev_time, _router_domains, _error_page_services
 
     from datetime import datetime, timezone
     from .models import TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
@@ -59,6 +69,7 @@ async def _do_scrape(app: "Quart") -> None:
         compute_status_codes,
         extract_certs,
         fetch_metrics,
+        fetch_error_page_services,
         fetch_router_domains,
     )
     from steward.core.alerts import MetricBatch
@@ -77,6 +88,10 @@ async def _do_scrape(app: "Quart") -> None:
         if _router_domains is not None:
             logger.warning("Traefik API stopped answering (%s); certificates shown without routers", exc)
         _router_domains = None
+    try:
+        _error_page_services = await fetch_error_page_services(api_base(metrics_url, cfg.get("api_url") or ""))
+    except Exception as exc:
+        logger.debug("Traefik middlewares not read (%s); keeping the last error-page services", exc)
 
     now = time.monotonic()
     elapsed = now - _prev_time if _prev_time else 60.0

plugins/traefik/scraper.py

@@ -341,6 +341,32 @@ async def fetch_router_domains(base: str) -> dict[str, list[str]]:
     return out
 
 
+async def fetch_error_page_services(base: str) -> frozenset[str]:
+    """The services Traefik's `errors` middlewares send error pages to, named
+    as the metrics name them ("error-pages-service@swarm"). Such a service
+    answers with the code of the error it covers for: its 4xx and 5xx are
+    other routers' errors, already graded on them."""
+    async with httpx.AsyncClient(timeout=10.0) as client:
+        response = await client.get(f"{base}/http/middlewares", params={"per_page": 1000})
+        response.raise_for_status()
+        return error_page_services(response.json())
+
+
+def error_page_services(middlewares: list[dict]) -> frozenset[str]:
+    """fetch_error_page_services' reading of /api/http/middlewares: a service
+    named without a provider belongs to its middleware's provider."""
+    out = set()
+    for m in middlewares:
+        service = (m.get("errors") or {}).get("service")
+        if not service:
+            continue
+        if "@" not in service:
+            provider = m.get("provider") or (m.get("name", "").partition("@")[2])
+            service = f"{service}@{provider}" if provider else service
+        out.add(service)
+    return frozenset(out)
+
+
 def cert_names(cn: str | None, sans: str | None) -> set[str]:
     """Every name a certificate covers: its CN and SANs, lower-cased."""
     names = {n.strip().lower() for n in (sans or "").split(",") if n.strip()}

tests/integration/test_host_maintenance_cadence.py

@@ -107,7 +107,9 @@ def test_hosts_go_one_at_a_time_and_a_failure_stops_the_run():
             async with s.begin():
                 plan = await s.get(models.HostMaintenancePlan, seed["plan_id"])
                 await cadence.start_run(s, plan, datetime.now(timezone.utc), manual=True)
-        await _tick(app)
+        first = await _tick(app)
+        # Before any job finishes, every agent host reads 0 rather than nothing.
+        assert first[seed["hosts"]["a"].name] == 0.0 and first[seed["hosts"]["b"].name] == 0.0
 
         # Only the first host (by name) has a job; the second waits.
         ta, tb = seed["tokens"]["a"], seed["tokens"]["b"]

tests/plugins/host_agent/test_expand_metrics.py

@@ -156,3 +156,13 @@ def test_the_registration_keeps_the_host_s_updates_and_reboot():
                         "reboot": {"required": False}}, now)
     assert reg.updates_pending == 4 and reg.updates_error == "apt is locked"
     assert reg.reboot_required is False and reg.reboot_required_since is None
+
+
+def test_the_page_lists_unwired_sensors_apart_hottest_first():
+    """VDNT-Util01's sensor list read SYSTIN 115 °C in red at the top, though
+    the host's temperature already left it out (#5759)."""
+    from plugins.host_agent.routes import split_temps
+    real, unwired = split_temps({"SYSTIN": 115.0, "AUXTIN0": 110.0, "Package id 0": 52.0,
+                                 "Composite": 48.9, "PCH_CHIP_TEMP": 0.0, "CPUTIN": 47.0, "gone": None})
+    assert list(real) == ["Package id 0", "Composite", "CPUTIN", "gone"]
+    assert list(unwired) == ["SYSTIN", "AUXTIN0", "PCH_CHIP_TEMP"]

tests/plugins/traefik/test_meaning.py

@@ -127,3 +127,24 @@ async def test_a_card_for_an_unknown_service_says_so(app):
     assert "Status codes not recorded yet" in card
     assert "The app is answering with server errors" in card
     assert "Steward can't tell which service serves this router" in card
+
+
+def test_the_error_pages_service_is_found_from_traefiks_errors_middlewares():
+    from plugins.traefik.scraper import error_page_services
+    got = error_page_services([
+        {"name": "error-pages@swarm", "provider": "swarm", "errors": {"service": "error-pages-service",
+                                                                      "status": ["400-599"]}},
+        {"name": "other@file", "errors": {"service": "pages@docker"}},      # already names its provider
+        {"name": "https-redirect@swarm", "redirectScheme": {"scheme": "https"}},
+    ])
+    assert got == {"error-pages-service@swarm", "pages@docker"}
+
+
+def test_the_error_pages_service_is_healthy_whatever_its_codes():
+    """It answers with the code of the error it covers for (#5780)."""
+    from plugins.traefik.routes import ERROR_PAGES_WHY, grade_routers
+    routers = [{"name": "error-pages-service@swarm", "latest": _latest(e4=90.0, e5=8.0)},
+               {"name": "app@swarm", "latest": _latest(e5=8.0)}]
+    grade_routers(routers, {}, TH, error_pages=frozenset({"error-pages-service@swarm"}))
+    assert (routers[0]["state"], routers[0]["why"], routers[0]["kind"]) == ("ok", ERROR_PAGES_WHY, None)
+    assert routers[1]["state"] == "crit" and routers[1]["kind"] == "5xx"