fix: error-pages graded healthy, unwired sensors listed apart, Maintenance failed reads 0 (#5780, #5781, #5782)

- #5780 "The error-pages service is graded on other routers' errors". The
  Traefik scrape reads which services an errors middleware sends to, and
  grades them healthy: their 4xx and 5xx are the codes of the errors they
  cover for, already graded on the routers that failed.
- #5781 "The Maintenance failed rules sit at no data until a host has had
  a maintenance job". Every host with an agent reads 0 until a job fails.
- #5782 "VDNT-Util01's host page lists SYSTIN at 115 °C in red". The host
  page lists sensors that counts_as_hottest leaves out (unwired board
  inputs, 0 °C or below) on one muted line, not as heat.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/routes.py

@@ -18,7 +18,7 @@ from steward.core.settings import floor_of
 from steward.models.metrics import PluginMetricLatest
 from . import meaning
 from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
-from .scheduler import router_domains
+from .scheduler import error_page_services, router_domains
 from .scraper import cert_names, covers
 
 traefik_bp = Blueprint("traefik", __name__, template_folder="templates")
@@ -175,12 +175,21 @@ async def status_codes(db, names: list[str], since: datetime) -> dict[str, list[
     return out
 
 
-def grade_routers(routers: list[dict], window: dict, thresholds: dict, norms: dict | None = None) -> None:
+ERROR_PAGES_WHY = "serves other routers' error pages: its 4xx and 5xx are theirs, graded on them"
+
+
+def grade_routers(routers: list[dict], window: dict, thresholds: dict, norms: dict | None = None,
+                  error_pages: frozenset[str] = frozenset()) -> None:
     """Set each router's state and why from its recent window, or from its
     latest scrape when the window holds nothing, against its normals
-    (router_norms') where it has them."""
+    (router_norms') where it has them. A service Traefik's errors middleware
+    sends error pages to (`error_pages`) is healthy whatever its codes: it
+    answers with the code of the error it covers for (#5780)."""
     norms = norms or {}
     for r in routers:
+        if r["name"] in error_pages:
+            r["state"], r["why"], r["kind"], r["normed"] = "ok", ERROR_PAGES_WHY, None, False
+            continue
         reading, own = window.get(r["name"]), norms.get(r["name"])
         r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS, own) if reading
                                 else router_state(r["latest"], thresholds, norms=own))
@@ -365,7 +374,7 @@ async def rows():
         history = await router_history(db, names, since, bucket_seconds(since))
         phases.mark("history")
         grade_routers(current, await recent_window(db, names, now), thresholds,
-                      await router_norms(db, names, now))
+                      await router_norms(db, names, now), error_page_services())
         phases.mark("window")
         # The routers' metric anomalies, last 30 days (#5741).
         anomalies = await anom.views(db, await anom.list_anomalies(
@@ -479,7 +488,7 @@ async def widget():
     served, _old = current_certs(certs, scrape)
     domains = router_domains()
     by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
-    grade_routers(current, window, thresholds, norms)
+    grade_routers(current, window, thresholds, norms, error_page_services())
     for r in current:
         r["cert"] = by_router.get(r["name"])
         if r["cert"]:

plugins/traefik/scheduler.py

@@ -26,6 +26,16 @@ def router_domains() -> dict[str, list[str]] | None:
     return _router_domains
 
 
+# The services Traefik's errors middlewares send error pages to (#5780), as
+# last read from its API. Kept through an API blip: that config rarely moves,
+# and dropping it would grade the error pages as failing for that while.
+_error_page_services: frozenset[str] = frozenset()
+
+
+def error_page_services() -> frozenset[str]:
+    return _error_page_services
+
+
 # Access log state
 _geoip_db = None
 _access_log_last_tick: "datetime | None" = None
@@ -48,7 +58,7 @@ def make_scrape_task(app: "Quart") -> ScheduledTask:
 
 async def _do_scrape(app: "Quart") -> None:
     """Fetch Traefik metrics, write to DB, and emit plugin_metrics."""
-    global _prev_metrics, _prev_time, _router_domains
+    global _prev_metrics, _prev_time, _router_domains, _error_page_services
 
     from datetime import datetime, timezone
     from .models import TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
@@ -59,6 +69,7 @@ async def _do_scrape(app: "Quart") -> None:
         compute_status_codes,
         extract_certs,
         fetch_metrics,
+        fetch_error_page_services,
         fetch_router_domains,
     )
     from steward.core.alerts import MetricBatch
@@ -77,6 +88,10 @@ async def _do_scrape(app: "Quart") -> None:
         if _router_domains is not None:
             logger.warning("Traefik API stopped answering (%s); certificates shown without routers", exc)
         _router_domains = None
+    try:
+        _error_page_services = await fetch_error_page_services(api_base(metrics_url, cfg.get("api_url") or ""))
+    except Exception as exc:
+        logger.debug("Traefik middlewares not read (%s); keeping the last error-page services", exc)
 
     now = time.monotonic()
     elapsed = now - _prev_time if _prev_time else 60.0

plugins/traefik/scraper.py

@@ -341,6 +341,32 @@ async def fetch_router_domains(base: str) -> dict[str, list[str]]:
     return out
 
 
+async def fetch_error_page_services(base: str) -> frozenset[str]:
+    """The services Traefik's `errors` middlewares send error pages to, named
+    as the metrics name them ("error-pages-service@swarm"). Such a service
+    answers with the code of the error it covers for: its 4xx and 5xx are
+    other routers' errors, already graded on them."""
+    async with httpx.AsyncClient(timeout=10.0) as client:
+        response = await client.get(f"{base}/http/middlewares", params={"per_page": 1000})
+        response.raise_for_status()
+        return error_page_services(response.json())
+
+
+def error_page_services(middlewares: list[dict]) -> frozenset[str]:
+    """fetch_error_page_services' reading of /api/http/middlewares: a service
+    named without a provider belongs to its middleware's provider."""
+    out = set()
+    for m in middlewares:
+        service = (m.get("errors") or {}).get("service")
+        if not service:
+            continue
+        if "@" not in service:
+            provider = m.get("provider") or (m.get("name", "").partition("@")[2])
+            service = f"{service}@{provider}" if provider else service
+        out.add(service)
+    return frozenset(out)
+
+
 def cert_names(cn: str | None, sans: str | None) -> set[str]:
     """Every name a certificate covers: its CN and SANs, lower-cased."""
     names = {n.strip().lower() for n in (sans or "").split(",") if n.strip()}