From 72e754b580dd55f114476aa1650550d793c18198 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 14 Oct 2026 21:15:44 -0400 Subject: [PATCH] fix: error-pages graded healthy, unwired sensors listed apart, Maintenance failed reads 0 (#5780, #5781, #5782) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - #5780 "The error-pages service is graded on other routers' errors". The Traefik scrape reads which services an errors middleware sends to, and grades them healthy: their 4xx and 5xx are the codes of the errors they cover for, already graded on the routers that failed. - #5781 "The Maintenance failed rules sit at no data until a host has had a maintenance job". Every host with an agent reads 0 until a job fails. - #5782 "VDNT-Util01's host page lists SYSTIN at 115 °C in red". The host page lists sensors that counts_as_hottest leaves out (unwired board inputs, 0 °C or below) on one muted line, not as heat. Co-Authored-By: Claude Opus 5.5 --- plugins/host_agent/cadence.py | 16 ++++++++---- plugins/host_agent/routes.py | 14 +++++++++- .../host_agent/templates/_host_metrics.html | 6 +++++ plugins/traefik/routes.py | 19 ++++++++++---- plugins/traefik/scheduler.py | 17 +++++++++++- plugins/traefik/scraper.py | 26 +++++++++++++++++++ .../test_host_maintenance_cadence.py | 4 ++- .../plugins/host_agent/test_expand_metrics.py | 10 +++++++ tests/plugins/traefik/test_meaning.py | 21 +++++++++++++++ 9 files changed, 120 insertions(+), 13 deletions(-) diff --git a/plugins/host_agent/cadence.py b/plugins/host_agent/cadence.py index 51195c8..465c15d 100644 --- a/plugins/host_agent/cadence.py +++ b/plugins/host_agent/cadence.py @@ -285,12 +285,17 @@ async def tick(session, now: datetime) -> list[str]: async def failed_hosts(session, now: datetime) -> dict[str, float]: - """{host name: 1 or 0}: whether each host's latest finished maintenance job - failed or timed out, for the maintenance_failed figure an alert rule - watches. A later job that succeeds puts it back to 0.""" + """{host name: 1 or 0} for every host with an agent: whether its latest + finished maintenance job failed or timed out, for the maintenance_failed + figure an alert rule watches. A host with no job yet reads 0, so a rule + on it reports rather than sitting at "no data". A later job that + succeeds puts it back to 0.""" from sqlalchemy import select from steward.models.hosts import Host - from .models import HostMaintenanceJob + from .models import HostAgentRegistration, HostMaintenanceJob + agents = (await session.execute( + select(Host.name).join(HostAgentRegistration, HostAgentRegistration.host_id == Host.id))).scalars() + out = {name: 0.0 for name in agents} rows = (await session.execute( select(Host.name, HostMaintenanceJob.state, HostMaintenanceJob.finished_at) .join(Host, Host.id == HostMaintenanceJob.host_id) @@ -300,4 +305,5 @@ async def failed_hosts(session, now: datetime) -> dict[str, float]: for name, state, finished in rows: if name not in latest or finished > latest[name][1]: latest[name] = (state, finished) - return {name: 0.0 if state == "done" else 1.0 for name, (state, _) in latest.items()} + out.update({name: 0.0 if state == "done" else 1.0 for name, (state, _) in latest.items()}) + return out diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 9e15da3..a845f0a 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -113,6 +113,17 @@ def counts_as_hottest(label: str, celsius: float) -> bool: return not (_BOARD_INPUT.match(label) and celsius >= BOARD_INPUT_CEILING_C) +def split_temps(temps: dict) -> tuple[dict, dict]: + """(real, unwired): a host's sensors, hottest first, apart from the ones + counts_as_hottest leaves out (a board input with nothing wired to it, a + reading at or below 0 °C). Those read a constant at any load, so the + page lists them apart instead of colouring them as the host's heat.""" + def by_heat(d: dict) -> dict: + return dict(sorted(d.items(), key=lambda kv: -(kv[1] if kv[1] is not None else -999))) + real = {k: c for k, c in temps.items() if c is None or counts_as_hottest(k, c)} + return by_heat(real), by_heat({k: c for k, c in temps.items() if k not in real}) + + def _ts_or(value, fallback: datetime) -> datetime: """An agent-sent ISO time, or `fallback` when it isn't one.""" try: @@ -856,12 +867,13 @@ async def host_detail_metrics(host_id: str): HostAgentRegistration.host_id == host_id))).scalar_one_or_none() latest = await _latest_metrics_for_host(session, host.name) hostlvl, cores, nets, disks_io, temps, mounts = _split_host_metrics(host.name, latest) + temps, unwired = split_temps(temps) ls = reg.last_seen_at if reg else None stale = ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds() return await render_template( "_host_metrics.html", host=host, reg=reg, stale=stale, hostlvl=hostlvl, - cores=cores, nets=nets, disks_io=disks_io, temps=temps, mounts=mounts, + cores=cores, nets=nets, disks_io=disks_io, temps=temps, unwired_temps=unwired, mounts=mounts, ) diff --git a/plugins/host_agent/templates/_host_metrics.html b/plugins/host_agent/templates/_host_metrics.html index 4ef1876..9734d00 100644 --- a/plugins/host_agent/templates/_host_metrics.html +++ b/plugins/host_agent/templates/_host_metrics.html @@ -159,6 +159,12 @@ {% endfor %} + {% if unwired_temps %} + {# Board inputs with nothing wired to them, and readings at or below 0 °C: + constant at any load, left out of the host's temperature (#5759). #} +

Not counted, reading a constant with nothing wired to them: + {% for label, c in unwired_temps.items() %}{{ label }} {{ "%.0f"|format(c) }}°C{{ ", " if not loop.last }}{% endfor %}.

+ {% endif %} {% endif %} diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py index a934760..1abd59f 100644 --- a/plugins/traefik/routes.py +++ b/plugins/traefik/routes.py @@ -18,7 +18,7 @@ from steward.core.settings import floor_of from steward.models.metrics import PluginMetricLatest from . import meaning from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode -from .scheduler import router_domains +from .scheduler import error_page_services, router_domains from .scraper import cert_names, covers traefik_bp = Blueprint("traefik", __name__, template_folder="templates") @@ -175,12 +175,21 @@ async def status_codes(db, names: list[str], since: datetime) -> dict[str, list[ return out -def grade_routers(routers: list[dict], window: dict, thresholds: dict, norms: dict | None = None) -> None: +ERROR_PAGES_WHY = "serves other routers' error pages: its 4xx and 5xx are theirs, graded on them" + + +def grade_routers(routers: list[dict], window: dict, thresholds: dict, norms: dict | None = None, + error_pages: frozenset[str] = frozenset()) -> None: """Set each router's state and why from its recent window, or from its latest scrape when the window holds nothing, against its normals - (router_norms') where it has them.""" + (router_norms') where it has them. A service Traefik's errors middleware + sends error pages to (`error_pages`) is healthy whatever its codes: it + answers with the code of the error it covers for (#5780).""" norms = norms or {} for r in routers: + if r["name"] in error_pages: + r["state"], r["why"], r["kind"], r["normed"] = "ok", ERROR_PAGES_WHY, None, False + continue reading, own = window.get(r["name"]), norms.get(r["name"]) r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS, own) if reading else router_state(r["latest"], thresholds, norms=own)) @@ -365,7 +374,7 @@ async def rows(): history = await router_history(db, names, since, bucket_seconds(since)) phases.mark("history") grade_routers(current, await recent_window(db, names, now), thresholds, - await router_norms(db, names, now)) + await router_norms(db, names, now), error_page_services()) phases.mark("window") # The routers' metric anomalies, last 30 days (#5741). anomalies = await anom.views(db, await anom.list_anomalies( @@ -479,7 +488,7 @@ async def widget(): served, _old = current_certs(certs, scrape) domains = router_domains() by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now) - grade_routers(current, window, thresholds, norms) + grade_routers(current, window, thresholds, norms, error_page_services()) for r in current: r["cert"] = by_router.get(r["name"]) if r["cert"]: diff --git a/plugins/traefik/scheduler.py b/plugins/traefik/scheduler.py index 77b467c..4271ae5 100644 --- a/plugins/traefik/scheduler.py +++ b/plugins/traefik/scheduler.py @@ -26,6 +26,16 @@ def router_domains() -> dict[str, list[str]] | None: return _router_domains +# The services Traefik's errors middlewares send error pages to (#5780), as +# last read from its API. Kept through an API blip: that config rarely moves, +# and dropping it would grade the error pages as failing for that while. +_error_page_services: frozenset[str] = frozenset() + + +def error_page_services() -> frozenset[str]: + return _error_page_services + + # Access log state _geoip_db = None _access_log_last_tick: "datetime | None" = None @@ -48,7 +58,7 @@ def make_scrape_task(app: "Quart") -> ScheduledTask: async def _do_scrape(app: "Quart") -> None: """Fetch Traefik metrics, write to DB, and emit plugin_metrics.""" - global _prev_metrics, _prev_time, _router_domains + global _prev_metrics, _prev_time, _router_domains, _error_page_services from datetime import datetime, timezone from .models import TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode @@ -59,6 +69,7 @@ async def _do_scrape(app: "Quart") -> None: compute_status_codes, extract_certs, fetch_metrics, + fetch_error_page_services, fetch_router_domains, ) from steward.core.alerts import MetricBatch @@ -77,6 +88,10 @@ async def _do_scrape(app: "Quart") -> None: if _router_domains is not None: logger.warning("Traefik API stopped answering (%s); certificates shown without routers", exc) _router_domains = None + try: + _error_page_services = await fetch_error_page_services(api_base(metrics_url, cfg.get("api_url") or "")) + except Exception as exc: + logger.debug("Traefik middlewares not read (%s); keeping the last error-page services", exc) now = time.monotonic() elapsed = now - _prev_time if _prev_time else 60.0 diff --git a/plugins/traefik/scraper.py b/plugins/traefik/scraper.py index 7d319e4..e44facc 100644 --- a/plugins/traefik/scraper.py +++ b/plugins/traefik/scraper.py @@ -341,6 +341,32 @@ async def fetch_router_domains(base: str) -> dict[str, list[str]]: return out +async def fetch_error_page_services(base: str) -> frozenset[str]: + """The services Traefik's `errors` middlewares send error pages to, named + as the metrics name them ("error-pages-service@swarm"). Such a service + answers with the code of the error it covers for: its 4xx and 5xx are + other routers' errors, already graded on them.""" + async with httpx.AsyncClient(timeout=10.0) as client: + response = await client.get(f"{base}/http/middlewares", params={"per_page": 1000}) + response.raise_for_status() + return error_page_services(response.json()) + + +def error_page_services(middlewares: list[dict]) -> frozenset[str]: + """fetch_error_page_services' reading of /api/http/middlewares: a service + named without a provider belongs to its middleware's provider.""" + out = set() + for m in middlewares: + service = (m.get("errors") or {}).get("service") + if not service: + continue + if "@" not in service: + provider = m.get("provider") or (m.get("name", "").partition("@")[2]) + service = f"{service}@{provider}" if provider else service + out.add(service) + return frozenset(out) + + def cert_names(cn: str | None, sans: str | None) -> set[str]: """Every name a certificate covers: its CN and SANs, lower-cased.""" names = {n.strip().lower() for n in (sans or "").split(",") if n.strip()} diff --git a/tests/integration/test_host_maintenance_cadence.py b/tests/integration/test_host_maintenance_cadence.py index 141d654..b6bb7a4 100644 --- a/tests/integration/test_host_maintenance_cadence.py +++ b/tests/integration/test_host_maintenance_cadence.py @@ -107,7 +107,9 @@ def test_hosts_go_one_at_a_time_and_a_failure_stops_the_run(): async with s.begin(): plan = await s.get(models.HostMaintenancePlan, seed["plan_id"]) await cadence.start_run(s, plan, datetime.now(timezone.utc), manual=True) - await _tick(app) + first = await _tick(app) + # Before any job finishes, every agent host reads 0 rather than nothing. + assert first[seed["hosts"]["a"].name] == 0.0 and first[seed["hosts"]["b"].name] == 0.0 # Only the first host (by name) has a job; the second waits. ta, tb = seed["tokens"]["a"], seed["tokens"]["b"] diff --git a/tests/plugins/host_agent/test_expand_metrics.py b/tests/plugins/host_agent/test_expand_metrics.py index 6f48fd4..8096614 100644 --- a/tests/plugins/host_agent/test_expand_metrics.py +++ b/tests/plugins/host_agent/test_expand_metrics.py @@ -156,3 +156,13 @@ def test_the_registration_keeps_the_host_s_updates_and_reboot(): "reboot": {"required": False}}, now) assert reg.updates_pending == 4 and reg.updates_error == "apt is locked" assert reg.reboot_required is False and reg.reboot_required_since is None + + +def test_the_page_lists_unwired_sensors_apart_hottest_first(): + """VDNT-Util01's sensor list read SYSTIN 115 °C in red at the top, though + the host's temperature already left it out (#5759).""" + from plugins.host_agent.routes import split_temps + real, unwired = split_temps({"SYSTIN": 115.0, "AUXTIN0": 110.0, "Package id 0": 52.0, + "Composite": 48.9, "PCH_CHIP_TEMP": 0.0, "CPUTIN": 47.0, "gone": None}) + assert list(real) == ["Package id 0", "Composite", "CPUTIN", "gone"] + assert list(unwired) == ["SYSTIN", "AUXTIN0", "PCH_CHIP_TEMP"] diff --git a/tests/plugins/traefik/test_meaning.py b/tests/plugins/traefik/test_meaning.py index 2e6afeb..74ed7ef 100644 --- a/tests/plugins/traefik/test_meaning.py +++ b/tests/plugins/traefik/test_meaning.py @@ -127,3 +127,24 @@ async def test_a_card_for_an_unknown_service_says_so(app): assert "Status codes not recorded yet" in card assert "The app is answering with server errors" in card assert "Steward can't tell which service serves this router" in card + + +def test_the_error_pages_service_is_found_from_traefiks_errors_middlewares(): + from plugins.traefik.scraper import error_page_services + got = error_page_services([ + {"name": "error-pages@swarm", "provider": "swarm", "errors": {"service": "error-pages-service", + "status": ["400-599"]}}, + {"name": "other@file", "errors": {"service": "pages@docker"}}, # already names its provider + {"name": "https-redirect@swarm", "redirectScheme": {"scheme": "https"}}, + ]) + assert got == {"error-pages-service@swarm", "pages@docker"} + + +def test_the_error_pages_service_is_healthy_whatever_its_codes(): + """It answers with the code of the error it covers for (#5780).""" + from plugins.traefik.routes import ERROR_PAGES_WHY, grade_routers + routers = [{"name": "error-pages-service@swarm", "latest": _latest(e4=90.0, e5=8.0)}, + {"name": "app@swarm", "latest": _latest(e5=8.0)}] + grade_routers(routers, {}, TH, error_pages=frozenset({"error-pages-service@swarm"})) + assert (routers[0]["state"], routers[0]["why"], routers[0]["kind"]) == ("ok", ERROR_PAGES_WHY, None) + assert routers[1]["state"] == "crit" and routers[1]["kind"] == "5xx"