"""What a router problem means, whether it is likely real, and what to do (#5712). Pure: a problem card hands in its grade, its why, the status codes it saw and its latest figures, and gets back the words for it. The wording lives here, in one place, so other views of a router (its normal, an alert) can say the same. """ from __future__ import annotations # A wide gap between p95 and p50 means a few slow requests, not a slow app. SLOW_TAIL_RATIO = 4.0 KINDS = { "5xx": "Server errors", "4xx": "Client errors", "slow": "Slow responses", } def problem_kind(state: str, why: str | None) -> str | None: """The kind of a graded router's problem: "5xx", "slow", "4xx", or None for a healthy router. routes.router_state grades 5xx as crit, and words a warning as either a p95 or a 4xx share.""" if state == "crit": return "5xx" if state == "warn": return "slow" if (why or "").startswith("p95") else "4xx" return None def _top(codes: list[tuple[str, int]], first: str) -> str | None: """The most-answered code of one class ("4" or "5"); codes come most first.""" return next((c for c, _n in codes if c[:1] == first), None) def _share(codes: list[tuple[str, int]], first: str, wanted: set[str]) -> float: total = sum(n for c, n in codes if c[:1] == first) return sum(n for c, n in codes if c in wanted) / total if total else 0.0 def explain(kind: str, codes: list[tuple[str, int]], p50_ms: float | None = None, p95_ms: float | None = None, normed: bool = False) -> dict: """{"label", "meaning", "likely", "action"} for a problem card: `meaning` what the codes or figures usually point to, `likely` whether it is usually a fault, `action` "act" (look at the app) or "tune" (the cutoff may be wrong for this router). `normed` says the router was graded against its own normal, so the problem is a departure from its usual.""" if kind == "5xx": top = _top(codes, "5") if top == "500": meaning = "Mostly 500: the app is failing requests. Its logs say why." elif top in ("502", "503", "504"): meaning = (f"Mostly {top}: Traefik couldn't reach the app, or it timed out. " "Check that its container is up and healthy.") elif top: meaning = f"Mostly {top}: the app is answering with server errors. Its logs say why." else: meaning = "The app is answering with server errors. Its logs say why." return {"label": KINDS[kind], "meaning": meaning, "likely": "Almost always a real fault.", "action": "act"} if kind == "4xx": top = _top(codes, "4") if top == "404": meaning = ("Mostly 404: usually scanners and bots probing for paths, or links to " "pages that moved. Usually harmless.") elif top in ("401", "403"): meaning = (f"Mostly {top}: a login wall doing its job, refusing requests without " "a session. A problem only if it's you being refused.") elif top == "429": meaning = "Mostly 429: rate limiting, telling clients to slow down." elif top == "499": meaning = ("Mostly 499: clients giving up before the app answered, " "often because the app is slow.") elif top: meaning = f"Mostly {top}: requests the app turned away as not allowed or not understood." else: meaning = "Requests the app turned away: often bots, scanners or a login wall." # A slow app behind the 499s is a fault; a login wall or scanners are not. if top == "499" and _share(codes, "4", {"499"}) >= 0.5: return {"label": KINDS[kind], "meaning": meaning, "likely": "Often a slow app rather than the clients.", "action": "act"} likely = ("Above its usual level for this hour, so something changed, " "but 4xx is still usually traffic rather than a fault." if normed else "Usually traffic rather than a fault: worth tuning, not fixing.") return {"label": KINDS[kind], "meaning": meaning, "likely": likely, "action": "tune"} # slow if p50_ms and p95_ms and p95_ms >= SLOW_TAIL_RATIO * p50_ms: meaning = (f"p95 is {p95_ms / p50_ms:.0f}× p50 ({p50_ms:.0f} ms): a few slow requests, " "likely one endpoint or a heavy page, not the whole app.") elif p50_ms and p95_ms: meaning = (f"p50 is close behind at {p50_ms:.0f} ms: the whole app is slow, " "not one endpoint.") else: meaning = "The app is slow overall, or one endpoint is." likely = ("Slower than usual for this hour." if normed else "Real if users feel it; some apps are simply slow.") return {"label": KINDS["slow"], "meaning": meaning, "likely": likely, "action": "act"} def swarm_service(router: str, services: list[str]) -> str | None: """The swarm service a router fronts, by name: "radarr@swarm" is served by "arr_radarr", "codex@swarm" by "codex_app" (its stack's app), and "fabled-searxng@swarm" by "fabled_search_searxng" (its last word). None when no single service matches, or the router isn't a swarm one.""" base, _, provider = router.partition("@") if provider not in ("swarm", "docker"): return None for name in dict.fromkeys((base, base.rsplit("-", 1)[-1])): for match in ([s for s in services if s == name], [s for s in services if s.endswith("_" + name)], [s for s in services if s == f"{name}_app"]): if len(match) == 1: return match[0] return None