FabledSteward/plugins/traefik/meaning.py

"""What a router problem means, whether it is likely real, and what to do (#5712).

Pure: a problem card hands in its grade, its why, the status codes it saw and
its latest figures, and gets back the words for it. The wording lives here, in
one place, so other views of a router (its normal, an alert) can say the same.
"""
from __future__ import annotations

# A wide gap between p95 and p50 means a few slow requests, not a slow app.
SLOW_TAIL_RATIO = 4.0

KINDS = {
    "5xx": "Server errors",
    "4xx": "Client errors",
    "slow": "Slow responses",
}


def problem_kind(state: str, why: str | None) -> str | None:
    """The kind of a graded router's problem: "5xx", "slow", "4xx", or None
    for a healthy router. routes.router_state grades 5xx as crit, and words a
    warning as either a p95 or a 4xx share."""
    if state == "crit":
        return "5xx"
    if state == "warn":
        return "slow" if (why or "").startswith("p95") else "4xx"
    return None


def _top(codes: list[tuple[str, int]], first: str) -> str | None:
    """The most-answered code of one class ("4" or "5"); codes come most first."""
    return next((c for c, _n in codes if c[:1] == first), None)


def _share(codes: list[tuple[str, int]], first: str, wanted: set[str]) -> float:
    total = sum(n for c, n in codes if c[:1] == first)
    return sum(n for c, n in codes if c in wanted) / total if total else 0.0


def explain(kind: str, codes: list[tuple[str, int]], p50_ms: float | None = None,
            p95_ms: float | None = None, normed: bool = False) -> dict:
    """{"label", "meaning", "likely", "action"} for a problem card:
    `meaning` what the codes or figures usually point to, `likely` whether
    it is usually a fault, `action` "act" (look at the app) or "tune" (the
    cutoff may be wrong for this router). `normed` says the router was graded
    against its own normal, so the problem is a departure from its usual."""
    if kind == "5xx":
        top = _top(codes, "5")
        if top == "500":
            meaning = "Mostly 500: the app is failing requests. Its logs say why."
        elif top in ("502", "503", "504"):
            meaning = (f"Mostly {top}: Traefik couldn't reach the app, or it timed out. "
                       "Check that its container is up and healthy.")
        elif top:
            meaning = f"Mostly {top}: the app is answering with server errors. Its logs say why."
        else:
            meaning = "The app is answering with server errors. Its logs say why."
        return {"label": KINDS[kind], "meaning": meaning,
                "likely": "Almost always a real fault.", "action": "act"}

    if kind == "4xx":
        top = _top(codes, "4")
        if top == "404":
            meaning = ("Mostly 404: usually scanners and bots probing for paths, or links to "
                       "pages that moved. Usually harmless.")
        elif top in ("401", "403"):
            meaning = (f"Mostly {top}: a login wall doing its job, refusing requests without "
                       "a session. A problem only if it's you being refused.")
        elif top == "429":
            meaning = "Mostly 429: rate limiting, telling clients to slow down."
        elif top == "499":
            meaning = ("Mostly 499: clients giving up before the app answered, "
                       "often because the app is slow.")
        elif top:
            meaning = f"Mostly {top}: requests the app turned away as not allowed or not understood."
        else:
            meaning = "Requests the app turned away: often bots, scanners or a login wall."
        # A slow app behind the 499s is a fault; a login wall or scanners are not.
        if top == "499" and _share(codes, "4", {"499"}) >= 0.5:
            return {"label": KINDS[kind], "meaning": meaning,
                    "likely": "Often a slow app rather than the clients.", "action": "act"}
        likely = ("Above its usual level for this hour, so something changed, "
                  "but 4xx is still usually traffic rather than a fault." if normed
                  else "Usually traffic rather than a fault: worth tuning, not fixing.")
        return {"label": KINDS[kind], "meaning": meaning, "likely": likely, "action": "tune"}

    # slow
    if p50_ms and p95_ms and p95_ms >= SLOW_TAIL_RATIO * p50_ms:
        meaning = (f"p95 is {p95_ms / p50_ms:.0f}× p50 ({p50_ms:.0f} ms): a few slow requests, "
                   "likely one endpoint or a heavy page, not the whole app.")
    elif p50_ms and p95_ms:
        meaning = (f"p50 is close behind at {p50_ms:.0f} ms: the whole app is slow, "
                   "not one endpoint.")
    else:
        meaning = "The app is slow overall, or one endpoint is."
    likely = ("Slower than usual for this hour." if normed
              else "Real if users feel it; some apps are simply slow.")
    return {"label": KINDS["slow"], "meaning": meaning, "likely": likely, "action": "act"}


def swarm_service(router: str, services: list[str]) -> str | None:
    """The swarm service a router fronts, by name: "radarr@swarm" is served
    by "arr_radarr", "codex@swarm" by "codex_app" (its stack's app), and
    "fabled-searxng@swarm" by "fabled_search_searxng" (its last word). None
    when no single service matches, or the router isn't a swarm one."""
    base, _, provider = router.partition("@")
    if provider not in ("swarm", "docker"):
        return None
    for name in dict.fromkeys((base, base.rsplit("-", 1)[-1])):
        for match in ([s for s in services if s == name],
                      [s for s in services if s.endswith("_" + name)],
                      [s for s in services if s == f"{name}_app"]):
            if len(match) == 1:
                return match[0]
    return None