"""Each router problem says what it means, whether it's real, and what to do (#5712): plugins/traefik/meaning.py's wording, and the problem card that carries it.""" import pytest from plugins.traefik import meaning from plugins.traefik.routes import STATE_WINDOW_WORDS, explain_problems, router_state from steward.core.settings import to_thresholds_cfg from tests._templates import render_plugin_template from tests.plugins.traefik.test_traefik_page import _latest TH = to_thresholds_cfg({}) @pytest.mark.parametrize("latest, kind", [ (_latest(e5=2.5), "5xx"), (_latest(p95=450.0), "slow"), (_latest(e4=5.0), "4xx"), (_latest(e5=2.5, e4=5.0, p95=450.0), "5xx"), # graded on its worst (_latest(p95=450.0, e4=5.0), "slow"), # router_state checks p95 before 4xx (_latest(), None), ]) def test_the_kind_follows_router_state_wording(latest, kind): for over in ("", STATE_WINDOW_WORDS): state, why = router_state(latest, TH, over) assert meaning.problem_kind(state, why) == kind @pytest.mark.parametrize("codes, says, action", [ ([("500", 30), ("502", 2)], "the app is failing requests", "act"), ([("502", 40), ("500", 1)], "couldn't reach the app", "act"), ([("504", 5)], "Mostly 504: Traefik couldn't reach the app, or it timed out", "act"), ([], "The app is answering with server errors", "act"), ]) def test_server_errors_are_real_and_point_at_the_app(codes, says, action): got = meaning.explain("5xx", codes) assert says in got["meaning"] and got["action"] == action and "real fault" in got["likely"] @pytest.mark.parametrize("codes, says, action", [ ([("404", 1812), ("401", 9)], "scanners and bots", "tune"), ([("401", 300), ("404", 2)], "a login wall doing its job", "tune"), ([("403", 30)], "Mostly 403: a login wall", "tune"), ([("429", 12)], "rate limiting", "tune"), ([("499", 50), ("404", 10)], "clients giving up", "act"), # mostly 499: a slow app ([("404", 50), ("499", 10)], "scanners and bots", "tune"), ([("400", 7)], "Mostly 400: requests the app turned away", "tune"), ([], "often bots, scanners or a login wall", "tune"), ]) def test_client_errors_say_which_kind_of_traffic(codes, says, action): got = meaning.explain("4xx", codes) assert says in got["meaning"] and got["action"] == action def test_a_4xx_past_its_own_normal_says_something_changed(): assert "Usually traffic" in meaning.explain("4xx", [("404", 9)])["likely"] assert "Above its usual level" in meaning.explain("4xx", [("404", 9)], normed=True)["likely"] def test_slow_compares_p95_with_p50(): tail = meaning.explain("slow", [], p50_ms=40.0, p95_ms=900.0) assert "p95 is 22× p50 (40 ms): a few slow requests" in tail["meaning"] and tail["action"] == "act" whole = meaning.explain("slow", [], p50_ms=600.0, p95_ms=900.0) assert "the whole app is slow" in whole["meaning"] assert meaning.explain("slow", [], p95_ms=900.0)["meaning"] == "The app is slow overall, or one endpoint is." assert "Slower than usual" in meaning.explain("slow", [], normed=True)["likely"] SERVICES = ["arr_radarr", "codex_app", "codex_db", "steward_app", "steward_db", "personal_nextcloud", "fabled_search_searxng", "traefik_router", "radarr_old_radarr"] @pytest.mark.parametrize("router, service", [ ("codex@swarm", "codex_app"), # its stack's app ("steward@swarm", "steward_app"), ("nextcloud@swarm", "personal_nextcloud"), # the one service ending in it ("fabled-searxng@swarm", "fabled_search_searxng"), # by its last word ("radarr@swarm", None), # two services end in _radarr: can't tell ("home@swarm", None), ("jellyfin@file", None), # not a swarm router ("traefik_router@docker", "traefik_router"), ]) def test_a_router_finds_the_swarm_service_behind_it(router, service): assert meaning.swarm_service(router, SERVICES) == service def _problem(latest, codes): state, why = router_state(latest, TH) return {"name": "app@swarm", "latest": latest, "state": state, "why": why, "codes": codes, "normed": False, **{f"sparkline_{k}": "" for k in ("req", "bytes", "p95", "4xx", "5xx")}} async def _card(app, router, codes_known=True): async with app.test_request_context("/plugins/traefik/rows"): page = await render_plugin_template( app, "traefik", "traefik/rows.html", problems=[router], healthy=[], gone=[], global_stat=None, certs=[], old_certs=[], range_key="24h", current_range="24h", error_4xx=1.0, error_5xx=0.1, codes_known=codes_known) page = " ".join(page.split()) return page[page.index(">app@swarm"):] async def test_a_404_heavy_card_says_scanners_and_points_to_tuning(app): r = _problem(_latest(e4=6.0), [("404", 1812), ("401", 9)]) explain_problems([r], ["app_app"]) card = await _card(app, r) assert "Client errors: 6.0% of responses are 4xx" in card assert "scanners and bots" in card and "Usually traffic rather than a fault" in card assert 'href="/settings/thresholds/#traefik"' in card and "/plugins/docker/service/" not in card assert card.index("scanners") < card.index("Server errors" in card and "couldn't reach the app" in card assert 'href="/plugins/docker/service/app_app">app_app: its tasks and logs' in card assert "/settings/thresholds/" not in card async def test_a_card_for_an_unknown_service_says_so(app): r = _problem(_latest(e5=2.5), []) explain_problems([r], []) card = await _card(app, r, codes_known=False) assert "Status codes not recorded yet" in card assert "The app is answering with server errors" in card assert "Steward can't tell which service serves this router" in card def test_the_error_pages_service_is_found_from_traefiks_errors_middlewares(): from plugins.traefik.scraper import error_page_services got = error_page_services([ {"name": "error-pages@swarm", "provider": "swarm", "errors": {"service": "error-pages-service", "status": ["400-599"]}}, {"name": "other@file", "errors": {"service": "pages@docker"}}, # already names its provider {"name": "https-redirect@swarm", "redirectScheme": {"scheme": "https"}}, ]) assert got == {"error-pages-service@swarm", "pages@docker"} def test_the_error_pages_service_is_healthy_whatever_its_codes(): """It answers with the code of the error it covers for (#5780).""" from plugins.traefik.routes import ERROR_PAGES_WHY, grade_routers routers = [{"name": "error-pages-service@swarm", "latest": _latest(e4=90.0, e5=8.0)}, {"name": "app@swarm", "latest": _latest(e5=8.0)}] grade_routers(routers, {}, TH, error_pages=frozenset({"error-pages-service@swarm"})) assert (routers[0]["state"], routers[0]["why"], routers[0]["kind"]) == ("ok", ERROR_PAGES_WHY, None) assert routers[1]["state"] == "crit" and routers[1]["kind"] == "5xx"