diff --git a/docs/plugins/traefik.md b/docs/plugins/traefik.md index 1981136..f21c10b 100644 --- a/docs/plugins/traefik.md +++ b/docs/plugins/traefik.md @@ -84,6 +84,11 @@ The full Traefik page at `/plugins/traefik/` reads top to bottom, for the time r 1. **The verdict.** It names the routers that need a look, says why, and says when the problem started ("api, 3.0% of responses are 5xx, started 2h ago"). When a router has been a problem for the whole range, it says so instead. Otherwise it reads "All N routers healthy". 2. **The request chart.** Requests per second across all routers and the share answered 5xx, the same chart as the dashboard widget. 3. **A card for each router that needs a look:** why, since when, the 4xx and 5xx status codes it answered in the last 15 minutes, most first ("404 ×1,812, 502 ×40"), and its request rate, bandwidth, p95/p99 latency and 4xx/5xx shares, each with its history. The code is what separates scanners probing for pages (404), a login wall (401/403) and an app that isn't answering (502/503/504). Steward records codes from the version that added them on; until then a card says they aren't recorded yet. + Each card then says what its problem usually means and what to do (`plugins/traefik/meaning.py`, #5712): + - **Server errors (5xx)** are almost always real. Mostly 500: the app is failing requests. Mostly 502/503/504: Traefik couldn't reach the app, or it timed out. + - **Client errors (4xx)** are usually traffic rather than a fault. Mostly 404: scanners and bots. 401/403: a login wall. 429: rate limiting. Mostly 499 is the exception: clients giving up on a slow app. + - **Slow responses** compare p95 with p50. When p95 is 4× p50 or more, a few requests are slow (one endpoint or a heavy page). When p50 is close behind, the whole app is slow. + - **Next:** a fault links to the swarm service behind the router (its tasks and logs) when Steward can match the name ("codex@swarm" → `codex_app`, "radarr@swarm" → `arr_radarr`), otherwise to Docker → Swarm. Traffic links to Settings → Thresholds. 4. **Traefik's own figures:** when its config last reloaded, open connections, memory. 5. **The healthy routers,** folded into one table, then the routers Traefik no longer reports, then TLS certificates by soonest expiry, then Traffic Origin when the access log is on. diff --git a/plugins/host_agent/maintenance.py b/plugins/host_agent/maintenance.py index 10cd06d..3dc2166 100644 --- a/plugins/host_agent/maintenance.py +++ b/plugins/host_agent/maintenance.py @@ -385,18 +385,9 @@ def summary(job) -> str: def docker_models(): - """The loaded Docker plugin's models module, or None when it isn't loaded. - Reached through the plugin manager: importing plugins.docker.models by its - path would define its tables a second time, and the app would refuse it.""" - import importlib - from steward.core.plugin_manager import get_loaded_plugin_modules - loaded = get_loaded_plugin_modules().get("docker") - if loaded is None: - return None - try: - return importlib.import_module(f"{loaded.__name__}.models") - except ImportError: - return None + """The loaded Docker plugin's models module, or None when it isn't loaded.""" + from steward.core.plugin_manager import loaded_plugin_submodule + return loaded_plugin_submodule("docker", "models") async def _swarm(session) -> tuple: diff --git a/plugins/traefik/meaning.py b/plugins/traefik/meaning.py new file mode 100644 index 0000000..b4f018a --- /dev/null +++ b/plugins/traefik/meaning.py @@ -0,0 +1,115 @@ +"""What a router problem means, whether it is likely real, and what to do (#5712). + +Pure: a problem card hands in its grade, its why, the status codes it saw and +its latest figures, and gets back the words for it. The wording lives here, in +one place, so other views of a router (its normal, an alert) can say the same. +""" +from __future__ import annotations + +# A wide gap between p95 and p50 means a few slow requests, not a slow app. +SLOW_TAIL_RATIO = 4.0 + +KINDS = { + "5xx": "Server errors", + "4xx": "Client errors", + "slow": "Slow responses", +} + + +def problem_kind(state: str, why: str | None) -> str | None: + """The kind of a graded router's problem: "5xx", "slow", "4xx", or None + for a healthy router. routes.router_state grades 5xx as crit, and words a + warning as either a p95 or a 4xx share.""" + if state == "crit": + return "5xx" + if state == "warn": + return "slow" if (why or "").startswith("p95") else "4xx" + return None + + +def _top(codes: list[tuple[str, int]], first: str) -> str | None: + """The most-answered code of one class ("4" or "5"); codes come most first.""" + return next((c for c, _n in codes if c[:1] == first), None) + + +def _share(codes: list[tuple[str, int]], first: str, wanted: set[str]) -> float: + total = sum(n for c, n in codes if c[:1] == first) + return sum(n for c, n in codes if c in wanted) / total if total else 0.0 + + +def explain(kind: str, codes: list[tuple[str, int]], p50_ms: float | None = None, + p95_ms: float | None = None, normed: bool = False) -> dict: + """{"label", "meaning", "likely", "action"} for a problem card: + `meaning` what the codes or figures usually point to, `likely` whether + it is usually a fault, `action` "act" (look at the app) or "tune" (the + cutoff may be wrong for this router). `normed` says the router was graded + against its own normal, so the problem is a departure from its usual.""" + if kind == "5xx": + top = _top(codes, "5") + if top == "500": + meaning = "Mostly 500: the app is failing requests. Its logs say why." + elif top in ("502", "503", "504"): + meaning = (f"Mostly {top}: Traefik couldn't reach the app, or it timed out. " + "Check that its container is up and healthy.") + elif top: + meaning = f"Mostly {top}: the app is answering with server errors. Its logs say why." + else: + meaning = "The app is answering with server errors. Its logs say why." + return {"label": KINDS[kind], "meaning": meaning, + "likely": "Almost always a real fault.", "action": "act"} + + if kind == "4xx": + top = _top(codes, "4") + if top == "404": + meaning = ("Mostly 404: usually scanners and bots probing for paths, or links to " + "pages that moved. Usually harmless.") + elif top in ("401", "403"): + meaning = (f"Mostly {top}: a login wall doing its job, refusing requests without " + "a session. A problem only if it's you being refused.") + elif top == "429": + meaning = "Mostly 429: rate limiting, telling clients to slow down." + elif top == "499": + meaning = ("Mostly 499: clients giving up before the app answered, " + "often because the app is slow.") + elif top: + meaning = f"Mostly {top}: requests the app turned away as not allowed or not understood." + else: + meaning = "Requests the app turned away: often bots, scanners or a login wall." + # A slow app behind the 499s is a fault; a login wall or scanners are not. + if top == "499" and _share(codes, "4", {"499"}) >= 0.5: + return {"label": KINDS[kind], "meaning": meaning, + "likely": "Often a slow app rather than the clients.", "action": "act"} + likely = ("Above its usual level for this hour, so something changed, " + "but 4xx is still usually traffic rather than a fault." if normed + else "Usually traffic rather than a fault: worth tuning, not fixing.") + return {"label": KINDS[kind], "meaning": meaning, "likely": likely, "action": "tune"} + + # slow + if p50_ms and p95_ms and p95_ms >= SLOW_TAIL_RATIO * p50_ms: + meaning = (f"p95 is {p95_ms / p50_ms:.0f}× p50 ({p50_ms:.0f} ms): a few slow requests, " + "likely one endpoint or a heavy page, not the whole app.") + elif p50_ms and p95_ms: + meaning = (f"p50 is close behind at {p50_ms:.0f} ms: the whole app is slow, " + "not one endpoint.") + else: + meaning = "The app is slow overall, or one endpoint is." + likely = ("Slower than usual for this hour." if normed + else "Real if users feel it; some apps are simply slow.") + return {"label": KINDS["slow"], "meaning": meaning, "likely": likely, "action": "act"} + + +def swarm_service(router: str, services: list[str]) -> str | None: + """The swarm service a router fronts, by name: "radarr@swarm" is served + by "arr_radarr", "codex@swarm" by "codex_app" (its stack's app), and + "fabled-searxng@swarm" by "fabled_search_searxng" (its last word). None + when no single service matches, or the router isn't a swarm one.""" + base, _, provider = router.partition("@") + if provider not in ("swarm", "docker"): + return None + for name in dict.fromkeys((base, base.rsplit("-", 1)[-1])): + for match in ([s for s in services if s == name], + [s for s in services if s.endswith("_" + name)], + [s for s in services if s == f"{name}_app"]): + if len(match) == 1: + return match[0] + return None diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py index 32645c0..1cebef0 100644 --- a/plugins/traefik/routes.py +++ b/plugins/traefik/routes.py @@ -16,6 +16,7 @@ from steward.core.formatting import cert_left, status_class from steward.core.norms import norms_at from steward.core.settings import floor_of from steward.models.metrics import PluginMetricLatest +from . import meaning from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode from .scheduler import router_domains from .scraper import cert_names, covers @@ -183,6 +184,30 @@ def grade_routers(routers: list[dict], window: dict, thresholds: dict, norms: di reading, own = window.get(r["name"]), norms.get(r["name"]) r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS, own) if reading else router_state(r["latest"], thresholds, norms=own)) + r["kind"] = meaning.problem_kind(r["state"], r["why"]) + r["normed"] = bool(own) and {"slow": "latency_p95_ms", "4xx": "error_rate_4xx_pct"}.get( + r["kind"], "") in own + + +async def swarm_service_names(db) -> list[str]: + """Every swarm service the Docker plugin has stored; [] without it.""" + from steward.core.plugin_manager import loaded_plugin_submodule + models = loaded_plugin_submodule("docker", "models") + if models is None: + return [] + return list((await db.execute(select(models.DockerSwarmService.service_name).distinct())).scalars()) + + +def explain_problems(problems: list[dict], services: list[str]) -> None: + """Give each problem card what it means and its one next step (#5712): + `meaning` (meaning.explain's) and `service`, the swarm service behind + the router when Steward can tell which.""" + for r in problems: + r["meaning"] = meaning.explain(r["kind"], r.get("codes", []), + getattr(r["latest"], "latency_p50_ms", None), + getattr(r["latest"], "latency_p95_ms", None), + r.get("normed", False)) + r["service"] = meaning.swarm_service(r["name"], services) # A router missing from this many scrapes in a row is no longer reported: @@ -362,6 +387,7 @@ async def rows(): # old, a router with none listed may simply predate the recording. first_code = await db.scalar(select(func.min(TraefikStatusCode.scraped_at))) codes_known = first_code is not None and first_code <= now - STATE_WINDOW + services = await swarm_service_names(db) if any(r["state"] != "ok" for r in current) else [] phases.mark("codes") # Latest global stat (always most recent, not range-filtered) @@ -400,6 +426,7 @@ async def rows(): certs = [cert_row(c) for c in served] old_certs = [cert_row(c) for c in old] problems, healthy = split_routers(current) + explain_problems(problems, services) phases.mark("cert_rows") error_5xx, error_4xx = error_cutoffs(thresholds) page = await render_template( diff --git a/plugins/traefik/templates/traefik/rows.html b/plugins/traefik/templates/traefik/rows.html index 3bbc585..175f418 100644 --- a/plugins/traefik/templates/traefik/rows.html +++ b/plugins/traefik/templates/traefik/rows.html @@ -45,7 +45,7 @@ {{ status_badge(r.state, "Failing" if r.state == 'crit' else "Warning") }}

{{ r.name }}

-

{{ r.why }} +

{% if r.meaning %}{{ r.meaning.label }}: {% endif %}{{ r.why }} {%- if r.since_before %}, all of the last {{ current_range }}{% elif r.since %}, started {{ r.since|ago }}{% endif %}

{% if r.codes %}

Last 15 min: @@ -56,6 +56,18 @@ {% else %}

Status codes not recorded yet: Steward keeps them from this version on.

{% endif %} + {% if r.meaning %} + {# What it means and the one next step (#5712; plugins/traefik/meaning.py). #} +

{{ r.meaning.meaning }} {{ r.meaning.likely }}

+

Next: + {%- if r.meaning.action == 'act' %} + {%- if r.service %} {{ r.service }}: its tasks and logs → + {%- else %} look at the app's container and logs under Docker → Swarm; Steward can't tell which service serves this router. + {%- endif %} + {%- else %} if this is normal for this router, raise the 4xx cutoff in Settings → Thresholds. It is one cutoff for every router + {%- if not r.normed %}; once Steward has learned this router's normal for the hour, it flags only departures from it{% endif %}. + {%- endif %}

+ {% endif %} diff --git a/steward/core/plugin_manager.py b/steward/core/plugin_manager.py index d2fe820..981c95b 100644 --- a/steward/core/plugin_manager.py +++ b/steward/core/plugin_manager.py @@ -77,6 +77,21 @@ def get_loaded_plugin_modules() -> dict[str, object]: return dict(_PLUGIN_MODULES) +def loaded_plugin_submodule(name: str, sub: str): + """A loaded plugin's submodule (e.g. ("docker", "models")), or None when + the plugin isn't loaded. Another plugin reaches its models this way: + importing plugins..models by path would define its tables a second + time, because the plugin was loaded as _steward_plugin_ (#5775).""" + import importlib + loaded = _PLUGIN_MODULES.get(name) + if loaded is None: + return None + try: + return importlib.import_module(f"{loaded.__name__}.{sub}") + except ImportError: + return None + + def _collect_plugin_module(app: "Quart", name: str, module) -> None: """Remember a loaded plugin module; hand it to a live MCP server if mounted. diff --git a/tests/plugins/traefik/test_meaning.py b/tests/plugins/traefik/test_meaning.py new file mode 100644 index 0000000..df3c0fb --- /dev/null +++ b/tests/plugins/traefik/test_meaning.py @@ -0,0 +1,129 @@ +"""Each router problem says what it means, whether it's real, and what to do +(#5712): plugins/traefik/meaning.py's wording, and the problem card that +carries it.""" + +import pytest + +from plugins.traefik import meaning +from plugins.traefik.routes import STATE_WINDOW_WORDS, explain_problems, router_state +from steward.core.settings import to_thresholds_cfg +from tests._templates import render_plugin_template +from tests.plugins.traefik.test_traefik_page import _latest + +TH = to_thresholds_cfg({}) + + +@pytest.mark.parametrize("latest, kind", [ + (_latest(e5=2.5), "5xx"), + (_latest(p95=450.0), "slow"), + (_latest(e4=5.0), "4xx"), + (_latest(e5=2.5, e4=5.0, p95=450.0), "5xx"), # graded on its worst + (_latest(p95=450.0, e4=5.0), "slow"), # router_state checks p95 before 4xx + (_latest(), None), +]) +def test_the_kind_follows_router_state_wording(latest, kind): + for over in ("", STATE_WINDOW_WORDS): + state, why = router_state(latest, TH, over) + assert meaning.problem_kind(state, why) == kind + + +@pytest.mark.parametrize("codes, says, action", [ + ([("500", 30), ("502", 2)], "the app is failing requests", "act"), + ([("502", 40), ("500", 1)], "couldn't reach the app", "act"), + ([("504", 5)], "Mostly 504: Traefik couldn't reach the app, or it timed out", "act"), + ([], "The app is answering with server errors", "act"), +]) +def test_server_errors_are_real_and_point_at_the_app(codes, says, action): + got = meaning.explain("5xx", codes) + assert says in got["meaning"] and got["action"] == action and "real fault" in got["likely"] + + +@pytest.mark.parametrize("codes, says, action", [ + ([("404", 1812), ("401", 9)], "scanners and bots", "tune"), + ([("401", 300), ("404", 2)], "a login wall doing its job", "tune"), + ([("403", 30)], "Mostly 403: a login wall", "tune"), + ([("429", 12)], "rate limiting", "tune"), + ([("499", 50), ("404", 10)], "clients giving up", "act"), # mostly 499: a slow app + ([("404", 50), ("499", 10)], "scanners and bots", "tune"), + ([("400", 7)], "Mostly 400: requests the app turned away", "tune"), + ([], "often bots, scanners or a login wall", "tune"), +]) +def test_client_errors_say_which_kind_of_traffic(codes, says, action): + got = meaning.explain("4xx", codes) + assert says in got["meaning"] and got["action"] == action + + +def test_a_4xx_past_its_own_normal_says_something_changed(): + assert "Usually traffic" in meaning.explain("4xx", [("404", 9)])["likely"] + assert "Above its usual level" in meaning.explain("4xx", [("404", 9)], normed=True)["likely"] + + +def test_slow_compares_p95_with_p50(): + tail = meaning.explain("slow", [], p50_ms=40.0, p95_ms=900.0) + assert "p95 is 22× p50 (40 ms): a few slow requests" in tail["meaning"] and tail["action"] == "act" + whole = meaning.explain("slow", [], p50_ms=600.0, p95_ms=900.0) + assert "the whole app is slow" in whole["meaning"] + assert meaning.explain("slow", [], p95_ms=900.0)["meaning"] == "The app is slow overall, or one endpoint is." + assert "Slower than usual" in meaning.explain("slow", [], normed=True)["likely"] + + +SERVICES = ["arr_radarr", "codex_app", "codex_db", "steward_app", "steward_db", + "personal_nextcloud", "fabled_search_searxng", "traefik_router", "radarr_old_radarr"] + + +@pytest.mark.parametrize("router, service", [ + ("codex@swarm", "codex_app"), # its stack's app + ("steward@swarm", "steward_app"), + ("nextcloud@swarm", "personal_nextcloud"), # the one service ending in it + ("fabled-searxng@swarm", "fabled_search_searxng"), # by its last word + ("radarr@swarm", None), # two services end in _radarr: can't tell + ("home@swarm", None), + ("jellyfin@file", None), # not a swarm router + ("traefik_router@docker", "traefik_router"), +]) +def test_a_router_finds_the_swarm_service_behind_it(router, service): + assert meaning.swarm_service(router, SERVICES) == service + + +def _problem(latest, codes): + state, why = router_state(latest, TH) + return {"name": "app@swarm", "latest": latest, "state": state, "why": why, "codes": codes, + "normed": False, **{f"sparkline_{k}": "" for k in ("req", "bytes", "p95", "4xx", "5xx")}} + + +async def _card(app, router, codes_known=True): + async with app.test_request_context("/plugins/traefik/rows"): + page = await render_plugin_template( + app, "traefik", "traefik/rows.html", problems=[router], healthy=[], gone=[], global_stat=None, + certs=[], old_certs=[], range_key="24h", current_range="24h", error_4xx=1.0, error_5xx=0.1, + codes_known=codes_known) + page = " ".join(page.split()) + return page[page.index(">app@swarm"):] + + +async def test_a_404_heavy_card_says_scanners_and_points_to_tuning(app): + r = _problem(_latest(e4=6.0), [("404", 1812), ("401", 9)]) + explain_problems([r], ["app_app"]) + card = await _card(app, r) + assert "Client errors: 6.0% of responses are 4xx" in card + assert "scanners and bots" in card and "Usually traffic rather than a fault" in card + assert 'href="/settings/thresholds/#traefik"' in card and "/plugins/docker/service/" not in card + assert card.index("scanners") < card.index("Server errors" in card and "couldn't reach the app" in card + assert 'href="/plugins/docker/service/app_app">app_app: its tasks and logs' in card + assert "/settings/thresholds/" not in card + + +async def test_a_card_for_an_unknown_service_says_so(app): + r = _problem(_latest(e5=2.5), []) + explain_problems([r], []) + card = await _card(app, r, codes_known=False) + assert "Status codes not recorded yet" in card + assert "The app is answering with server errors" in card + assert "Steward can't tell which service serves this router" in card
Req/s{{ "%.2f"|format(r.latest.request_rate) }}{{ r.sparkline_req|safe }}
Bandwidth{{ r.latest.response_bytes_rate|rate }}{{ r.sparkline_bytes|safe }}