feat(traefik): each router problem says what it means, whether it's real, and what to do (#5712)

#5712 "Each router problem says what it means, whether it's real, and what
to do". A problem card now names its kind and reads its codes:
- Server errors: 500 means the app is failing; 502/503/504 means Traefik
  couldn't reach it or it timed out.
- Client errors: 404 scanners, a 401/403 login wall, 429 rate limiting, or
  499 clients giving up on a slow app.
- Slow: p95 against p50.
Each card says whether the problem is likely real, and ends with one next
step. A fault links to the swarm service behind the router, matched by
name, and traffic links to Settings → Thresholds. The wording lives in
plugins/traefik/meaning.py.

The plugin manager now hands out a loaded plugin's submodule
(loaded_plugin_submodule), so Traefik and host_agent reach Docker's
models the same way (#5775).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/maintenance.py

@@ -385,18 +385,9 @@ def summary(job) -> str:
 
 
 def docker_models():
-    """The loaded Docker plugin's models module, or None when it isn't loaded.
-    Reached through the plugin manager: importing plugins.docker.models by its
-    path would define its tables a second time, and the app would refuse it."""
-    import importlib
-    from steward.core.plugin_manager import get_loaded_plugin_modules
-    loaded = get_loaded_plugin_modules().get("docker")
-    if loaded is None:
-        return None
-    try:
-        return importlib.import_module(f"{loaded.__name__}.models")
-    except ImportError:
-        return None
+    """The loaded Docker plugin's models module, or None when it isn't loaded."""
+    from steward.core.plugin_manager import loaded_plugin_submodule
+    return loaded_plugin_submodule("docker", "models")
 
 
 async def _swarm(session) -> tuple:

plugins/traefik/meaning.py

@@ -0,0 +1,115 @@
+"""What a router problem means, whether it is likely real, and what to do (#5712).
+
+Pure: a problem card hands in its grade, its why, the status codes it saw and
+its latest figures, and gets back the words for it. The wording lives here, in
+one place, so other views of a router (its normal, an alert) can say the same.
+"""
+from __future__ import annotations
+
+# A wide gap between p95 and p50 means a few slow requests, not a slow app.
+SLOW_TAIL_RATIO = 4.0
+
+KINDS = {
+    "5xx": "Server errors",
+    "4xx": "Client errors",
+    "slow": "Slow responses",
+}
+
+
+def problem_kind(state: str, why: str | None) -> str | None:
+    """The kind of a graded router's problem: "5xx", "slow", "4xx", or None
+    for a healthy router. routes.router_state grades 5xx as crit, and words a
+    warning as either a p95 or a 4xx share."""
+    if state == "crit":
+        return "5xx"
+    if state == "warn":
+        return "slow" if (why or "").startswith("p95") else "4xx"
+    return None
+
+
+def _top(codes: list[tuple[str, int]], first: str) -> str | None:
+    """The most-answered code of one class ("4" or "5"); codes come most first."""
+    return next((c for c, _n in codes if c[:1] == first), None)
+
+
+def _share(codes: list[tuple[str, int]], first: str, wanted: set[str]) -> float:
+    total = sum(n for c, n in codes if c[:1] == first)
+    return sum(n for c, n in codes if c in wanted) / total if total else 0.0
+
+
+def explain(kind: str, codes: list[tuple[str, int]], p50_ms: float | None = None,
+            p95_ms: float | None = None, normed: bool = False) -> dict:
+    """{"label", "meaning", "likely", "action"} for a problem card:
+    `meaning` what the codes or figures usually point to, `likely` whether
+    it is usually a fault, `action` "act" (look at the app) or "tune" (the
+    cutoff may be wrong for this router). `normed` says the router was graded
+    against its own normal, so the problem is a departure from its usual."""
+    if kind == "5xx":
+        top = _top(codes, "5")
+        if top == "500":
+            meaning = "Mostly 500: the app is failing requests. Its logs say why."
+        elif top in ("502", "503", "504"):
+            meaning = (f"Mostly {top}: Traefik couldn't reach the app, or it timed out. "
+                       "Check that its container is up and healthy.")
+        elif top:
+            meaning = f"Mostly {top}: the app is answering with server errors. Its logs say why."
+        else:
+            meaning = "The app is answering with server errors. Its logs say why."
+        return {"label": KINDS[kind], "meaning": meaning,
+                "likely": "Almost always a real fault.", "action": "act"}
+
+    if kind == "4xx":
+        top = _top(codes, "4")
+        if top == "404":
+            meaning = ("Mostly 404: usually scanners and bots probing for paths, or links to "
+                       "pages that moved. Usually harmless.")
+        elif top in ("401", "403"):
+            meaning = (f"Mostly {top}: a login wall doing its job, refusing requests without "
+                       "a session. A problem only if it's you being refused.")
+        elif top == "429":
+            meaning = "Mostly 429: rate limiting, telling clients to slow down."
+        elif top == "499":
+            meaning = ("Mostly 499: clients giving up before the app answered, "
+                       "often because the app is slow.")
+        elif top:
+            meaning = f"Mostly {top}: requests the app turned away as not allowed or not understood."
+        else:
+            meaning = "Requests the app turned away: often bots, scanners or a login wall."
+        # A slow app behind the 499s is a fault; a login wall or scanners are not.
+        if top == "499" and _share(codes, "4", {"499"}) >= 0.5:
+            return {"label": KINDS[kind], "meaning": meaning,
+                    "likely": "Often a slow app rather than the clients.", "action": "act"}
+        likely = ("Above its usual level for this hour, so something changed, "
+                  "but 4xx is still usually traffic rather than a fault." if normed
+                  else "Usually traffic rather than a fault: worth tuning, not fixing.")
+        return {"label": KINDS[kind], "meaning": meaning, "likely": likely, "action": "tune"}
+
+    # slow
+    if p50_ms and p95_ms and p95_ms >= SLOW_TAIL_RATIO * p50_ms:
+        meaning = (f"p95 is {p95_ms / p50_ms:.0f}× p50 ({p50_ms:.0f} ms): a few slow requests, "
+                   "likely one endpoint or a heavy page, not the whole app.")
+    elif p50_ms and p95_ms:
+        meaning = (f"p50 is close behind at {p50_ms:.0f} ms: the whole app is slow, "
+                   "not one endpoint.")
+    else:
+        meaning = "The app is slow overall, or one endpoint is."
+    likely = ("Slower than usual for this hour." if normed
+              else "Real if users feel it; some apps are simply slow.")
+    return {"label": KINDS["slow"], "meaning": meaning, "likely": likely, "action": "act"}
+
+
+def swarm_service(router: str, services: list[str]) -> str | None:
+    """The swarm service a router fronts, by name: "radarr@swarm" is served
+    by "arr_radarr", "codex@swarm" by "codex_app" (its stack's app), and
+    "fabled-searxng@swarm" by "fabled_search_searxng" (its last word). None
+    when no single service matches, or the router isn't a swarm one."""
+    base, _, provider = router.partition("@")
+    if provider not in ("swarm", "docker"):
+        return None
+    for name in dict.fromkeys((base, base.rsplit("-", 1)[-1])):
+        for match in ([s for s in services if s == name],
+                      [s for s in services if s.endswith("_" + name)],
+                      [s for s in services if s == f"{name}_app"]):
+            if len(match) == 1:
+                return match[0]
+    return None

plugins/traefik/routes.py

@@ -16,6 +16,7 @@ from steward.core.formatting import cert_left, status_class
 from steward.core.norms import norms_at
 from steward.core.settings import floor_of
 from steward.models.metrics import PluginMetricLatest
+from . import meaning
 from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
 from .scheduler import router_domains
 from .scraper import cert_names, covers
@@ -183,6 +184,30 @@ def grade_routers(routers: list[dict], window: dict, thresholds: dict, norms: di
         reading, own = window.get(r["name"]), norms.get(r["name"])
         r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS, own) if reading
                                 else router_state(r["latest"], thresholds, norms=own))
+        r["kind"] = meaning.problem_kind(r["state"], r["why"])
+        r["normed"] = bool(own) and {"slow": "latency_p95_ms", "4xx": "error_rate_4xx_pct"}.get(
+            r["kind"], "") in own
+
+
+async def swarm_service_names(db) -> list[str]:
+    """Every swarm service the Docker plugin has stored; [] without it."""
+    from steward.core.plugin_manager import loaded_plugin_submodule
+    models = loaded_plugin_submodule("docker", "models")
+    if models is None:
+        return []
+    return list((await db.execute(select(models.DockerSwarmService.service_name).distinct())).scalars())
+
+
+def explain_problems(problems: list[dict], services: list[str]) -> None:
+    """Give each problem card what it means and its one next step (#5712):
+    `meaning` (meaning.explain's) and `service`, the swarm service behind
+    the router when Steward can tell which."""
+    for r in problems:
+        r["meaning"] = meaning.explain(r["kind"], r.get("codes", []),
+                                       getattr(r["latest"], "latency_p50_ms", None),
+                                       getattr(r["latest"], "latency_p95_ms", None),
+                                       r.get("normed", False))
+        r["service"] = meaning.swarm_service(r["name"], services)
 
 
 # A router missing from this many scrapes in a row is no longer reported:
@@ -362,6 +387,7 @@ async def rows():
         # old, a router with none listed may simply predate the recording.
         first_code = await db.scalar(select(func.min(TraefikStatusCode.scraped_at)))
         codes_known = first_code is not None and first_code <= now - STATE_WINDOW
+        services = await swarm_service_names(db) if any(r["state"] != "ok" for r in current) else []
         phases.mark("codes")
 
         # Latest global stat (always most recent, not range-filtered)
@@ -400,6 +426,7 @@ async def rows():
     certs = [cert_row(c) for c in served]
     old_certs = [cert_row(c) for c in old]
     problems, healthy = split_routers(current)
+    explain_problems(problems, services)
     phases.mark("cert_rows")
     error_5xx, error_4xx = error_cutoffs(thresholds)
     page = await render_template(

plugins/traefik/templates/traefik/rows.html

@@ -45,7 +45,7 @@
       {{ status_badge(r.state, "Failing" if r.state == 'crit' else "Warning") }}
       <h3 class="section-title" style="margin:0;word-break:break-all;">{{ r.name }}</h3>
     </div>
-    <p class="problem-detail" style="margin-bottom:0.5rem;">{{ r.why }}
+    <p class="problem-detail" style="margin-bottom:0.5rem;">{% if r.meaning %}<strong>{{ r.meaning.label }}</strong>: {% endif %}{{ r.why }}
       {%- if r.since_before %}, all of the last {{ current_range }}{% elif r.since %}, started {{ r.since|ago }}{% endif %}</p>
     {% if r.codes %}
     <p class="field-hint" style="margin:0 0 0.5rem;">Last 15 min:
@@ -56,6 +56,18 @@
     {% else %}
     <p class="field-hint" style="margin:0 0 0.5rem;">Status codes not recorded yet: Steward keeps them from this version on.</p>
     {% endif %}
+    {% if r.meaning %}
+    {# What it means and the one next step (#5712; plugins/traefik/meaning.py). #}
+    <p style="margin:0 0 0.3rem;font-size:0.85rem;">{{ r.meaning.meaning }} <span class="tone-muted">{{ r.meaning.likely }}</span></p>
+    <p class="field-hint" style="margin:0 0 0.5rem;">Next:
+      {%- if r.meaning.action == 'act' %}
+        {%- if r.service %} <a href="/plugins/docker/service/{{ r.service|urlencode }}">{{ r.service }}: its tasks and logs →</a>
+        {%- else %} look at the app's container and logs under <a href="/plugins/docker/swarm">Docker → Swarm</a>; Steward can't tell which service serves this router.
+        {%- endif %}
+      {%- else %} if this is normal for this router, raise the 4xx cutoff in <a href="/settings/thresholds/#traefik">Settings → Thresholds</a>. It is one cutoff for every router
+        {%- if not r.normed %}; once Steward has learned this router's normal for the hour, it flags only departures from it{% endif %}.
+      {%- endif %}</p>
+    {% endif %}
     <table class="table" style="font-size:0.85rem;">
         <tr><td class="field-hint">Req/s</td><td>{{ "%.2f"|format(r.latest.request_rate) }}</td><td>{{ r.sparkline_req|safe }}</td></tr>
         <tr><td class="field-hint">Bandwidth</td><td>{{ r.latest.response_bytes_rate|rate }}</td><td>{{ r.sparkline_bytes|safe }}</td></tr>