feat(traefik): each router problem says what it means, whether it's real, and what to do (#5712)
#5712 "Each router problem says what it means, whether it's real, and what to do". A problem card now names its kind and reads its codes: - Server errors: 500 means the app is failing; 502/503/504 means Traefik couldn't reach it or it timed out. - Client errors: 404 scanners, a 401/403 login wall, 429 rate limiting, or 499 clients giving up on a slow app. - Slow: p95 against p50. Each card says whether the problem is likely real, and ends with one next step. A fault links to the swarm service behind the router, matched by name, and traffic links to Settings → Thresholds. The wording lives in plugins/traefik/meaning.py. The plugin manager now hands out a loaded plugin's submodule (loaded_plugin_submodule), so Traefik and host_agent reach Docker's models the same way (#5775). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -84,6 +84,11 @@ The full Traefik page at `/plugins/traefik/` reads top to bottom, for the time r
|
||||
1. **The verdict.** It names the routers that need a look, says why, and says when the problem started ("api, 3.0% of responses are 5xx, started 2h ago"). When a router has been a problem for the whole range, it says so instead. Otherwise it reads "All N routers healthy".
|
||||
2. **The request chart.** Requests per second across all routers and the share answered 5xx, the same chart as the dashboard widget.
|
||||
3. **A card for each router that needs a look:** why, since when, the 4xx and 5xx status codes it answered in the last 15 minutes, most first ("404 ×1,812, 502 ×40"), and its request rate, bandwidth, p95/p99 latency and 4xx/5xx shares, each with its history. The code is what separates scanners probing for pages (404), a login wall (401/403) and an app that isn't answering (502/503/504). Steward records codes from the version that added them on; until then a card says they aren't recorded yet.
|
||||
Each card then says what its problem usually means and what to do (`plugins/traefik/meaning.py`, #5712):
|
||||
- **Server errors (5xx)** are almost always real. Mostly 500: the app is failing requests. Mostly 502/503/504: Traefik couldn't reach the app, or it timed out.
|
||||
- **Client errors (4xx)** are usually traffic rather than a fault. Mostly 404: scanners and bots. 401/403: a login wall. 429: rate limiting. Mostly 499 is the exception: clients giving up on a slow app.
|
||||
- **Slow responses** compare p95 with p50. When p95 is 4× p50 or more, a few requests are slow (one endpoint or a heavy page). When p50 is close behind, the whole app is slow.
|
||||
- **Next:** a fault links to the swarm service behind the router (its tasks and logs) when Steward can match the name ("codex@swarm" → `codex_app`, "radarr@swarm" → `arr_radarr`), otherwise to Docker → Swarm. Traffic links to Settings → Thresholds.
|
||||
4. **Traefik's own figures:** when its config last reloaded, open connections, memory.
|
||||
5. **The healthy routers,** folded into one table, then the routers Traefik no longer reports, then TLS certificates by soonest expiry, then Traffic Origin when the access log is on.
|
||||
|
||||
|
||||
@@ -385,18 +385,9 @@ def summary(job) -> str:
|
||||
|
||||
|
||||
def docker_models():
|
||||
"""The loaded Docker plugin's models module, or None when it isn't loaded.
|
||||
Reached through the plugin manager: importing plugins.docker.models by its
|
||||
path would define its tables a second time, and the app would refuse it."""
|
||||
import importlib
|
||||
from steward.core.plugin_manager import get_loaded_plugin_modules
|
||||
loaded = get_loaded_plugin_modules().get("docker")
|
||||
if loaded is None:
|
||||
return None
|
||||
try:
|
||||
return importlib.import_module(f"{loaded.__name__}.models")
|
||||
except ImportError:
|
||||
return None
|
||||
"""The loaded Docker plugin's models module, or None when it isn't loaded."""
|
||||
from steward.core.plugin_manager import loaded_plugin_submodule
|
||||
return loaded_plugin_submodule("docker", "models")
|
||||
|
||||
|
||||
async def _swarm(session) -> tuple:
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
"""What a router problem means, whether it is likely real, and what to do (#5712).
|
||||
|
||||
Pure: a problem card hands in its grade, its why, the status codes it saw and
|
||||
its latest figures, and gets back the words for it. The wording lives here, in
|
||||
one place, so other views of a router (its normal, an alert) can say the same.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
# A wide gap between p95 and p50 means a few slow requests, not a slow app.
|
||||
SLOW_TAIL_RATIO = 4.0
|
||||
|
||||
KINDS = {
|
||||
"5xx": "Server errors",
|
||||
"4xx": "Client errors",
|
||||
"slow": "Slow responses",
|
||||
}
|
||||
|
||||
|
||||
def problem_kind(state: str, why: str | None) -> str | None:
|
||||
"""The kind of a graded router's problem: "5xx", "slow", "4xx", or None
|
||||
for a healthy router. routes.router_state grades 5xx as crit, and words a
|
||||
warning as either a p95 or a 4xx share."""
|
||||
if state == "crit":
|
||||
return "5xx"
|
||||
if state == "warn":
|
||||
return "slow" if (why or "").startswith("p95") else "4xx"
|
||||
return None
|
||||
|
||||
|
||||
def _top(codes: list[tuple[str, int]], first: str) -> str | None:
|
||||
"""The most-answered code of one class ("4" or "5"); codes come most first."""
|
||||
return next((c for c, _n in codes if c[:1] == first), None)
|
||||
|
||||
|
||||
def _share(codes: list[tuple[str, int]], first: str, wanted: set[str]) -> float:
|
||||
total = sum(n for c, n in codes if c[:1] == first)
|
||||
return sum(n for c, n in codes if c in wanted) / total if total else 0.0
|
||||
|
||||
|
||||
def explain(kind: str, codes: list[tuple[str, int]], p50_ms: float | None = None,
|
||||
p95_ms: float | None = None, normed: bool = False) -> dict:
|
||||
"""{"label", "meaning", "likely", "action"} for a problem card:
|
||||
`meaning` what the codes or figures usually point to, `likely` whether
|
||||
it is usually a fault, `action` "act" (look at the app) or "tune" (the
|
||||
cutoff may be wrong for this router). `normed` says the router was graded
|
||||
against its own normal, so the problem is a departure from its usual."""
|
||||
if kind == "5xx":
|
||||
top = _top(codes, "5")
|
||||
if top == "500":
|
||||
meaning = "Mostly 500: the app is failing requests. Its logs say why."
|
||||
elif top in ("502", "503", "504"):
|
||||
meaning = (f"Mostly {top}: Traefik couldn't reach the app, or it timed out. "
|
||||
"Check that its container is up and healthy.")
|
||||
elif top:
|
||||
meaning = f"Mostly {top}: the app is answering with server errors. Its logs say why."
|
||||
else:
|
||||
meaning = "The app is answering with server errors. Its logs say why."
|
||||
return {"label": KINDS[kind], "meaning": meaning,
|
||||
"likely": "Almost always a real fault.", "action": "act"}
|
||||
|
||||
if kind == "4xx":
|
||||
top = _top(codes, "4")
|
||||
if top == "404":
|
||||
meaning = ("Mostly 404: usually scanners and bots probing for paths, or links to "
|
||||
"pages that moved. Usually harmless.")
|
||||
elif top in ("401", "403"):
|
||||
meaning = (f"Mostly {top}: a login wall doing its job, refusing requests without "
|
||||
"a session. A problem only if it's you being refused.")
|
||||
elif top == "429":
|
||||
meaning = "Mostly 429: rate limiting, telling clients to slow down."
|
||||
elif top == "499":
|
||||
meaning = ("Mostly 499: clients giving up before the app answered, "
|
||||
"often because the app is slow.")
|
||||
elif top:
|
||||
meaning = f"Mostly {top}: requests the app turned away as not allowed or not understood."
|
||||
else:
|
||||
meaning = "Requests the app turned away: often bots, scanners or a login wall."
|
||||
# A slow app behind the 499s is a fault; a login wall or scanners are not.
|
||||
if top == "499" and _share(codes, "4", {"499"}) >= 0.5:
|
||||
return {"label": KINDS[kind], "meaning": meaning,
|
||||
"likely": "Often a slow app rather than the clients.", "action": "act"}
|
||||
likely = ("Above its usual level for this hour, so something changed, "
|
||||
"but 4xx is still usually traffic rather than a fault." if normed
|
||||
else "Usually traffic rather than a fault: worth tuning, not fixing.")
|
||||
return {"label": KINDS[kind], "meaning": meaning, "likely": likely, "action": "tune"}
|
||||
|
||||
# slow
|
||||
if p50_ms and p95_ms and p95_ms >= SLOW_TAIL_RATIO * p50_ms:
|
||||
meaning = (f"p95 is {p95_ms / p50_ms:.0f}× p50 ({p50_ms:.0f} ms): a few slow requests, "
|
||||
"likely one endpoint or a heavy page, not the whole app.")
|
||||
elif p50_ms and p95_ms:
|
||||
meaning = (f"p50 is close behind at {p50_ms:.0f} ms: the whole app is slow, "
|
||||
"not one endpoint.")
|
||||
else:
|
||||
meaning = "The app is slow overall, or one endpoint is."
|
||||
likely = ("Slower than usual for this hour." if normed
|
||||
else "Real if users feel it; some apps are simply slow.")
|
||||
return {"label": KINDS["slow"], "meaning": meaning, "likely": likely, "action": "act"}
|
||||
|
||||
|
||||
def swarm_service(router: str, services: list[str]) -> str | None:
|
||||
"""The swarm service a router fronts, by name: "radarr@swarm" is served
|
||||
by "arr_radarr", "codex@swarm" by "codex_app" (its stack's app), and
|
||||
"fabled-searxng@swarm" by "fabled_search_searxng" (its last word). None
|
||||
when no single service matches, or the router isn't a swarm one."""
|
||||
base, _, provider = router.partition("@")
|
||||
if provider not in ("swarm", "docker"):
|
||||
return None
|
||||
for name in dict.fromkeys((base, base.rsplit("-", 1)[-1])):
|
||||
for match in ([s for s in services if s == name],
|
||||
[s for s in services if s.endswith("_" + name)],
|
||||
[s for s in services if s == f"{name}_app"]):
|
||||
if len(match) == 1:
|
||||
return match[0]
|
||||
return None
|
||||
@@ -16,6 +16,7 @@ from steward.core.formatting import cert_left, status_class
|
||||
from steward.core.norms import norms_at
|
||||
from steward.core.settings import floor_of
|
||||
from steward.models.metrics import PluginMetricLatest
|
||||
from . import meaning
|
||||
from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
|
||||
from .scheduler import router_domains
|
||||
from .scraper import cert_names, covers
|
||||
@@ -183,6 +184,30 @@ def grade_routers(routers: list[dict], window: dict, thresholds: dict, norms: di
|
||||
reading, own = window.get(r["name"]), norms.get(r["name"])
|
||||
r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS, own) if reading
|
||||
else router_state(r["latest"], thresholds, norms=own))
|
||||
r["kind"] = meaning.problem_kind(r["state"], r["why"])
|
||||
r["normed"] = bool(own) and {"slow": "latency_p95_ms", "4xx": "error_rate_4xx_pct"}.get(
|
||||
r["kind"], "") in own
|
||||
|
||||
|
||||
async def swarm_service_names(db) -> list[str]:
|
||||
"""Every swarm service the Docker plugin has stored; [] without it."""
|
||||
from steward.core.plugin_manager import loaded_plugin_submodule
|
||||
models = loaded_plugin_submodule("docker", "models")
|
||||
if models is None:
|
||||
return []
|
||||
return list((await db.execute(select(models.DockerSwarmService.service_name).distinct())).scalars())
|
||||
|
||||
|
||||
def explain_problems(problems: list[dict], services: list[str]) -> None:
|
||||
"""Give each problem card what it means and its one next step (#5712):
|
||||
`meaning` (meaning.explain's) and `service`, the swarm service behind
|
||||
the router when Steward can tell which."""
|
||||
for r in problems:
|
||||
r["meaning"] = meaning.explain(r["kind"], r.get("codes", []),
|
||||
getattr(r["latest"], "latency_p50_ms", None),
|
||||
getattr(r["latest"], "latency_p95_ms", None),
|
||||
r.get("normed", False))
|
||||
r["service"] = meaning.swarm_service(r["name"], services)
|
||||
|
||||
|
||||
# A router missing from this many scrapes in a row is no longer reported:
|
||||
@@ -362,6 +387,7 @@ async def rows():
|
||||
# old, a router with none listed may simply predate the recording.
|
||||
first_code = await db.scalar(select(func.min(TraefikStatusCode.scraped_at)))
|
||||
codes_known = first_code is not None and first_code <= now - STATE_WINDOW
|
||||
services = await swarm_service_names(db) if any(r["state"] != "ok" for r in current) else []
|
||||
phases.mark("codes")
|
||||
|
||||
# Latest global stat (always most recent, not range-filtered)
|
||||
@@ -400,6 +426,7 @@ async def rows():
|
||||
certs = [cert_row(c) for c in served]
|
||||
old_certs = [cert_row(c) for c in old]
|
||||
problems, healthy = split_routers(current)
|
||||
explain_problems(problems, services)
|
||||
phases.mark("cert_rows")
|
||||
error_5xx, error_4xx = error_cutoffs(thresholds)
|
||||
page = await render_template(
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
{{ status_badge(r.state, "Failing" if r.state == 'crit' else "Warning") }}
|
||||
<h3 class="section-title" style="margin:0;word-break:break-all;">{{ r.name }}</h3>
|
||||
</div>
|
||||
<p class="problem-detail" style="margin-bottom:0.5rem;">{{ r.why }}
|
||||
<p class="problem-detail" style="margin-bottom:0.5rem;">{% if r.meaning %}<strong>{{ r.meaning.label }}</strong>: {% endif %}{{ r.why }}
|
||||
{%- if r.since_before %}, all of the last {{ current_range }}{% elif r.since %}, started {{ r.since|ago }}{% endif %}</p>
|
||||
{% if r.codes %}
|
||||
<p class="field-hint" style="margin:0 0 0.5rem;">Last 15 min:
|
||||
@@ -56,6 +56,18 @@
|
||||
{% else %}
|
||||
<p class="field-hint" style="margin:0 0 0.5rem;">Status codes not recorded yet: Steward keeps them from this version on.</p>
|
||||
{% endif %}
|
||||
{% if r.meaning %}
|
||||
{# What it means and the one next step (#5712; plugins/traefik/meaning.py). #}
|
||||
<p style="margin:0 0 0.3rem;font-size:0.85rem;">{{ r.meaning.meaning }} <span class="tone-muted">{{ r.meaning.likely }}</span></p>
|
||||
<p class="field-hint" style="margin:0 0 0.5rem;">Next:
|
||||
{%- if r.meaning.action == 'act' %}
|
||||
{%- if r.service %} <a href="/plugins/docker/service/{{ r.service|urlencode }}">{{ r.service }}: its tasks and logs →</a>
|
||||
{%- else %} look at the app's container and logs under <a href="/plugins/docker/swarm">Docker → Swarm</a>; Steward can't tell which service serves this router.
|
||||
{%- endif %}
|
||||
{%- else %} if this is normal for this router, raise the 4xx cutoff in <a href="/settings/thresholds/#traefik">Settings → Thresholds</a>. It is one cutoff for every router
|
||||
{%- if not r.normed %}; once Steward has learned this router's normal for the hour, it flags only departures from it{% endif %}.
|
||||
{%- endif %}</p>
|
||||
{% endif %}
|
||||
<table class="table" style="font-size:0.85rem;">
|
||||
<tr><td class="field-hint">Req/s</td><td>{{ "%.2f"|format(r.latest.request_rate) }}</td><td>{{ r.sparkline_req|safe }}</td></tr>
|
||||
<tr><td class="field-hint">Bandwidth</td><td>{{ r.latest.response_bytes_rate|rate }}</td><td>{{ r.sparkline_bytes|safe }}</td></tr>
|
||||
|
||||
@@ -77,6 +77,21 @@ def get_loaded_plugin_modules() -> dict[str, object]:
|
||||
return dict(_PLUGIN_MODULES)
|
||||
|
||||
|
||||
def loaded_plugin_submodule(name: str, sub: str):
|
||||
"""A loaded plugin's submodule (e.g. ("docker", "models")), or None when
|
||||
the plugin isn't loaded. Another plugin reaches its models this way:
|
||||
importing plugins.<name>.models by path would define its tables a second
|
||||
time, because the plugin was loaded as _steward_plugin_<name> (#5775)."""
|
||||
import importlib
|
||||
loaded = _PLUGIN_MODULES.get(name)
|
||||
if loaded is None:
|
||||
return None
|
||||
try:
|
||||
return importlib.import_module(f"{loaded.__name__}.{sub}")
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
|
||||
def _collect_plugin_module(app: "Quart", name: str, module) -> None:
|
||||
"""Remember a loaded plugin module; hand it to a live MCP server if mounted.
|
||||
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Each router problem says what it means, whether it's real, and what to do
|
||||
(#5712): plugins/traefik/meaning.py's wording, and the problem card that
|
||||
carries it."""
|
||||
|
||||
import pytest
|
||||
|
||||
from plugins.traefik import meaning
|
||||
from plugins.traefik.routes import STATE_WINDOW_WORDS, explain_problems, router_state
|
||||
from steward.core.settings import to_thresholds_cfg
|
||||
from tests._templates import render_plugin_template
|
||||
from tests.plugins.traefik.test_traefik_page import _latest
|
||||
|
||||
TH = to_thresholds_cfg({})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("latest, kind", [
|
||||
(_latest(e5=2.5), "5xx"),
|
||||
(_latest(p95=450.0), "slow"),
|
||||
(_latest(e4=5.0), "4xx"),
|
||||
(_latest(e5=2.5, e4=5.0, p95=450.0), "5xx"), # graded on its worst
|
||||
(_latest(p95=450.0, e4=5.0), "slow"), # router_state checks p95 before 4xx
|
||||
(_latest(), None),
|
||||
])
|
||||
def test_the_kind_follows_router_state_wording(latest, kind):
|
||||
for over in ("", STATE_WINDOW_WORDS):
|
||||
state, why = router_state(latest, TH, over)
|
||||
assert meaning.problem_kind(state, why) == kind
|
||||
|
||||
|
||||
@pytest.mark.parametrize("codes, says, action", [
|
||||
([("500", 30), ("502", 2)], "the app is failing requests", "act"),
|
||||
([("502", 40), ("500", 1)], "couldn't reach the app", "act"),
|
||||
([("504", 5)], "Mostly 504: Traefik couldn't reach the app, or it timed out", "act"),
|
||||
([], "The app is answering with server errors", "act"),
|
||||
])
|
||||
def test_server_errors_are_real_and_point_at_the_app(codes, says, action):
|
||||
got = meaning.explain("5xx", codes)
|
||||
assert says in got["meaning"] and got["action"] == action and "real fault" in got["likely"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("codes, says, action", [
|
||||
([("404", 1812), ("401", 9)], "scanners and bots", "tune"),
|
||||
([("401", 300), ("404", 2)], "a login wall doing its job", "tune"),
|
||||
([("403", 30)], "Mostly 403: a login wall", "tune"),
|
||||
([("429", 12)], "rate limiting", "tune"),
|
||||
([("499", 50), ("404", 10)], "clients giving up", "act"), # mostly 499: a slow app
|
||||
([("404", 50), ("499", 10)], "scanners and bots", "tune"),
|
||||
([("400", 7)], "Mostly 400: requests the app turned away", "tune"),
|
||||
([], "often bots, scanners or a login wall", "tune"),
|
||||
])
|
||||
def test_client_errors_say_which_kind_of_traffic(codes, says, action):
|
||||
got = meaning.explain("4xx", codes)
|
||||
assert says in got["meaning"] and got["action"] == action
|
||||
|
||||
|
||||
def test_a_4xx_past_its_own_normal_says_something_changed():
|
||||
assert "Usually traffic" in meaning.explain("4xx", [("404", 9)])["likely"]
|
||||
assert "Above its usual level" in meaning.explain("4xx", [("404", 9)], normed=True)["likely"]
|
||||
|
||||
|
||||
def test_slow_compares_p95_with_p50():
|
||||
tail = meaning.explain("slow", [], p50_ms=40.0, p95_ms=900.0)
|
||||
assert "p95 is 22× p50 (40 ms): a few slow requests" in tail["meaning"] and tail["action"] == "act"
|
||||
whole = meaning.explain("slow", [], p50_ms=600.0, p95_ms=900.0)
|
||||
assert "the whole app is slow" in whole["meaning"]
|
||||
assert meaning.explain("slow", [], p95_ms=900.0)["meaning"] == "The app is slow overall, or one endpoint is."
|
||||
assert "Slower than usual" in meaning.explain("slow", [], normed=True)["likely"]
|
||||
|
||||
|
||||
SERVICES = ["arr_radarr", "codex_app", "codex_db", "steward_app", "steward_db",
|
||||
"personal_nextcloud", "fabled_search_searxng", "traefik_router", "radarr_old_radarr"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("router, service", [
|
||||
("codex@swarm", "codex_app"), # its stack's app
|
||||
("steward@swarm", "steward_app"),
|
||||
("nextcloud@swarm", "personal_nextcloud"), # the one service ending in it
|
||||
("fabled-searxng@swarm", "fabled_search_searxng"), # by its last word
|
||||
("radarr@swarm", None), # two services end in _radarr: can't tell
|
||||
("home@swarm", None),
|
||||
("jellyfin@file", None), # not a swarm router
|
||||
("traefik_router@docker", "traefik_router"),
|
||||
])
|
||||
def test_a_router_finds_the_swarm_service_behind_it(router, service):
|
||||
assert meaning.swarm_service(router, SERVICES) == service
|
||||
|
||||
|
||||
def _problem(latest, codes):
|
||||
state, why = router_state(latest, TH)
|
||||
return {"name": "app@swarm", "latest": latest, "state": state, "why": why, "codes": codes,
|
||||
"normed": False, **{f"sparkline_{k}": "" for k in ("req", "bytes", "p95", "4xx", "5xx")}}
|
||||
|
||||
|
||||
async def _card(app, router, codes_known=True):
|
||||
async with app.test_request_context("/plugins/traefik/rows"):
|
||||
page = await render_plugin_template(
|
||||
app, "traefik", "traefik/rows.html", problems=[router], healthy=[], gone=[], global_stat=None,
|
||||
certs=[], old_certs=[], range_key="24h", current_range="24h", error_4xx=1.0, error_5xx=0.1,
|
||||
codes_known=codes_known)
|
||||
page = " ".join(page.split())
|
||||
return page[page.index(">app@swarm</h3>"):]
|
||||
|
||||
|
||||
async def test_a_404_heavy_card_says_scanners_and_points_to_tuning(app):
|
||||
r = _problem(_latest(e4=6.0), [("404", 1812), ("401", 9)])
|
||||
explain_problems([r], ["app_app"])
|
||||
card = await _card(app, r)
|
||||
assert "<strong>Client errors</strong>: 6.0% of responses are 4xx" in card
|
||||
assert "scanners and bots" in card and "Usually traffic rather than a fault" in card
|
||||
assert 'href="/settings/thresholds/#traefik"' in card and "/plugins/docker/service/" not in card
|
||||
assert card.index("scanners") < card.index("<table")
|
||||
|
||||
|
||||
async def test_a_502_card_points_to_its_service(app):
|
||||
r = _problem(_latest(e5=2.5), [("502", 40)])
|
||||
explain_problems([r], ["app_app", "app_db"])
|
||||
card = await _card(app, r)
|
||||
assert "<strong>Server errors</strong>" in card and "couldn't reach the app" in card
|
||||
assert 'href="/plugins/docker/service/app_app">app_app: its tasks and logs' in card
|
||||
assert "/settings/thresholds/" not in card
|
||||
|
||||
|
||||
async def test_a_card_for_an_unknown_service_says_so(app):
|
||||
r = _problem(_latest(e5=2.5), [])
|
||||
explain_problems([r], [])
|
||||
card = await _card(app, r, codes_known=False)
|
||||
assert "Status codes not recorded yet" in card
|
||||
assert "The app is answering with server errors" in card
|
||||
assert "Steward can't tell which service serves this router" in card
|
||||
Reference in New Issue
Block a user