diff --git a/docs/plugins/traefik.md b/docs/plugins/traefik.md index 5da484c..55cda69 100644 --- a/docs/plugins/traefik.md +++ b/docs/plugins/traefik.md @@ -79,7 +79,15 @@ The widget auto-refreshes via HTMX polling every `poll_interval` seconds. Fragme ### Detail Page -The full Traefik page at `/plugins/traefik/` shows all routers with sparkline history charts (last 20 data points) for request rate, p95 latency, and 5xx error rate. +The full Traefik page at `/plugins/traefik/` reads top to bottom, for the time range picked at the top: + +1. **The verdict.** It names the routers that need a look, says why, and says when the problem started ("api, 3.0% of responses are 5xx, started 2h ago"). When a router has been a problem for the whole range, it says so instead. Otherwise it reads "All N routers healthy". +2. **The request chart.** Requests per second across all routers and the share answered 5xx, the same chart as the dashboard widget. +3. **A card for each router that needs a look:** why, since when, and its request rate, bandwidth, p95/p99 latency and 4xx/5xx shares, each with its history. +4. **Traefik's own figures:** when its config last reloaded, open connections, memory. +5. **The healthy routers,** folded into one table, then the routers Traefik no longer reports, then TLS certificates by soonest expiry, then Traffic Origin when the access log is on. + +A router is graded on its last 15 minutes against Settings → Thresholds. When a problem started is read from the router's history over the range. --- diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py index 96f8d60..195c2c2 100644 --- a/plugins/traefik/routes.py +++ b/plugins/traefik/routes.py @@ -137,6 +137,23 @@ def gone_routers(routers: list[dict], scrape_seconds: int) -> tuple[list[dict], return current, gone +def problem_since(history: list, thresholds: dict) -> tuple[datetime | None, bool]: + """(start, before_range) of a router's current problem, read from its + history buckets (router_history's, oldest first): the start of the + trailing run of buckets that grade as a problem. (None, False) when the + newest bucket reads ok, as for a problem too recent or too brief for a + bucket to show. `before_range` says the run reaches back to the first + bucket, so the problem began before the range shown.""" + start = None + for point in reversed(history): + if router_state(point, thresholds)[0] == "ok": + break + start = point + if start is None: + return None, False + return start.bucket, start is history[0] + + def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]: """(problems, healthy): failing routers before slow or noisy ones, each group by name; healthy routers by name.""" @@ -259,6 +276,8 @@ async def rows(): phases.mark("window") for r in current: points = history[r["name"]] + r["since"], r["since_before"] = (problem_since(points, thresholds) if r["state"] != "ok" + else (None, False)) r["sparkline_req"] = _sparkline([p.request_rate for p in points]) r["sparkline_bytes"] = _sparkline([p.response_bytes_rate for p in points]) r["sparkline_p95"] = _sparkline([p.latency_p95_ms for p in points]) @@ -309,6 +328,7 @@ async def rows(): problems=problems, healthy=healthy, gone=gone, + current_range=range_key, error_4xx=error_4xx, error_5xx=error_5xx, global_stat=global_stat, @@ -524,11 +544,15 @@ async def widget_request_chart(): share of them answered 5xx, over the last few hours (#5256). Each scrape is summed over routers first (the total), then averaged per bucket; the 5xx share is weighted by each router's traffic, not averaged per router.""" - hours = max(1, min(24, int(request.args.get("hours", 6) or 6))) widget_id = request.args.get("wid", "0") - - since = datetime.now(timezone.utc) - timedelta(hours=hours) - b_secs = max(60, (hours * 3600) // 80) # ~80 buckets + if request.args.get("range"): + # The Traefik page: the page's own time range (#5710). + since, span = parse_range(request.args.get("range")) + b_secs = bucket_seconds(since) + else: + hours = max(1, min(24, int(request.args.get("hours", 6) or 6))) + since, span = datetime.now(timezone.utc) - timedelta(hours=hours), f"{hours}h" + b_secs = max(60, (hours * 3600) // 80) # ~80 buckets async with current_app.db_sessionmaker() as db: history = await request_history(db, since, b_secs) @@ -543,7 +567,8 @@ async def widget_request_chart(): req_rate_json=json.dumps(req_rates), error_rate_json=json.dumps(err_pcts), widget_id=widget_id, - hours=hours, + span=span, + days=(datetime.now(timezone.utc) - since) > timedelta(hours=24), now_req=req_rates[-1] if req_rates else None, peak_req=max(req_rates) if req_rates else None, peak_err=max(err_pcts) if err_pcts else None, diff --git a/plugins/traefik/templates/traefik/index.html b/plugins/traefik/templates/traefik/index.html index ce44ceb..4614c9d 100644 --- a/plugins/traefik/templates/traefik/index.html +++ b/plugins/traefik/templates/traefik/index.html @@ -15,6 +15,19 @@ +{# State first (#5710): the verdict, which /rows fills out of band each poll; + then requests and 5xx over the page's range; then the routers. #} +

Reading routers…

+ +
+
+
+
+
- {{ stat_tile("Needs attention", problems|length, state="crit" if problems else "inactive") }} - {{ stat_tile("Healthy routers", healthy|length, state="up" if healthy else "inactive") }} - {% if global_stat %} - {% if global_stat.config_last_reload_at %} - {{ stat_tile("Config reloaded", global_stat.config_last_reload_at|ago, - hint=(global_stat.config_reloads_total|int ~ " reloads") if global_stat.config_reloads_total is not none else none) }} - {% endif %} - {% if global_stat.open_conns_total is not none %}{{ stat_tile("Open connections", global_stat.open_conns_total|int) }}{% endif %} - {% if global_stat.process_memory_bytes is not none %}{{ stat_tile("Memory", global_stat.process_memory_bytes|bytes) }}{% endif %} - {% endif %} -
+{# ── Verdict: what needs a look and since when, then the counts ─────────── #} +{% set shown = 3 %} +

+ {%- if problems %}{{ problems|length }} router{{ "s" if problems|length != 1 }} need{{ "s" if problems|length == 1 }} a look + {%- for r in problems[:shown] %}{{ ":" if loop.first else ";" }} {{ r.name }}, {{ r.why }} + {%- if r.since_before %}, all of the last {{ current_range }}{% elif r.since %}, started {{ r.since|ago }}{% endif %} + {%- endfor %} + {%- if problems|length > shown %}; {{ problems|length - shown }} more below{% endif %}. + {%- if healthy %} {{ healthy|length }} healthy.{% endif %} + {%- elif healthy %}All {{ healthy|length }} router{{ "s" if healthy|length != 1 }} healthy + {%- else %}No routers reported yet. + {%- endif %} + {%- if gone %} {{ gone|length }} no longer reported.{% endif %} +

{% macro router_card(r) %}
@@ -23,7 +25,8 @@ {{ status_badge(r.state, "Failing" if r.state == 'crit' else "Warning") }}

{{ r.name }}

-

{{ r.why }}

+

{{ r.why }} + {%- if r.since_before %}, all of the last {{ current_range }}{% elif r.since %}, started {{ r.since|ago }}{% endif %}

@@ -46,6 +49,19 @@ {% endif %} +{# ── The proxy's own figures ─────────────────────────────────────────────── #} +{% if global_stat and (global_stat.config_last_reload_at or global_stat.open_conns_total is not none + or global_stat.process_memory_bytes is not none) %} +
+ {% if global_stat.config_last_reload_at %} + {{ stat_tile("Config reloaded", global_stat.config_last_reload_at|ago, + hint=(global_stat.config_reloads_total|int ~ " reloads") if global_stat.config_reloads_total is not none else none) }} + {% endif %} + {% if global_stat.open_conns_total is not none %}{{ stat_tile("Open connections", global_stat.open_conns_total|int) }}{% endif %} + {% if global_stat.process_memory_bytes is not none %}{{ stat_tile("Memory", global_stat.process_memory_bytes|bytes) }}{% endif %} +
+{% endif %} + {# ── Healthy routers, folded ─────────────────────────────────────────────── #} {% if healthy %}
diff --git a/plugins/traefik/templates/traefik/widget_request_chart.html b/plugins/traefik/templates/traefik/widget_request_chart.html index 0df306c..f9f89cf 100644 --- a/plugins/traefik/templates/traefik/widget_request_chart.html +++ b/plugins/traefik/templates/traefik/widget_request_chart.html @@ -1,16 +1,17 @@ {# traefik/widget_request_chart.html — requests per second across all routers - and the share answered 5xx, over the last few hours (#5256). The verdict + and the share answered 5xx, over the last few hours (#5256), or over the + Traefik page's time range there (#5710). The verdict carries the figures, so the widget still says something if the chart can't draw; a chart that fails says why in place of the canvas. #} {% if not labels_json or labels_json == '[]' %} -
No Traefik requests recorded in the last {{ hours }}h.
+
No Traefik requests recorded in the last {{ span }}.
{% else %}
Now {{ "%.2f"|format(now_req) }} req/s across all routers, peak {{ "%.2f"|format(peak_req) }}. {%- if peak_err %} 5xx peaked at {{ "%.2f"|format(peak_err) }}% of responses.{% else %} No 5xx.{% endif %}
- +
-
last {{ hours }}h, your local time
+
last {{ span }}, your local time
{% endif %} diff --git a/tests/plugins/traefik/test_traefik_page.py b/tests/plugins/traefik/test_traefik_page.py index 43fbfdf..0489688 100644 --- a/tests/plugins/traefik/test_traefik_page.py +++ b/tests/plugins/traefik/test_traefik_page.py @@ -242,10 +242,10 @@ async def test_the_request_chart_says_its_figures_before_the_chart(app): async with app.test_request_context("/plugins/traefik/widget/request_chart"): page = await render_plugin_template( app, "traefik", "traefik/widget_request_chart.html", labels_json="[1, 2]", - req_rate_json="[1.5, 4.25]", error_rate_json="[0, 0.5]", widget_id="7", hours=6, - now_req=4.25, peak_req=4.25, peak_err=0.5) + req_rate_json="[1.5, 4.25]", error_rate_json="[0, 0.5]", widget_id="7", span="6h", + days=False, now_req=4.25, peak_req=4.25, peak_err=0.5) empty = await render_plugin_template( - app, "traefik", "traefik/widget_request_chart.html", labels_json="[]", hours=6) + app, "traefik", "traefik/widget_request_chart.html", labels_json="[]", span="6h") page = " ".join(page.split()) assert "Now 4.25 req/s across all routers, peak 4.25. 5xx peaked at 0.50% of responses." in page assert page.index("w-verdict") < page.index("")] + assert 'hx-swap-oob="true"' in verdict + assert '2 routers need a look' in verdict + assert "api, 3.0% of responses are 5xx, started , p95 latency is 450 ms, all of the last 24h" in verdict + assert "1 healthy." in verdict and "1 no longer reported." in verdict + # The card says since when too; the proxy's figures follow the cards, not the verdict. + card = page[page.index(">api"):] + assert card.index("3.0% of responses are 5xx, started ") + assert page.index(">api") < page.index("Open connections") < page.index('id="traefik-healthy"') + assert "Needs attention" not in page and "Healthy routers" not in page # the verdict says them + + +async def test_the_verdict_when_all_is_well_or_nothing_is_reported(app): + well = await _rows(app, [_router("web", _latest()), _router("api", _latest())]) + assert 'All 2 routers healthy' in well + assert "stat-strip" not in well # no figures, no empty strip + nothing = await _rows(app, []) + assert "No routers reported yet." in nothing + + +async def test_the_verdict_lists_three_problems_then_counts_the_rest(app): + page = await _rows(app, [_router(f"r{i}", _latest(e5=3.0)) for i in range(5)]) + verdict = page[page.index('id="traefik-verdict"'):page.index("

")] + assert "r2" in verdict and "r3" not in verdict and "; 2 more below." in verdict + + +async def test_the_page_puts_the_verdict_then_the_request_chart_above_the_routers(app): + from quart import session + async with app.test_request_context("/plugins/traefik/"): + session["user_role"] = "viewer" + page = await render_plugin_template(app, "traefik", "traefik/index.html", poll_interval=30, + access_log_enabled=True, current_range="7d") + assert (page.index('id="traefik-verdict"') < page.index('id="traefik-chart"') + < page.index('id="traefik-rows"') < page.index('id="traefik-traffic"')) + chart = page[page.index('id="traefik-chart"'):page.index('id="traefik-rows"')] + assert 'hx-get="/plugins/traefik/widget/request_chart?wid=page"' in chart + assert 'hx-include="#time-range"' in chart and "rangeChange from:body" in chart + + +async def test_the_request_chart_over_days_labels_its_times_with_the_day(app): + kw = dict(labels_json="[1, 2]", req_rate_json="[1, 2]", error_rate_json="[0, 0]", widget_id="page", + span="7d", now_req=2, peak_req=2, peak_err=0) + async with app.test_request_context("/plugins/traefik/widget/request_chart"): + week = await render_plugin_template(app, "traefik", "traefik/widget_request_chart.html", days=True, **kw) + hours = await render_plugin_template(app, "traefik", "traefik/widget_request_chart.html", days=False, **kw) + assert '{ weekday: "short", day: "numeric", hour: "2-digit", minute: "2-digit", hourCycle: "h23" }' in week + assert '{ hour: "2-digit", minute: "2-digit", hourCycle: "h23" }' in hours + assert "last 7d, your local time" in week
Req/s{{ "%.2f"|format(r.latest.request_rate) }}{{ r.sparkline_req|safe }}
Bandwidth{{ r.latest.response_bytes_rate|rate }}{{ r.sparkline_bytes|safe }}