feat(traefik): request chart shows the total, a time axis and a weighted 5xx share; says why if it can't draw (#5256)

#5256 "Traefik — Request chart: blank on the live dashboard; time axis and honest labels".

- "Req/s" was the average of the routers, not the total. Each scrape is now
  summed over routers, then averaged per bucket (request_history).
- The 5xx share is now weighted by each router's traffic. The axis is no
  longer pinned at 0–100 (suggestedMax 5).
- The x axis shows times, in the viewer's local time. The tooltip shows
  both series at that time.
- A verdict line comes first: now / peak req/s and the peak 5xx share. The
  figures read even if the chart doesn't.
- The blank live chart's cause is still unknown. Chart.js and
  chart-colors.js both load on the dashboard. A missing library or a thrown
  error is now written into the widget instead of leaving it empty, so the
  cause will show on the page.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/routes.py

@@ -375,38 +375,57 @@ async def widget_access_log():
                                  summary=summary, widget_id=widget_id)
 
 
+async def request_history(db, since, bucket_seconds: int) -> list[dict]:
+    """[{"at", "req_rate", "err_pct"}] per time bucket since `since`: each
+    scrape summed over routers (the total req/s), averaged over the bucket;
+    the 5xx share weighted by each router's traffic, so a quiet router's
+    errors don't count as much as a busy one's."""
+    per_scrape = (
+        select(
+            TraefikMetric.scraped_at.label("at"),
+            func.sum(TraefikMetric.request_rate).label("req"),
+            func.sum(TraefikMetric.request_rate * TraefikMetric.error_rate_5xx_pct).label("err_weighted"),
+        )
+        .where(TraefikMetric.scraped_at >= since)
+        .group_by(TraefikMetric.scraped_at)
+        .subquery()
+    )
+    bucket_col = (cast(func.extract("epoch", per_scrape.c.at), Integer) / bucket_seconds).label("bucket")
+    rows = (await db.execute(
+        select(
+            func.avg(per_scrape.c.req).label("req_rate"),
+            func.sum(per_scrape.c.err_weighted).label("err_weighted"),
+            func.sum(per_scrape.c.req).label("req_sum"),
+            func.min(per_scrape.c.at).label("at"),
+            bucket_col,
+        )
+        .group_by(bucket_col)
+        .order_by(bucket_col)
+    )).all()
+    return [{"at": r.at, "req_rate": round(r.req_rate or 0, 3),
+             "err_pct": round((r.err_weighted or 0) / r.req_sum, 2) if r.req_sum else 0.0}
+            for r in rows]
+
+
 @traefik_bp.get("/widget/request_chart")
 @require_role(UserRole.viewer)
 async def widget_request_chart():
-    """HTMX dashboard widget: Chart.js request rate + error % over time."""
-    from datetime import timedelta
+    """HTMX dashboard widget: requests per second across all routers, and the
+    share of them answered 5xx, over the last few hours (#5256). Each scrape
+    is summed over routers first (the total), then averaged per bucket; the
+    5xx share is weighted by each router's traffic, not averaged per router."""
     hours = max(1, min(24, int(request.args.get("hours", 6) or 6)))
     widget_id = request.args.get("wid", "0")
 
     since = datetime.now(timezone.utc) - timedelta(hours=hours)
     b_secs = max(60, (hours * 3600) // 80)  # ~80 buckets
 
-    bucket_col = (
-        cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs
-    ).label("bucket")
-
     async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
-            select(
-                func.avg(TraefikMetric.request_rate).label("req_rate"),
-                func.avg(TraefikMetric.error_rate_5xx_pct).label("err_pct"),
-                func.min(TraefikMetric.scraped_at).label("scraped_at"),
-                bucket_col,
-            )
-            .where(TraefikMetric.scraped_at >= since)
-            .group_by(bucket_col)
-            .order_by(bucket_col)
-        )
-        history = result.all()
+        history = await request_history(db, since, b_secs)
 
-    labels = list(range(len(history)))
-    req_rates = [round(r.req_rate or 0, 3) for r in history]
-    err_pcts = [round(r.err_pct or 0, 2) for r in history]
+    labels = [int(r["at"].timestamp() * 1000) for r in history]   # epoch ms; the browser shows local time
+    req_rates = [r["req_rate"] for r in history]
+    err_pcts = [r["err_pct"] for r in history]
 
     return await render_template(
         "traefik/widget_request_chart.html",
@@ -415,6 +434,9 @@ async def widget_request_chart():
         error_rate_json=json.dumps(err_pcts),
         widget_id=widget_id,
         hours=hours,
+        now_req=req_rates[-1] if req_rates else None,
+        peak_req=max(req_rates) if req_rates else None,
+        peak_err=max(err_pcts) if err_pcts else None,
     )
 
 

plugins/traefik/templates/traefik/widget_request_chart.html

@@ -1,74 +1,92 @@
-{# traefik/widget_request_chart.html — Chart.js request rate + error % over time #}
+{# traefik/widget_request_chart.html — requests per second across all routers
+   and the share answered 5xx, over the last few hours (#5256). The verdict
+   carries the figures, so the widget still says something if the chart
+   can't draw; a chart that fails says why in place of the canvas. #}
 {% if not labels_json or labels_json == '[]' %}
-<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No data for last {{ hours }}h.</div>
+<div class="w-verdict" style="color:var(--text-muted);">No Traefik requests recorded in the last {{ hours }}h.</div>
 {% else %}
-<div style="position:relative;height:160px;">
-  <canvas id="chart-traefik-req-{{ widget_id }}"></canvas>
+<div class="w-verdict {{ 'tone-crit' if peak_err and peak_err >= 1 else ('tone-warn' if peak_err else 'tone-ok') }}">
+  Now {{ "%.2f"|format(now_req) }} req/s across all routers, peak {{ "%.2f"|format(peak_req) }}.
+  {%- if peak_err %} 5xx peaked at {{ "%.2f"|format(peak_err) }}% of responses.{% else %} No 5xx.{% endif %}
+</div>
+<div style="position:relative;height:160px;" id="chart-traefik-req-box-{{ widget_id }}">
+  <canvas id="chart-traefik-req-{{ widget_id }}" aria-label="Requests per second and 5xx share, last {{ hours }} hours"></canvas>
 </div>
 <script>
 (function() {
+  var box = document.getElementById("chart-traefik-req-box-{{ widget_id }}");
   var ctx = document.getElementById("chart-traefik-req-{{ widget_id }}");
-  if (!ctx || !window.Chart) return;
-  var existing = Chart.getChart(ctx);
-  if (existing) existing.destroy();
-  new Chart(ctx.getContext("2d"), {
-    type: "line",
-    data: {
-      labels: {{ labels_json | safe }},
-      datasets: [
-        {
-          label: "Req/s",
-          data: {{ req_rate_json | safe }},
-          borderColor: stewardColor("--blue"),
-          backgroundColor: stewardColor("--blue-dim"),
-          fill: true,
-          tension: 0.3,
-          pointRadius: 0,
-          borderWidth: 1.5,
-          yAxisID: "y",
-        },
-        {
-          label: "5xx %",
-          data: {{ error_rate_json | safe }},
-          borderColor: stewardColor("--red"),
-          fill: false,
-          tension: 0.3,
-          pointRadius: 0,
-          borderWidth: 1.5,
-          yAxisID: "y2",
-        },
-      ]
-    },
-    options: {
-      responsive: true,
-      maintainAspectRatio: false,
-      animation: false,
-      plugins: {
-        legend: {
-          labels: { boxWidth: 10, font: { size: 10 } }
-        }
+  if (!ctx) return;
+  function fail(why) {
+    box.style.height = "auto";
+    box.innerHTML = '<div class="w-note"></div>';
+    box.firstChild.textContent = "The chart couldn't be drawn: " + why;
+  }
+  if (!window.Chart) { fail("the chart library didn't load."); return; }
+  try {
+    var existing = Chart.getChart(ctx);
+    if (existing) existing.destroy();
+    var times = {{ labels_json | safe }};
+    var hhmm = function (ms) {
+      return new Date(ms).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" });
+    };
+    new Chart(ctx.getContext("2d"), {
+      type: "line",
+      data: {
+        labels: times,
+        datasets: [
+          {
+            label: "Requests/s (all routers)",
+            data: {{ req_rate_json | safe }},
+            borderColor: stewardColor("--blue"),
+            backgroundColor: stewardColor("--blue-dim"),
+            fill: true, tension: 0.3, pointRadius: 0, borderWidth: 1.5, yAxisID: "y",
+          },
+          {
+            label: "5xx share %",
+            data: {{ error_rate_json | safe }},
+            borderColor: stewardColor("--red"),
+            fill: false, tension: 0.3, pointRadius: 0, borderWidth: 1.5, yAxisID: "y2",
+          },
+        ]
       },
-      scales: {
-        x: { display: false },
-        y: {
-          min: 0,
-          ticks: { font: { size: 10 } },
-          title: { display: true, text: "req/s", font: { size: 9 } },
-        },
-        y2: {
-          position: "right",
-          min: 0,
-          max: 100,
-          grid: { drawOnChartArea: false },
-          ticks: { color: stewardColor("--red"), font: { size: 10 } },
-          title: { display: true, text: "5xx%", color: stewardColor("--red"), font: { size: 9 } },
+      options: {
+        responsive: true,
+        maintainAspectRatio: false,
+        animation: false,
+        interaction: { mode: "index", intersect: false },
+        plugins: {
+          legend: { labels: { boxWidth: 10, font: { size: 10 } } },
+          tooltip: { callbacks: { title: function (items) { return items.length ? hhmm(times[items[0].dataIndex]) : ""; } } },
         },
+        scales: {
+          x: {
+            ticks: {
+              font: { size: 10 }, maxRotation: 0, autoSkip: true, maxTicksLimit: 6,
+              callback: function (value) { return hhmm(times[value]); },
+            },
+            grid: { display: false },
+          },
+          y: {
+            min: 0,
+            ticks: { font: { size: 10 } },
+            title: { display: true, text: "req/s", font: { size: 9 } },
+          },
+          y2: {
+            position: "right",
+            min: 0,
+            suggestedMax: 5,
+            grid: { drawOnChartArea: false },
+            ticks: { color: stewardColor("--red"), font: { size: 10 } },
+            title: { display: true, text: "5xx %", color: stewardColor("--red"), font: { size: 9 } },
+          },
+        }
       }
-    }
-  });
+    });
+  } catch (err) {
+    fail(err && err.message ? err.message : String(err));
+  }
 })();
 </script>
-<div style="font-size:0.72rem;color:var(--text-muted);margin-top:0.35rem;text-align:right;">
-  last {{ hours }}h
-</div>
+<div class="w-note" style="text-align:right;">last {{ hours }}h, your local time</div>
 {% endif %}

tests/integration/test_traefik_latest.py

@@ -44,3 +44,28 @@ def test_each_router_reads_as_its_newest_row(app):
     got = asyncio.run(go())
     assert got[a].request_rate == 2.0 and got[a].scraped_at == now - timedelta(minutes=1)
     assert got[b].request_rate == 4.0
+
+
+def test_the_request_chart_sums_routers_and_weights_the_5xx_share(app):
+    """#5256: two routers in one scrape are 4 req/s in total, not an average
+    of 2; a 5xx share of 50% on the 3 req/s router is 37.5% of all traffic."""
+    from plugins.traefik.models import TraefikMetric
+    from plugins.traefik.routes import request_history
+
+    # Far ahead of every other test's rows, so only these fall in the window.
+    at = datetime(2099, 1, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=uuid.uuid4().int % 10_000)
+
+    def row(name, rate, e5):
+        return TraefikMetric(router_name=name, scraped_at=at, request_rate=rate, error_rate_4xx_pct=0.0,
+                             error_rate_5xx_pct=e5, latency_p50_ms=1.0, latency_p95_ms=2.0,
+                             latency_p99_ms=3.0, response_bytes_rate=0.0)
+
+    async def go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add_all([row("quiet@docker", 1.0, 0.0), row("busy@docker", 3.0, 50.0)])
+            async with s.begin():
+                return await request_history(s, at, 60)
+
+    got = asyncio.run(go())
+    assert got == [{"at": at, "req_rate": 4.0, "err_pct": 37.5}]

tests/plugins/traefik/test_traefik_page.py

@@ -230,3 +230,19 @@ async def test_the_page_sets_certificates_traefik_no_longer_serves_apart(app):
     page = " ".join(page.split())
     assert "also www.app.example" in page and ">app</td>" in page and ">not in use</td>" in page
     assert page.index("spare.example") < page.index("No longer served (1)") < page.index("gone.example")
+
+
+async def test_the_request_chart_says_its_figures_before_the_chart(app):
+    async with app.test_request_context("/plugins/traefik/widget/request_chart"):
+        page = await render_plugin_template(
+            app, "traefik", "traefik/widget_request_chart.html", labels_json="[1, 2]",
+            req_rate_json="[1.5, 4.25]", error_rate_json="[0, 0.5]", widget_id="7", hours=6,
+            now_req=4.25, peak_req=4.25, peak_err=0.5)
+        empty = await render_plugin_template(
+            app, "traefik", "traefik/widget_request_chart.html", labels_json="[]", hours=6)
+    page = " ".join(page.split())
+    assert "Now 4.25 req/s across all routers, peak 4.25. 5xx peaked at 0.50% of responses." in page
+    assert page.index("w-verdict") < page.index("<canvas")
+    assert "Requests/s (all routers)" in page and "max: 100" not in page
+    assert "couldn't be drawn" in page                    # a failure says why in place of the chart
+    assert "No Traefik requests recorded in the last 6h." in empty