fix(traefik): latency is each scrape's own percentile, every label set summed (#5755)

#5755 "Traefik latency was a percentile of every response since Traefik
started, over unsummed label sets". _percentile_ms read the duration
histogram as it stood, which counts from Traefik's start, so p50/p95/p99 were
lifetime figures that barely moved. It also walked every code/method label
set's buckets as one unsummed list, so equal counts from two sets returned
an edge exactly: nextcloud's flat 100.0 ms p95.

service_buckets() now takes each bucket's rise since the previous scrape,
summed per le over every label set. A scrape with no responses, and the
first scrape, record no latency: traefik_007_latency_nullable lets the
columns be NULL, the batch skips the figure, and the page shows "—".

Found while preparing #5741 "Traefik routers are graded against their own
normal", which grades p95 against a learned normal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/migrations/versions/traefik_007_latency_nullable.py

@@ -0,0 +1,30 @@
+"""A router's latency may be absent
+
+Latency is now each scrape's own percentile (#5741), and a scrape in which a
+router answered nothing has none: those rows carry NULL rather than a
+made-up 0 or the old lifetime figure.
+
+Revision ID: traefik_007_latency_nullable
+Revises: traefik_006_status_codes
+Create Date: 2026-10-14
+"""
+from typing import Sequence, Union
+from alembic import op
+
+revision: str = "traefik_007_latency_nullable"
+down_revision: Union[str, None] = "traefik_006_status_codes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+_COLUMNS = ("latency_p50_ms", "latency_p95_ms", "latency_p99_ms")
+
+
+def upgrade() -> None:
+    for column in _COLUMNS:
+        op.alter_column("traefik_metrics", column, nullable=True)
+
+
+def downgrade() -> None:
+    for column in _COLUMNS:
+        op.execute(f"UPDATE traefik_metrics SET {column} = 0 WHERE {column} IS NULL")
+        op.alter_column("traefik_metrics", column, nullable=False)

plugins/traefik/models.py

@@ -18,9 +18,10 @@ class TraefikMetric(Base):
     request_rate: Mapped[float] = mapped_column(Float, nullable=False)
     error_rate_4xx_pct: Mapped[float] = mapped_column(Float, nullable=False)
     error_rate_5xx_pct: Mapped[float] = mapped_column(Float, nullable=False)
-    latency_p50_ms: Mapped[float] = mapped_column(Float, nullable=False)
-    latency_p95_ms: Mapped[float] = mapped_column(Float, nullable=False)
-    latency_p99_ms: Mapped[float] = mapped_column(Float, nullable=False)
+    # None for a scrape with no responses: an idle router has no latency (#5741).
+    latency_p50_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
+    latency_p95_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
+    latency_p99_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
     response_bytes_rate: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
 
 

plugins/traefik/routes.py

@@ -204,7 +204,7 @@ async def latest_per_router(db) -> list[SimpleNamespace]:
     for r in rows:
         by_router.setdefault(r.resource_name, {})[r.metric_name] = r
     return [SimpleNamespace(router_name=name, scraped_at=max(m.recorded_at for m in figures.values()),
-                            **{k: figures[k].value if k in figures else 0.0 for k in ROUTER_METRICS})
+                            **{k: figures[k].value if k in figures else None for k in ROUTER_METRICS})
             for name, figures in by_router.items() if "request_rate" in figures]
 
 
@@ -299,7 +299,7 @@ async def rows():
                                              else (None, False))
             r["sparkline_req"] = _sparkline([p.request_rate for p in points])
             r["sparkline_bytes"] = _sparkline([p.response_bytes_rate for p in points])
-            r["sparkline_p95"] = _sparkline([p.latency_p95_ms for p in points])
+            r["sparkline_p95"] = _sparkline([p.latency_p95_ms for p in points if p.latency_p95_ms is not None])
             r["sparkline_4xx"] = _sparkline([p.error_rate_4xx_pct for p in points])
             r["sparkline_5xx"] = _sparkline([p.error_rate_5xx_pct for p in points])
         phases.mark("sparklines")

plugins/traefik/scheduler.py

@@ -117,6 +117,8 @@ async def _do_scrape(app: "Quart") -> None:
                     ("latency_p99_ms", metrics["latency_p99_ms"]),
                     ("response_bytes_rate", metrics["response_bytes_rate"]),
                 ]:
+                    if value is None:       # no responses since the last scrape: no latency
+                        continue
                     # Stamped with the scrape, so a router's latest figures
                     # (routes.latest_per_router) all carry the same moment.
                     batch.add(

plugins/traefik/scraper.py

@@ -84,9 +84,9 @@ def compute_router_metrics(
             "request_rate": float,           # requests/sec
             "error_rate_4xx_pct": float,     # % 4xx of total
             "error_rate_5xx_pct": float,     # % 5xx of total
-            "latency_p50_ms": float,
-            "latency_p95_ms": float,
-            "latency_p99_ms": float,
+            "latency_p50_ms": float | None,  # None: no response since the last scrape
+            "latency_p95_ms": float | None,
+            "latency_p99_ms": float | None,
             "response_bytes_rate": float,    # bytes/sec
         }}
     """
@@ -128,6 +128,7 @@ def compute_router_metrics(
 
     # ── Histogram buckets ─────────────────────────────────────────────────────
     hist_buckets = current.get("traefik_service_request_duration_seconds_bucket", {})
+    prev_hist = None if first else prev.get("traefik_service_request_duration_seconds_bucket", {})
 
     # Collect all known services
     all_services: set[str] = set(service_total.keys()) | set(service_bytes.keys())
@@ -140,15 +141,16 @@ def compute_router_metrics(
 
     for service in all_services:
         total = service_total.get(service, 0.0)
+        latency = service_buckets(hist_buckets, prev_hist, service)
         result[service] = {
             "request_rate": total / elapsed,
             "error_rate_4xx_pct": (service_4xx.get(service, 0.0) / total * 100.0)
             if total > 0 else 0.0,
             "error_rate_5xx_pct": (service_5xx.get(service, 0.0) / total * 100.0)
             if total > 0 else 0.0,
-            "latency_p50_ms": _percentile_ms(hist_buckets, service, 0.50),
-            "latency_p95_ms": _percentile_ms(hist_buckets, service, 0.95),
-            "latency_p99_ms": _percentile_ms(hist_buckets, service, 0.99),
+            "latency_p50_ms": _percentile_ms(latency, 0.50),
+            "latency_p95_ms": _percentile_ms(latency, 0.95),
+            "latency_p99_ms": _percentile_ms(latency, 0.99),
             "response_bytes_rate": service_bytes.get(service, 0.0) / elapsed,
         }
 
@@ -230,48 +232,52 @@ def extract_certs(current: ParsedMetrics) -> list[dict]:
     return certs
 
 
-def _percentile_ms(
-    buckets: dict[frozenset, float],
-    service: str,
-    pct: float,
-) -> float:
-    """Linear interpolation of a Prometheus histogram percentile, returned in ms."""
-    router_buckets: list[tuple[float, float]] = []
-    for labels, count in buckets.items():
+def service_buckets(current: dict[frozenset, float], previous: dict[frozenset, float] | None,
+                    service: str) -> list[tuple[float, float]]:
+    """The service's latency histogram since the previous scrape, as sorted
+    (le, responses) pairs: each bucket's rise, summed over every label set
+    (code, method, protocol) that shares its `le`. Empty on the first scrape.
+
+    Traefik's buckets count from its start, so reading them as they stand
+    gave a percentile of every response since then, a figure that barely
+    moved (nextcloud's flat 100 ms p95); and walking every label set's
+    buckets as one list stopped at whichever came first (#5741)."""
+    if previous is None:
+        return []
+    by_le: dict[float, float] = defaultdict(float)
+    for labels, count in current.items():
         label_dict = dict(labels)
         if label_dict.get("service") != service:
             continue
         le_str = label_dict.get("le", "")
         try:
             le = float("inf") if le_str == "+Inf" else float(le_str)
-            router_buckets.append((le, count))
         except ValueError:
             continue
+        by_le[le] += _rise(previous, labels, count)
+    return sorted(by_le.items())
 
-    if not router_buckets:
-        return 0.0
-
-    router_buckets.sort(key=lambda t: t[0])
-    total = router_buckets[-1][1]  # +Inf bucket count == total requests
-    if total == 0.0:
-        return 0.0
 
+def _percentile_ms(buckets: list[tuple[float, float]], pct: float) -> float | None:
+    """Linear interpolation of a percentile of `buckets` (service_buckets'),
+    in ms; None when no response arrived since the previous scrape, as an
+    idle router has no latency to report. A percentile past the last finite
+    bucket reads as that bucket's edge: at least that long."""
+    total = buckets[-1][1] if buckets else 0.0   # the +Inf bucket counts every response
+    if total <= 0.0:
+        return None
     target = total * pct
-    for i, (le, count) in enumerate(router_buckets):
+    for i, (le, count) in enumerate(buckets):
         if count < target:
             continue
         if i == 0:
-            ratio = target / count if count > 0 else 0.0
-            return le * ratio * 1000.0
-        prev_le, prev_count = router_buckets[i - 1]
-        if count == prev_count:
-            return prev_le * 1000.0
+            return le * (target / count) * 1000.0
+        prev_le, prev_count = buckets[i - 1]
         if math.isinf(le):
             return prev_le * 1000.0
         fraction = (target - prev_count) / (count - prev_count)
         return (prev_le + fraction * (le - prev_le)) * 1000.0
-
-    return 0.0
+    return None
 
 
 # ──────────────────────────────────────────────────────────────────────────────

plugins/traefik/templates/traefik/rows.html

@@ -34,6 +34,10 @@
 {% endif %}
 {% endmacro %}
 
+{# A router that has answered nothing since Steward started has no latency yet. #}
+{% macro ms(v) %}{% if v is none %}—{% else %}{{ "%.0f"|format(v) }} ms{% endif %}{% endmacro %}
+{% macro latency_cell(v) %}<td{% if v is not none %} class="tone-{{ status_info(status_class(v, 'latency'))[0] }}"{% endif %}>{{ ms(v) }}</td>{% endmacro %}
+
 {% macro router_card(r) %}
 <div class="card">
     <div class="host-health-head">
@@ -54,8 +58,8 @@
     <table class="table" style="font-size:0.85rem;">
         <tr><td class="field-hint">Req/s</td><td>{{ "%.2f"|format(r.latest.request_rate) }}</td><td>{{ r.sparkline_req|safe }}</td></tr>
         <tr><td class="field-hint">Bandwidth</td><td>{{ r.latest.response_bytes_rate|rate }}</td><td>{{ r.sparkline_bytes|safe }}</td></tr>
-        <tr><td class="field-hint">p95</td><td class="tone-{{ status_info(status_class(r.latest.latency_p95_ms, 'latency'))[0] }}">{{ "%.0f"|format(r.latest.latency_p95_ms) }} ms</td><td>{{ r.sparkline_p95|safe }}</td></tr>
-        <tr><td class="field-hint">p99</td><td>{{ "%.0f"|format(r.latest.latency_p99_ms) }} ms</td><td></td></tr>
+        <tr><td class="field-hint">p95</td>{{ latency_cell(r.latest.latency_p95_ms) }}<td>{{ r.sparkline_p95|safe }}</td></tr>
+        <tr><td class="field-hint">p99</td><td>{{ ms(r.latest.latency_p99_ms) }}</td><td></td></tr>
         <tr><td class="field-hint">4xx</td><td class="{{ 'tone-warn' if r.latest.error_rate_4xx_pct > error_4xx else '' }}">{{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%</td><td>{{ r.sparkline_4xx|safe }}</td></tr>
         <tr><td class="field-hint">5xx</td><td class="{{ 'tone-crit' if r.latest.error_rate_5xx_pct > error_5xx else '' }}">{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%</td><td>{{ r.sparkline_5xx|safe }}</td></tr>
     </table>
@@ -88,7 +92,7 @@
       <td style="word-break:break-all;">{{ r.name }}</td>
       <td>{{ "%.2f"|format(r.latest.request_rate) }}</td>
       <td>{{ r.latest.response_bytes_rate|rate }}</td>
-      <td class="tone-{{ status_info(status_class(r.latest.latency_p95_ms, 'latency'))[0] }}">{{ "%.0f"|format(r.latest.latency_p95_ms) }} ms</td>
+      {{ latency_cell(r.latest.latency_p95_ms) }}
       <td>{{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%</td>
       <td>{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%</td>
     </tr>