fix(traefik): latency is each scrape's own percentile, every label set summed (#5755)

#5755 "Traefik latency was a percentile of every response since Traefik
started, over unsummed label sets". _percentile_ms read the duration
histogram as it stood, which counts from Traefik's start, so p50/p95/p99 were
lifetime figures that barely moved. It also walked every code/method label
set's buckets as one unsummed list, so equal counts from two sets returned
an edge exactly: nextcloud's flat 100.0 ms p95.

service_buckets() now takes each bucket's rise since the previous scrape,
summed per le over every label set. A scrape with no responses, and the
first scrape, record no latency: traefik_007_latency_nullable lets the
columns be NULL, the batch skips the figure, and the page shows "—".

Found while preparing #5741 "Traefik routers are graded against their own
normal", which grades p95 against a learned normal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/scraper.py

@@ -84,9 +84,9 @@ def compute_router_metrics(
             "request_rate": float,           # requests/sec
             "error_rate_4xx_pct": float,     # % 4xx of total
             "error_rate_5xx_pct": float,     # % 5xx of total
-            "latency_p50_ms": float,
-            "latency_p95_ms": float,
-            "latency_p99_ms": float,
+            "latency_p50_ms": float | None,  # None: no response since the last scrape
+            "latency_p95_ms": float | None,
+            "latency_p99_ms": float | None,
             "response_bytes_rate": float,    # bytes/sec
         }}
     """
@@ -128,6 +128,7 @@ def compute_router_metrics(
 
     # ── Histogram buckets ─────────────────────────────────────────────────────
     hist_buckets = current.get("traefik_service_request_duration_seconds_bucket", {})
+    prev_hist = None if first else prev.get("traefik_service_request_duration_seconds_bucket", {})
 
     # Collect all known services
     all_services: set[str] = set(service_total.keys()) | set(service_bytes.keys())
@@ -140,15 +141,16 @@ def compute_router_metrics(
 
     for service in all_services:
         total = service_total.get(service, 0.0)
+        latency = service_buckets(hist_buckets, prev_hist, service)
         result[service] = {
             "request_rate": total / elapsed,
             "error_rate_4xx_pct": (service_4xx.get(service, 0.0) / total * 100.0)
             if total > 0 else 0.0,
             "error_rate_5xx_pct": (service_5xx.get(service, 0.0) / total * 100.0)
             if total > 0 else 0.0,
-            "latency_p50_ms": _percentile_ms(hist_buckets, service, 0.50),
-            "latency_p95_ms": _percentile_ms(hist_buckets, service, 0.95),
-            "latency_p99_ms": _percentile_ms(hist_buckets, service, 0.99),
+            "latency_p50_ms": _percentile_ms(latency, 0.50),
+            "latency_p95_ms": _percentile_ms(latency, 0.95),
+            "latency_p99_ms": _percentile_ms(latency, 0.99),
             "response_bytes_rate": service_bytes.get(service, 0.0) / elapsed,
         }
 
@@ -230,48 +232,52 @@ def extract_certs(current: ParsedMetrics) -> list[dict]:
     return certs
 
 
-def _percentile_ms(
-    buckets: dict[frozenset, float],
-    service: str,
-    pct: float,
-) -> float:
-    """Linear interpolation of a Prometheus histogram percentile, returned in ms."""
-    router_buckets: list[tuple[float, float]] = []
-    for labels, count in buckets.items():
+def service_buckets(current: dict[frozenset, float], previous: dict[frozenset, float] | None,
+                    service: str) -> list[tuple[float, float]]:
+    """The service's latency histogram since the previous scrape, as sorted
+    (le, responses) pairs: each bucket's rise, summed over every label set
+    (code, method, protocol) that shares its `le`. Empty on the first scrape.
+
+    Traefik's buckets count from its start, so reading them as they stand
+    gave a percentile of every response since then, a figure that barely
+    moved (nextcloud's flat 100 ms p95); and walking every label set's
+    buckets as one list stopped at whichever came first (#5741)."""
+    if previous is None:
+        return []
+    by_le: dict[float, float] = defaultdict(float)
+    for labels, count in current.items():
         label_dict = dict(labels)
         if label_dict.get("service") != service:
             continue
         le_str = label_dict.get("le", "")
         try:
             le = float("inf") if le_str == "+Inf" else float(le_str)
-            router_buckets.append((le, count))
         except ValueError:
             continue
+        by_le[le] += _rise(previous, labels, count)
+    return sorted(by_le.items())
 
-    if not router_buckets:
-        return 0.0
-
-    router_buckets.sort(key=lambda t: t[0])
-    total = router_buckets[-1][1]  # +Inf bucket count == total requests
-    if total == 0.0:
-        return 0.0
 
+def _percentile_ms(buckets: list[tuple[float, float]], pct: float) -> float | None:
+    """Linear interpolation of a percentile of `buckets` (service_buckets'),
+    in ms; None when no response arrived since the previous scrape, as an
+    idle router has no latency to report. A percentile past the last finite
+    bucket reads as that bucket's edge: at least that long."""
+    total = buckets[-1][1] if buckets else 0.0   # the +Inf bucket counts every response
+    if total <= 0.0:
+        return None
     target = total * pct
-    for i, (le, count) in enumerate(router_buckets):
+    for i, (le, count) in enumerate(buckets):
         if count < target:
             continue
         if i == 0:
-            ratio = target / count if count > 0 else 0.0
-            return le * ratio * 1000.0
-        prev_le, prev_count = router_buckets[i - 1]
-        if count == prev_count:
-            return prev_le * 1000.0
+            return le * (target / count) * 1000.0
+        prev_le, prev_count = buckets[i - 1]
         if math.isinf(le):
             return prev_le * 1000.0
         fraction = (target - prev_count) / (count - prev_count)
         return (prev_le + fraction * (le - prev_le)) * 1000.0
-
-    return 0.0
+    return None
 
 
 # ──────────────────────────────────────────────────────────────────────────────