fix(traefik): latency is each scrape's own percentile, every label set summed (#5755)

#5755 "Traefik latency was a percentile of every response since Traefik
started, over unsummed label sets". _percentile_ms read the duration
histogram as it stood, which counts from Traefik's start, so p50/p95/p99 were
lifetime figures that barely moved. It also walked every code/method label
set's buckets as one unsummed list, so equal counts from two sets returned
an edge exactly: nextcloud's flat 100.0 ms p95.

service_buckets() now takes each bucket's rise since the previous scrape,
summed per le over every label set. A scrape with no responses, and the
first scrape, record no latency: traefik_007_latency_nullable lets the
columns be NULL, the batch skips the figure, and the page shows "—".

Found while preparing #5741 "Traefik routers are graded against their own
normal", which grades p95 against a learned normal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/plugins/traefik/test_latency.py

@@ -0,0 +1,61 @@
+"""A router's latency is each scrape's own percentile (#5741): the rise in
+Traefik's histogram since the last scrape, every label set's buckets summed
+per `le`. Read as it stood, the histogram gave a percentile of every
+response since Traefik started, which barely moved (nextcloud's flat 100 ms
+p95)."""
+import pytest
+
+from plugins.traefik.scraper import compute_router_metrics, service_buckets
+
+BUCKET = "traefik_service_request_duration_seconds_bucket"
+EDGES = ("0.1", "0.3", "1.2", "5", "+Inf")       # Traefik's default buckets
+
+
+def _hist(service, code, method, counts):
+    """{labels: count} for one label set's cumulative buckets, in EDGES order."""
+    return {frozenset({"service": service, "code": code, "method": method, "le": le}.items()): n
+            for le, n in zip(EDGES, counts)}
+
+
+def _scrape(*sets):
+    out = {}
+    for s in sets:
+        out.update(_hist(*s))
+    return {BUCKET: out}
+
+
+def test_latency_reads_the_responses_since_the_last_scrape_not_since_start():
+    # A million fast responses since start, then 100 slow ones (all 0.3–1.2 s).
+    before = _scrape(("cloud@docker", "200", "GET", (1_000_000, 1_000_000, 1_000_000, 1_000_000, 1_000_000)))
+    after = _scrape(("cloud@docker", "200", "GET", (1_000_000, 1_000_000, 1_000_100, 1_000_100, 1_000_100)))
+    got = compute_router_metrics(after, before, 60.0)["cloud@docker"]
+    assert 300.0 < got["latency_p50_ms"] < 1200.0
+    assert got["latency_p95_ms"] == pytest.approx(300.0 + 0.95 * 900.0)
+
+
+def test_every_label_set_counts_toward_one_histogram():
+    # GET answered fast and POST slowly: 90 + 10 responses, so p95 is a POST's.
+    before = _scrape(("app@docker", "200", "GET", (0, 0, 0, 0, 0)),
+                     ("app@docker", "200", "POST", (0, 0, 0, 0, 0)))
+    after = _scrape(("app@docker", "200", "GET", (90, 90, 90, 90, 90)),
+                    ("app@docker", "200", "POST", (0, 0, 0, 10, 10)))
+    assert service_buckets(after[BUCKET], before[BUCKET], "app@docker") == [
+        (0.1, 90.0), (0.3, 90.0), (1.2, 90.0), (5.0, 100.0), (float("inf"), 100.0)]
+    got = compute_router_metrics(after, before, 60.0)["app@docker"]
+    assert got["latency_p50_ms"] < 100.0
+    assert 1200.0 < got["latency_p95_ms"] < 5000.0
+
+
+def test_an_idle_scrape_and_the_first_scrape_have_no_latency():
+    still = _scrape(("idle@docker", "200", "GET", (50, 60, 60, 60, 60)))
+    idle = compute_router_metrics(still, still, 60.0)["idle@docker"]
+    first = compute_router_metrics(still, None, 60.0)["idle@docker"]
+    for got in (idle, first):
+        assert got["latency_p50_ms"] is None and got["latency_p95_ms"] is None
+        assert got["latency_p99_ms"] is None
+
+
+def test_a_percentile_past_the_last_edge_reads_as_at_least_that_edge():
+    before = _scrape(("ws@docker", "101", "GET", (0, 0, 0, 0, 0)))
+    after = _scrape(("ws@docker", "101", "GET", (0, 0, 0, 0, 20)))     # all longer than 5 s
+    assert compute_router_metrics(after, before, 60.0)["ws@docker"]["latency_p95_ms"] == 5000.0