bvandeusenandClaude Opus 5.5
98a0ced5eb
#5755 "Traefik latency was a percentile of every response since Traefik started, over unsummed label sets". _percentile_ms read the duration histogram as it stood, which counts from Traefik's start, so p50/p95/p99 were lifetime figures that barely moved. It also walked every code/method label set's buckets as one unsummed list, so equal counts from two sets returned an edge exactly: nextcloud's flat 100.0 ms p95. service_buckets() now takes each bucket's rise since the previous scrape, summed per le over every label set. A scrape with no responses, and the first scrape, record no latency: traefik_007_latency_nullable lets the columns be NULL, the batch skips the figure, and the page shows "—". Found while preparing #5741 "Traefik routers are graded against their own normal", which grades p95 against a learned normal. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
62 lines
2.9 KiB
Python
62 lines
2.9 KiB
Python
"""A router's latency is each scrape's own percentile (#5741): the rise in
|
||
Traefik's histogram since the last scrape, every label set's buckets summed
|
||
per `le`. Read as it stood, the histogram gave a percentile of every
|
||
response since Traefik started, which barely moved (nextcloud's flat 100 ms
|
||
p95)."""
|
||
import pytest
|
||
|
||
from plugins.traefik.scraper import compute_router_metrics, service_buckets
|
||
|
||
BUCKET = "traefik_service_request_duration_seconds_bucket"
|
||
EDGES = ("0.1", "0.3", "1.2", "5", "+Inf") # Traefik's default buckets
|
||
|
||
|
||
def _hist(service, code, method, counts):
|
||
"""{labels: count} for one label set's cumulative buckets, in EDGES order."""
|
||
return {frozenset({"service": service, "code": code, "method": method, "le": le}.items()): n
|
||
for le, n in zip(EDGES, counts)}
|
||
|
||
|
||
def _scrape(*sets):
|
||
out = {}
|
||
for s in sets:
|
||
out.update(_hist(*s))
|
||
return {BUCKET: out}
|
||
|
||
|
||
def test_latency_reads_the_responses_since_the_last_scrape_not_since_start():
|
||
# A million fast responses since start, then 100 slow ones (all 0.3–1.2 s).
|
||
before = _scrape(("cloud@docker", "200", "GET", (1_000_000, 1_000_000, 1_000_000, 1_000_000, 1_000_000)))
|
||
after = _scrape(("cloud@docker", "200", "GET", (1_000_000, 1_000_000, 1_000_100, 1_000_100, 1_000_100)))
|
||
got = compute_router_metrics(after, before, 60.0)["cloud@docker"]
|
||
assert 300.0 < got["latency_p50_ms"] < 1200.0
|
||
assert got["latency_p95_ms"] == pytest.approx(300.0 + 0.95 * 900.0)
|
||
|
||
|
||
def test_every_label_set_counts_toward_one_histogram():
|
||
# GET answered fast and POST slowly: 90 + 10 responses, so p95 is a POST's.
|
||
before = _scrape(("app@docker", "200", "GET", (0, 0, 0, 0, 0)),
|
||
("app@docker", "200", "POST", (0, 0, 0, 0, 0)))
|
||
after = _scrape(("app@docker", "200", "GET", (90, 90, 90, 90, 90)),
|
||
("app@docker", "200", "POST", (0, 0, 0, 10, 10)))
|
||
assert service_buckets(after[BUCKET], before[BUCKET], "app@docker") == [
|
||
(0.1, 90.0), (0.3, 90.0), (1.2, 90.0), (5.0, 100.0), (float("inf"), 100.0)]
|
||
got = compute_router_metrics(after, before, 60.0)["app@docker"]
|
||
assert got["latency_p50_ms"] < 100.0
|
||
assert 1200.0 < got["latency_p95_ms"] < 5000.0
|
||
|
||
|
||
def test_an_idle_scrape_and_the_first_scrape_have_no_latency():
|
||
still = _scrape(("idle@docker", "200", "GET", (50, 60, 60, 60, 60)))
|
||
idle = compute_router_metrics(still, still, 60.0)["idle@docker"]
|
||
first = compute_router_metrics(still, None, 60.0)["idle@docker"]
|
||
for got in (idle, first):
|
||
assert got["latency_p50_ms"] is None and got["latency_p95_ms"] is None
|
||
assert got["latency_p99_ms"] is None
|
||
|
||
|
||
def test_a_percentile_past_the_last_edge_reads_as_at_least_that_edge():
|
||
before = _scrape(("ws@docker", "101", "GET", (0, 0, 0, 0, 0)))
|
||
after = _scrape(("ws@docker", "101", "GET", (0, 0, 0, 0, 20))) # all longer than 5 s
|
||
assert compute_router_metrics(after, before, 60.0)["ws@docker"]["latency_p95_ms"] == 5000.0
|