fix(traefik): latency is each scrape's own percentile, every label set summed (#5755)

#5755 "Traefik latency was a percentile of every response since Traefik
started, over unsummed label sets". _percentile_ms read the duration
histogram as it stood, which counts from Traefik's start, so p50/p95/p99 were
lifetime figures that barely moved. It also walked every code/method label
set's buckets as one unsummed list, so equal counts from two sets returned
an edge exactly: nextcloud's flat 100.0 ms p95.

service_buckets() now takes each bucket's rise since the previous scrape,
summed per le over every label set. A scrape with no responses, and the
first scrape, record no latency: traefik_007_latency_nullable lets the
columns be NULL, the batch skips the figure, and the page shows "—".

Found while preparing #5741 "Traefik routers are graded against their own
normal", which grades p95 against a learned normal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/plugins/traefik.md

@@ -12,7 +12,7 @@ On each scrape:
 3. Writes computed metrics to the `traefik_metrics` history table
 4. Calls `record_metric()` for each metric, making them available to alert rules
 
-Request rates and error rates are computed as deltas between the current and previous scrape, divided by elapsed time. Latency percentiles (p50, p95, p99) are approximated via linear interpolation over histogram buckets — these are estimates, not exact percentiles.
+Request rates and error rates are computed as deltas between the current and previous scrape, divided by elapsed time. Latency percentiles (p50, p95, p99) are approximated via linear interpolation over histogram buckets — these are estimates, not exact percentiles. They cover the responses since the previous scrape (each bucket's rise, summed over every status code and method), not every response since Traefik started. A scrape in which a router answered nothing records no latency, shown as "—"; one past the last bucket (5 s by default) reads as that bucket's edge.
 
 ---
 

plugins/traefik/migrations/versions/traefik_007_latency_nullable.py

@@ -0,0 +1,30 @@
+"""A router's latency may be absent
+
+Latency is now each scrape's own percentile (#5741), and a scrape in which a
+router answered nothing has none: those rows carry NULL rather than a
+made-up 0 or the old lifetime figure.
+
+Revision ID: traefik_007_latency_nullable
+Revises: traefik_006_status_codes
+Create Date: 2026-10-14
+"""
+from typing import Sequence, Union
+from alembic import op
+
+revision: str = "traefik_007_latency_nullable"
+down_revision: Union[str, None] = "traefik_006_status_codes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+_COLUMNS = ("latency_p50_ms", "latency_p95_ms", "latency_p99_ms")
+
+
+def upgrade() -> None:
+    for column in _COLUMNS:
+        op.alter_column("traefik_metrics", column, nullable=True)
+
+
+def downgrade() -> None:
+    for column in _COLUMNS:
+        op.execute(f"UPDATE traefik_metrics SET {column} = 0 WHERE {column} IS NULL")
+        op.alter_column("traefik_metrics", column, nullable=False)

plugins/traefik/models.py

@@ -18,9 +18,10 @@ class TraefikMetric(Base):
     request_rate: Mapped[float] = mapped_column(Float, nullable=False)
     error_rate_4xx_pct: Mapped[float] = mapped_column(Float, nullable=False)
     error_rate_5xx_pct: Mapped[float] = mapped_column(Float, nullable=False)
-    latency_p50_ms: Mapped[float] = mapped_column(Float, nullable=False)
-    latency_p95_ms: Mapped[float] = mapped_column(Float, nullable=False)
-    latency_p99_ms: Mapped[float] = mapped_column(Float, nullable=False)
+    # None for a scrape with no responses: an idle router has no latency (#5741).
+    latency_p50_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
+    latency_p95_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
+    latency_p99_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
     response_bytes_rate: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
 
 

plugins/traefik/routes.py

@@ -204,7 +204,7 @@ async def latest_per_router(db) -> list[SimpleNamespace]:
     for r in rows:
         by_router.setdefault(r.resource_name, {})[r.metric_name] = r
     return [SimpleNamespace(router_name=name, scraped_at=max(m.recorded_at for m in figures.values()),
-                            **{k: figures[k].value if k in figures else 0.0 for k in ROUTER_METRICS})
+                            **{k: figures[k].value if k in figures else None for k in ROUTER_METRICS})
             for name, figures in by_router.items() if "request_rate" in figures]
 
 
@@ -299,7 +299,7 @@ async def rows():
                                              else (None, False))
             r["sparkline_req"] = _sparkline([p.request_rate for p in points])
             r["sparkline_bytes"] = _sparkline([p.response_bytes_rate for p in points])
-            r["sparkline_p95"] = _sparkline([p.latency_p95_ms for p in points])
+            r["sparkline_p95"] = _sparkline([p.latency_p95_ms for p in points if p.latency_p95_ms is not None])
             r["sparkline_4xx"] = _sparkline([p.error_rate_4xx_pct for p in points])
             r["sparkline_5xx"] = _sparkline([p.error_rate_5xx_pct for p in points])
         phases.mark("sparklines")

plugins/traefik/scheduler.py

@@ -117,6 +117,8 @@ async def _do_scrape(app: "Quart") -> None:
                     ("latency_p99_ms", metrics["latency_p99_ms"]),
                     ("response_bytes_rate", metrics["response_bytes_rate"]),
                 ]:
+                    if value is None:       # no responses since the last scrape: no latency
+                        continue
                     # Stamped with the scrape, so a router's latest figures
                     # (routes.latest_per_router) all carry the same moment.
                     batch.add(

plugins/traefik/scraper.py

@@ -84,9 +84,9 @@ def compute_router_metrics(
             "request_rate": float,           # requests/sec
             "error_rate_4xx_pct": float,     # % 4xx of total
             "error_rate_5xx_pct": float,     # % 5xx of total
-            "latency_p50_ms": float,
-            "latency_p95_ms": float,
-            "latency_p99_ms": float,
+            "latency_p50_ms": float | None,  # None: no response since the last scrape
+            "latency_p95_ms": float | None,
+            "latency_p99_ms": float | None,
             "response_bytes_rate": float,    # bytes/sec
         }}
     """
@@ -128,6 +128,7 @@ def compute_router_metrics(
 
     # ── Histogram buckets ─────────────────────────────────────────────────────
     hist_buckets = current.get("traefik_service_request_duration_seconds_bucket", {})
+    prev_hist = None if first else prev.get("traefik_service_request_duration_seconds_bucket", {})
 
     # Collect all known services
     all_services: set[str] = set(service_total.keys()) | set(service_bytes.keys())
@@ -140,15 +141,16 @@ def compute_router_metrics(
 
     for service in all_services:
         total = service_total.get(service, 0.0)
+        latency = service_buckets(hist_buckets, prev_hist, service)
         result[service] = {
             "request_rate": total / elapsed,
             "error_rate_4xx_pct": (service_4xx.get(service, 0.0) / total * 100.0)
             if total > 0 else 0.0,
             "error_rate_5xx_pct": (service_5xx.get(service, 0.0) / total * 100.0)
             if total > 0 else 0.0,
-            "latency_p50_ms": _percentile_ms(hist_buckets, service, 0.50),
-            "latency_p95_ms": _percentile_ms(hist_buckets, service, 0.95),
-            "latency_p99_ms": _percentile_ms(hist_buckets, service, 0.99),
+            "latency_p50_ms": _percentile_ms(latency, 0.50),
+            "latency_p95_ms": _percentile_ms(latency, 0.95),
+            "latency_p99_ms": _percentile_ms(latency, 0.99),
             "response_bytes_rate": service_bytes.get(service, 0.0) / elapsed,
         }
 
@@ -230,48 +232,52 @@ def extract_certs(current: ParsedMetrics) -> list[dict]:
     return certs
 
 
-def _percentile_ms(
-    buckets: dict[frozenset, float],
-    service: str,
-    pct: float,
-) -> float:
-    """Linear interpolation of a Prometheus histogram percentile, returned in ms."""
-    router_buckets: list[tuple[float, float]] = []
-    for labels, count in buckets.items():
+def service_buckets(current: dict[frozenset, float], previous: dict[frozenset, float] | None,
+                    service: str) -> list[tuple[float, float]]:
+    """The service's latency histogram since the previous scrape, as sorted
+    (le, responses) pairs: each bucket's rise, summed over every label set
+    (code, method, protocol) that shares its `le`. Empty on the first scrape.
+
+    Traefik's buckets count from its start, so reading them as they stand
+    gave a percentile of every response since then, a figure that barely
+    moved (nextcloud's flat 100 ms p95); and walking every label set's
+    buckets as one list stopped at whichever came first (#5741)."""
+    if previous is None:
+        return []
+    by_le: dict[float, float] = defaultdict(float)
+    for labels, count in current.items():
         label_dict = dict(labels)
         if label_dict.get("service") != service:
             continue
         le_str = label_dict.get("le", "")
         try:
             le = float("inf") if le_str == "+Inf" else float(le_str)
-            router_buckets.append((le, count))
         except ValueError:
             continue
+        by_le[le] += _rise(previous, labels, count)
+    return sorted(by_le.items())
 
-    if not router_buckets:
-        return 0.0
-
-    router_buckets.sort(key=lambda t: t[0])
-    total = router_buckets[-1][1]  # +Inf bucket count == total requests
-    if total == 0.0:
-        return 0.0
 
+def _percentile_ms(buckets: list[tuple[float, float]], pct: float) -> float | None:
+    """Linear interpolation of a percentile of `buckets` (service_buckets'),
+    in ms; None when no response arrived since the previous scrape, as an
+    idle router has no latency to report. A percentile past the last finite
+    bucket reads as that bucket's edge: at least that long."""
+    total = buckets[-1][1] if buckets else 0.0   # the +Inf bucket counts every response
+    if total <= 0.0:
+        return None
     target = total * pct
-    for i, (le, count) in enumerate(router_buckets):
+    for i, (le, count) in enumerate(buckets):
         if count < target:
             continue
         if i == 0:
-            ratio = target / count if count > 0 else 0.0
-            return le * ratio * 1000.0
-        prev_le, prev_count = router_buckets[i - 1]
-        if count == prev_count:
-            return prev_le * 1000.0
+            return le * (target / count) * 1000.0
+        prev_le, prev_count = buckets[i - 1]
         if math.isinf(le):
             return prev_le * 1000.0
         fraction = (target - prev_count) / (count - prev_count)
         return (prev_le + fraction * (le - prev_le)) * 1000.0
-
-    return 0.0
+    return None
 
 
 # ──────────────────────────────────────────────────────────────────────────────

plugins/traefik/templates/traefik/rows.html

@@ -34,6 +34,10 @@
 {% endif %}
 {% endmacro %}
 
+{# A router that has answered nothing since Steward started has no latency yet. #}
+{% macro ms(v) %}{% if v is none %}—{% else %}{{ "%.0f"|format(v) }} ms{% endif %}{% endmacro %}
+{% macro latency_cell(v) %}<td{% if v is not none %} class="tone-{{ status_info(status_class(v, 'latency'))[0] }}"{% endif %}>{{ ms(v) }}</td>{% endmacro %}
+
 {% macro router_card(r) %}
 <div class="card">
     <div class="host-health-head">
@@ -54,8 +58,8 @@
     <table class="table" style="font-size:0.85rem;">
         <tr><td class="field-hint">Req/s</td><td>{{ "%.2f"|format(r.latest.request_rate) }}</td><td>{{ r.sparkline_req|safe }}</td></tr>
         <tr><td class="field-hint">Bandwidth</td><td>{{ r.latest.response_bytes_rate|rate }}</td><td>{{ r.sparkline_bytes|safe }}</td></tr>
-        <tr><td class="field-hint">p95</td><td class="tone-{{ status_info(status_class(r.latest.latency_p95_ms, 'latency'))[0] }}">{{ "%.0f"|format(r.latest.latency_p95_ms) }} ms</td><td>{{ r.sparkline_p95|safe }}</td></tr>
-        <tr><td class="field-hint">p99</td><td>{{ "%.0f"|format(r.latest.latency_p99_ms) }} ms</td><td></td></tr>
+        <tr><td class="field-hint">p95</td>{{ latency_cell(r.latest.latency_p95_ms) }}<td>{{ r.sparkline_p95|safe }}</td></tr>
+        <tr><td class="field-hint">p99</td><td>{{ ms(r.latest.latency_p99_ms) }}</td><td></td></tr>
         <tr><td class="field-hint">4xx</td><td class="{{ 'tone-warn' if r.latest.error_rate_4xx_pct > error_4xx else '' }}">{{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%</td><td>{{ r.sparkline_4xx|safe }}</td></tr>
         <tr><td class="field-hint">5xx</td><td class="{{ 'tone-crit' if r.latest.error_rate_5xx_pct > error_5xx else '' }}">{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%</td><td>{{ r.sparkline_5xx|safe }}</td></tr>
     </table>
@@ -88,7 +92,7 @@
       <td style="word-break:break-all;">{{ r.name }}</td>
       <td>{{ "%.2f"|format(r.latest.request_rate) }}</td>
       <td>{{ r.latest.response_bytes_rate|rate }}</td>
-      <td class="tone-{{ status_info(status_class(r.latest.latency_p95_ms, 'latency'))[0] }}">{{ "%.0f"|format(r.latest.latency_p95_ms) }} ms</td>
+      {{ latency_cell(r.latest.latency_p95_ms) }}
       <td>{{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%</td>
       <td>{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%</td>
     </tr>

tests/plugins/traefik/test_latency.py

@@ -0,0 +1,61 @@
+"""A router's latency is each scrape's own percentile (#5741): the rise in
+Traefik's histogram since the last scrape, every label set's buckets summed
+per `le`. Read as it stood, the histogram gave a percentile of every
+response since Traefik started, which barely moved (nextcloud's flat 100 ms
+p95)."""
+import pytest
+
+from plugins.traefik.scraper import compute_router_metrics, service_buckets
+
+BUCKET = "traefik_service_request_duration_seconds_bucket"
+EDGES = ("0.1", "0.3", "1.2", "5", "+Inf")       # Traefik's default buckets
+
+
+def _hist(service, code, method, counts):
+    """{labels: count} for one label set's cumulative buckets, in EDGES order."""
+    return {frozenset({"service": service, "code": code, "method": method, "le": le}.items()): n
+            for le, n in zip(EDGES, counts)}
+
+
+def _scrape(*sets):
+    out = {}
+    for s in sets:
+        out.update(_hist(*s))
+    return {BUCKET: out}
+
+
+def test_latency_reads_the_responses_since_the_last_scrape_not_since_start():
+    # A million fast responses since start, then 100 slow ones (all 0.3–1.2 s).
+    before = _scrape(("cloud@docker", "200", "GET", (1_000_000, 1_000_000, 1_000_000, 1_000_000, 1_000_000)))
+    after = _scrape(("cloud@docker", "200", "GET", (1_000_000, 1_000_000, 1_000_100, 1_000_100, 1_000_100)))
+    got = compute_router_metrics(after, before, 60.0)["cloud@docker"]
+    assert 300.0 < got["latency_p50_ms"] < 1200.0
+    assert got["latency_p95_ms"] == pytest.approx(300.0 + 0.95 * 900.0)
+
+
+def test_every_label_set_counts_toward_one_histogram():
+    # GET answered fast and POST slowly: 90 + 10 responses, so p95 is a POST's.
+    before = _scrape(("app@docker", "200", "GET", (0, 0, 0, 0, 0)),
+                     ("app@docker", "200", "POST", (0, 0, 0, 0, 0)))
+    after = _scrape(("app@docker", "200", "GET", (90, 90, 90, 90, 90)),
+                    ("app@docker", "200", "POST", (0, 0, 0, 10, 10)))
+    assert service_buckets(after[BUCKET], before[BUCKET], "app@docker") == [
+        (0.1, 90.0), (0.3, 90.0), (1.2, 90.0), (5.0, 100.0), (float("inf"), 100.0)]
+    got = compute_router_metrics(after, before, 60.0)["app@docker"]
+    assert got["latency_p50_ms"] < 100.0
+    assert 1200.0 < got["latency_p95_ms"] < 5000.0
+
+
+def test_an_idle_scrape_and_the_first_scrape_have_no_latency():
+    still = _scrape(("idle@docker", "200", "GET", (50, 60, 60, 60, 60)))
+    idle = compute_router_metrics(still, still, 60.0)["idle@docker"]
+    first = compute_router_metrics(still, None, 60.0)["idle@docker"]
+    for got in (idle, first):
+        assert got["latency_p50_ms"] is None and got["latency_p95_ms"] is None
+        assert got["latency_p99_ms"] is None
+
+
+def test_a_percentile_past_the_last_edge_reads_as_at_least_that_edge():
+    before = _scrape(("ws@docker", "101", "GET", (0, 0, 0, 0, 0)))
+    after = _scrape(("ws@docker", "101", "GET", (0, 0, 0, 0, 20)))     # all longer than 5 s
+    assert compute_router_metrics(after, before, 60.0)["ws@docker"]["latency_p95_ms"] == 5000.0