diff --git a/docs/plugins/traefik.md b/docs/plugins/traefik.md index 0295be6..b1e75c2 100644 --- a/docs/plugins/traefik.md +++ b/docs/plugins/traefik.md @@ -12,7 +12,7 @@ On each scrape: 3. Writes computed metrics to the `traefik_metrics` history table 4. Calls `record_metric()` for each metric, making them available to alert rules -Request rates and error rates are computed as deltas between the current and previous scrape, divided by elapsed time. Latency percentiles (p50, p95, p99) are approximated via linear interpolation over histogram buckets — these are estimates, not exact percentiles. +Request rates and error rates are computed as deltas between the current and previous scrape, divided by elapsed time. Latency percentiles (p50, p95, p99) are approximated via linear interpolation over histogram buckets — these are estimates, not exact percentiles. They cover the responses since the previous scrape (each bucket's rise, summed over every status code and method), not every response since Traefik started. A scrape in which a router answered nothing records no latency, shown as "—"; one past the last bucket (5 s by default) reads as that bucket's edge. --- diff --git a/plugins/traefik/migrations/versions/traefik_007_latency_nullable.py b/plugins/traefik/migrations/versions/traefik_007_latency_nullable.py new file mode 100644 index 0000000..aa8dc15 --- /dev/null +++ b/plugins/traefik/migrations/versions/traefik_007_latency_nullable.py @@ -0,0 +1,30 @@ +"""A router's latency may be absent + +Latency is now each scrape's own percentile (#5741), and a scrape in which a +router answered nothing has none: those rows carry NULL rather than a +made-up 0 or the old lifetime figure. + +Revision ID: traefik_007_latency_nullable +Revises: traefik_006_status_codes +Create Date: 2026-10-14 +""" +from typing import Sequence, Union +from alembic import op + +revision: str = "traefik_007_latency_nullable" +down_revision: Union[str, None] = "traefik_006_status_codes" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_COLUMNS = ("latency_p50_ms", "latency_p95_ms", "latency_p99_ms") + + +def upgrade() -> None: + for column in _COLUMNS: + op.alter_column("traefik_metrics", column, nullable=True) + + +def downgrade() -> None: + for column in _COLUMNS: + op.execute(f"UPDATE traefik_metrics SET {column} = 0 WHERE {column} IS NULL") + op.alter_column("traefik_metrics", column, nullable=False) diff --git a/plugins/traefik/models.py b/plugins/traefik/models.py index 1e70cca..f7e4735 100644 --- a/plugins/traefik/models.py +++ b/plugins/traefik/models.py @@ -18,9 +18,10 @@ class TraefikMetric(Base): request_rate: Mapped[float] = mapped_column(Float, nullable=False) error_rate_4xx_pct: Mapped[float] = mapped_column(Float, nullable=False) error_rate_5xx_pct: Mapped[float] = mapped_column(Float, nullable=False) - latency_p50_ms: Mapped[float] = mapped_column(Float, nullable=False) - latency_p95_ms: Mapped[float] = mapped_column(Float, nullable=False) - latency_p99_ms: Mapped[float] = mapped_column(Float, nullable=False) + # None for a scrape with no responses: an idle router has no latency (#5741). + latency_p50_ms: Mapped[float | None] = mapped_column(Float, nullable=True) + latency_p95_ms: Mapped[float | None] = mapped_column(Float, nullable=True) + latency_p99_ms: Mapped[float | None] = mapped_column(Float, nullable=True) response_bytes_rate: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py index 434d57b..88c2023 100644 --- a/plugins/traefik/routes.py +++ b/plugins/traefik/routes.py @@ -204,7 +204,7 @@ async def latest_per_router(db) -> list[SimpleNamespace]: for r in rows: by_router.setdefault(r.resource_name, {})[r.metric_name] = r return [SimpleNamespace(router_name=name, scraped_at=max(m.recorded_at for m in figures.values()), - **{k: figures[k].value if k in figures else 0.0 for k in ROUTER_METRICS}) + **{k: figures[k].value if k in figures else None for k in ROUTER_METRICS}) for name, figures in by_router.items() if "request_rate" in figures] @@ -299,7 +299,7 @@ async def rows(): else (None, False)) r["sparkline_req"] = _sparkline([p.request_rate for p in points]) r["sparkline_bytes"] = _sparkline([p.response_bytes_rate for p in points]) - r["sparkline_p95"] = _sparkline([p.latency_p95_ms for p in points]) + r["sparkline_p95"] = _sparkline([p.latency_p95_ms for p in points if p.latency_p95_ms is not None]) r["sparkline_4xx"] = _sparkline([p.error_rate_4xx_pct for p in points]) r["sparkline_5xx"] = _sparkline([p.error_rate_5xx_pct for p in points]) phases.mark("sparklines") diff --git a/plugins/traefik/scheduler.py b/plugins/traefik/scheduler.py index 519f5e1..77b467c 100644 --- a/plugins/traefik/scheduler.py +++ b/plugins/traefik/scheduler.py @@ -117,6 +117,8 @@ async def _do_scrape(app: "Quart") -> None: ("latency_p99_ms", metrics["latency_p99_ms"]), ("response_bytes_rate", metrics["response_bytes_rate"]), ]: + if value is None: # no responses since the last scrape: no latency + continue # Stamped with the scrape, so a router's latest figures # (routes.latest_per_router) all carry the same moment. batch.add( diff --git a/plugins/traefik/scraper.py b/plugins/traefik/scraper.py index 026d7be..7d319e4 100644 --- a/plugins/traefik/scraper.py +++ b/plugins/traefik/scraper.py @@ -84,9 +84,9 @@ def compute_router_metrics( "request_rate": float, # requests/sec "error_rate_4xx_pct": float, # % 4xx of total "error_rate_5xx_pct": float, # % 5xx of total - "latency_p50_ms": float, - "latency_p95_ms": float, - "latency_p99_ms": float, + "latency_p50_ms": float | None, # None: no response since the last scrape + "latency_p95_ms": float | None, + "latency_p99_ms": float | None, "response_bytes_rate": float, # bytes/sec }} """ @@ -128,6 +128,7 @@ def compute_router_metrics( # ── Histogram buckets ───────────────────────────────────────────────────── hist_buckets = current.get("traefik_service_request_duration_seconds_bucket", {}) + prev_hist = None if first else prev.get("traefik_service_request_duration_seconds_bucket", {}) # Collect all known services all_services: set[str] = set(service_total.keys()) | set(service_bytes.keys()) @@ -140,15 +141,16 @@ def compute_router_metrics( for service in all_services: total = service_total.get(service, 0.0) + latency = service_buckets(hist_buckets, prev_hist, service) result[service] = { "request_rate": total / elapsed, "error_rate_4xx_pct": (service_4xx.get(service, 0.0) / total * 100.0) if total > 0 else 0.0, "error_rate_5xx_pct": (service_5xx.get(service, 0.0) / total * 100.0) if total > 0 else 0.0, - "latency_p50_ms": _percentile_ms(hist_buckets, service, 0.50), - "latency_p95_ms": _percentile_ms(hist_buckets, service, 0.95), - "latency_p99_ms": _percentile_ms(hist_buckets, service, 0.99), + "latency_p50_ms": _percentile_ms(latency, 0.50), + "latency_p95_ms": _percentile_ms(latency, 0.95), + "latency_p99_ms": _percentile_ms(latency, 0.99), "response_bytes_rate": service_bytes.get(service, 0.0) / elapsed, } @@ -230,48 +232,52 @@ def extract_certs(current: ParsedMetrics) -> list[dict]: return certs -def _percentile_ms( - buckets: dict[frozenset, float], - service: str, - pct: float, -) -> float: - """Linear interpolation of a Prometheus histogram percentile, returned in ms.""" - router_buckets: list[tuple[float, float]] = [] - for labels, count in buckets.items(): +def service_buckets(current: dict[frozenset, float], previous: dict[frozenset, float] | None, + service: str) -> list[tuple[float, float]]: + """The service's latency histogram since the previous scrape, as sorted + (le, responses) pairs: each bucket's rise, summed over every label set + (code, method, protocol) that shares its `le`. Empty on the first scrape. + + Traefik's buckets count from its start, so reading them as they stand + gave a percentile of every response since then, a figure that barely + moved (nextcloud's flat 100 ms p95); and walking every label set's + buckets as one list stopped at whichever came first (#5741).""" + if previous is None: + return [] + by_le: dict[float, float] = defaultdict(float) + for labels, count in current.items(): label_dict = dict(labels) if label_dict.get("service") != service: continue le_str = label_dict.get("le", "") try: le = float("inf") if le_str == "+Inf" else float(le_str) - router_buckets.append((le, count)) except ValueError: continue + by_le[le] += _rise(previous, labels, count) + return sorted(by_le.items()) - if not router_buckets: - return 0.0 - - router_buckets.sort(key=lambda t: t[0]) - total = router_buckets[-1][1] # +Inf bucket count == total requests - if total == 0.0: - return 0.0 +def _percentile_ms(buckets: list[tuple[float, float]], pct: float) -> float | None: + """Linear interpolation of a percentile of `buckets` (service_buckets'), + in ms; None when no response arrived since the previous scrape, as an + idle router has no latency to report. A percentile past the last finite + bucket reads as that bucket's edge: at least that long.""" + total = buckets[-1][1] if buckets else 0.0 # the +Inf bucket counts every response + if total <= 0.0: + return None target = total * pct - for i, (le, count) in enumerate(router_buckets): + for i, (le, count) in enumerate(buckets): if count < target: continue if i == 0: - ratio = target / count if count > 0 else 0.0 - return le * ratio * 1000.0 - prev_le, prev_count = router_buckets[i - 1] - if count == prev_count: - return prev_le * 1000.0 + return le * (target / count) * 1000.0 + prev_le, prev_count = buckets[i - 1] if math.isinf(le): return prev_le * 1000.0 fraction = (target - prev_count) / (count - prev_count) return (prev_le + fraction * (le - prev_le)) * 1000.0 - - return 0.0 + return None # ────────────────────────────────────────────────────────────────────────────── diff --git a/plugins/traefik/templates/traefik/rows.html b/plugins/traefik/templates/traefik/rows.html index e13fb89..f0f1278 100644 --- a/plugins/traefik/templates/traefik/rows.html +++ b/plugins/traefik/templates/traefik/rows.html @@ -34,6 +34,10 @@ {% endif %} {% endmacro %} +{# A router that has answered nothing since Steward started has no latency yet. #} +{% macro ms(v) %}{% if v is none %}—{% else %}{{ "%.0f"|format(v) }} ms{% endif %}{% endmacro %} +{% macro latency_cell(v) %}{{ ms(v) }}{% endmacro %} + {% macro router_card(r) %}
@@ -54,8 +58,8 @@ - - + {{ latency_cell(r.latest.latency_p95_ms) }} +
Req/s{{ "%.2f"|format(r.latest.request_rate) }}{{ r.sparkline_req|safe }}
Bandwidth{{ r.latest.response_bytes_rate|rate }}{{ r.sparkline_bytes|safe }}
p95{{ "%.0f"|format(r.latest.latency_p95_ms) }} ms{{ r.sparkline_p95|safe }}
p99{{ "%.0f"|format(r.latest.latency_p99_ms) }} ms
p95{{ r.sparkline_p95|safe }}
p99{{ ms(r.latest.latency_p99_ms) }}
4xx{{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%{{ r.sparkline_4xx|safe }}
5xx{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%{{ r.sparkline_5xx|safe }}
@@ -88,7 +92,7 @@ {{ r.name }} {{ "%.2f"|format(r.latest.request_rate) }} {{ r.latest.response_bytes_rate|rate }} - {{ "%.0f"|format(r.latest.latency_p95_ms) }} ms + {{ latency_cell(r.latest.latency_p95_ms) }} {{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}% {{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}% diff --git a/tests/plugins/traefik/test_latency.py b/tests/plugins/traefik/test_latency.py new file mode 100644 index 0000000..ae0ee55 --- /dev/null +++ b/tests/plugins/traefik/test_latency.py @@ -0,0 +1,61 @@ +"""A router's latency is each scrape's own percentile (#5741): the rise in +Traefik's histogram since the last scrape, every label set's buckets summed +per `le`. Read as it stood, the histogram gave a percentile of every +response since Traefik started, which barely moved (nextcloud's flat 100 ms +p95).""" +import pytest + +from plugins.traefik.scraper import compute_router_metrics, service_buckets + +BUCKET = "traefik_service_request_duration_seconds_bucket" +EDGES = ("0.1", "0.3", "1.2", "5", "+Inf") # Traefik's default buckets + + +def _hist(service, code, method, counts): + """{labels: count} for one label set's cumulative buckets, in EDGES order.""" + return {frozenset({"service": service, "code": code, "method": method, "le": le}.items()): n + for le, n in zip(EDGES, counts)} + + +def _scrape(*sets): + out = {} + for s in sets: + out.update(_hist(*s)) + return {BUCKET: out} + + +def test_latency_reads_the_responses_since_the_last_scrape_not_since_start(): + # A million fast responses since start, then 100 slow ones (all 0.3–1.2 s). + before = _scrape(("cloud@docker", "200", "GET", (1_000_000, 1_000_000, 1_000_000, 1_000_000, 1_000_000))) + after = _scrape(("cloud@docker", "200", "GET", (1_000_000, 1_000_000, 1_000_100, 1_000_100, 1_000_100))) + got = compute_router_metrics(after, before, 60.0)["cloud@docker"] + assert 300.0 < got["latency_p50_ms"] < 1200.0 + assert got["latency_p95_ms"] == pytest.approx(300.0 + 0.95 * 900.0) + + +def test_every_label_set_counts_toward_one_histogram(): + # GET answered fast and POST slowly: 90 + 10 responses, so p95 is a POST's. + before = _scrape(("app@docker", "200", "GET", (0, 0, 0, 0, 0)), + ("app@docker", "200", "POST", (0, 0, 0, 0, 0))) + after = _scrape(("app@docker", "200", "GET", (90, 90, 90, 90, 90)), + ("app@docker", "200", "POST", (0, 0, 0, 10, 10))) + assert service_buckets(after[BUCKET], before[BUCKET], "app@docker") == [ + (0.1, 90.0), (0.3, 90.0), (1.2, 90.0), (5.0, 100.0), (float("inf"), 100.0)] + got = compute_router_metrics(after, before, 60.0)["app@docker"] + assert got["latency_p50_ms"] < 100.0 + assert 1200.0 < got["latency_p95_ms"] < 5000.0 + + +def test_an_idle_scrape_and_the_first_scrape_have_no_latency(): + still = _scrape(("idle@docker", "200", "GET", (50, 60, 60, 60, 60))) + idle = compute_router_metrics(still, still, 60.0)["idle@docker"] + first = compute_router_metrics(still, None, 60.0)["idle@docker"] + for got in (idle, first): + assert got["latency_p50_ms"] is None and got["latency_p95_ms"] is None + assert got["latency_p99_ms"] is None + + +def test_a_percentile_past_the_last_edge_reads_as_at_least_that_edge(): + before = _scrape(("ws@docker", "101", "GET", (0, 0, 0, 0, 0))) + after = _scrape(("ws@docker", "101", "GET", (0, 0, 0, 0, 20))) # all longer than 5 s + assert compute_router_metrics(after, before, 60.0)["ws@docker"]["latency_p95_ms"] == 5000.0