bvandeusenandClaude Opus 5.5
98440d8de6
One metrics URL can reach several Traefik processes: a swarm service's replicas behind the routing mesh or its VIP, a Kubernetes Service's pods, a keepalived pair after failover, or the same Traefik after a restart. Each keeps its own counters, so comparing one with another read the second one's lifetime totals as new traffic: retired routers flashed 100% 502 and an idle router showed 834 req/s whenever the scrape moved to the other replica. Each scrape is now keyed by Traefik's process_start_time_seconds and compared only with the same process's previous scrape. A process seen for the first time (or not in the last hour) sets its baseline and counts nothing. Without a start time every scrape is one process, as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
58 lines
2.5 KiB
Python
58 lines
2.5 KiB
Python
"""#5844: one metrics URL can reach several Traefik processes (swarm replicas,
|
|
pods, a keepalived pair, a restart). Each is counted only against its own last
|
|
scrape, so one replica's lifetime totals never read as another's new traffic."""
|
|
from plugins.traefik.scraper import BASELINE_TTL_SECONDS, Baselines, compute_router_metrics, instance_of
|
|
|
|
REQ = "traefik_service_requests_total"
|
|
|
|
|
|
def _scrape(started: float | None, **counts):
|
|
"""A scrape from the process started at `started`, with {code: total} for qbittorrent@file."""
|
|
out = {REQ: {frozenset({"service": "qbittorrent@file", "code": c, "method": "GET"}.items()): v
|
|
for c, v in counts.items()}}
|
|
if started is not None:
|
|
out["process_start_time_seconds"] = {frozenset(): started}
|
|
return out
|
|
|
|
|
|
def test_each_process_is_compared_only_with_itself():
|
|
b = Baselines()
|
|
a1, b1 = _scrape(1000.0, **{"200": 10}), _scrape(2000.0, **{"502": 336})
|
|
assert b.swap(a1, 0.0) == (None, 0.0)
|
|
# The other replica answers: it has no baseline yet, so nothing is counted.
|
|
assert b.swap(b1, 60.0) == (None, 0.0)
|
|
# Back to the first: compared with its own scrape two minutes ago.
|
|
a2 = _scrape(1000.0, **{"200": 22})
|
|
assert b.swap(a2, 120.0) == (a1, 120.0)
|
|
assert b.swap(_scrape(2000.0, **{"502": 336}), 180.0) == (b1, 120.0)
|
|
|
|
|
|
def test_landing_on_the_other_replica_no_longer_reads_its_totals_as_traffic():
|
|
"""The live symptom: an idle retired router flashing 336 502s, 100% 5xx."""
|
|
b = Baselines()
|
|
b.swap(_scrape(1000.0, **{"200": 10}), 0.0)
|
|
previous, elapsed = b.swap(_scrape(2000.0, **{"502": 336}), 60.0)
|
|
got = compute_router_metrics(_scrape(2000.0, **{"502": 336}), previous, elapsed)
|
|
assert got["qbittorrent@file"]["request_rate"] == 0.0
|
|
assert got["qbittorrent@file"]["error_rate_5xx_pct"] == 0.0
|
|
|
|
|
|
def test_a_restarted_traefik_starts_a_new_baseline():
|
|
b = Baselines()
|
|
b.swap(_scrape(1000.0, **{"200": 5000}), 0.0)
|
|
assert b.swap(_scrape(3000.0, **{"200": 3}), 60.0) == (None, 0.0)
|
|
|
|
|
|
def test_without_a_start_time_every_scrape_is_one_process_as_before():
|
|
b = Baselines()
|
|
first = _scrape(None, **{"200": 1})
|
|
assert instance_of(first) == ""
|
|
b.swap(first, 0.0)
|
|
assert b.swap(_scrape(None, **{"200": 2}), 60.0) == (first, 60.0)
|
|
|
|
|
|
def test_a_baseline_older_than_the_ttl_is_dropped_not_compared():
|
|
b = Baselines()
|
|
b.swap(_scrape(1000.0, **{"200": 1}), 0.0)
|
|
assert b.swap(_scrape(1000.0, **{"200": 900}), BASELINE_TTL_SECONDS + 1) == (None, 0.0)
|