fix(traefik): count each Traefik process only against its own last scrape (#5844)
One metrics URL can reach several Traefik processes: a swarm service's replicas behind the routing mesh or its VIP, a Kubernetes Service's pods, a keepalived pair after failover, or the same Traefik after a restart. Each keeps its own counters, so comparing one with another read the second one's lifetime totals as new traffic: retired routers flashed 100% 502 and an idle router showed 834 req/s whenever the scrape moved to the other replica. Each scrape is now keyed by Traefik's process_start_time_seconds and compared only with the same process's previous scrape. A process seen for the first time (or not in the last hour) sets its baseline and counts nothing. Without a start time every scrape is one process, as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -12,9 +12,8 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Module-level state: previous scrape data for delta computation
|
||||
_prev_metrics: dict = {}
|
||||
_prev_time: float = 0.0
|
||||
# Each Traefik process's last scrape, the baseline its counters rise from (#5844).
|
||||
_baselines = None
|
||||
|
||||
# Router name -> host names from Traefik's API, refreshed each scrape (#5254).
|
||||
# None until the API has answered once, and again whenever it stops answering,
|
||||
@@ -58,11 +57,12 @@ def make_scrape_task(app: "Quart") -> ScheduledTask:
|
||||
|
||||
async def _do_scrape(app: "Quart") -> None:
|
||||
"""Fetch Traefik metrics, write to DB, and emit plugin_metrics."""
|
||||
global _prev_metrics, _prev_time, _router_domains, _error_page_services
|
||||
global _baselines, _router_domains, _error_page_services
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from .models import TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
|
||||
from .scraper import (
|
||||
Baselines,
|
||||
api_base,
|
||||
compute_global_stats,
|
||||
compute_router_metrics,
|
||||
@@ -93,16 +93,14 @@ async def _do_scrape(app: "Quart") -> None:
|
||||
except Exception as exc:
|
||||
logger.debug("Traefik middlewares not read (%s); keeping the last error-page services", exc)
|
||||
|
||||
now = time.monotonic()
|
||||
elapsed = now - _prev_time if _prev_time else 60.0
|
||||
router_metrics = compute_router_metrics(current, _prev_metrics or None, elapsed)
|
||||
global_stats = compute_global_stats(current, _prev_metrics or None)
|
||||
status_codes = compute_status_codes(current, _prev_metrics or None)
|
||||
if _baselines is None:
|
||||
_baselines = Baselines()
|
||||
previous, elapsed = _baselines.swap(current, time.monotonic())
|
||||
router_metrics = compute_router_metrics(current, previous, elapsed)
|
||||
global_stats = compute_global_stats(current, previous)
|
||||
status_codes = compute_status_codes(current, previous)
|
||||
cert_list = extract_certs(current)
|
||||
|
||||
_prev_metrics = current
|
||||
_prev_time = now
|
||||
|
||||
scraped_at = datetime.now(timezone.utc)
|
||||
|
||||
async with app.db_sessionmaker() as session:
|
||||
|
||||
Reference in New Issue
Block a user