fix(traefik): count each Traefik process only against its own last scrape (#5844)

One metrics URL can reach several Traefik processes: a swarm service's
replicas behind the routing mesh or its VIP, a Kubernetes Service's pods,
a keepalived pair after failover, or the same Traefik after a restart.
Each keeps its own counters, so comparing one with another read the second
one's lifetime totals as new traffic: retired routers flashed 100% 502 and
an idle router showed 834 req/s whenever the scrape moved to the other
replica.

Each scrape is now keyed by Traefik's process_start_time_seconds and
compared only with the same process's previous scrape. A process seen for
the first time (or not in the last hour) sets its baseline and counts
nothing. Without a start time every scrape is one process, as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/scheduler.py

@@ -12,9 +12,8 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
-# Module-level state: previous scrape data for delta computation
-_prev_metrics: dict = {}
-_prev_time: float = 0.0
+# Each Traefik process's last scrape, the baseline its counters rise from (#5844).
+_baselines = None
 
 # Router name -> host names from Traefik's API, refreshed each scrape (#5254).
 # None until the API has answered once, and again whenever it stops answering,
@@ -58,11 +57,12 @@ def make_scrape_task(app: "Quart") -> ScheduledTask:
 
 async def _do_scrape(app: "Quart") -> None:
     """Fetch Traefik metrics, write to DB, and emit plugin_metrics."""
-    global _prev_metrics, _prev_time, _router_domains, _error_page_services
+    global _baselines, _router_domains, _error_page_services
 
     from datetime import datetime, timezone
     from .models import TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
     from .scraper import (
+        Baselines,
         api_base,
         compute_global_stats,
         compute_router_metrics,
@@ -93,16 +93,14 @@ async def _do_scrape(app: "Quart") -> None:
     except Exception as exc:
         logger.debug("Traefik middlewares not read (%s); keeping the last error-page services", exc)
 
-    now = time.monotonic()
-    elapsed = now - _prev_time if _prev_time else 60.0
-    router_metrics = compute_router_metrics(current, _prev_metrics or None, elapsed)
-    global_stats = compute_global_stats(current, _prev_metrics or None)
-    status_codes = compute_status_codes(current, _prev_metrics or None)
+    if _baselines is None:
+        _baselines = Baselines()
+    previous, elapsed = _baselines.swap(current, time.monotonic())
+    router_metrics = compute_router_metrics(current, previous, elapsed)
+    global_stats = compute_global_stats(current, previous)
+    status_codes = compute_status_codes(current, previous)
     cert_list = extract_certs(current)
 
-    _prev_metrics = current
-    _prev_time = now
-
     scraped_at = datetime.now(timezone.utc)
 
     async with app.db_sessionmaker() as session:

plugins/traefik/scraper.py

@@ -39,6 +39,53 @@ def parse_prometheus(text: str) -> ParsedMetrics:
     return metrics
 
 
+# ──────────────────────────────────────────────────────────────────────────────
+# Which Traefik answered (#5844)
+# ──────────────────────────────────────────────────────────────────────────────
+#
+# One metrics URL can reach more than one Traefik: a swarm service's replicas
+# behind the routing mesh or its VIP, a Kubernetes Service's pods, a keepalived
+# pair after failover, or the same Traefik after a restart. Each process keeps
+# its own counters, so comparing one process's scrape with another's reads the
+# second one's lifetime totals as new traffic (a retired router flashing 100%
+# 502, an idle one at 800 req/s). Each process is told apart by its start time,
+# which Traefik has exposed since v2.0, and is only ever compared with itself.
+
+# A process not heard from for this long has restarted or gone; its baseline
+# is dropped rather than kept forever.
+BASELINE_TTL_SECONDS = 3600
+
+
+def instance_of(current: ParsedMetrics) -> str:
+    """The Traefik process a scrape came from: its process_start_time_seconds,
+    or "" when the exposition doesn't carry one (then every scrape is taken to
+    be the same process, as before #5844)."""
+    start = current.get("process_start_time_seconds", {})
+    return f"{next(iter(start.values())):.2f}" if start else ""
+
+
+class Baselines:
+    """Each Traefik process's last scrape and when it was taken."""
+
+    def __init__(self) -> None:
+        self._seen: dict[str, tuple[ParsedMetrics, float]] = {}
+
+    def swap(self, current: ParsedMetrics, now: float) -> tuple[ParsedMetrics | None, float]:
+        """(the previous scrape of the process `current` came from, seconds
+        since it), and keep `current` as that process's newest. A process seen
+        for the first time has no previous scrape: (None, 0.0), so nothing is
+        counted until it has a baseline of its own."""
+        key = instance_of(current)
+        prev = self._seen.get(key)
+        self._seen[key] = (current, now)
+        for k, (_, at) in list(self._seen.items()):
+            if now - at > BASELINE_TTL_SECONDS:
+                del self._seen[k]
+        if prev is None or now - prev[1] > BASELINE_TTL_SECONDS:
+            return None, 0.0
+        return prev[0], now - prev[1]
+
+
 # ──────────────────────────────────────────────────────────────────────────────
 # Per-router metric computation
 # ──────────────────────────────────────────────────────────────────────────────