feat(traefik): scrape every Traefik replica behind the metrics URL and add them up (#5844)

Through one address Steward read whichever replica answered, and with
host-mode ports behind a keepalived VIP that can be the replica serving
nothing: since the 18:43 restart every scrape landed on Docker03's idle
Traefik, which has no traefik_service_* series, so no router was stored.

A metrics_url host name that resolves to several addresses (a swarm
service's tasks.<service>, a Kubernetes headless Service, round-robin DNS)
is now scraped address by address, with the original Host header and TLS
name. Answers from one process are kept once; a replica that doesn't answer
is skipped with a warning. Requests, bytes and latency buckets are summed
over the replicas that have a baseline, so the sum's rise is the sum of each
one's own rise; certificates are listed once; Traefik's own figures add
connections and memory and keep the newest reload. A name with one address
is one scrape, as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/plugins/traefik/test_baselines.py

@@ -1,7 +1,16 @@
 """#5844: one metrics URL can reach several Traefik processes (swarm replicas,
 pods, a keepalived pair, a restart). Each is counted only against its own last
 scrape, so one replica's lifetime totals never read as another's new traffic."""
-from plugins.traefik.scraper import BASELINE_TTL_SECONDS, Baselines, compute_router_metrics, instance_of
+from datetime import datetime, timezone
+from functools import partial
+
+import httpx
+import pytest
+
+from plugins.traefik import scraper
+from plugins.traefik.scraper import (
+    BASELINE_TTL_SECONDS, Baselines, combine, compute_router_metrics, instance_of, merge_global_stats, unique_certs,
+)
 
 REQ = "traefik_service_requests_total"
 
@@ -55,3 +64,95 @@ def test_a_baseline_older_than_the_ttl_is_dropped_not_compared():
     b = Baselines()
     b.swap(_scrape(1000.0, **{"200": 1}), 0.0)
     assert b.swap(_scrape(1000.0, **{"200": 900}), BASELINE_TTL_SECONDS + 1) == (None, 0.0)
+
+
+# ── Every replica, added up (#5844) ──────────────────────────────────────────
+
+def _router(started, service, **counts):
+    out = {REQ: {frozenset({"service": service, "code": c, "method": "GET"}.items()): v
+                 for c, v in counts.items()}}
+    out["process_start_time_seconds"] = {frozenset(): started}
+    return out
+
+
+def test_two_replicas_add_up_and_a_new_one_waits_for_its_baseline():
+    a0, a1 = _router(1.0, "app@swarm", **{"200": 100}), _router(1.0, "app@swarm", **{"200": 160})
+    b0, b1 = _router(2.0, "app@swarm", **{"200": 50}), _router(2.0, "app@swarm", **{"200": 70, "502": 4})
+    c1 = _router(3.0, "app@swarm", **{"200": 9000})                 # first seen: no baseline
+    current, previous, elapsed = combine([(a1, a0, 60.0), (b1, b0, 60.0), (c1, None, 0.0)])
+    got = compute_router_metrics(current, previous, elapsed)["app@swarm"]
+    # (160-100) + (70-50) + 4 new 502s = 84 requests in 60 s; C's 9000 are not traffic.
+    assert got["request_rate"] == pytest.approx(84 / 60)
+    assert got["error_rate_5xx_pct"] == pytest.approx(4 / 84 * 100)
+
+
+def test_the_busy_replica_counts_though_the_idle_one_has_no_routers():
+    """The live case: the VIP sends everything to one replica; the other has
+    no traefik_service_* series at all."""
+    busy0, busy1 = _router(1.0, "roundtable@swarm", **{"200": 10}), _router(1.0, "roundtable@swarm", **{"200": 40})
+    idle = {"process_start_time_seconds": {frozenset(): 2.0}}
+    current, previous, elapsed = combine([(busy1, busy0, 60.0), (idle, idle, 60.0)])
+    assert compute_router_metrics(current, previous, elapsed)["roundtable@swarm"]["request_rate"] == 0.5
+
+
+def test_with_no_baseline_yet_every_router_is_listed_at_zero():
+    current, previous, _ = combine([(_router(1.0, "a@swarm", **{"200": 5}), None, 0.0),
+                                    (_router(2.0, "b@swarm", **{"200": 7}), None, 0.0)])
+    assert previous is None
+    got = compute_router_metrics(current, previous, 0.0)
+    assert set(got) == {"a@swarm", "b@swarm"} and got["a@swarm"]["request_rate"] == 0.0
+
+
+def test_traefiks_own_figures_add_up_or_take_the_newest():
+    t1, t2 = datetime(2026, 10, 15, 18, 0, tzinfo=timezone.utc), datetime(2026, 10, 15, 18, 43, tzinfo=timezone.utc)
+    got = merge_global_stats([
+        {"open_conns_total": 40.0, "config_reloads_total": 70.0, "config_last_reload_at": t1, "process_memory_bytes": 2e8},
+        {"open_conns_total": 1.0, "config_reloads_total": 72.0, "config_last_reload_at": t2, "process_memory_bytes": 1e8},
+    ])
+    assert got == {"open_conns_total": 41.0, "config_reloads_total": 72.0, "config_last_reload_at": t2,
+                   "process_memory_bytes": 3e8}
+
+
+def test_a_certificate_both_replicas_serve_is_listed_once():
+    cert = {"traefik_tls_certs_not_after": {frozenset({"serial": "ab", "cn": "x.example", "sans": ""}.items()): 2e9}}
+    assert [c["serial"] for c in unique_certs([cert, cert])] == ["ab"]
+
+
+EXPO = {
+    "10.0.2.5": "process_start_time_seconds 1.0\ntraefik_service_requests_total{service=\"a@swarm\",code=\"200\",method=\"GET\"} 10\n",
+    "10.0.2.6": "process_start_time_seconds 2.0\n",
+    "10.0.2.7": "process_start_time_seconds 1.0\n",                # the same process again
+}
+
+
+async def test_a_name_for_several_replicas_is_scraped_address_by_address(monkeypatch):
+    seen_hosts = []
+
+    def answer(request: httpx.Request) -> httpx.Response:
+        seen_hosts.append(request.headers["host"])
+        if request.url.host == "10.0.2.8":
+            return httpx.Response(503)
+        return httpx.Response(200, text=EXPO[request.url.host])
+
+    async def addresses(host, port):
+        assert (host, port) == ("tasks.traefik_router", 8080)
+        return ["10.0.2.5", "10.0.2.6", "10.0.2.7", "10.0.2.8"]
+
+    monkeypatch.setattr(scraper, "_addresses", addresses)
+    monkeypatch.setattr(scraper.httpx, "AsyncClient",
+                        partial(httpx.AsyncClient, transport=httpx.MockTransport(answer)))
+    got = await scraper.fetch_instances("http://tasks.traefik_router:8080/metrics")
+    # 10.0.2.8 didn't answer and 10.0.2.7 is 10.0.2.5's process: two Traefiks.
+    assert sorted(instance_of(g) for g in got) == ["1.00", "2.00"]
+    assert set(seen_hosts) == {"tasks.traefik_router:8080"}
+
+
+async def test_a_name_with_one_address_is_one_scrape_as_before(monkeypatch):
+    async def addresses(host, port):
+        return ["10.0.2.5"]
+
+    monkeypatch.setattr(scraper, "_addresses", addresses)
+    monkeypatch.setattr(scraper.httpx, "AsyncClient", partial(
+        httpx.AsyncClient, transport=httpx.MockTransport(lambda r: httpx.Response(200, text=EXPO["10.0.2.5"]))))
+    [got] = await scraper.fetch_instances("http://trfk.traefik.internal/metrics")
+    assert instance_of(got) == "1.00"