feat(traefik): scrape every Traefik replica behind the metrics URL and add them up (#5844)

Through one address Steward read whichever replica answered, and with
host-mode ports behind a keepalived VIP that can be the replica serving
nothing: since the 18:43 restart every scrape landed on Docker03's idle
Traefik, which has no traefik_service_* series, so no router was stored.

A metrics_url host name that resolves to several addresses (a swarm
service's tasks.<service>, a Kubernetes headless Service, round-robin DNS)
is now scraped address by address, with the original Host header and TLS
name. Answers from one process are kept once; a replica that doesn't answer
is skipped with a warning. Requests, bytes and latency buckets are summed
over the replicas that have a baseline, so the sum's rise is the sum of each
one's own rise; certificates are listed once; Traefik's own figures add
connections and memory and keep the newest reload. A name with one address
is one scrape, as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/plugin.yaml

@@ -13,6 +13,9 @@ tags:
   - access-log
 
 config:
+  # A host name that resolves to several Traefiks (a swarm service's
+  # tasks.<service>, a Kubernetes headless Service) is scraped address by
+  # address and added up, so every replica's traffic counts.
   metrics_url: "http://localhost:8080/metrics"
   # Traefik's API, for which hosts each router serves (so a certificate's
   # expiry shows on its router). Empty: /api on the metrics URL's host.

plugins/traefik/scheduler.py

@@ -64,11 +64,13 @@ async def _do_scrape(app: "Quart") -> None:
     from .scraper import (
         Baselines,
         api_base,
+        combine,
         compute_global_stats,
         compute_router_metrics,
         compute_status_codes,
-        extract_certs,
-        fetch_metrics,
+        fetch_instances,
+        merge_global_stats,
+        unique_certs,
         fetch_error_page_services,
         fetch_router_domains,
     )
@@ -77,7 +79,7 @@ async def _do_scrape(app: "Quart") -> None:
     metrics_url: str = app.config["PLUGINS"]["traefik"]["metrics_url"]
 
     try:
-        current = await fetch_metrics(metrics_url)
+        scrapes = await fetch_instances(metrics_url)
     except Exception:
         logger.exception("Traefik scrape failed (url=%s)", metrics_url)
         return
@@ -95,11 +97,12 @@ async def _do_scrape(app: "Quart") -> None:
 
     if _baselines is None:
         _baselines = Baselines()
-    previous, elapsed = _baselines.swap(current, time.monotonic())
+    now = time.monotonic()
+    current, previous, elapsed = combine([(scrape, *_baselines.swap(scrape, now)) for scrape in scrapes])
     router_metrics = compute_router_metrics(current, previous, elapsed)
-    global_stats = compute_global_stats(current, previous)
+    global_stats = merge_global_stats([compute_global_stats(scrape, None) for scrape in scrapes])
     status_codes = compute_status_codes(current, previous)
-    cert_list = extract_certs(current)
+    cert_list = unique_certs(scrapes)
 
     scraped_at = datetime.now(timezone.utc)
 

plugins/traefik/scraper.py

@@ -11,8 +11,11 @@ Traefik exposes Prometheus metrics at /metrics. This module:
 5. Computes process-level and config health stats
 """
 from __future__ import annotations
+import asyncio
+import logging
 import math
 import re
+import socket
 from collections import defaultdict
 from datetime import datetime, timezone
 
@@ -20,6 +23,8 @@ import httpx
 
 from steward.core import prometheus
 
+logger = logging.getLogger(__name__)
+
 
 # ──────────────────────────────────────────────────────────────────────────────
 # Prometheus text format parser
@@ -86,6 +91,63 @@ class Baselines:
         return prev[0], now - prev[1]
 
 
+# The counter families the per-router figures are computed from. Across
+# several Traefiks these add up (requests, bytes and latency buckets are each
+# replica's share); everything else in an exposition is a gauge or a
+# timestamp, which summing would turn into nonsense.
+SUMMED = ("traefik_service_requests_total", "traefik_service_responses_bytes_total",
+          "traefik_service_request_duration_seconds_bucket")
+
+
+def _sum(scrapes: list[ParsedMetrics]) -> ParsedMetrics:
+    out: ParsedMetrics = {}
+    for scrape in scrapes:
+        for name in SUMMED:
+            into = out.setdefault(name, {})
+            for labels, value in scrape.get(name, {}).items():
+                into[labels] = into.get(labels, 0.0) + value
+    return out
+
+
+def combine(pairs: list[tuple[ParsedMetrics, ParsedMetrics | None, float]],
+            ) -> tuple[ParsedMetrics, ParsedMetrics | None, float]:
+    """One (current, previous, elapsed) for the router arithmetic out of each
+    Traefik's (current, its own previous scrape, seconds since it).
+
+    Only the Traefiks that have a baseline are added up, current and previous
+    alike, so the sum's rise is the sum of each one's own rise: a replica seen
+    for the first time contributes from its next scrape on, and is never read
+    as traffic. When none has a baseline yet, every current comes back with no
+    previous, so each router is listed at zero."""
+    based = [(cur, prev, el) for cur, prev, el in pairs if prev is not None]
+    if not based:
+        return _sum([cur for cur, _, _ in pairs]), None, 0.0
+    return (_sum([cur for cur, _, _ in based]), _sum([prev for _, prev, _ in based]),
+            sum(el for _, _, el in based) / len(based))
+
+
+def merge_global_stats(stats: list[dict]) -> dict:
+    """Traefik's own figures across its replicas: connections and memory add
+    up; reloads and the last reload are the newest any replica reports."""
+    def values(key):
+        return [s[key] for s in stats if s.get(key) is not None]
+    conns, mem = values("open_conns_total"), values("process_memory_bytes")
+    reloads, last = values("config_reloads_total"), values("config_last_reload_at")
+    return {"open_conns_total": sum(conns) if conns else None,
+            "config_reloads_total": max(reloads) if reloads else None,
+            "config_last_reload_at": max(last) if last else None,
+            "process_memory_bytes": sum(mem) if mem else None}
+
+
+def unique_certs(scrapes: list[ParsedMetrics]) -> list[dict]:
+    """The certificates any replica serves, each once (by serial)."""
+    seen: dict[str, dict] = {}
+    for scrape in scrapes:
+        for cert in extract_certs(scrape):
+            seen.setdefault(cert["serial"], cert)
+    return list(seen.values())
+
+
 # ──────────────────────────────────────────────────────────────────────────────
 # Per-router metric computation
 # ──────────────────────────────────────────────────────────────────────────────
@@ -339,6 +401,53 @@ async def fetch_metrics(metrics_url: str) -> ParsedMetrics:
     return parse_prometheus(response.text)
 
 
+async def _addresses(host: str, port: int) -> list[str]:
+    """Every address `host` resolves to, in the order given, each once."""
+    infos = await asyncio.get_running_loop().getaddrinfo(host, port, type=socket.SOCK_STREAM)
+    return list(dict.fromkeys(info[4][0] for info in infos))
+
+
+async def fetch_instances(metrics_url: str) -> list[ParsedMetrics]:
+    """A scrape of every Traefik the metrics URL's host name resolves to.
+
+    A name for several replicas (a swarm service's `tasks.<service>`, a
+    Kubernetes headless Service, round-robin DNS) is scraped address by
+    address, each with the original Host header and TLS name, so every
+    replica's traffic is counted (#5844). A name with one address is one
+    scrape, as before. Two answers from the same process (localhost over IPv4
+    and IPv6) are kept once. A replica that doesn't answer is left out with a
+    warning; none answering raises."""
+    url = httpx.URL(metrics_url)
+    host = url.host
+    port = url.port or (443 if url.scheme == "https" else 80)
+    try:
+        addresses = await _addresses(host, port)
+    except OSError:
+        addresses = [host]
+    if len(addresses) <= 1:
+        return [await fetch_metrics(metrics_url)]
+
+    async def one(address: str) -> ParsedMetrics:
+        target = url.copy_with(host=address)
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            response = await client.get(target, headers={"Host": url.netloc.decode()},
+                                        extensions={"sni_hostname": host})
+            response.raise_for_status()
+        return parse_prometheus(response.text)
+
+    results = await asyncio.gather(*(one(a) for a in addresses), return_exceptions=True)
+    scrapes: dict[str, ParsedMetrics] = {}
+    for address, result in zip(addresses, results):
+        if isinstance(result, BaseException):
+            logger.warning("Traefik at %s (%s) didn't answer: %s", address, host, result)
+            continue
+        key = instance_of(result) or address
+        scrapes.setdefault(key, result)
+    if not scrapes:
+        raise RuntimeError(f"no Traefik behind {host} answered ({len(addresses)} addresses)")
+    return list(scrapes.values())
+
+
 # ──────────────────────────────────────────────────────────────────────────────
 # Router domains (Traefik API)
 # ──────────────────────────────────────────────────────────────────────────────