feat(traefik): keep each router's 4xx and 5xx responses by status code (#5711)

The scraper folded traefik_service_requests_total{code} into two shares and
dropped the code, so a problem couldn't say whether it was scanners (404), a
login wall (401/403) or the app down (502/503/504). Each scrape now stores the
responses per router and code that occurred (traefik_status_codes, migration
traefik_006), pruned with the router metrics and left out of backups (System
180 ruling: Traefik history is not backed up). A problem card lists its codes
over the last 15 minutes, most first, or says none occurred, or that codes
aren't recorded yet.

A counter series new since the previous scrape now counts from 0 instead of
being skipped: Traefik creates a router's 502 series at its first 502, so a
first burst of 5xx read as 0% until it happened again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/scraper.py

@@ -43,6 +43,33 @@ def parse_prometheus(text: str) -> ParsedMetrics:
 # Per-router metric computation
 # ──────────────────────────────────────────────────────────────────────────────
 
+def _rise(prev: dict[frozenset, float] | None, labels: frozenset, value: float) -> float:
+    """How far a counter rose since the previous scrape. On the first scrape
+    (`prev` None) nothing can be compared, so 0. A series the previous scrape
+    didn't have started from 0 since then, so all of it counts: Traefik makes
+    a router's first 502 series when the first 502 happens (#5711)."""
+    if prev is None:
+        return 0.0
+    return max(0.0, value - prev.get(labels, 0.0))
+
+
+def compute_status_codes(current: ParsedMetrics, previous: ParsedMetrics | None) -> dict[str, dict[str, int]]:
+    """{service: {code: responses since the previous scrape}} for each 4xx and
+    5xx code that occurred, from traefik_service_requests_total{code} (#5711).
+    Codes that didn't occur are left out; the first scrape has none."""
+    prev = None if previous is None else previous.get("traefik_service_requests_total", {})
+    out: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+    for labels, value in current.get("traefik_service_requests_total", {}).items():
+        label_dict = dict(labels)
+        service, code = label_dict.get("service", ""), label_dict.get("code", "")
+        if not service or code[:1] not in ("4", "5"):
+            continue
+        count = round(_rise(prev, labels, value))
+        if count:
+            out[service][code] += count
+    return {service: dict(codes) for service, codes in out.items()}
+
+
 def compute_router_metrics(
     current: ParsedMetrics,
     previous: ParsedMetrics | None,
@@ -63,12 +90,13 @@ def compute_router_metrics(
             "response_bytes_rate": float,    # bytes/sec
         }}
     """
+    first = not previous
     prev = previous or {}
     elapsed = max(elapsed_seconds, 1.0)
 
     # ── Request counters ──────────────────────────────────────────────────────
     req_samples = current.get("traefik_service_requests_total", {})
-    prev_req = prev.get("traefik_service_requests_total", {})
+    prev_req = None if first else prev.get("traefik_service_requests_total", {})
 
     service_total: dict[str, float] = defaultdict(float)
     service_4xx: dict[str, float] = defaultdict(float)
@@ -80,8 +108,7 @@ def compute_router_metrics(
         if not service:
             continue
         code = label_dict.get("code", "")
-        prev_value = prev_req.get(labels, value)  # no delta on first scrape
-        delta = max(0.0, value - prev_value)
+        delta = _rise(prev_req, labels, value)
         service_total[service] += delta
         if code.startswith("4"):
             service_4xx[service] += delta
@@ -90,15 +117,14 @@ def compute_router_metrics(
 
     # ── Response bytes counters ───────────────────────────────────────────────
     bytes_samples = current.get("traefik_service_responses_bytes_total", {})
-    prev_bytes = prev.get("traefik_service_responses_bytes_total", {})
+    prev_bytes = None if first else prev.get("traefik_service_responses_bytes_total", {})
 
     service_bytes: dict[str, float] = defaultdict(float)
     for labels, value in bytes_samples.items():
         service = dict(labels).get("service", "")
         if not service:
             continue
-        prev_value = prev_bytes.get(labels, value)
-        service_bytes[service] += max(0.0, value - prev_value)
+        service_bytes[service] += _rise(prev_bytes, labels, value)
 
     # ── Histogram buckets ─────────────────────────────────────────────────────
     hist_buckets = current.get("traefik_service_request_duration_seconds_bucket", {})