feat(traefik): keep each router's 4xx and 5xx responses by status code (#5711)

The scraper folded traefik_service_requests_total{code} into two shares and
dropped the code, so a problem couldn't say whether it was scanners (404), a
login wall (401/403) or the app down (502/503/504). Each scrape now stores the
responses per router and code that occurred (traefik_status_codes, migration
traefik_006), pruned with the router metrics and left out of backups (System
180 ruling: Traefik history is not backed up). A problem card lists its codes
over the last 15 minutes, most first, or says none occurred, or that codes
aren't recorded yet.

A counter series new since the previous scrape now counts from 0 instead of
being skipped: Traefik creates a router's 502 series at its first 502, so a
first burst of 5xx read as 0% until it happened again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/plugins/traefik/test_status_codes.py

@@ -0,0 +1,78 @@
+"""Each router's 4xx and 5xx responses by code (#5711): counted from
+traefik_service_requests_total{code} between scrapes, and shown on the
+problem card so it can say whether it's scanners, a login wall or the app."""
+from datetime import datetime, timedelta, timezone
+
+from plugins.traefik.routes import router_state
+from plugins.traefik.scraper import compute_router_metrics, compute_status_codes
+from steward.core.settings import to_thresholds_cfg
+from tests._templates import render_plugin_template
+from tests.plugins.traefik.test_traefik_page import _latest
+
+REQ = "traefik_service_requests_total"
+
+
+def _req(*series):
+    """{REQ: {labels: value}} from (service, code, method, value) tuples."""
+    return {REQ: {frozenset({"service": s, "code": c, "method": m}.items()): v for s, c, m, v in series}}
+
+
+def test_the_codes_since_the_last_scrape_are_counted_per_router():
+    before = _req(("app@docker", "200", "GET", 1000), ("app@docker", "404", "GET", 100),
+                  ("app@docker", "404", "POST", 10), ("api@docker", "401", "GET", 5))
+    after = _req(("app@docker", "200", "GET", 1500), ("app@docker", "404", "GET", 160),
+                 ("app@docker", "404", "POST", 12), ("api@docker", "401", "GET", 5),
+                 ("app@docker", "502", "GET", 3))                 # a series the last scrape didn't have
+    got = compute_status_codes(after, before)
+    # 2xx is left out; 404 sums over methods; a new 502 series counts whole; api's 401 didn't move.
+    assert got == {"app@docker": {"404": 62, "502": 3}}
+
+
+def test_the_first_scrape_counts_nothing_and_a_reset_counts_nothing():
+    series = _req(("app@docker", "500", "GET", 40))
+    assert compute_status_codes(series, None) == {}
+    assert compute_status_codes(_req(("app@docker", "500", "GET", 2)), series) == {}   # Traefik restarted
+
+
+def test_a_router_first_502s_count_in_its_5xx_share():
+    """A series new since the last scrape started from 0: before #5711 its
+    whole first burst was dropped, so a router's first 502s read as 0%."""
+    before = _req(("app@docker", "200", "GET", 100))
+    after = _req(("app@docker", "200", "GET", 190), ("app@docker", "502", "GET", 10))
+    got = compute_router_metrics(after, before, 60.0)["app@docker"]
+    assert got["error_rate_5xx_pct"] == 10.0 and abs(got["request_rate"] - 100 / 60) < 1e-9
+
+
+TH = to_thresholds_cfg({})
+
+
+def _problem(**codes_ctx):
+    latest = _latest(e4=6.0)
+    state, why = router_state(latest, TH)
+    return {"name": "app", "latest": latest, "state": state, "why": why, **codes_ctx,
+            **{f"sparkline_{k}": "" for k in ("req", "bytes", "p95", "4xx", "5xx")}}
+
+
+async def _card(app, router, codes_known=True):
+    async with app.test_request_context("/plugins/traefik/rows"):
+        page = await render_plugin_template(
+            app, "traefik", "traefik/rows.html", problems=[router], healthy=[], gone=[], global_stat=None,
+            certs=[], old_certs=[], range_key="24h", current_range="24h", error_4xx=1.0, error_5xx=0.1,
+            codes_known=codes_known)
+    page = " ".join(page.split())
+    return page[page.index(">app</h3>"):]
+
+
+async def test_the_card_lists_the_codes_most_first_before_its_figures(app):
+    codes = [("404", 1812), ("502", 40), ("401", 9), ("403", 3), ("429", 2), ("499", 1), ("500", 1)]
+    card = await _card(app, _problem(codes=codes))
+    assert card.index('Last 15 min: <span class="tone-warn">404</span> ×1,812, <span class="tone-crit">502</span> ×40') \
+        < card.index("<table")
+    assert "429</span> ×2, 2 more codes" in card and "499" not in card
+
+
+async def test_a_card_without_codes_says_whether_they_were_recorded(app):
+    none = await _card(app, _problem(codes=[]), codes_known=True)
+    assert "No 4xx or 5xx responses in the last 15 min." in none
+    early = await _card(app, _problem(codes=[]), codes_known=False)
+    assert "Status codes not recorded yet" in early