feat(traefik): keep each router's 4xx and 5xx responses by status code (#5711)

The scraper folded traefik_service_requests_total{code} into two shares and
dropped the code, so a problem couldn't say whether it was scanners (404), a
login wall (401/403) or the app down (502/503/504). Each scrape now stores the
responses per router and code that occurred (traefik_status_codes, migration
traefik_006), pruned with the router metrics and left out of backups (System
180 ruling: Traefik history is not backed up). A problem card lists its codes
over the last 15 minutes, most first, or says none occurred, or that codes
aren't recorded yet.

A counter series new since the previous scrape now counts from 0 instead of
being skipped: Traefik creates a router's 502 series at its first 502, so a
first burst of 5xx read as 0% until it happened again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/routes.py

@@ -13,7 +13,7 @@ from steward.models.users import UserRole
 from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample, time_bucket
 from steward.core.formatting import cert_left, status_class
 from steward.models.metrics import PluginMetricLatest
-from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric
+from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
 from .scheduler import router_domains
 from .scraper import cert_names, covers
 
@@ -110,6 +110,25 @@ async def recent_window(db, names: list[str], now: datetime) -> dict[str, Simple
     return out
 
 
+async def status_codes(db, names: list[str], since: datetime) -> dict[str, list[tuple[str, int]]]:
+    """{router: [(code, responses), …]} for the 4xx and 5xx codes each router
+    answered since `since`, most first (#5711). A router with none is absent."""
+    if not names:
+        return {}
+    total = func.sum(TraefikStatusCode.count)
+    rows = (await db.execute(
+        select(TraefikStatusCode.router_name, TraefikStatusCode.code, total.label("n"))
+        .where(TraefikStatusCode.router_name.in_(names))
+        .where(TraefikStatusCode.scraped_at >= since)
+        .group_by(TraefikStatusCode.router_name, TraefikStatusCode.code)
+        .order_by(TraefikStatusCode.router_name, total.desc(), TraefikStatusCode.code)
+    )).all()
+    out: dict[str, list[tuple[str, int]]] = {}
+    for r in rows:
+        out.setdefault(r.router_name, []).append((r.code, int(r.n)))
+    return out
+
+
 def grade_routers(routers: list[dict], window: dict, thresholds: dict) -> None:
     """Set each router's state and why from its recent window, or from its
     latest scrape when the window holds nothing."""
@@ -284,6 +303,15 @@ async def rows():
             r["sparkline_4xx"] = _sparkline([p.error_rate_4xx_pct for p in points])
             r["sparkline_5xx"] = _sparkline([p.error_rate_5xx_pct for p in points])
         phases.mark("sparklines")
+        # The codes behind each problem, over the window it is graded on.
+        codes = await status_codes(db, [r["name"] for r in current if r["state"] != "ok"], now - STATE_WINDOW)
+        for r in current:
+            r["codes"] = codes.get(r["name"], [])
+        # Codes are kept from #5711 on: until the first of them is a window
+        # old, a router with none listed may simply predate the recording.
+        first_code = await db.scalar(select(func.min(TraefikStatusCode.scraped_at)))
+        codes_known = first_code is not None and first_code <= now - STATE_WINDOW
+        phases.mark("codes")
 
         # Latest global stat (always most recent, not range-filtered)
         result = await db.execute(
@@ -329,6 +357,7 @@ async def rows():
         healthy=healthy,
         gone=gone,
         current_range=range_key,
+        codes_known=codes_known,
         error_4xx=error_4xx,
         error_5xx=error_5xx,
         global_stat=global_stat,