feat(traefik): keep each router's 4xx and 5xx responses by status code (#5711)

The scraper folded traefik_service_requests_total{code} into two shares and
dropped the code, so a problem couldn't say whether it was scanners (404), a
login wall (401/403) or the app down (502/503/504). Each scrape now stores the
responses per router and code that occurred (traefik_status_codes, migration
traefik_006), pruned with the router metrics and left out of backups (System
180 ruling: Traefik history is not backed up). A problem card lists its codes
over the last 15 minutes, most first, or says none occurred, or that codes
aren't recorded yet.

A counter series new since the previous scrape now counts from 0 instead of
being skipped: Traefik creates a router's 502 series at its first 502, so a
first burst of 5xx read as 0% until it happened again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/__init__.py

@@ -11,10 +11,13 @@ _app: "Quart | None" = None
 def setup(app: "Quart") -> None:
     global _app
     _app = app
-    from .models import TraefikMetric, TraefikCert, TraefikGlobalStat, TraefikAccessSummary  # noqa: F401
+    from .models import (  # noqa: F401
+        TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode,
+    )
     from steward.core.cleanup import prune_by_age
     # Samples older than data.retention_days are pruned by the cleanup (#5155).
     prune_by_age(TraefikMetric.scraped_at)
+    prune_by_age(TraefikStatusCode.scraped_at)
     prune_by_age(TraefikGlobalStat.scraped_at)
     prune_by_age(TraefikAccessSummary.period_end)
 

plugins/traefik/migrations/versions/traefik_006_status_codes.py

@@ -0,0 +1,40 @@
+"""Each router's 4xx and 5xx responses by status code
+
+The scraper folded traefik_service_requests_total{code} into a 4xx share and
+a 5xx share and dropped the code, so a problem couldn't say whether it was
+scanners probing (404), a login wall (401/403) or the app down (502/503/504)
+(#5711). One row per router, scrape and code that occurred; the page reads a
+router's codes over a time window, answered from the index alone.
+
+Revision ID: traefik_006_status_codes
+Revises: traefik_005_reload_at
+Create Date: 2026-10-14
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "traefik_006_status_codes"
+down_revision: Union[str, None] = "traefik_005_reload_at"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "traefik_status_codes",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("router_name", sa.String(255), nullable=False),
+        sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("code", sa.String(8), nullable=False),
+        sa.Column("count", sa.Integer, nullable=False),
+    )
+    op.create_index("ix_traefik_status_codes_router_scraped", "traefik_status_codes",
+                    ["router_name", "scraped_at"], postgresql_include=["code", "count"])
+    op.create_index("ix_traefik_status_codes_scraped", "traefik_status_codes", ["scraped_at"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_traefik_status_codes_scraped", "traefik_status_codes")
+    op.drop_index("ix_traefik_status_codes_router_scraped", "traefik_status_codes")
+    op.drop_table("traefik_status_codes")

plugins/traefik/models.py

@@ -67,3 +67,17 @@ class TraefikGlobalStat(Base):
     # traefik_config_last_reload_success gauge is that moment, not a 0/1 flag.
     config_last_reload_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
     process_memory_bytes: Mapped[float | None] = mapped_column(Float, nullable=True)
+
+
+class TraefikStatusCode(Base):
+    """How many responses a router answered with one 4xx or 5xx code in one
+    scrape interval (#5711): what lets a problem say whether it is scanners
+    probing (404), a login wall (401/403) or the app down (502/503/504).
+    Only codes that occurred are stored."""
+    __tablename__ = "traefik_status_codes"
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
+    router_name: Mapped[str] = mapped_column(String(255), nullable=False)
+    scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    code: Mapped[str] = mapped_column(String(8), nullable=False)
+    count: Mapped[int] = mapped_column(Integer, nullable=False)

plugins/traefik/routes.py

@@ -13,7 +13,7 @@ from steward.models.users import UserRole
 from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample, time_bucket
 from steward.core.formatting import cert_left, status_class
 from steward.models.metrics import PluginMetricLatest
-from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric
+from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
 from .scheduler import router_domains
 from .scraper import cert_names, covers
 
@@ -110,6 +110,25 @@ async def recent_window(db, names: list[str], now: datetime) -> dict[str, Simple
     return out
 
 
+async def status_codes(db, names: list[str], since: datetime) -> dict[str, list[tuple[str, int]]]:
+    """{router: [(code, responses), …]} for the 4xx and 5xx codes each router
+    answered since `since`, most first (#5711). A router with none is absent."""
+    if not names:
+        return {}
+    total = func.sum(TraefikStatusCode.count)
+    rows = (await db.execute(
+        select(TraefikStatusCode.router_name, TraefikStatusCode.code, total.label("n"))
+        .where(TraefikStatusCode.router_name.in_(names))
+        .where(TraefikStatusCode.scraped_at >= since)
+        .group_by(TraefikStatusCode.router_name, TraefikStatusCode.code)
+        .order_by(TraefikStatusCode.router_name, total.desc(), TraefikStatusCode.code)
+    )).all()
+    out: dict[str, list[tuple[str, int]]] = {}
+    for r in rows:
+        out.setdefault(r.router_name, []).append((r.code, int(r.n)))
+    return out
+
+
 def grade_routers(routers: list[dict], window: dict, thresholds: dict) -> None:
     """Set each router's state and why from its recent window, or from its
     latest scrape when the window holds nothing."""
@@ -284,6 +303,15 @@ async def rows():
             r["sparkline_4xx"] = _sparkline([p.error_rate_4xx_pct for p in points])
             r["sparkline_5xx"] = _sparkline([p.error_rate_5xx_pct for p in points])
         phases.mark("sparklines")
+        # The codes behind each problem, over the window it is graded on.
+        codes = await status_codes(db, [r["name"] for r in current if r["state"] != "ok"], now - STATE_WINDOW)
+        for r in current:
+            r["codes"] = codes.get(r["name"], [])
+        # Codes are kept from #5711 on: until the first of them is a window
+        # old, a router with none listed may simply predate the recording.
+        first_code = await db.scalar(select(func.min(TraefikStatusCode.scraped_at)))
+        codes_known = first_code is not None and first_code <= now - STATE_WINDOW
+        phases.mark("codes")
 
         # Latest global stat (always most recent, not range-filtered)
         result = await db.execute(
@@ -329,6 +357,7 @@ async def rows():
         healthy=healthy,
         gone=gone,
         current_range=range_key,
+        codes_known=codes_known,
         error_4xx=error_4xx,
         error_5xx=error_5xx,
         global_stat=global_stat,

plugins/traefik/scheduler.py

@@ -51,11 +51,12 @@ async def _do_scrape(app: "Quart") -> None:
     global _prev_metrics, _prev_time, _router_domains
 
     from datetime import datetime, timezone
-    from .models import TraefikCert, TraefikGlobalStat, TraefikMetric
+    from .models import TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
     from .scraper import (
         api_base,
         compute_global_stats,
         compute_router_metrics,
+        compute_status_codes,
         extract_certs,
         fetch_metrics,
         fetch_router_domains,
@@ -81,6 +82,7 @@ async def _do_scrape(app: "Quart") -> None:
     elapsed = now - _prev_time if _prev_time else 60.0
     router_metrics = compute_router_metrics(current, _prev_metrics or None, elapsed)
     global_stats = compute_global_stats(current, _prev_metrics or None)
+    status_codes = compute_status_codes(current, _prev_metrics or None)
     cert_list = extract_certs(current)
 
     _prev_metrics = current
@@ -125,6 +127,12 @@ async def _do_scrape(app: "Quart") -> None:
                         recorded_at=scraped_at,
                     )
 
+            # ── 4xx and 5xx responses by code (#5711) ─────────────────────────
+            for router_name, codes in status_codes.items():
+                for code, count in codes.items():
+                    session.add(TraefikStatusCode(router_name=router_name, scraped_at=scraped_at,
+                                                  code=code, count=count))
+
             # ── Global stats ──────────────────────────────────────────────────
             session.add(TraefikGlobalStat(scraped_at=scraped_at, **global_stats))
 

plugins/traefik/scraper.py

@@ -43,6 +43,33 @@ def parse_prometheus(text: str) -> ParsedMetrics:
 # Per-router metric computation
 # ──────────────────────────────────────────────────────────────────────────────
 
+def _rise(prev: dict[frozenset, float] | None, labels: frozenset, value: float) -> float:
+    """How far a counter rose since the previous scrape. On the first scrape
+    (`prev` None) nothing can be compared, so 0. A series the previous scrape
+    didn't have started from 0 since then, so all of it counts: Traefik makes
+    a router's first 502 series when the first 502 happens (#5711)."""
+    if prev is None:
+        return 0.0
+    return max(0.0, value - prev.get(labels, 0.0))
+
+
+def compute_status_codes(current: ParsedMetrics, previous: ParsedMetrics | None) -> dict[str, dict[str, int]]:
+    """{service: {code: responses since the previous scrape}} for each 4xx and
+    5xx code that occurred, from traefik_service_requests_total{code} (#5711).
+    Codes that didn't occur are left out; the first scrape has none."""
+    prev = None if previous is None else previous.get("traefik_service_requests_total", {})
+    out: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+    for labels, value in current.get("traefik_service_requests_total", {}).items():
+        label_dict = dict(labels)
+        service, code = label_dict.get("service", ""), label_dict.get("code", "")
+        if not service or code[:1] not in ("4", "5"):
+            continue
+        count = round(_rise(prev, labels, value))
+        if count:
+            out[service][code] += count
+    return {service: dict(codes) for service, codes in out.items()}
+
+
 def compute_router_metrics(
     current: ParsedMetrics,
     previous: ParsedMetrics | None,
@@ -63,12 +90,13 @@ def compute_router_metrics(
             "response_bytes_rate": float,    # bytes/sec
         }}
     """
+    first = not previous
     prev = previous or {}
     elapsed = max(elapsed_seconds, 1.0)
 
     # ── Request counters ──────────────────────────────────────────────────────
     req_samples = current.get("traefik_service_requests_total", {})
-    prev_req = prev.get("traefik_service_requests_total", {})
+    prev_req = None if first else prev.get("traefik_service_requests_total", {})
 
     service_total: dict[str, float] = defaultdict(float)
     service_4xx: dict[str, float] = defaultdict(float)
@@ -80,8 +108,7 @@ def compute_router_metrics(
         if not service:
             continue
         code = label_dict.get("code", "")
-        prev_value = prev_req.get(labels, value)  # no delta on first scrape
-        delta = max(0.0, value - prev_value)
+        delta = _rise(prev_req, labels, value)
         service_total[service] += delta
         if code.startswith("4"):
             service_4xx[service] += delta
@@ -90,15 +117,14 @@ def compute_router_metrics(
 
     # ── Response bytes counters ───────────────────────────────────────────────
     bytes_samples = current.get("traefik_service_responses_bytes_total", {})
-    prev_bytes = prev.get("traefik_service_responses_bytes_total", {})
+    prev_bytes = None if first else prev.get("traefik_service_responses_bytes_total", {})
 
     service_bytes: dict[str, float] = defaultdict(float)
     for labels, value in bytes_samples.items():
         service = dict(labels).get("service", "")
         if not service:
             continue
-        prev_value = prev_bytes.get(labels, value)
-        service_bytes[service] += max(0.0, value - prev_value)
+        service_bytes[service] += _rise(prev_bytes, labels, value)
 
     # ── Histogram buckets ─────────────────────────────────────────────────────
     hist_buckets = current.get("traefik_service_request_duration_seconds_bucket", {})

plugins/traefik/templates/traefik/rows.html

@@ -42,6 +42,15 @@
     </div>
     <p class="problem-detail" style="margin-bottom:0.5rem;">{{ r.why }}
       {%- if r.since_before %}, all of the last {{ current_range }}{% elif r.since %}, started {{ r.since|ago }}{% endif %}</p>
+    {% if r.codes %}
+    <p class="field-hint" style="margin:0 0 0.5rem;">Last 15 min:
+      {% for code, n in r.codes[:5] %}<span class="tone-{{ 'crit' if code[:1] == '5' else 'warn' }}">{{ code }}</span> ×{{ "{:,}".format(n) }}{{ ", " if not loop.last }}{% endfor %}
+      {%- if r.codes|length > 5 %}, {{ r.codes|length - 5 }} more code{{ "s" if r.codes|length - 5 != 1 }}{% endif %}</p>
+    {% elif codes_known %}
+    <p class="field-hint" style="margin:0 0 0.5rem;">No 4xx or 5xx responses in the last 15 min.</p>
+    {% else %}
+    <p class="field-hint" style="margin:0 0 0.5rem;">Status codes not recorded yet: Steward keeps them from this version on.</p>
+    {% endif %}
     <table class="table" style="font-size:0.85rem;">
         <tr><td class="field-hint">Req/s</td><td>{{ "%.2f"|format(r.latest.request_rate) }}</td><td>{{ r.sparkline_req|safe }}</td></tr>
         <tr><td class="field-hint">Bandwidth</td><td>{{ r.latest.response_bytes_rate|rate }}</td><td>{{ r.sparkline_bytes|safe }}</td></tr>