feat(traefik): keep each router's 4xx and 5xx responses by status code (#5711)

The scraper folded traefik_service_requests_total{code} into two shares and
dropped the code, so a problem couldn't say whether it was scanners (404), a
login wall (401/403) or the app down (502/503/504). Each scrape now stores the
responses per router and code that occurred (traefik_status_codes, migration
traefik_006), pruned with the router metrics and left out of backups (System
180 ruling: Traefik history is not backed up). A problem card lists its codes
over the last 15 minutes, most first, or says none occurred, or that codes
aren't recorded yet.

A counter series new since the previous scrape now counts from 0 instead of
being skipped: Traefik creates a router's 502 series at its first 502, so a
first burst of 5xx read as 0% until it happened again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/plugins/traefik.md

@@ -83,7 +83,7 @@ The full Traefik page at `/plugins/traefik/` reads top to bottom, for the time r
 
 1. **The verdict.** It names the routers that need a look, says why, and says when the problem started ("api, 3.0% of responses are 5xx, started 2h ago"). When a router has been a problem for the whole range, it says so instead. Otherwise it reads "All N routers healthy".
 2. **The request chart.** Requests per second across all routers and the share answered 5xx, the same chart as the dashboard widget.
-3. **A card for each router that needs a look:** why, since when, and its request rate, bandwidth, p95/p99 latency and 4xx/5xx shares, each with its history.
+3. **A card for each router that needs a look:** why, since when, the 4xx and 5xx status codes it answered in the last 15 minutes, most first ("404 ×1,812, 502 ×40"), and its request rate, bandwidth, p95/p99 latency and 4xx/5xx shares, each with its history. The code is what separates scanners probing for pages (404), a login wall (401/403) and an app that isn't answering (502/503/504). Steward records codes from the version that added them on; until then a card says they aren't recorded yet.
 4. **Traefik's own figures:** when its config last reloaded, open connections, memory.
 5. **The healthy routers,** folded into one table, then the routers Traefik no longer reports, then TLS certificates by soonest expiry, then Traffic Origin when the access log is on.
 

plugins/traefik/__init__.py

@@ -11,10 +11,13 @@ _app: "Quart | None" = None
 def setup(app: "Quart") -> None:
     global _app
     _app = app
-    from .models import TraefikMetric, TraefikCert, TraefikGlobalStat, TraefikAccessSummary  # noqa: F401
+    from .models import (  # noqa: F401
+        TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode,
+    )
     from steward.core.cleanup import prune_by_age
     # Samples older than data.retention_days are pruned by the cleanup (#5155).
     prune_by_age(TraefikMetric.scraped_at)
+    prune_by_age(TraefikStatusCode.scraped_at)
     prune_by_age(TraefikGlobalStat.scraped_at)
     prune_by_age(TraefikAccessSummary.period_end)
 

plugins/traefik/migrations/versions/traefik_006_status_codes.py

@@ -0,0 +1,40 @@
+"""Each router's 4xx and 5xx responses by status code
+
+The scraper folded traefik_service_requests_total{code} into a 4xx share and
+a 5xx share and dropped the code, so a problem couldn't say whether it was
+scanners probing (404), a login wall (401/403) or the app down (502/503/504)
+(#5711). One row per router, scrape and code that occurred; the page reads a
+router's codes over a time window, answered from the index alone.
+
+Revision ID: traefik_006_status_codes
+Revises: traefik_005_reload_at
+Create Date: 2026-10-14
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "traefik_006_status_codes"
+down_revision: Union[str, None] = "traefik_005_reload_at"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "traefik_status_codes",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("router_name", sa.String(255), nullable=False),
+        sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("code", sa.String(8), nullable=False),
+        sa.Column("count", sa.Integer, nullable=False),
+    )
+    op.create_index("ix_traefik_status_codes_router_scraped", "traefik_status_codes",
+                    ["router_name", "scraped_at"], postgresql_include=["code", "count"])
+    op.create_index("ix_traefik_status_codes_scraped", "traefik_status_codes", ["scraped_at"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_traefik_status_codes_scraped", "traefik_status_codes")
+    op.drop_index("ix_traefik_status_codes_router_scraped", "traefik_status_codes")
+    op.drop_table("traefik_status_codes")

plugins/traefik/models.py

@@ -67,3 +67,17 @@ class TraefikGlobalStat(Base):
     # traefik_config_last_reload_success gauge is that moment, not a 0/1 flag.
     config_last_reload_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
     process_memory_bytes: Mapped[float | None] = mapped_column(Float, nullable=True)
+
+
+class TraefikStatusCode(Base):
+    """How many responses a router answered with one 4xx or 5xx code in one
+    scrape interval (#5711): what lets a problem say whether it is scanners
+    probing (404), a login wall (401/403) or the app down (502/503/504).
+    Only codes that occurred are stored."""
+    __tablename__ = "traefik_status_codes"
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
+    router_name: Mapped[str] = mapped_column(String(255), nullable=False)
+    scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    code: Mapped[str] = mapped_column(String(8), nullable=False)
+    count: Mapped[int] = mapped_column(Integer, nullable=False)

plugins/traefik/routes.py

@@ -13,7 +13,7 @@ from steward.models.users import UserRole
 from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample, time_bucket
 from steward.core.formatting import cert_left, status_class
 from steward.models.metrics import PluginMetricLatest
-from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric
+from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
 from .scheduler import router_domains
 from .scraper import cert_names, covers
 
@@ -110,6 +110,25 @@ async def recent_window(db, names: list[str], now: datetime) -> dict[str, Simple
     return out
 
 
+async def status_codes(db, names: list[str], since: datetime) -> dict[str, list[tuple[str, int]]]:
+    """{router: [(code, responses), …]} for the 4xx and 5xx codes each router
+    answered since `since`, most first (#5711). A router with none is absent."""
+    if not names:
+        return {}
+    total = func.sum(TraefikStatusCode.count)
+    rows = (await db.execute(
+        select(TraefikStatusCode.router_name, TraefikStatusCode.code, total.label("n"))
+        .where(TraefikStatusCode.router_name.in_(names))
+        .where(TraefikStatusCode.scraped_at >= since)
+        .group_by(TraefikStatusCode.router_name, TraefikStatusCode.code)
+        .order_by(TraefikStatusCode.router_name, total.desc(), TraefikStatusCode.code)
+    )).all()
+    out: dict[str, list[tuple[str, int]]] = {}
+    for r in rows:
+        out.setdefault(r.router_name, []).append((r.code, int(r.n)))
+    return out
+
+
 def grade_routers(routers: list[dict], window: dict, thresholds: dict) -> None:
     """Set each router's state and why from its recent window, or from its
     latest scrape when the window holds nothing."""
@@ -284,6 +303,15 @@ async def rows():
             r["sparkline_4xx"] = _sparkline([p.error_rate_4xx_pct for p in points])
             r["sparkline_5xx"] = _sparkline([p.error_rate_5xx_pct for p in points])
         phases.mark("sparklines")
+        # The codes behind each problem, over the window it is graded on.
+        codes = await status_codes(db, [r["name"] for r in current if r["state"] != "ok"], now - STATE_WINDOW)
+        for r in current:
+            r["codes"] = codes.get(r["name"], [])
+        # Codes are kept from #5711 on: until the first of them is a window
+        # old, a router with none listed may simply predate the recording.
+        first_code = await db.scalar(select(func.min(TraefikStatusCode.scraped_at)))
+        codes_known = first_code is not None and first_code <= now - STATE_WINDOW
+        phases.mark("codes")
 
         # Latest global stat (always most recent, not range-filtered)
         result = await db.execute(
@@ -329,6 +357,7 @@ async def rows():
         healthy=healthy,
         gone=gone,
         current_range=range_key,
+        codes_known=codes_known,
         error_4xx=error_4xx,
         error_5xx=error_5xx,
         global_stat=global_stat,

plugins/traefik/scheduler.py

@@ -51,11 +51,12 @@ async def _do_scrape(app: "Quart") -> None:
     global _prev_metrics, _prev_time, _router_domains
 
     from datetime import datetime, timezone
-    from .models import TraefikCert, TraefikGlobalStat, TraefikMetric
+    from .models import TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
     from .scraper import (
         api_base,
         compute_global_stats,
         compute_router_metrics,
+        compute_status_codes,
         extract_certs,
         fetch_metrics,
         fetch_router_domains,
@@ -81,6 +82,7 @@ async def _do_scrape(app: "Quart") -> None:
     elapsed = now - _prev_time if _prev_time else 60.0
     router_metrics = compute_router_metrics(current, _prev_metrics or None, elapsed)
     global_stats = compute_global_stats(current, _prev_metrics or None)
+    status_codes = compute_status_codes(current, _prev_metrics or None)
     cert_list = extract_certs(current)
 
     _prev_metrics = current
@@ -125,6 +127,12 @@ async def _do_scrape(app: "Quart") -> None:
                         recorded_at=scraped_at,
                     )
 
+            # ── 4xx and 5xx responses by code (#5711) ─────────────────────────
+            for router_name, codes in status_codes.items():
+                for code, count in codes.items():
+                    session.add(TraefikStatusCode(router_name=router_name, scraped_at=scraped_at,
+                                                  code=code, count=count))
+
             # ── Global stats ──────────────────────────────────────────────────
             session.add(TraefikGlobalStat(scraped_at=scraped_at, **global_stats))
 

plugins/traefik/scraper.py

@@ -43,6 +43,33 @@ def parse_prometheus(text: str) -> ParsedMetrics:
 # Per-router metric computation
 # ──────────────────────────────────────────────────────────────────────────────
 
+def _rise(prev: dict[frozenset, float] | None, labels: frozenset, value: float) -> float:
+    """How far a counter rose since the previous scrape. On the first scrape
+    (`prev` None) nothing can be compared, so 0. A series the previous scrape
+    didn't have started from 0 since then, so all of it counts: Traefik makes
+    a router's first 502 series when the first 502 happens (#5711)."""
+    if prev is None:
+        return 0.0
+    return max(0.0, value - prev.get(labels, 0.0))
+
+
+def compute_status_codes(current: ParsedMetrics, previous: ParsedMetrics | None) -> dict[str, dict[str, int]]:
+    """{service: {code: responses since the previous scrape}} for each 4xx and
+    5xx code that occurred, from traefik_service_requests_total{code} (#5711).
+    Codes that didn't occur are left out; the first scrape has none."""
+    prev = None if previous is None else previous.get("traefik_service_requests_total", {})
+    out: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+    for labels, value in current.get("traefik_service_requests_total", {}).items():
+        label_dict = dict(labels)
+        service, code = label_dict.get("service", ""), label_dict.get("code", "")
+        if not service or code[:1] not in ("4", "5"):
+            continue
+        count = round(_rise(prev, labels, value))
+        if count:
+            out[service][code] += count
+    return {service: dict(codes) for service, codes in out.items()}
+
+
 def compute_router_metrics(
     current: ParsedMetrics,
     previous: ParsedMetrics | None,
@@ -63,12 +90,13 @@ def compute_router_metrics(
             "response_bytes_rate": float,    # bytes/sec
         }}
     """
+    first = not previous
     prev = previous or {}
     elapsed = max(elapsed_seconds, 1.0)
 
     # ── Request counters ──────────────────────────────────────────────────────
     req_samples = current.get("traefik_service_requests_total", {})
-    prev_req = prev.get("traefik_service_requests_total", {})
+    prev_req = None if first else prev.get("traefik_service_requests_total", {})
 
     service_total: dict[str, float] = defaultdict(float)
     service_4xx: dict[str, float] = defaultdict(float)
@@ -80,8 +108,7 @@ def compute_router_metrics(
         if not service:
             continue
         code = label_dict.get("code", "")
-        prev_value = prev_req.get(labels, value)  # no delta on first scrape
+        delta = _rise(prev_req, labels, value)
-        delta = max(0.0, value - prev_value)
         service_total[service] += delta
         if code.startswith("4"):
             service_4xx[service] += delta
@@ -90,15 +117,14 @@ def compute_router_metrics(
 
     # ── Response bytes counters ───────────────────────────────────────────────
     bytes_samples = current.get("traefik_service_responses_bytes_total", {})
-    prev_bytes = prev.get("traefik_service_responses_bytes_total", {})
+    prev_bytes = None if first else prev.get("traefik_service_responses_bytes_total", {})
 
     service_bytes: dict[str, float] = defaultdict(float)
     for labels, value in bytes_samples.items():
         service = dict(labels).get("service", "")
         if not service:
             continue
-        prev_value = prev_bytes.get(labels, value)
+        service_bytes[service] += _rise(prev_bytes, labels, value)
-        service_bytes[service] += max(0.0, value - prev_value)
 
     # ── Histogram buckets ─────────────────────────────────────────────────────
     hist_buckets = current.get("traefik_service_request_duration_seconds_bucket", {})

plugins/traefik/templates/traefik/rows.html

@@ -42,6 +42,15 @@
     </div>
     <p class="problem-detail" style="margin-bottom:0.5rem;">{{ r.why }}
       {%- if r.since_before %}, all of the last {{ current_range }}{% elif r.since %}, started {{ r.since|ago }}{% endif %}</p>
+    {% if r.codes %}
+    <p class="field-hint" style="margin:0 0 0.5rem;">Last 15 min:
+      {% for code, n in r.codes[:5] %}<span class="tone-{{ 'crit' if code[:1] == '5' else 'warn' }}">{{ code }}</span> ×{{ "{:,}".format(n) }}{{ ", " if not loop.last }}{% endfor %}
+      {%- if r.codes|length > 5 %}, {{ r.codes|length - 5 }} more code{{ "s" if r.codes|length - 5 != 1 }}{% endif %}</p>
+    {% elif codes_known %}
+    <p class="field-hint" style="margin:0 0 0.5rem;">No 4xx or 5xx responses in the last 15 min.</p>
+    {% else %}
+    <p class="field-hint" style="margin:0 0 0.5rem;">Status codes not recorded yet: Steward keeps them from this version on.</p>
+    {% endif %}
     <table class="table" style="font-size:0.85rem;">
         <tr><td class="field-hint">Req/s</td><td>{{ "%.2f"|format(r.latest.request_rate) }}</td><td>{{ r.sparkline_req|safe }}</td></tr>
         <tr><td class="field-hint">Bandwidth</td><td>{{ r.latest.response_bytes_rate|rate }}</td><td>{{ r.sparkline_bytes|safe }}</td></tr>

steward/backup/contents.py

@@ -38,6 +38,7 @@ EXCLUDED: frozenset[str] = frozenset({
     "docker_images", "docker_logs", "docker_metrics", "docker_metrics_hourly",
     "docker_swarm_nodes", "docker_swarm_services",
     "traefik_access_summaries", "traefik_certs", "traefik_global_stats", "traefik_metrics",
+    "traefik_status_codes",
     "unifi_alarms", "unifi_client_snapshots", "unifi_devices", "unifi_dpi_snapshots",
     "unifi_events", "unifi_fw_rules", "unifi_networks", "unifi_port_forwards",
     "unifi_speedtest_results", "unifi_wan_stats", "unifi_wlan_stats",

tests/integration/test_traefik_latest.py

@@ -152,3 +152,28 @@ def test_recent_window_weights_shares_by_traffic_and_keeps_to_the_window(app):
     assert set(got) == {busy}                                         # nothing in empty's window
     assert got[busy].error_rate_4xx_pct == pytest.approx(1.0)         # 0.1 × 100 over 10 req/s
     assert got[busy].latency_p95_ms == pytest.approx(20.0)
+
+
+def test_status_codes_sum_each_routers_codes_in_the_window_most_first(app):
+    """#5711: the codes behind a router's problem, summed over the window it
+    is graded on; an older row and a router not asked for are not read."""
+    from plugins.traefik.models import TraefikStatusCode
+    from plugins.traefik.routes import status_codes
+
+    at = datetime(2097, 6, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=5 * (uuid.uuid4().int % 2_000))
+    tag = uuid.uuid4().hex[:8]
+    a, b, other = (f"{x}-{tag}@docker" for x in ("a", "b", "other"))
+
+    def row(name, minute, code, count):
+        return TraefikStatusCode(router_name=name, scraped_at=at + timedelta(minutes=minute), code=code, count=count)
+
+    async def go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add_all([row(a, 1, "404", 30), row(a, 2, "404", 12), row(a, 2, "502", 50),
+                           row(a, -20, "500", 999), row(b, 3, "401", 4), row(other, 3, "404", 7)])
+            async with s.begin():
+                return await status_codes(s, [a, b], at)
+
+    got = asyncio.run(go())
+    assert got == {a: [("502", 50), ("404", 42)], b: [("401", 4)]}

tests/plugins/traefik/test_status_codes.py

@@ -0,0 +1,78 @@
+"""Each router's 4xx and 5xx responses by code (#5711): counted from
+traefik_service_requests_total{code} between scrapes, and shown on the
+problem card so it can say whether it's scanners, a login wall or the app."""
+from datetime import datetime, timedelta, timezone
+
+from plugins.traefik.routes import router_state
+from plugins.traefik.scraper import compute_router_metrics, compute_status_codes
+from steward.core.settings import to_thresholds_cfg
+from tests._templates import render_plugin_template
+from tests.plugins.traefik.test_traefik_page import _latest
+
+REQ = "traefik_service_requests_total"
+
+
+def _req(*series):
+    """{REQ: {labels: value}} from (service, code, method, value) tuples."""
+    return {REQ: {frozenset({"service": s, "code": c, "method": m}.items()): v for s, c, m, v in series}}
+
+
+def test_the_codes_since_the_last_scrape_are_counted_per_router():
+    before = _req(("app@docker", "200", "GET", 1000), ("app@docker", "404", "GET", 100),
+                  ("app@docker", "404", "POST", 10), ("api@docker", "401", "GET", 5))
+    after = _req(("app@docker", "200", "GET", 1500), ("app@docker", "404", "GET", 160),
+                 ("app@docker", "404", "POST", 12), ("api@docker", "401", "GET", 5),
+                 ("app@docker", "502", "GET", 3))                 # a series the last scrape didn't have
+    got = compute_status_codes(after, before)
+    # 2xx is left out; 404 sums over methods; a new 502 series counts whole; api's 401 didn't move.
+    assert got == {"app@docker": {"404": 62, "502": 3}}
+
+
+def test_the_first_scrape_counts_nothing_and_a_reset_counts_nothing():
+    series = _req(("app@docker", "500", "GET", 40))
+    assert compute_status_codes(series, None) == {}
+    assert compute_status_codes(_req(("app@docker", "500", "GET", 2)), series) == {}   # Traefik restarted
+
+
+def test_a_router_first_502s_count_in_its_5xx_share():
+    """A series new since the last scrape started from 0: before #5711 its
+    whole first burst was dropped, so a router's first 502s read as 0%."""
+    before = _req(("app@docker", "200", "GET", 100))
+    after = _req(("app@docker", "200", "GET", 190), ("app@docker", "502", "GET", 10))
+    got = compute_router_metrics(after, before, 60.0)["app@docker"]
+    assert got["error_rate_5xx_pct"] == 10.0 and abs(got["request_rate"] - 100 / 60) < 1e-9
+
+
+TH = to_thresholds_cfg({})
+
+
+def _problem(**codes_ctx):
+    latest = _latest(e4=6.0)
+    state, why = router_state(latest, TH)
+    return {"name": "app", "latest": latest, "state": state, "why": why, **codes_ctx,
+            **{f"sparkline_{k}": "" for k in ("req", "bytes", "p95", "4xx", "5xx")}}
+
+
+async def _card(app, router, codes_known=True):
+    async with app.test_request_context("/plugins/traefik/rows"):
+        page = await render_plugin_template(
+            app, "traefik", "traefik/rows.html", problems=[router], healthy=[], gone=[], global_stat=None,
+            certs=[], old_certs=[], range_key="24h", current_range="24h", error_4xx=1.0, error_5xx=0.1,
+            codes_known=codes_known)
+    page = " ".join(page.split())
+    return page[page.index(">app</h3>"):]
+
+
+async def test_the_card_lists_the_codes_most_first_before_its_figures(app):
+    codes = [("404", 1812), ("502", 40), ("401", 9), ("403", 3), ("429", 2), ("499", 1), ("500", 1)]
+    card = await _card(app, _problem(codes=codes))
+    assert card.index('Last 15 min: <span class="tone-warn">404</span> ×1,812, <span class="tone-crit">502</span> ×40') \
+        < card.index("<table")
+    assert "429</span> ×2, 2 more codes" in card and "499" not in card
+
+
+async def test_a_card_without_codes_says_whether_they_were_recorded(app):
+    none = await _card(app, _problem(codes=[]), codes_known=True)
+    assert "No 4xx or 5xx responses in the last 15 min." in none
+    early = await _card(app, _problem(codes=[]), codes_known=False)
+    assert "Status codes not recorded yet" in early