fix(traefik): grade routers on their last 15 minutes; the Config tile reads Traefik's reload time (#5355)

The operator: problem cards "can disappear so quickly". Each router was
graded on its latest scrape, so a 4xx share near the 1% cutoff crossed it
on one scrape and fell back on the next. recent_window reads every
router's last 15 minutes in one query, the error shares weighted by
traffic; grade_routers uses it on the page and the dashboard widget, and
the why says "over the last 15 min".

traefik_config_last_reload_success is the Unix time of the last good
reload, not a 0/1 flag, so the tile said "Reload failed" on every scrape.
traefik_005 stores it as config_last_reload_at, and the tile reads
"Config reloaded 3m ago". The router index also carries the 4xx share.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/migrations/versions/traefik_005_reload_at.py

@@ -0,0 +1,41 @@
+"""Traefik's last good config reload is a moment; routers are graded on a window
+
+- traefik_config_last_reload_success is the Unix time of the last successful
+  reload. It was stored as a float and read as 1.0 = success, so the page said
+  "Reload failed" on every scrape (#5355). The column becomes the moment; the
+  old values were timestamps misread, so they go with it.
+- Routers are now graded on their last 15 minutes (routes.recent_window),
+  which reads the 4xx share too: the covering router index carries it, so
+  that read stays index-only.
+
+Revision ID: traefik_005_reload_at
+Revises: traefik_004_covering_indexes
+Create Date: 2026-10-08
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "traefik_005_reload_at"
+down_revision: Union[str, None] = "traefik_004_covering_indexes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+_CHARTED = ["request_rate", "latency_p95_ms", "error_rate_5xx_pct"]
+
+
+def upgrade() -> None:
+    op.drop_column("traefik_global_stats", "config_last_reload_success")
+    op.add_column("traefik_global_stats",
+                  sa.Column("config_last_reload_at", sa.DateTime(timezone=True), nullable=True))
+    op.drop_index("ix_traefik_metrics_router_scraped", "traefik_metrics")
+    op.create_index("ix_traefik_metrics_router_scraped", "traefik_metrics", ["router_name", "scraped_at"],
+                    postgresql_include=_CHARTED + ["error_rate_4xx_pct"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_traefik_metrics_router_scraped", "traefik_metrics")
+    op.create_index("ix_traefik_metrics_router_scraped", "traefik_metrics", ["router_name", "scraped_at"],
+                    postgresql_include=_CHARTED)
+    op.drop_column("traefik_global_stats", "config_last_reload_at")
+    op.add_column("traefik_global_stats", sa.Column("config_last_reload_success", sa.Float, nullable=True))

plugins/traefik/models.py

@@ -63,5 +63,7 @@ class TraefikGlobalStat(Base):
     )
     open_conns_total: Mapped[float | None] = mapped_column(Float, nullable=True)
     config_reloads_total: Mapped[float | None] = mapped_column(Float, nullable=True)
-    config_last_reload_success: Mapped[float | None] = mapped_column(Float, nullable=True)
+    # When Traefik last loaded its configuration successfully: its
+    # traefik_config_last_reload_success gauge is that moment, not a 0/1 flag.
+    config_last_reload_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
     process_memory_bytes: Mapped[float | None] = mapped_column(Float, nullable=True)

plugins/traefik/routes.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 from collections import defaultdict
 from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
 from quart import Blueprint, current_app, render_template, request
 from sqlalchemy import func, select, text
 import json
@@ -46,20 +47,71 @@ ERROR_4XX_PCT = 1.0
 _GRADE_ORDER = {"crit": 0, "warn": 1, "ok": 2}
 
 
-def router_state(latest, thresholds: dict) -> tuple[str, str | None]:
-    """(grade, why) for a router's latest scrape: crit while it serves 5xx,
-    warn on a p95 past the latency critical cutoff (Settings → Thresholds)
-    or a high 4xx share, otherwise ok with nothing to say."""
-    if (latest.error_rate_5xx_pct or 0) > ERROR_5XX_PCT:
-        return "crit", f"{latest.error_rate_5xx_pct:.1f}% of responses are 5xx"
-    p95 = latest.latency_p95_ms
+def router_state(reading, thresholds: dict, over: str = "") -> tuple[str, str | None]:
+    """(grade, why) for a router's reading (recent_window's, or one scrape's):
+    crit while it serves 5xx, warn on a p95 past the latency critical cutoff
+    (Settings → Thresholds) or a high 4xx share, otherwise ok with nothing to
+    say. `over` names the span the reading covers, for the why."""
+    if (reading.error_rate_5xx_pct or 0) > ERROR_5XX_PCT:
+        return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}"
+    p95 = reading.latency_p95_ms
     if p95 is not None and status_class(p95, "latency", thresholds) == "crit":
-        return "warn", f"p95 latency is {p95:.0f} ms"
-    if (latest.error_rate_4xx_pct or 0) > ERROR_4XX_PCT:
-        return "warn", f"{latest.error_rate_4xx_pct:.1f}% of responses are 4xx"
+        return "warn", f"p95 latency is {p95:.0f} ms{over}"
+    if (reading.error_rate_4xx_pct or 0) > ERROR_4XX_PCT:
+        return "warn", f"{reading.error_rate_4xx_pct:.1f}% of responses are 4xx{over}"
     return "ok", None
 
 
+# A router is graded on its last STATE_WINDOW of traffic, not on one scrape:
+# a share near a cutoff crossed it on one scrape and fell back on the next,
+# so problem cards came and went every poll (#5214). Over the window a real
+# problem stays listed until it has been clean a while, and a one-scrape
+# blip is diluted rather than shown.
+STATE_WINDOW = timedelta(minutes=15)
+STATE_WINDOW_WORDS = " over the last 15 min"
+
+
+async def recent_window(db, names: list[str], now: datetime) -> dict[str, SimpleNamespace]:
+    """{router: one reading of its last STATE_WINDOW}: the 4xx and 5xx shares
+    weighted by each scrape's traffic (an idle scrape's 100% counts for
+    little), p95 averaged. A router with no rows in the window is absent."""
+    if not names:
+        return {}
+    rate = TraefikMetric.request_rate
+    result = await db.execute(
+        select(
+            TraefikMetric.router_name,
+            func.sum(rate).label("req"),
+            func.sum(rate * TraefikMetric.error_rate_4xx_pct).label("e4w"),
+            func.sum(rate * TraefikMetric.error_rate_5xx_pct).label("e5w"),
+            func.avg(TraefikMetric.error_rate_4xx_pct).label("e4"),
+            func.avg(TraefikMetric.error_rate_5xx_pct).label("e5"),
+            func.avg(TraefikMetric.latency_p95_ms).label("p95"),
+        )
+        .where(TraefikMetric.router_name.in_(names))
+        .where(TraefikMetric.scraped_at >= now - STATE_WINDOW)
+        .group_by(TraefikMetric.router_name)
+    )
+    out = {}
+    for r in result.all():
+        req = r.req or 0.0
+        out[r.router_name] = SimpleNamespace(
+            error_rate_4xx_pct=r.e4w / req if req > 0 else r.e4,
+            error_rate_5xx_pct=r.e5w / req if req > 0 else r.e5,
+            latency_p95_ms=r.p95,
+        )
+    return out
+
+
+def grade_routers(routers: list[dict], window: dict, thresholds: dict) -> None:
+    """Set each router's state and why from its recent window, or from its
+    latest scrape when the window holds nothing."""
+    for r in routers:
+        reading = window.get(r["name"])
+        r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS) if reading
+                                else router_state(r["latest"], thresholds))
+
+
 # A router missing from this many scrapes in a row is no longer reported:
 # Traefik dropped it (a removed service), so its last state is history.
 GONE_AFTER_SCRAPES = 3
@@ -188,10 +240,11 @@ async def rows():
         newest = await latest_per_router(db)
         current, gone = gone_routers([{"name": m.router_name, "latest": m} for m in newest], scrape)
 
-        history = await router_history(db, [r["name"] for r in current], since, bucket_seconds(since))
+        names = [r["name"] for r in current]
+        history = await router_history(db, names, since, bucket_seconds(since))
+        grade_routers(current, await recent_window(db, names, now), thresholds)
         for r in current:
             points = history[r["name"]]
-            r["state"], r["why"] = router_state(r["latest"], thresholds)
             r["sparkline_req"] = _sparkline([p.request_rate for p in points])
             r["sparkline_p95"] = _sparkline([p.latency_p95_ms for p in points])
             r["sparkline_5xx"] = _sparkline([p.error_rate_5xx_pct for p in points])
@@ -259,14 +312,15 @@ async def widget():
 
     async with current_app.db_sessionmaker() as db:
         latest = await latest_per_router(db)
+        current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
+        window = await recent_window(db, [r["name"] for r in current], now)
         certs = (await db.execute(select(TraefikCert))).scalars().all()
 
-    current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
     served, _old = current_certs(certs, scrape)
     domains = router_domains()
     by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
+    grade_routers(current, window, thresholds)
     for r in current:
-        r["state"], r["why"] = router_state(r["latest"], thresholds)
         r["cert"] = by_router.get(r["name"])
         if r["cert"]:
             grade = status_class(r["cert"]["days"], "tls_days", thresholds)

plugins/traefik/scraper.py

@@ -152,21 +152,21 @@ def compute_router_metrics(
 def compute_global_stats(
     current: ParsedMetrics,
     previous: ParsedMetrics | None,
-) -> dict[str, float | None]:
+) -> dict[str, float | datetime | None]:
     """Extract process-level and config health metrics from a scrape.
 
     Returns:
         {
             "open_conns_total": float | None,
             "config_reloads_total": float | None,
-            "config_last_reload_success": float | None,   # 1.0 = success, 0.0 = failure
+            "config_last_reload_at": datetime | None,     # last successful reload
             "process_memory_bytes": float | None,
         }
     """
-    stats: dict[str, float | None] = {
+    stats: dict[str, float | datetime | None] = {
         "open_conns_total": None,
         "config_reloads_total": None,
-        "config_last_reload_success": None,
+        "config_last_reload_at": None,
         "process_memory_bytes": None,
     }
 
@@ -180,11 +180,15 @@ def compute_global_stats(
     if reloads:
         stats["config_reloads_total"] = sum(reloads.values())
 
-    # Last reload success gauge
+    # The moment of the last successful reload, as a Unix timestamp: Traefik
+    # names it "last_reload_success", but it is a time, not a 0/1 flag. Read
+    # as a flag, every scrape said "Reload failed" (#5355).
     last_success = current.get("traefik_config_last_reload_success", {})
     if last_success:
         # Usually a single sample with no labels
-        stats["config_last_reload_success"] = next(iter(last_success.values()))
+        ts = next(iter(last_success.values()))
+        if ts > 0:
+            stats["config_last_reload_at"] = datetime.fromtimestamp(ts, timezone.utc)
 
     # Process RSS memory
     mem = current.get("process_resident_memory_bytes", {})

plugins/traefik/templates/traefik/rows.html

@@ -8,9 +8,8 @@
   {{ stat_tile("Needs attention", problems|length, state="crit" if problems else "inactive") }}
   {{ stat_tile("Healthy routers", healthy|length, state="up" if healthy else "inactive") }}
   {% if global_stat %}
-  {% if global_stat.config_last_reload_success is not none %}
-  {{ stat_tile("Config", "OK" if global_stat.config_last_reload_success == 1.0 else "Reload failed",
-               state="up" if global_stat.config_last_reload_success == 1.0 else "crit",
+  {% if global_stat.config_last_reload_at %}
+  {{ stat_tile("Config reloaded", global_stat.config_last_reload_at|ago,
                hint=(global_stat.config_reloads_total|int ~ " reloads") if global_stat.config_reloads_total is not none else none) }}
   {% endif %}
   {% if global_stat.open_conns_total is not none %}{{ stat_tile("Open connections", global_stat.open_conns_total|int) }}{% endif %}