fix(traefik): grade routers on their last 15 minutes; the Config tile reads Traefik's reload time (#5355)

The operator: problem cards "can disappear so quickly". Each router was
graded on its latest scrape, so a 4xx share near the 1% cutoff crossed it
on one scrape and fell back on the next. recent_window reads every
router's last 15 minutes in one query, the error shares weighted by
traffic; grade_routers uses it on the page and the dashboard widget, and
the why says "over the last 15 min".

traefik_config_last_reload_success is the Unix time of the last good
reload, not a 0/1 flag, so the tile said "Reload failed" on every scrape.
traefik_005 stores it as config_last_reload_at, and the tile reads
"Config reloaded 3m ago". The router index also carries the 4xx share.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/routes.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 from collections import defaultdict
 from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
 from quart import Blueprint, current_app, render_template, request
 from sqlalchemy import func, select, text
 import json
@@ -46,20 +47,71 @@ ERROR_4XX_PCT = 1.0
 _GRADE_ORDER = {"crit": 0, "warn": 1, "ok": 2}
 
 
-def router_state(latest, thresholds: dict) -> tuple[str, str | None]:
-    """(grade, why) for a router's latest scrape: crit while it serves 5xx,
-    warn on a p95 past the latency critical cutoff (Settings → Thresholds)
-    or a high 4xx share, otherwise ok with nothing to say."""
-    if (latest.error_rate_5xx_pct or 0) > ERROR_5XX_PCT:
-        return "crit", f"{latest.error_rate_5xx_pct:.1f}% of responses are 5xx"
-    p95 = latest.latency_p95_ms
+def router_state(reading, thresholds: dict, over: str = "") -> tuple[str, str | None]:
+    """(grade, why) for a router's reading (recent_window's, or one scrape's):
+    crit while it serves 5xx, warn on a p95 past the latency critical cutoff
+    (Settings → Thresholds) or a high 4xx share, otherwise ok with nothing to
+    say. `over` names the span the reading covers, for the why."""
+    if (reading.error_rate_5xx_pct or 0) > ERROR_5XX_PCT:
+        return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}"
+    p95 = reading.latency_p95_ms
     if p95 is not None and status_class(p95, "latency", thresholds) == "crit":
-        return "warn", f"p95 latency is {p95:.0f} ms"
-    if (latest.error_rate_4xx_pct or 0) > ERROR_4XX_PCT:
-        return "warn", f"{latest.error_rate_4xx_pct:.1f}% of responses are 4xx"
+        return "warn", f"p95 latency is {p95:.0f} ms{over}"
+    if (reading.error_rate_4xx_pct or 0) > ERROR_4XX_PCT:
+        return "warn", f"{reading.error_rate_4xx_pct:.1f}% of responses are 4xx{over}"
     return "ok", None
 
 
+# A router is graded on its last STATE_WINDOW of traffic, not on one scrape:
+# a share near a cutoff crossed it on one scrape and fell back on the next,
+# so problem cards came and went every poll (#5214). Over the window a real
+# problem stays listed until it has been clean a while, and a one-scrape
+# blip is diluted rather than shown.
+STATE_WINDOW = timedelta(minutes=15)
+STATE_WINDOW_WORDS = " over the last 15 min"
+
+
+async def recent_window(db, names: list[str], now: datetime) -> dict[str, SimpleNamespace]:
+    """{router: one reading of its last STATE_WINDOW}: the 4xx and 5xx shares
+    weighted by each scrape's traffic (an idle scrape's 100% counts for
+    little), p95 averaged. A router with no rows in the window is absent."""
+    if not names:
+        return {}
+    rate = TraefikMetric.request_rate
+    result = await db.execute(
+        select(
+            TraefikMetric.router_name,
+            func.sum(rate).label("req"),
+            func.sum(rate * TraefikMetric.error_rate_4xx_pct).label("e4w"),
+            func.sum(rate * TraefikMetric.error_rate_5xx_pct).label("e5w"),
+            func.avg(TraefikMetric.error_rate_4xx_pct).label("e4"),
+            func.avg(TraefikMetric.error_rate_5xx_pct).label("e5"),
+            func.avg(TraefikMetric.latency_p95_ms).label("p95"),
+        )
+        .where(TraefikMetric.router_name.in_(names))
+        .where(TraefikMetric.scraped_at >= now - STATE_WINDOW)
+        .group_by(TraefikMetric.router_name)
+    )
+    out = {}
+    for r in result.all():
+        req = r.req or 0.0
+        out[r.router_name] = SimpleNamespace(
+            error_rate_4xx_pct=r.e4w / req if req > 0 else r.e4,
+            error_rate_5xx_pct=r.e5w / req if req > 0 else r.e5,
+            latency_p95_ms=r.p95,
+        )
+    return out
+
+
+def grade_routers(routers: list[dict], window: dict, thresholds: dict) -> None:
+    """Set each router's state and why from its recent window, or from its
+    latest scrape when the window holds nothing."""
+    for r in routers:
+        reading = window.get(r["name"])
+        r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS) if reading
+                                else router_state(r["latest"], thresholds))
+
+
 # A router missing from this many scrapes in a row is no longer reported:
 # Traefik dropped it (a removed service), so its last state is history.
 GONE_AFTER_SCRAPES = 3
@@ -188,10 +240,11 @@ async def rows():
         newest = await latest_per_router(db)
         current, gone = gone_routers([{"name": m.router_name, "latest": m} for m in newest], scrape)
 
-        history = await router_history(db, [r["name"] for r in current], since, bucket_seconds(since))
+        names = [r["name"] for r in current]
+        history = await router_history(db, names, since, bucket_seconds(since))
+        grade_routers(current, await recent_window(db, names, now), thresholds)
         for r in current:
             points = history[r["name"]]
-            r["state"], r["why"] = router_state(r["latest"], thresholds)
             r["sparkline_req"] = _sparkline([p.request_rate for p in points])
             r["sparkline_p95"] = _sparkline([p.latency_p95_ms for p in points])
             r["sparkline_5xx"] = _sparkline([p.error_rate_5xx_pct for p in points])
@@ -259,14 +312,15 @@ async def widget():
 
     async with current_app.db_sessionmaker() as db:
         latest = await latest_per_router(db)
+        current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
+        window = await recent_window(db, [r["name"] for r in current], now)
         certs = (await db.execute(select(TraefikCert))).scalars().all()
 
-    current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
     served, _old = current_certs(certs, scrape)
     domains = router_domains()
     by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
+    grade_routers(current, window, thresholds)
     for r in current:
-        r["state"], r["why"] = router_state(r["latest"], thresholds)
         r["cert"] = by_router.get(r["name"])
         if r["cert"]:
             grade = status_class(r["cert"]["days"], "tls_days", thresholds)