feat(traefik): Routers widget shows current routers only, one header row, certs on the routers they serve (#5254)

#5254 "Traefik — Routers: current routers only, header row, certs on the router they serve".

- Router hosts come from Traefik's API (/api/http|tcp/routers), read
  best-effort on each scrape. An optional api_url overrides it; the default
  is /api on the metrics host. A certificate's expiry then shows in a TLS
  column on the router whose host it covers. An expiring one makes that
  router a problem, graded by Settings → Thresholds.
- Only current: routers and certificates must have appeared in the last
  three scrapes. Certificates are upserted by serial and never deleted, so
  renewed or retired ones lingered on the widget.
- Widget order: verdict, problem routers with the reason, then the busiest
  healthy routers under one header (Router / Req/s / Bandwidth / p95 / TLS).
  A missing value shows "—".
- If the API is unreachable, expiring certificates are listed after the
  routers, not above them.
- The page splits certificates into served (with the routers using each,
  plus SANs) and "No longer served".
- Latest row per router: an index walk (recursive CTE + LATERAL) replaces
  DISTINCT plus a query per router.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/routes.py

@@ -3,14 +3,16 @@ from __future__ import annotations
 from collections import defaultdict
 from datetime import datetime, timedelta, timezone
 from quart import Blueprint, current_app, render_template, request
-from sqlalchemy import Integer, cast, func, select
+from sqlalchemy import Integer, cast, func, select, text
 import json
 
 from steward.auth.middleware import require_role
 from steward.models.users import UserRole
 from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample
-from steward.core.formatting import status_class
+from steward.core.formatting import cert_left, status_class
 from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric
+from .scheduler import router_domains
+from .scraper import cert_names, covers
 
 traefik_bp = Blueprint("traefik", __name__, template_folder="templates")
 
@@ -85,6 +87,71 @@ def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]:
     return problems, healthy
 
 
+# Newest row per router. Postgres has no skip scan, so DISTINCT or max() GROUP
+# BY reads every row in retention; this walks the (router_name, scraped_at)
+# index one router at a time instead: a step per router, not per row.
+_LATEST_PER_ROUTER = text("""
+    WITH RECURSIVE names AS (
+        SELECT min(router_name) AS name FROM traefik_metrics
+        UNION ALL
+        SELECT (SELECT min(router_name) FROM traefik_metrics WHERE router_name > names.name)
+        FROM names WHERE names.name IS NOT NULL
+    )
+    SELECT m.* FROM names
+    JOIN LATERAL (
+        SELECT * FROM traefik_metrics WHERE router_name = names.name
+        ORDER BY scraped_at DESC LIMIT 1
+    ) m ON true
+""")
+
+
+async def latest_per_router(db) -> list:
+    return list((await db.execute(
+        select(TraefikMetric).from_statement(_LATEST_PER_ROUTER))).scalars().all())
+
+
+def current_certs(certs, scrape_seconds: int) -> tuple[list, list]:
+    """(served, old): a certificate is served while Traefik still reports it.
+    Rows are upserted by serial and never deleted, so a renewed or retired
+    certificate lingers; one missing from the last GONE_AFTER_SCRAPES
+    scrapes is old (#5254)."""
+    certs = list(certs)
+    if not certs:
+        return [], []
+    newest = max(c.scraped_at for c in certs)
+    cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
+    return ([c for c in certs if c.scraped_at >= cutoff],
+            sorted((c for c in certs if c.scraped_at < cutoff), key=lambda c: c.scraped_at, reverse=True))
+
+
+def router_hosts(name: str, domains: dict[str, list[str]]) -> list[str]:
+    """The hosts a metrics router name serves. The API names a router
+    "app@docker"; metrics may carry it as "websecure-app@docker", one per
+    entrypoint, so a name not found as-is is tried without that prefix."""
+    if name in domains:
+        return domains[name]
+    for api_name, hosts in domains.items():
+        if name.endswith("-" + api_name):
+            return hosts
+    return []
+
+
+def certs_by_router(routers: list[str], domains: dict[str, list[str]], certs, now) -> dict[str, dict]:
+    """{router: {"cn", "days"}}: the soonest-expiring served certificate valid
+    for one of the router's hosts. A router with no hosts, or none covered,
+    is absent."""
+    out: dict[str, dict] = {}
+    names = [(c, cert_names(c.cn, c.sans)) for c in certs]
+    for router in routers:
+        for host in router_hosts(router, domains):
+            for c, cn in names:
+                if covers(cn, host):
+                    days = (c.not_after - now).total_seconds() / 86400.0
+                    if router not in out or days < out[router]["days"]:
+                        out[router] = {"cn": c.cn or host, "days": days}
+    return out
+
+
 def _access_log() -> dict:
     """The plugin's access_log settings ({} when unset or malformed)."""
     cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {})
@@ -114,13 +181,9 @@ async def rows():
     thresholds = current_app.config.get("THRESHOLDS", {})
 
     async with current_app.db_sessionmaker() as db:
-        # All known routers
-        result = await db.execute(
-            select(TraefikMetric.router_name)
-            .distinct()
-            .order_by(TraefikMetric.router_name)
-        )
-        routers = [row[0] for row in result.all()]
+        # Every known router, with its most recent raw row
+        newest = {m.router_name: m for m in await latest_per_router(db)}
+        routers = sorted(newest)
 
         b_secs = bucket_seconds(since)
         bucket_col = (
@@ -145,16 +208,7 @@ async def rows():
             )
             history = result.all()
 
-            # Latest value — always the most recent raw row
-            result = await db.execute(
-                select(TraefikMetric)
-                .where(TraefikMetric.router_name == router)
-                .order_by(TraefikMetric.scraped_at.desc())
-                .limit(1)
-            )
-            latest = result.scalar_one_or_none()
-            if not latest:
-                continue
+            latest = newest[router]
 
             state, why = router_state(latest, thresholds)
             router_data.append({
@@ -181,19 +235,28 @@ async def rows():
         )
         raw_certs = result.scalars().all()
 
-    certs = []
-    for c in raw_certs:
-        days = (c.not_after - now).total_seconds() / 86400.0
-        certs.append({
+    scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
+    current, gone = gone_routers(router_data, scrape)
+    served, old = current_certs(raw_certs, scrape)
+    domains = router_domains()
+
+    def cert_row(c):
+        names = cert_names(c.cn, c.sans)
+        return {
             "serial": c.serial,
             "cn": c.cn or c.serial,
-            "sans": c.sans,
+            "sans": [n for n in names if n != c.cn],
             "not_after": c.not_after,
-            "days_remaining": days,
-        })
+            "days_remaining": (c.not_after - now).total_seconds() / 86400.0,
+            # The routers whose hosts it covers; None when Traefik's API
+            # couldn't say which hosts each router serves.
+            "routers": None if domains is None else sorted(
+                {r["name"].split("@")[0] for r in current
+                 for h in router_hosts(r["name"], domains) if covers(names, h)}),
+        }
 
-    current, gone = gone_routers(
-        router_data, current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"])
+    certs = [cert_row(c) for c in served]
+    old_certs = [cert_row(c) for c in old]
     problems, healthy = split_routers(current)
     return await render_template(
         "traefik/rows.html",
@@ -204,6 +267,7 @@ async def rows():
         error_5xx=ERROR_5XX_PCT,
         global_stat=global_stat,
         certs=certs,
+        old_certs=old_certs,
         range_key=range_key,
     )
 
@@ -211,53 +275,53 @@ async def rows():
 @traefik_bp.get("/widget")
 @require_role(UserRole.viewer)
 async def widget():
-    """HTMX dashboard widget fragment."""
+    """The dashboard Routers widget (#5254), in the widget-shape order: a
+    verdict, routers in trouble with why, then the busiest healthy ones under
+    one header. Only routers and certificates Traefik still reports. A
+    certificate's expiry shows on the router it serves, and makes that router
+    a problem once it is inside the Settings → Thresholds TLS window."""
     now = datetime.now(timezone.utc)
+    thresholds = current_app.config.get("THRESHOLDS", {})
+    scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
 
     async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
-            select(TraefikMetric.router_name).distinct().order_by(TraefikMetric.router_name)
-        )
-        routers = [row[0] for row in result.all()]
-        router_data = []
-        for router in routers:
-            result = await db.execute(
-                select(TraefikMetric)
-                .where(TraefikMetric.router_name == router)
-                .order_by(TraefikMetric.scraped_at.desc())
-                .limit(1)
-            )
-            latest = result.scalar_one_or_none()
-            if latest:
-                router_data.append({"name": router, "latest": latest})
-
-        router_data.sort(key=lambda r: (
-            -(1 if r["latest"].error_rate_5xx_pct > 0.1 else 0),
-            -(1 if r["latest"].error_rate_4xx_pct > 5.0 else 0),
-            -r["latest"].request_rate,
-        ))
-        total_routers = len(router_data)
-        router_data = router_data[:5]
-
-        result = await db.execute(
-            select(TraefikCert).order_by(TraefikCert.not_after)
-        )
-        raw_certs = result.scalars().all()
-
-    expiring_certs = []
-    for c in raw_certs:
-        days = (c.not_after - now).total_seconds() / 86400.0
-        if days < 30:
-            expiring_certs.append({"cn": c.cn or c.serial, "days_remaining": days})
+        latest = await latest_per_router(db)
+        certs = (await db.execute(select(TraefikCert))).scalars().all()
 
+    current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
+    served, _old = current_certs(certs, scrape)
+    domains = router_domains()
+    by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
+    for r in current:
+        r["state"], r["why"] = router_state(r["latest"], thresholds)
+        r["cert"] = by_router.get(r["name"])
+        if r["cert"]:
+            grade = status_class(r["cert"]["days"], "tls_days", thresholds)
+            r["cert"]["grade"] = grade
+            if _GRADE_ORDER.get(grade, 2) < _GRADE_ORDER[r["state"]]:
+                left = cert_left(r["cert"]["days"])     # "12 days", "expires today", "expired …"
+                r["state"], r["why"] = grade, (
+                    f"certificate expires in {left}" if left[0].isdigit() else f"certificate {left}")
+    problems, healthy = split_routers(current)
+    healthy.sort(key=lambda r: -(r["latest"].request_rate or 0))
+    # With no router domains to go on, certificates can't sit on their
+    # routers: the expiring ones are listed after the routers instead.
+    loose = [] if domains is not None else sorted(
+        ({"cn": c.cn or c.serial, "days": (c.not_after - now).total_seconds() / 86400.0} for c in served),
+        key=lambda c: c["days"])
+    loose = [c for c in loose if status_class(c["days"], "tls_days", thresholds) in ("warn", "crit")]
     return await render_template(
         "traefik/widget.html",
-        router_data=router_data,
-        expiring_certs=expiring_certs,
-        total_routers=total_routers,
+        problems=problems, healthy=healthy[:ROUTERS_SHOWN],
+        healthy_more=max(0, len(healthy) - ROUTERS_SHOWN),
+        total=len(current), loose_certs=loose, domains_known=domains is not None,
     )
 
 
+# Healthy routers on the dashboard widget (the busiest); the page lists all.
+ROUTERS_SHOWN = 6
+
+
 @traefik_bp.get("/widget/access_log")
 @require_role(UserRole.viewer)
 async def widget_access_log():