# plugins/traefik/routes.py from __future__ import annotations from collections import defaultdict from datetime import datetime, timedelta, timezone from types import SimpleNamespace from quart import Blueprint, current_app, render_template, request from sqlalchemy import func, select import json import time from steward.auth.middleware import require_role from steward.models.users import UserRole from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample, time_bucket from steward.core.formatting import cert_left, status_class from steward.models.metrics import PluginMetricLatest from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric from .scheduler import router_domains from .scraper import cert_names, covers traefik_bp = Blueprint("traefik", __name__, template_folder="templates") def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: """Generate an inline SVG sparkline from a list of float values.""" if len(values) < 2: return f'' mn, mx = min(values), max(values) if mx == mn: mx = mn + 1.0 step = width / (len(values) - 1) pts = [] for i, v in enumerate(values): x = i * step y = height - (v - mn) / (mx - mn) * (height - 2) - 1 pts.append(f"{x:.1f},{y:.1f}") poly = " ".join(pts) return ( f'' f'' f'' ) _GRADE_ORDER = {"crit": 0, "warn": 1, "ok": 2} def error_cutoffs(thresholds: dict) -> tuple[float, float]: """(5xx, 4xx): the error shares, in percent, past which a router is failing or worth a look. Settings → Thresholds, Traefik routers (#5380).""" return (float(thresholds.get("error_5xx", {}).get("crit", 0.1)), float(thresholds.get("error_4xx", {}).get("warn", 1.0))) def router_state(reading, thresholds: dict, over: str = "") -> tuple[str, str | None]: """(grade, why) for a router's reading (recent_window's, or one scrape's): crit while it serves 5xx past its cutoff, warn on a p95 past the latency critical cutoff or a 4xx share past its cutoff (all Settings → Thresholds), otherwise ok with nothing to say. `over` names the span the reading covers, for the why.""" error_5xx, error_4xx = error_cutoffs(thresholds) if (reading.error_rate_5xx_pct or 0) > error_5xx: return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}" p95 = reading.latency_p95_ms if p95 is not None and status_class(p95, "latency", thresholds) == "crit": return "warn", f"p95 latency is {p95:.0f} ms{over}" if (reading.error_rate_4xx_pct or 0) > error_4xx: return "warn", f"{reading.error_rate_4xx_pct:.1f}% of responses are 4xx{over}" return "ok", None # A router is graded on its last STATE_WINDOW of traffic, not on one scrape: # a share near a cutoff crossed it on one scrape and fell back on the next, # so problem cards came and went every poll (#5214). Over the window a real # problem stays listed until it has been clean a while, and a one-scrape # blip is diluted rather than shown. STATE_WINDOW = timedelta(minutes=15) STATE_WINDOW_WORDS = " over the last 15 min" async def recent_window(db, names: list[str], now: datetime) -> dict[str, SimpleNamespace]: """{router: one reading of its last STATE_WINDOW}: the 4xx and 5xx shares weighted by each scrape's traffic (an idle scrape's 100% counts for little), p95 averaged. A router with no rows in the window is absent.""" if not names: return {} rate = TraefikMetric.request_rate result = await db.execute( select( TraefikMetric.router_name, func.sum(rate).label("req"), func.sum(rate * TraefikMetric.error_rate_4xx_pct).label("e4w"), func.sum(rate * TraefikMetric.error_rate_5xx_pct).label("e5w"), func.avg(TraefikMetric.error_rate_4xx_pct).label("e4"), func.avg(TraefikMetric.error_rate_5xx_pct).label("e5"), func.avg(TraefikMetric.latency_p95_ms).label("p95"), ) .where(TraefikMetric.router_name.in_(names)) .where(TraefikMetric.scraped_at >= now - STATE_WINDOW) .group_by(TraefikMetric.router_name) ) out = {} for r in result.all(): req = r.req or 0.0 out[r.router_name] = SimpleNamespace( error_rate_4xx_pct=r.e4w / req if req > 0 else r.e4, error_rate_5xx_pct=r.e5w / req if req > 0 else r.e5, latency_p95_ms=r.p95, ) return out def grade_routers(routers: list[dict], window: dict, thresholds: dict) -> None: """Set each router's state and why from its recent window, or from its latest scrape when the window holds nothing.""" for r in routers: reading = window.get(r["name"]) r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS) if reading else router_state(r["latest"], thresholds)) # A router missing from this many scrapes in a row is no longer reported: # Traefik dropped it (a removed service), so its last state is history. GONE_AFTER_SCRAPES = 3 def gone_routers(routers: list[dict], scrape_seconds: int) -> tuple[list[dict], list[dict]]: """(current, gone): a router whose newest row trails the newest scrape of any router by GONE_AFTER_SCRAPES intervals is gone, newest-seen first.""" if not routers: return [], [] newest = max(r["latest"].scraped_at for r in routers) cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1)) current = [r for r in routers if r["latest"].scraped_at >= cutoff] gone = sorted((r for r in routers if r["latest"].scraped_at < cutoff), key=lambda r: r["latest"].scraped_at, reverse=True) return current, gone def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]: """(problems, healthy): failing routers before slow or noisy ones, each group by name; healthy routers by name.""" problems = sorted((r for r in routers if r["state"] != "ok"), key=lambda r: (_GRADE_ORDER[r["state"]], r["name"].lower())) healthy = sorted((r for r in routers if r["state"] == "ok"), key=lambda r: r["name"].lower()) return problems, healthy # The figures the scraper records for every router (scheduler._do_scrape). ROUTER_METRICS = ("request_rate", "error_rate_4xx_pct", "error_rate_5xx_pct", "latency_p50_ms", "latency_p95_ms", "latency_p99_ms", "response_bytes_rate") async def latest_per_router(db) -> list[SimpleNamespace]: """Each router's newest scrape, read from plugin_metric_latest: the scraper records every figure through MetricBatch, which keeps one row per router and figure there (latest_by_resource's table, #5208). A walk of traefik_metrics' index, one step per router, took 7 s on the live instance (#5355); each step lands on the router's oldest entries, which retention deletes every minute, so it likely waded through dead ones. Each router reads as one scrape, with the attributes a TraefikMetric row has; certificates, recorded under the same source, are left out.""" rows = (await db.execute(select(PluginMetricLatest).where( PluginMetricLatest.source_module == "traefik", PluginMetricLatest.metric_name.in_(ROUTER_METRICS)))).scalars() by_router: dict[str, dict] = {} for r in rows: by_router.setdefault(r.resource_name, {})[r.metric_name] = r return [SimpleNamespace(router_name=name, scraped_at=max(m.recorded_at for m in figures.values()), **{k: figures[k].value if k in figures else 0.0 for k in ROUTER_METRICS}) for name, figures in by_router.items() if "request_rate" in figures] def current_certs(certs, scrape_seconds: int) -> tuple[list, list]: """(served, old): a certificate is served while Traefik still reports it. Rows are upserted by serial and never deleted, so a renewed or retired certificate lingers; one missing from the last GONE_AFTER_SCRAPES scrapes is old (#5254).""" certs = list(certs) if not certs: return [], [] newest = max(c.scraped_at for c in certs) cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1)) return ([c for c in certs if c.scraped_at >= cutoff], sorted((c for c in certs if c.scraped_at < cutoff), key=lambda c: c.scraped_at, reverse=True)) def router_hosts(name: str, domains: dict[str, list[str]]) -> list[str]: """The hosts a metrics router name serves. The API names a router "app@docker"; metrics may carry it as "websecure-app@docker", one per entrypoint, so a name not found as-is is tried without that prefix.""" if name in domains: return domains[name] for api_name, hosts in domains.items(): if name.endswith("-" + api_name): return hosts return [] def certs_by_router(routers: list[str], domains: dict[str, list[str]], certs, now) -> dict[str, dict]: """{router: {"cn", "days"}}: the soonest-expiring served certificate valid for one of the router's hosts. A router with no hosts, or none covered, is absent.""" out: dict[str, dict] = {} names = [(c, cert_names(c.cn, c.sans)) for c in certs] for router in routers: for host in router_hosts(router, domains): for c, cn in names: if covers(cn, host): days = (c.not_after - now).total_seconds() / 86400.0 if router not in out or days < out[router]["days"]: out[router] = {"cn": c.cn or host, "days": days} return out def _access_log() -> dict: """The plugin's access_log settings ({} when unset or malformed).""" cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {}) return cfg if isinstance(cfg, dict) else {} @traefik_bp.get("/") @require_role(UserRole.viewer) async def index(): poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) access_log_enabled = bool(_access_log().get("enabled", False)) current_range = request.args.get("range", DEFAULT_RANGE) return await render_template( "traefik/index.html", poll_interval=poll_interval, access_log_enabled=access_log_enabled, current_range=current_range, ) @traefik_bp.get("/rows") @require_role(UserRole.viewer) async def rows(): """HTMX fragment: service cards with sparklines scoped to selected time range.""" since, range_key = parse_range(request.args.get("range")) now = datetime.now(timezone.utc) thresholds = current_app.config.get("THRESHOLDS", {}) scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"] phases = _Phases() async with current_app.db_sessionmaker() as db: # Every known router, with its most recent raw row; a router Traefik # stopped reporting gets no sparklines, so its history is not read. newest = await latest_per_router(db) current, gone = gone_routers([{"name": m.router_name, "latest": m} for m in newest], scrape) phases.mark("latest") names = [r["name"] for r in current] history = await router_history(db, names, since, bucket_seconds(since)) phases.mark("history") grade_routers(current, await recent_window(db, names, now), thresholds) phases.mark("window") for r in current: points = history[r["name"]] r["sparkline_req"] = _sparkline([p.request_rate for p in points]) r["sparkline_bytes"] = _sparkline([p.response_bytes_rate for p in points]) r["sparkline_p95"] = _sparkline([p.latency_p95_ms for p in points]) r["sparkline_4xx"] = _sparkline([p.error_rate_4xx_pct for p in points]) r["sparkline_5xx"] = _sparkline([p.error_rate_5xx_pct for p in points]) phases.mark("sparklines") # Latest global stat (always most recent, not range-filtered) result = await db.execute( select(TraefikGlobalStat) .order_by(TraefikGlobalStat.scraped_at.desc()) .limit(1) ) global_stat = result.scalar_one_or_none() # All certs, sorted by soonest expiry result = await db.execute( select(TraefikCert).order_by(TraefikCert.not_after) ) raw_certs = result.scalars().all() phases.mark("stats_certs") served, old = current_certs(raw_certs, scrape) domains = router_domains() def cert_row(c): names = cert_names(c.cn, c.sans) return { "serial": c.serial, "cn": c.cn or c.serial, "sans": [n for n in names if n != c.cn], "not_after": c.not_after, "days_remaining": (c.not_after - now).total_seconds() / 86400.0, # The routers whose hosts it covers; None when Traefik's API # couldn't say which hosts each router serves. "routers": None if domains is None else sorted( {r["name"].split("@")[0] for r in current for h in router_hosts(r["name"], domains) if covers(names, h)}), } certs = [cert_row(c) for c in served] old_certs = [cert_row(c) for c in old] problems, healthy = split_routers(current) phases.mark("cert_rows") error_5xx, error_4xx = error_cutoffs(thresholds) page = await render_template( "traefik/rows.html", problems=problems, healthy=healthy, gone=gone, error_4xx=error_4xx, error_5xx=error_5xx, global_stat=global_stat, certs=certs, old_certs=old_certs, range_key=range_key, ) phases.mark("render") phases.report("rows", f"routers={len(current)} range={range_key}") return page @traefik_bp.get("/widget") @require_role(UserRole.viewer) async def widget(): """The dashboard Routers widget (#5254), in the widget-shape order: a verdict, routers in trouble with why, then the busiest healthy ones under one header. Only routers and certificates Traefik still reports. A certificate's expiry shows on the router it serves, and makes that router a problem once it is inside the Settings → Thresholds TLS window.""" now = datetime.now(timezone.utc) thresholds = current_app.config.get("THRESHOLDS", {}) scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"] phases = _Phases() async with current_app.db_sessionmaker() as db: latest = await latest_per_router(db) phases.mark("latest") current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape) window = await recent_window(db, [r["name"] for r in current], now) phases.mark("window") certs = (await db.execute(select(TraefikCert))).scalars().all() phases.mark("certs") phases.report("widget", f"routers={len(current)}") served, _old = current_certs(certs, scrape) domains = router_domains() by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now) grade_routers(current, window, thresholds) for r in current: r["cert"] = by_router.get(r["name"]) if r["cert"]: grade = status_class(r["cert"]["days"], "tls_days", thresholds) r["cert"]["grade"] = grade if _GRADE_ORDER.get(grade, 2) < _GRADE_ORDER[r["state"]]: left = cert_left(r["cert"]["days"]) # "12 days", "expires today", "expired …" r["state"], r["why"] = grade, ( f"certificate expires in {left}" if left[0].isdigit() else f"certificate {left}") problems, healthy = split_routers(current) healthy.sort(key=lambda r: -(r["latest"].request_rate or 0)) # With no router domains to go on, certificates can't sit on their # routers: the expiring ones are listed after the routers instead. loose = [] if domains is not None else sorted( ({"cn": c.cn or c.serial, "days": (c.not_after - now).total_seconds() / 86400.0} for c in served), key=lambda c: c["days"]) loose = [c for c in loose if status_class(c["days"], "tls_days", thresholds) in ("warn", "crit")] return await render_template( "traefik/widget.html", problems=problems, healthy=healthy[:ROUTERS_SHOWN], healthy_more=max(0, len(healthy) - ROUTERS_SHOWN), total=len(current), loose_certs=loose, domains_known=domains is not None, ) # Healthy routers on the dashboard widget (the busiest); the page lists all. ROUTERS_SHOWN = 6 @traefik_bp.get("/widget/access_log") @require_role(UserRole.viewer) async def widget_access_log(): """HTMX dashboard widget: traffic origin summary with IP filter.""" from collections import defaultdict ip_filter = request.args.get("ip_filter", "all") limit = max(1, min(50, int(request.args.get("limit", 10) or 10))) widget_id = request.args.get("wid", "0") async with current_app.db_sessionmaker() as db: result = await db.execute( select(TraefikAccessSummary) .order_by(TraefikAccessSummary.period_end.desc()) .limit(12) # last ~1 hour of 5-min windows ) rows = result.scalars().all() if not rows: return await render_template("traefik/widget_access_log.html", summary=None, widget_id=widget_id) total = sum(r.total_requests for r in rows) internal = sum(r.internal_requests for r in rows) external = sum(r.external_requests for r in rows) ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True}) for row in rows: for entry in json.loads(row.top_ips_json): k = entry["ip"] ip_counts[k]["count"] += entry["count"] ip_counts[k]["internal"] = entry.get("internal", True) all_ips = sorted( [{"ip": k, **v} for k, v in ip_counts.items()], key=lambda x: x["count"], reverse=True, ) if ip_filter == "internal": all_ips = [e for e in all_ips if e["internal"]] elif ip_filter == "external": all_ips = [e for e in all_ips if not e["internal"]] summary = { "total": total, "internal": internal, "external": external, "external_pct": (external / total * 100.0) if total else 0.0, "top_ips": all_ips[:limit], "ip_filter": ip_filter, } return await render_template("traefik/widget_access_log.html", summary=summary, widget_id=widget_id) # A rows or widget request past this logs where its time went, phase by # phase, so a slow query names itself (#5355: 8 s on every poll, warm). SLOW_PAGE_SECONDS = 1.0 class _Phases: """Wall-clock per phase of one request; report() logs them when slow.""" def __init__(self): self.start = self.last = time.monotonic() self.took: dict[str, float] = {} def mark(self, name: str) -> None: now = time.monotonic() self.took[name] = now - self.last self.last = now def report(self, what: str, detail: str = "") -> None: total = time.monotonic() - self.start if total >= SLOW_PAGE_SECONDS: current_app.logger.warning("traefik %s: slow %.1fs (%s) %s", what, total, " ".join(f"{k}={v:.2f}s" for k, v in self.took.items()), detail) async def router_history(db, names: list[str], since, bucket_secs: int) -> dict[str, list]: """{router: [rows of request_rate, latency_p95_ms, error_rate_5xx_pct, error_rate_4xx_pct, response_bytes_rate]} per `bucket_secs` window since `since`, oldest first: one query for every router, answered from the covering (router_name, scraped_at) index (traefik_004). It was a query per router, each row a heap read, which took the rows page to 12-28 s.""" history: dict[str, list] = {n: [] for n in names} if not history: return history bucket_col = time_bucket(TraefikMetric.scraped_at, bucket_secs) result = await db.execute( select( TraefikMetric.router_name, func.avg(TraefikMetric.request_rate).label("request_rate"), func.avg(TraefikMetric.latency_p95_ms).label("latency_p95_ms"), func.avg(TraefikMetric.error_rate_5xx_pct).label("error_rate_5xx_pct"), func.avg(TraefikMetric.error_rate_4xx_pct).label("error_rate_4xx_pct"), func.avg(TraefikMetric.response_bytes_rate).label("response_bytes_rate"), bucket_col, ) .where(TraefikMetric.router_name.in_(names)) .where(TraefikMetric.scraped_at >= since) .group_by(TraefikMetric.router_name, bucket_col) .order_by(TraefikMetric.router_name, bucket_col) ) for row in result.all(): history[row.router_name].append(row) return history async def request_history(db, since, bucket_seconds: int) -> list[dict]: """[{"at", "req_rate", "err_pct"}] per time bucket since `since`: each scrape summed over routers (the total req/s), averaged over the bucket; the 5xx share weighted by each router's traffic, so a quiet router's errors don't count as much as a busy one's.""" per_scrape = ( select( TraefikMetric.scraped_at.label("at"), func.sum(TraefikMetric.request_rate).label("req"), func.sum(TraefikMetric.request_rate * TraefikMetric.error_rate_5xx_pct).label("err_weighted"), ) .where(TraefikMetric.scraped_at >= since) .group_by(TraefikMetric.scraped_at) .subquery() ) bucket_col = time_bucket(per_scrape.c.at, bucket_seconds) rows = (await db.execute( select( func.avg(per_scrape.c.req).label("req_rate"), func.sum(per_scrape.c.err_weighted).label("err_weighted"), func.sum(per_scrape.c.req).label("req_sum"), func.min(per_scrape.c.at).label("at"), bucket_col, ) .group_by(bucket_col) .order_by(bucket_col) )).all() return [{"at": r.at, "req_rate": round(r.req_rate or 0, 3), "err_pct": round((r.err_weighted or 0) / r.req_sum, 2) if r.req_sum else 0.0} for r in rows] @traefik_bp.get("/widget/request_chart") @require_role(UserRole.viewer) async def widget_request_chart(): """HTMX dashboard widget: requests per second across all routers, and the share of them answered 5xx, over the last few hours (#5256). Each scrape is summed over routers first (the total), then averaged per bucket; the 5xx share is weighted by each router's traffic, not averaged per router.""" hours = max(1, min(24, int(request.args.get("hours", 6) or 6))) widget_id = request.args.get("wid", "0") since = datetime.now(timezone.utc) - timedelta(hours=hours) b_secs = max(60, (hours * 3600) // 80) # ~80 buckets async with current_app.db_sessionmaker() as db: history = await request_history(db, since, b_secs) labels = [int(r["at"].timestamp() * 1000) for r in history] # epoch ms; the browser shows local time req_rates = [r["req_rate"] for r in history] err_pcts = [r["err_pct"] for r in history] return await render_template( "traefik/widget_request_chart.html", labels_json=json.dumps(labels), req_rate_json=json.dumps(req_rates), error_rate_json=json.dumps(err_pcts), widget_id=widget_id, hours=hours, now_req=req_rates[-1] if req_rates else None, peak_req=max(req_rates) if req_rates else None, peak_err=max(err_pcts) if err_pcts else None, ) @traefik_bp.get("/traffic") @require_role(UserRole.viewer) async def traffic(): """HTMX fragment: traffic origin summary scoped to selected time range.""" since, range_key = parse_range(request.args.get("range")) async with current_app.db_sessionmaker() as db: result = await db.execute( select(TraefikAccessSummary) .where(TraefikAccessSummary.period_end >= since) .order_by(TraefikAccessSummary.period_end.asc()) # No LIMIT — all summaries in range needed for accurate totals; # sparkline subsamples to 80 points afterwards ) rows = result.scalars().all() if not rows: al = _access_log() return await render_template( "traefik/traffic.html", summary=None, history=[], range_key=range_key, access_log_enabled=bool(al.get("enabled", False)), access_log_path=al.get("path", ""), ) total = sum(r.total_requests for r in rows) internal = sum(r.internal_requests for r in rows) external = sum(r.external_requests for r in rows) total_bytes = sum(r.total_bytes for r in rows) ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True, "country": None}) country_counts: dict[str, int] = defaultdict(int) router_counts: dict[str, int] = defaultdict(int) for row in rows: for entry in json.loads(row.top_ips_json): k = entry["ip"] ip_counts[k]["count"] += entry["count"] ip_counts[k]["internal"] = entry["internal"] ip_counts[k]["country"] = entry.get("country") for entry in json.loads(row.top_countries_json): country_counts[entry["country"]] += entry["count"] for entry in json.loads(row.top_routers_json): router_counts[entry["router"]] += entry["count"] top_ips = sorted( [{"ip": k, **v} for k, v in ip_counts.items()], key=lambda x: x["count"], reverse=True )[:10] top_countries = sorted( [{"country": k, "count": v} for k, v in country_counts.items()], key=lambda x: x["count"], reverse=True )[:10] top_routers = sorted( [{"router": k, "count": v} for k, v in router_counts.items()], key=lambda x: x["count"], reverse=True )[:10] history = [ { "period_end": r.period_end, "external_pct": (r.external_requests / r.total_requests * 100.0) if r.total_requests else 0.0, } for r in rows ] summary = { "total": total, "internal": internal, "external": external, "total_bytes": total_bytes, "external_pct": (external / total * 100.0) if total else 0.0, "top_ips": top_ips, "top_countries": top_countries, "top_routers": top_routers, "sparkline_external": _sparkline( [h["external_pct"] for h in subsample(history, 80)] ), } return await render_template( "traefik/traffic.html", summary=summary, history=history, range_key=range_key )