bvandeusenandClaude Opus 5.5
398312b8be
A Traefik that hasn't restarted keeps reporting the certificate it replaced, so codex.fabledsword.com's old one (21 days left) was listed beside its renewal (81 days) and would have raised an expiry warning on 2026-10-22. A reported certificate whose every name a later-expiring one covers is now old, on the page, the widget, the overview and MCP. list_certificates takes `limit` (default 100) with total/truncated, and says how many served certificates no HTTP router uses (`unrouted`). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
808 lines
36 KiB
Python
808 lines
36 KiB
Python
# plugins/traefik/routes.py
|
|
from __future__ import annotations
|
|
from collections import defaultdict
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace
|
|
from quart import Blueprint, current_app, render_template, request
|
|
from sqlalchemy import func, select
|
|
import json
|
|
import time
|
|
|
|
from steward.auth.middleware import require_role
|
|
from steward.models.users import UserRole
|
|
from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample, time_bucket
|
|
from steward.core import anomalies as anom
|
|
from steward.core.formatting import cert_left, status_class
|
|
from steward.core.norms import norms_at
|
|
from steward.core.settings import floor_of
|
|
from steward.models.metrics import PluginMetricLatest
|
|
from . import meaning
|
|
from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
|
|
from .scheduler import error_page_services, router_domains
|
|
from .scraper import cert_names, covers
|
|
|
|
traefik_bp = Blueprint("traefik", __name__, template_folder="templates")
|
|
|
|
|
|
def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
|
|
"""Generate an inline SVG sparkline from a list of float values."""
|
|
if len(values) < 2:
|
|
return f'<svg width="{width}" height="{height}"></svg>'
|
|
mn, mx = min(values), max(values)
|
|
if mx == mn:
|
|
mx = mn + 1.0
|
|
step = width / (len(values) - 1)
|
|
pts = []
|
|
for i, v in enumerate(values):
|
|
x = i * step
|
|
y = height - (v - mn) / (mx - mn) * (height - 2) - 1
|
|
pts.append(f"{x:.1f},{y:.1f}")
|
|
poly = " ".join(pts)
|
|
return (
|
|
f'<svg width="{width}" height="{height}" viewBox="0 0 {width} {height}" '
|
|
f'style="vertical-align:middle;">'
|
|
f'<polyline points="{poly}" fill="none" stroke="#6060c0" stroke-width="1.5"/>'
|
|
f'</svg>'
|
|
)
|
|
|
|
|
|
_GRADE_ORDER = {"crit": 0, "warn": 1, "ok": 2}
|
|
|
|
|
|
def error_cutoffs(thresholds: dict) -> tuple[float, float]:
|
|
"""(5xx, 4xx): the error shares, in percent, past which a router is failing
|
|
or worth a look. Settings → Thresholds, Traefik routers (#5380)."""
|
|
return (float(thresholds.get("error_5xx", {}).get("crit", 0.1)),
|
|
float(thresholds.get("error_4xx", {}).get("warn", 1.0)))
|
|
|
|
|
|
# Each router's 4xx share and p95 are judged against its own normal once it
|
|
# has one (#5741, System 182): an error-pages service answers 4xx all day and
|
|
# that is its normal, not a problem. Departing by less than the floor
|
|
# (Settings → Thresholds, norms.floor.router_*) is never news. A share over
|
|
# fewer than min_requests() in the window is not graded: one 404 of three
|
|
# requests is 33%, two 500s of sixteen 12%. The 5xx share is a failure and
|
|
# keeps its fixed cutoff whatever the history; it shares the 4xx share's
|
|
# minimum of traffic (the operator, #5859), and is failing only once
|
|
# min_5xx() errors were answered: one 502 of 70 requests is 1.4% (#5865).
|
|
NORMED = ("error_rate_4xx_pct", "latency_p95_ms")
|
|
|
|
|
|
def min_requests(thresholds: dict) -> float:
|
|
"""Fewest requests over the STATE_WINDOW for a 4xx or 5xx share to be
|
|
graded (Settings → Thresholds, traefik.threshold.min_requests)."""
|
|
return float(thresholds.get("router_requests", {}).get("warn", 30))
|
|
|
|
|
|
def min_5xx(thresholds: dict) -> float:
|
|
"""Fewest 5xx answered over the STATE_WINDOW for a 5xx share past its
|
|
cutoff to be failing (Settings → Thresholds, traefik.threshold.min_5xx)."""
|
|
return float(thresholds.get("router_5xx", {}).get("warn", 3))
|
|
|
|
|
|
def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = None) -> tuple[str, str | None]:
|
|
"""(grade, why) for a router's reading (recent_window's, or one scrape's):
|
|
crit while it serves 5xx past its cutoff, warn on a p95 past the latency
|
|
critical cutoff or a 4xx share past its cutoff (all Settings → Thresholds),
|
|
otherwise ok. `norms` ({metric: Norm}) are the router's normals for this
|
|
hour: a p95 or 4xx share past its cutoff but within its normal is ok, and
|
|
`why` then says it is usual. `over` names the span the reading covers.
|
|
Neither share is graded over fewer than min_requests() requests, and a
|
|
5xx share is failing only once about min_5xx() errors were answered."""
|
|
norms = norms or {}
|
|
error_5xx, error_4xx = error_cutoffs(thresholds)
|
|
requests = getattr(reading, "requests", None)
|
|
enough = requests is None or requests >= min_requests(thresholds)
|
|
e5 = reading.error_rate_5xx_pct or 0
|
|
errors = None if requests is None else round(requests * e5 / 100.0)
|
|
if enough and e5 > error_5xx and (errors is None or errors >= min_5xx(thresholds)):
|
|
return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}"
|
|
usual = []
|
|
p95 = reading.latency_p95_ms
|
|
if p95 is not None and status_class(p95, "latency", thresholds) == "crit":
|
|
out, said = anom.against(p95, norms.get("latency_p95_ms"),
|
|
floor_of(thresholds, "router_p95_ms"), " ms")
|
|
if out:
|
|
return "warn", f"p95 latency is {p95:.0f} ms{over}" + (f", {said}" if said else "")
|
|
usual.append(f"p95 {p95:.0f} ms is {said}")
|
|
e4 = reading.error_rate_4xx_pct or 0
|
|
if enough and e4 > error_4xx:
|
|
out, said = anom.against(e4, norms.get("error_rate_4xx_pct"),
|
|
floor_of(thresholds, "router_4xx_pct"), "%")
|
|
if out:
|
|
return "warn", f"{e4:.1f}% of responses are 4xx{over}" + (f", {said}" if said else "")
|
|
usual.append(f"4xx {e4:.1f}% is {said}")
|
|
return "ok", ("; ".join(usual) or None)
|
|
|
|
|
|
async def router_norms(db, names: list[str], now: datetime) -> dict[str, dict]:
|
|
"""{router: {metric: Norm}}: each router's learned normals for this hour.
|
|
A router without enough history yet is absent, and graded on the cutoffs."""
|
|
got = await norms_at(db, [("traefik", n, m) for n in names for m in NORMED], now)
|
|
out: dict[str, dict] = {}
|
|
for (_s, router, metric), norm in got.items():
|
|
out.setdefault(router, {})[metric] = norm
|
|
return out
|
|
|
|
|
|
# A router is graded on its last STATE_WINDOW of traffic, not on one scrape:
|
|
# a share near a cutoff crossed it on one scrape and fell back on the next,
|
|
# so problem cards came and went every poll (#5214). Over the window a real
|
|
# problem stays listed until it has been clean a while, and a one-scrape
|
|
# blip is diluted rather than shown.
|
|
STATE_WINDOW = timedelta(minutes=15)
|
|
STATE_WINDOW_WORDS = " over the last 15 min"
|
|
|
|
|
|
async def recent_window(db, names: list[str], now: datetime) -> dict[str, SimpleNamespace]:
|
|
"""{router: one reading of its last STATE_WINDOW}: the 4xx and 5xx shares
|
|
weighted by each scrape's traffic (an idle scrape's 100% counts for
|
|
little), p95 averaged, and about how many requests it answered. A router
|
|
with no rows in the window is absent."""
|
|
if not names:
|
|
return {}
|
|
rate = TraefikMetric.request_rate
|
|
result = await db.execute(
|
|
select(
|
|
TraefikMetric.router_name,
|
|
func.sum(rate).label("req"),
|
|
func.sum(rate * TraefikMetric.error_rate_4xx_pct).label("e4w"),
|
|
func.sum(rate * TraefikMetric.error_rate_5xx_pct).label("e5w"),
|
|
func.avg(TraefikMetric.error_rate_4xx_pct).label("e4"),
|
|
func.avg(TraefikMetric.error_rate_5xx_pct).label("e5"),
|
|
func.avg(TraefikMetric.latency_p95_ms).label("p95"),
|
|
func.avg(rate).label("rate"),
|
|
)
|
|
.where(TraefikMetric.router_name.in_(names))
|
|
.where(TraefikMetric.scraped_at >= now - STATE_WINDOW)
|
|
.group_by(TraefikMetric.router_name)
|
|
)
|
|
out = {}
|
|
for r in result.all():
|
|
req = r.req or 0.0
|
|
out[r.router_name] = SimpleNamespace(
|
|
error_rate_4xx_pct=r.e4w / req if req > 0 else r.e4,
|
|
error_rate_5xx_pct=r.e5w / req if req > 0 else r.e5,
|
|
latency_p95_ms=r.p95,
|
|
requests=(r.rate or 0.0) * STATE_WINDOW.total_seconds(),
|
|
)
|
|
return out
|
|
|
|
|
|
async def status_codes(db, names: list[str], since: datetime) -> dict[str, list[tuple[str, int]]]:
|
|
"""{router: [(code, responses), …]} for the 4xx and 5xx codes each router
|
|
answered since `since`, most first (#5711). A router with none is absent."""
|
|
if not names:
|
|
return {}
|
|
total = func.sum(TraefikStatusCode.count)
|
|
rows = (await db.execute(
|
|
select(TraefikStatusCode.router_name, TraefikStatusCode.code, total.label("n"))
|
|
.where(TraefikStatusCode.router_name.in_(names))
|
|
.where(TraefikStatusCode.scraped_at >= since)
|
|
.group_by(TraefikStatusCode.router_name, TraefikStatusCode.code)
|
|
.order_by(TraefikStatusCode.router_name, total.desc(), TraefikStatusCode.code)
|
|
)).all()
|
|
out: dict[str, list[tuple[str, int]]] = {}
|
|
for r in rows:
|
|
out.setdefault(r.router_name, []).append((r.code, int(r.n)))
|
|
return out
|
|
|
|
|
|
ERROR_PAGES_WHY = "serves other routers' error pages: its 4xx and 5xx are theirs, graded on them"
|
|
|
|
|
|
def grade_routers(routers: list[dict], window: dict, thresholds: dict, norms: dict | None = None,
|
|
error_pages: frozenset[str] = frozenset()) -> None:
|
|
"""Set each router's state and why from its recent window, or from its
|
|
latest scrape when the window holds nothing, against its normals
|
|
(router_norms') where it has them. A service Traefik's errors middleware
|
|
sends error pages to (`error_pages`) is healthy whatever its codes: it
|
|
answers with the code of the error it covers for (#5780)."""
|
|
norms = norms or {}
|
|
for r in routers:
|
|
if r["name"] in error_pages:
|
|
r["state"], r["why"], r["kind"], r["normed"] = "ok", ERROR_PAGES_WHY, None, False
|
|
continue
|
|
reading, own = window.get(r["name"]), norms.get(r["name"])
|
|
r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS, own) if reading
|
|
else router_state(r["latest"], thresholds, norms=own))
|
|
r["kind"] = meaning.problem_kind(r["state"], r["why"])
|
|
r["normed"] = bool(own) and {"slow": "latency_p95_ms", "4xx": "error_rate_4xx_pct"}.get(
|
|
r["kind"], "") in own
|
|
|
|
|
|
async def swarm_service_names(db) -> list[str]:
|
|
"""Every swarm service the Docker plugin has stored; [] without it."""
|
|
from steward.core.plugin_manager import loaded_plugin_submodule
|
|
models = loaded_plugin_submodule("docker", "models")
|
|
if models is None:
|
|
return []
|
|
return list((await db.execute(select(models.DockerSwarmService.service_name).distinct())).scalars())
|
|
|
|
|
|
def explain_problems(problems: list[dict], services: list[str]) -> None:
|
|
"""Give each problem card what it means and its one next step (#5712):
|
|
`meaning` (meaning.explain's) and `service`, the swarm service behind
|
|
the router when Steward can tell which."""
|
|
for r in problems:
|
|
kind = r.get("kind") or meaning.problem_kind(r["state"], r["why"])
|
|
r["meaning"] = meaning.explain(kind, r.get("codes", []),
|
|
getattr(r["latest"], "latency_p50_ms", None),
|
|
getattr(r["latest"], "latency_p95_ms", None),
|
|
r.get("normed", False))
|
|
r["service"] = meaning.swarm_service(r["name"], services)
|
|
|
|
|
|
# A router missing from this many scrapes in a row is no longer reported:
|
|
# Traefik dropped it (a removed service), so its last state is history.
|
|
GONE_AFTER_SCRAPES = 3
|
|
|
|
|
|
def gone_routers(routers: list[dict], scrape_seconds: int) -> tuple[list[dict], list[dict]]:
|
|
"""(current, gone): a router whose newest row trails the newest scrape of
|
|
any router by GONE_AFTER_SCRAPES intervals is gone, newest-seen first."""
|
|
if not routers:
|
|
return [], []
|
|
newest = max(r["latest"].scraped_at for r in routers)
|
|
cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
|
|
current = [r for r in routers if r["latest"].scraped_at >= cutoff]
|
|
gone = sorted((r for r in routers if r["latest"].scraped_at < cutoff),
|
|
key=lambda r: r["latest"].scraped_at, reverse=True)
|
|
return current, gone
|
|
|
|
|
|
def problem_since(history: list, thresholds: dict) -> tuple[datetime | None, bool]:
|
|
"""(start, before_range) of a router's current problem, read from its
|
|
history buckets (router_history's, oldest first): the start of the
|
|
trailing run of buckets that grade as a problem. (None, False) when the
|
|
newest bucket reads ok, as for a problem too recent or too brief for a
|
|
bucket to show. `before_range` says the run reaches back to the first
|
|
bucket, so the problem began before the range shown."""
|
|
start = None
|
|
for point in reversed(history):
|
|
if router_state(point, thresholds)[0] == "ok":
|
|
break
|
|
start = point
|
|
if start is None:
|
|
return None, False
|
|
return start.bucket, start is history[0]
|
|
|
|
|
|
def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]:
|
|
"""(problems, healthy): failing routers before slow or noisy ones, each
|
|
group by name; healthy routers by name."""
|
|
problems = sorted((r for r in routers if r["state"] != "ok"),
|
|
key=lambda r: (_GRADE_ORDER[r["state"]], r["name"].lower()))
|
|
healthy = sorted((r for r in routers if r["state"] == "ok"), key=lambda r: r["name"].lower())
|
|
return problems, healthy
|
|
|
|
|
|
# The figures the scraper records for every router (scheduler._do_scrape).
|
|
ROUTER_METRICS = ("request_rate", "error_rate_4xx_pct", "error_rate_5xx_pct", "latency_p50_ms",
|
|
"latency_p95_ms", "latency_p99_ms", "response_bytes_rate")
|
|
|
|
|
|
async def latest_per_router(db) -> list[SimpleNamespace]:
|
|
"""Each router's newest scrape, read from plugin_metric_latest: the scraper
|
|
records every figure through MetricBatch, which keeps one row per router
|
|
and figure there (latest_by_resource's table, #5208). A walk of
|
|
traefik_metrics' index, one step per router, took 7 s on the live
|
|
instance (#5355); each step lands on the router's oldest entries, which
|
|
retention deletes every minute, so it likely waded through dead ones.
|
|
|
|
Each router reads as one scrape, with the attributes a TraefikMetric row
|
|
has; certificates, recorded under the same source, are left out."""
|
|
rows = (await db.execute(select(PluginMetricLatest).where(
|
|
PluginMetricLatest.source_module == "traefik",
|
|
PluginMetricLatest.metric_name.in_(ROUTER_METRICS)))).scalars()
|
|
by_router: dict[str, dict] = {}
|
|
for r in rows:
|
|
by_router.setdefault(r.resource_name, {})[r.metric_name] = r
|
|
return [SimpleNamespace(router_name=name, scraped_at=max(m.recorded_at for m in figures.values()),
|
|
**{k: figures[k].value if k in figures else None for k in ROUTER_METRICS})
|
|
for name, figures in by_router.items() if "request_rate" in figures]
|
|
|
|
|
|
def current_certs(certs, scrape_seconds: int) -> tuple[list, list]:
|
|
"""(served, old): a certificate is served while Traefik still reports it.
|
|
Rows are upserted by serial and never deleted, so a renewed or retired
|
|
certificate lingers; one missing from the last GONE_AFTER_SCRAPES
|
|
scrapes is old (#5254). So is one renewed: a Traefik that hasn't
|
|
restarted since keeps reporting the certificate it replaced, so one
|
|
whose every name a later-expiring reported certificate covers is old
|
|
too (#5864); otherwise it reads as expiring while the renewal serves."""
|
|
certs = list(certs)
|
|
if not certs:
|
|
return [], []
|
|
newest = max(c.scraped_at for c in certs)
|
|
cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
|
|
reported = [c for c in certs if c.scraped_at >= cutoff]
|
|
names = {c.serial: cert_names(c.cn, c.sans) for c in reported}
|
|
renewed = {c.serial for c in reported for n in reported
|
|
if n.serial != c.serial and n.not_after > c.not_after and names[c.serial] <= names[n.serial]}
|
|
return ([c for c in reported if c.serial not in renewed],
|
|
sorted((c for c in certs if c.scraped_at < cutoff or c.serial in renewed),
|
|
key=lambda c: c.scraped_at, reverse=True))
|
|
|
|
|
|
def router_hosts(name: str, domains: dict[str, list[str]]) -> list[str]:
|
|
"""The hosts a metrics router name serves. The API names a router
|
|
"app@docker"; metrics may carry it as "websecure-app@docker", one per
|
|
entrypoint, so a name not found as-is is tried without that prefix."""
|
|
if name in domains:
|
|
return domains[name]
|
|
for api_name, hosts in domains.items():
|
|
if name.endswith("-" + api_name):
|
|
return hosts
|
|
return []
|
|
|
|
|
|
def certs_by_router(routers: list[str], domains: dict[str, list[str]], certs, now) -> dict[str, dict]:
|
|
"""{router: {"cn", "days"}}: the soonest-expiring served certificate valid
|
|
for one of the router's hosts. A router with no hosts, or none covered,
|
|
is absent."""
|
|
out: dict[str, dict] = {}
|
|
names = [(c, cert_names(c.cn, c.sans)) for c in certs]
|
|
for router in routers:
|
|
for host in router_hosts(router, domains):
|
|
for c, cn in names:
|
|
if covers(cn, host):
|
|
days = (c.not_after - now).total_seconds() / 86400.0
|
|
if router not in out or days < out[router]["days"]:
|
|
out[router] = {"cn": c.cn or host, "days": days}
|
|
return out
|
|
|
|
|
|
def _access_log() -> dict:
|
|
"""The plugin's access_log settings ({} when unset or malformed)."""
|
|
cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {})
|
|
return cfg if isinstance(cfg, dict) else {}
|
|
|
|
|
|
@traefik_bp.get("/")
|
|
@require_role(UserRole.viewer)
|
|
async def index():
|
|
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
|
|
access_log_enabled = bool(_access_log().get("enabled", False))
|
|
current_range = request.args.get("range", DEFAULT_RANGE)
|
|
return await render_template(
|
|
"traefik/index.html",
|
|
poll_interval=poll_interval,
|
|
access_log_enabled=access_log_enabled,
|
|
current_range=current_range,
|
|
)
|
|
|
|
|
|
@traefik_bp.get("/rows")
|
|
@require_role(UserRole.viewer)
|
|
async def rows():
|
|
"""HTMX fragment: service cards with sparklines scoped to selected time range."""
|
|
since, range_key = parse_range(request.args.get("range"))
|
|
now = datetime.now(timezone.utc)
|
|
thresholds = current_app.config.get("THRESHOLDS", {})
|
|
|
|
scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
|
|
phases = _Phases()
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
# Every known router, with its most recent raw row; a router Traefik
|
|
# stopped reporting gets no sparklines, so its history is not read.
|
|
newest = await latest_per_router(db)
|
|
current, gone = gone_routers([{"name": m.router_name, "latest": m} for m in newest], scrape)
|
|
phases.mark("latest")
|
|
|
|
names = [r["name"] for r in current]
|
|
history = await router_history(db, names, since, bucket_seconds(since))
|
|
phases.mark("history")
|
|
grade_routers(current, await recent_window(db, names, now), thresholds,
|
|
await router_norms(db, names, now), error_page_services())
|
|
phases.mark("window")
|
|
# The routers' metric anomalies, last 30 days (#5741).
|
|
anomalies = await anom.views(db, await anom.list_anomalies(
|
|
db, source="traefik", since=now - timedelta(days=30), limit=20), links=False)
|
|
for r in current:
|
|
points = history[r["name"]]
|
|
r["since"], r["since_before"] = (problem_since(points, thresholds) if r["state"] != "ok"
|
|
else (None, False))
|
|
r["sparkline_req"] = _sparkline([p.request_rate for p in points])
|
|
r["sparkline_bytes"] = _sparkline([p.response_bytes_rate for p in points])
|
|
r["sparkline_p95"] = _sparkline([p.latency_p95_ms for p in points if p.latency_p95_ms is not None])
|
|
r["sparkline_4xx"] = _sparkline([p.error_rate_4xx_pct for p in points])
|
|
r["sparkline_5xx"] = _sparkline([p.error_rate_5xx_pct for p in points])
|
|
phases.mark("sparklines")
|
|
# The codes behind each problem, over the window it is graded on.
|
|
codes = await status_codes(db, [r["name"] for r in current if r["state"] != "ok"], now - STATE_WINDOW)
|
|
for r in current:
|
|
r["codes"] = codes.get(r["name"], [])
|
|
# Codes are kept from #5711 on: until the first of them is a window
|
|
# old, a router with none listed may simply predate the recording.
|
|
first_code = await db.scalar(select(func.min(TraefikStatusCode.scraped_at)))
|
|
codes_known = first_code is not None and first_code <= now - STATE_WINDOW
|
|
services = await swarm_service_names(db) if any(r["state"] != "ok" for r in current) else []
|
|
phases.mark("codes")
|
|
|
|
# Latest global stat (always most recent, not range-filtered)
|
|
result = await db.execute(
|
|
select(TraefikGlobalStat)
|
|
.order_by(TraefikGlobalStat.scraped_at.desc())
|
|
.limit(1)
|
|
)
|
|
global_stat = result.scalar_one_or_none()
|
|
|
|
# All certs, sorted by soonest expiry
|
|
result = await db.execute(
|
|
select(TraefikCert).order_by(TraefikCert.not_after)
|
|
)
|
|
raw_certs = result.scalars().all()
|
|
phases.mark("stats_certs")
|
|
|
|
served, old = current_certs(raw_certs, scrape)
|
|
domains = router_domains()
|
|
|
|
def cert_row(c):
|
|
names = cert_names(c.cn, c.sans)
|
|
return {
|
|
"serial": c.serial,
|
|
"cn": c.cn or c.serial,
|
|
"sans": [n for n in names if n != c.cn],
|
|
"not_after": c.not_after,
|
|
"days_remaining": (c.not_after - now).total_seconds() / 86400.0,
|
|
# The routers whose hosts it covers; None when Traefik's API
|
|
# couldn't say which hosts each router serves.
|
|
"routers": None if domains is None else sorted(
|
|
{r["name"].split("@")[0] for r in current
|
|
for h in router_hosts(r["name"], domains) if covers(names, h)}),
|
|
}
|
|
|
|
certs = [cert_row(c) for c in served]
|
|
old_certs = [cert_row(c) for c in old]
|
|
problems, healthy = split_routers(current)
|
|
explain_problems(problems, services)
|
|
phases.mark("cert_rows")
|
|
error_5xx, error_4xx = error_cutoffs(thresholds)
|
|
page = await render_template(
|
|
"traefik/rows.html",
|
|
problems=problems,
|
|
healthy=healthy,
|
|
gone=gone,
|
|
current_range=range_key,
|
|
codes_known=codes_known,
|
|
error_4xx=error_4xx,
|
|
error_5xx=error_5xx,
|
|
global_stat=global_stat,
|
|
certs=certs,
|
|
old_certs=old_certs,
|
|
range_key=range_key,
|
|
anomalies=anomalies,
|
|
back="/plugins/traefik/",
|
|
base_resource=None,
|
|
)
|
|
phases.mark("render")
|
|
phases.report("rows", f"routers={len(current)} range={range_key}")
|
|
return page
|
|
|
|
|
|
@traefik_bp.get("/widget")
|
|
@require_role(UserRole.viewer)
|
|
async def widget():
|
|
"""The dashboard Routers widget (#5254), in the widget-shape order: a
|
|
verdict, routers in trouble with why, then the busiest healthy ones under
|
|
one header. Only routers and certificates Traefik still reports. A
|
|
certificate's expiry shows on the router it serves, and makes that router
|
|
a problem once it is inside the Settings → Thresholds TLS window."""
|
|
now = datetime.now(timezone.utc)
|
|
thresholds = current_app.config.get("THRESHOLDS", {})
|
|
scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
|
|
|
|
phases = _Phases()
|
|
async with current_app.db_sessionmaker() as db:
|
|
latest = await latest_per_router(db)
|
|
phases.mark("latest")
|
|
current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
|
|
window = await recent_window(db, [r["name"] for r in current], now)
|
|
norms = await router_norms(db, [r["name"] for r in current], now)
|
|
phases.mark("window")
|
|
certs = (await db.execute(select(TraefikCert))).scalars().all()
|
|
phases.mark("certs")
|
|
phases.report("widget", f"routers={len(current)}")
|
|
|
|
served, _old = current_certs(certs, scrape)
|
|
domains = router_domains()
|
|
by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
|
|
grade_routers(current, window, thresholds, norms, error_page_services())
|
|
for r in current:
|
|
r["cert"] = by_router.get(r["name"])
|
|
if r["cert"]:
|
|
grade = status_class(r["cert"]["days"], "tls_days", thresholds)
|
|
r["cert"]["grade"] = grade
|
|
if _GRADE_ORDER.get(grade, 2) < _GRADE_ORDER[r["state"]]:
|
|
left = cert_left(r["cert"]["days"]) # "12 days", "expires today", "expired …"
|
|
r["by_cert"] = True
|
|
r["state"], r["why"] = grade, (
|
|
f"certificate expires in {left}" if left[0].isdigit() else f"certificate {left}")
|
|
problems, healthy = split_routers(current)
|
|
healthy.sort(key=lambda r: -(r["latest"].request_rate or 0))
|
|
# Since when, from the day's history, for problems the traffic shows (a
|
|
# certificate's says its own date). Only the routers in trouble are read.
|
|
traffic = [r for r in problems if not r.get("by_cert")]
|
|
if traffic:
|
|
since = now - WIDGET_SINCE
|
|
async with current_app.db_sessionmaker() as db:
|
|
history = await router_history(db, [r["name"] for r in traffic], since,
|
|
bucket_seconds(since))
|
|
for r in traffic:
|
|
r["since"], r["since_before"] = problem_since(history[r["name"]], thresholds)
|
|
# With no router domains to go on, certificates can't sit on their
|
|
# routers: the expiring ones are listed after the routers instead.
|
|
loose = [] if domains is not None else sorted(
|
|
({"cn": c.cn or c.serial, "days": (c.not_after - now).total_seconds() / 86400.0} for c in served),
|
|
key=lambda c: c["days"])
|
|
loose = [c for c in loose if status_class(c["days"], "tls_days", thresholds) in ("warn", "crit")]
|
|
return await render_template(
|
|
"traefik/widget.html",
|
|
problems=problems, healthy=healthy[:ROUTERS_SHOWN],
|
|
healthy_more=max(0, len(healthy) - ROUTERS_SHOWN),
|
|
total=len(current), loose_certs=loose, domains_known=domains is not None,
|
|
)
|
|
|
|
|
|
# Healthy routers on the dashboard widget (the busiest); the page lists all.
|
|
ROUTERS_SHOWN = 6
|
|
# How far back the widget looks for when a router's problem began.
|
|
WIDGET_SINCE = timedelta(hours=24)
|
|
|
|
|
|
@traefik_bp.get("/widget/access_log")
|
|
@require_role(UserRole.viewer)
|
|
async def widget_access_log():
|
|
"""HTMX dashboard widget: traffic origin summary with IP filter."""
|
|
from collections import defaultdict
|
|
ip_filter = request.args.get("ip_filter", "all")
|
|
limit = max(1, min(50, int(request.args.get("limit", 10) or 10)))
|
|
widget_id = request.args.get("wid", "0")
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(
|
|
select(TraefikAccessSummary)
|
|
.order_by(TraefikAccessSummary.period_end.desc())
|
|
.limit(12) # last ~1 hour of 5-min windows
|
|
)
|
|
rows = result.scalars().all()
|
|
|
|
if not rows:
|
|
return await render_template("traefik/widget_access_log.html",
|
|
summary=None, widget_id=widget_id)
|
|
|
|
total = sum(r.total_requests for r in rows)
|
|
internal = sum(r.internal_requests for r in rows)
|
|
external = sum(r.external_requests for r in rows)
|
|
|
|
ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True})
|
|
for row in rows:
|
|
for entry in json.loads(row.top_ips_json):
|
|
k = entry["ip"]
|
|
ip_counts[k]["count"] += entry["count"]
|
|
ip_counts[k]["internal"] = entry.get("internal", True)
|
|
|
|
all_ips = sorted(
|
|
[{"ip": k, **v} for k, v in ip_counts.items()],
|
|
key=lambda x: x["count"], reverse=True,
|
|
)
|
|
if ip_filter == "internal":
|
|
all_ips = [e for e in all_ips if e["internal"]]
|
|
elif ip_filter == "external":
|
|
all_ips = [e for e in all_ips if not e["internal"]]
|
|
|
|
summary = {
|
|
"total": total,
|
|
"internal": internal,
|
|
"external": external,
|
|
"external_pct": (external / total * 100.0) if total else 0.0,
|
|
"top_ips": all_ips[:limit],
|
|
"ip_filter": ip_filter,
|
|
}
|
|
return await render_template("traefik/widget_access_log.html",
|
|
summary=summary, widget_id=widget_id)
|
|
|
|
|
|
# A rows or widget request past this logs where its time went, phase by
|
|
# phase, so a slow query names itself (#5355: 8 s on every poll, warm).
|
|
SLOW_PAGE_SECONDS = 1.0
|
|
|
|
|
|
class _Phases:
|
|
"""Wall-clock per phase of one request; report() logs them when slow."""
|
|
|
|
def __init__(self):
|
|
self.start = self.last = time.monotonic()
|
|
self.took: dict[str, float] = {}
|
|
|
|
def mark(self, name: str) -> None:
|
|
now = time.monotonic()
|
|
self.took[name] = now - self.last
|
|
self.last = now
|
|
|
|
def report(self, what: str, detail: str = "") -> None:
|
|
total = time.monotonic() - self.start
|
|
if total >= SLOW_PAGE_SECONDS:
|
|
current_app.logger.warning("traefik %s: slow %.1fs (%s) %s", what, total,
|
|
" ".join(f"{k}={v:.2f}s" for k, v in self.took.items()), detail)
|
|
|
|
|
|
async def router_history(db, names: list[str], since, bucket_secs: int) -> dict[str, list]:
|
|
"""{router: [rows of request_rate, latency_p95_ms, error_rate_5xx_pct,
|
|
error_rate_4xx_pct, response_bytes_rate]}
|
|
per `bucket_secs` window since `since`, oldest first: one query for every
|
|
router, answered from the covering (router_name, scraped_at) index
|
|
(traefik_004). It was a query per router, each row a heap read, which
|
|
took the rows page to 12-28 s."""
|
|
history: dict[str, list] = {n: [] for n in names}
|
|
if not history:
|
|
return history
|
|
bucket_col = time_bucket(TraefikMetric.scraped_at, bucket_secs)
|
|
result = await db.execute(
|
|
select(
|
|
TraefikMetric.router_name,
|
|
func.avg(TraefikMetric.request_rate).label("request_rate"),
|
|
func.avg(TraefikMetric.latency_p95_ms).label("latency_p95_ms"),
|
|
func.avg(TraefikMetric.error_rate_5xx_pct).label("error_rate_5xx_pct"),
|
|
func.avg(TraefikMetric.error_rate_4xx_pct).label("error_rate_4xx_pct"),
|
|
func.avg(TraefikMetric.response_bytes_rate).label("response_bytes_rate"),
|
|
bucket_col,
|
|
)
|
|
.where(TraefikMetric.router_name.in_(names))
|
|
.where(TraefikMetric.scraped_at >= since)
|
|
.group_by(TraefikMetric.router_name, bucket_col)
|
|
.order_by(TraefikMetric.router_name, bucket_col)
|
|
)
|
|
for row in result.all():
|
|
history[row.router_name].append(row)
|
|
return history
|
|
|
|
|
|
async def request_history(db, since, bucket_seconds: int) -> list[dict]:
|
|
"""[{"at", "req_rate", "err_pct"}] per time bucket since `since`: each
|
|
scrape summed over routers (the total req/s), averaged over the bucket;
|
|
the 5xx share weighted by each router's traffic, so a quiet router's
|
|
errors don't count as much as a busy one's."""
|
|
per_scrape = (
|
|
select(
|
|
TraefikMetric.scraped_at.label("at"),
|
|
func.sum(TraefikMetric.request_rate).label("req"),
|
|
func.sum(TraefikMetric.request_rate * TraefikMetric.error_rate_5xx_pct).label("err_weighted"),
|
|
)
|
|
.where(TraefikMetric.scraped_at >= since)
|
|
.group_by(TraefikMetric.scraped_at)
|
|
.subquery()
|
|
)
|
|
bucket_col = time_bucket(per_scrape.c.at, bucket_seconds)
|
|
rows = (await db.execute(
|
|
select(
|
|
func.avg(per_scrape.c.req).label("req_rate"),
|
|
func.sum(per_scrape.c.err_weighted).label("err_weighted"),
|
|
func.sum(per_scrape.c.req).label("req_sum"),
|
|
func.min(per_scrape.c.at).label("at"),
|
|
bucket_col,
|
|
)
|
|
.group_by(bucket_col)
|
|
.order_by(bucket_col)
|
|
)).all()
|
|
return [{"at": r.at, "req_rate": round(r.req_rate or 0, 3),
|
|
"err_pct": round((r.err_weighted or 0) / r.req_sum, 2) if r.req_sum else 0.0}
|
|
for r in rows]
|
|
|
|
|
|
@traefik_bp.get("/widget/request_chart")
|
|
@require_role(UserRole.viewer)
|
|
async def widget_request_chart():
|
|
"""HTMX dashboard widget: requests per second across all routers, and the
|
|
share of them answered 5xx, over the last few hours (#5256). Each scrape
|
|
is summed over routers first (the total), then averaged per bucket; the
|
|
5xx share is weighted by each router's traffic, not averaged per router."""
|
|
widget_id = request.args.get("wid", "0")
|
|
if request.args.get("range"):
|
|
# The Traefik page: the page's own time range (#5710).
|
|
since, span = parse_range(request.args.get("range"))
|
|
b_secs = bucket_seconds(since)
|
|
else:
|
|
hours = max(1, min(24, int(request.args.get("hours", 6) or 6)))
|
|
since, span = datetime.now(timezone.utc) - timedelta(hours=hours), f"{hours}h"
|
|
b_secs = max(60, (hours * 3600) // 80) # ~80 buckets
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
history = await request_history(db, since, b_secs)
|
|
|
|
labels = [int(r["at"].timestamp() * 1000) for r in history] # epoch ms; the browser shows local time
|
|
req_rates = [r["req_rate"] for r in history]
|
|
err_pcts = [r["err_pct"] for r in history]
|
|
|
|
return await render_template(
|
|
"traefik/widget_request_chart.html",
|
|
labels_json=json.dumps(labels),
|
|
req_rate_json=json.dumps(req_rates),
|
|
error_rate_json=json.dumps(err_pcts),
|
|
widget_id=widget_id,
|
|
span=span,
|
|
days=(datetime.now(timezone.utc) - since) > timedelta(hours=24),
|
|
now_req=req_rates[-1] if req_rates else None,
|
|
peak_req=max(req_rates) if req_rates else None,
|
|
peak_err=max(err_pcts) if err_pcts else None,
|
|
)
|
|
|
|
|
|
def summarize_access(rows, top: int = 10) -> dict:
|
|
"""The access-log summaries in a range added up: totals, the internal and
|
|
external split, and the top client IPs, countries and routers."""
|
|
total = sum(r.total_requests for r in rows)
|
|
external = sum(r.external_requests for r in rows)
|
|
ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True, "country": None})
|
|
country_counts: dict[str, int] = defaultdict(int)
|
|
router_counts: dict[str, int] = defaultdict(int)
|
|
for row in rows:
|
|
for entry in json.loads(row.top_ips_json):
|
|
k = entry["ip"]
|
|
ip_counts[k]["count"] += entry["count"]
|
|
ip_counts[k]["internal"] = entry["internal"]
|
|
ip_counts[k]["country"] = entry.get("country")
|
|
for entry in json.loads(row.top_countries_json):
|
|
country_counts[entry["country"]] += entry["count"]
|
|
for entry in json.loads(row.top_routers_json):
|
|
router_counts[entry["router"]] += entry["count"]
|
|
|
|
def ranked(counts: dict, label: str) -> list[dict]:
|
|
return sorted(({label: k, "count": v} for k, v in counts.items()),
|
|
key=lambda x: x["count"], reverse=True)[:top]
|
|
|
|
return {
|
|
"total": total,
|
|
"internal": sum(r.internal_requests for r in rows),
|
|
"external": external,
|
|
"total_bytes": sum(r.total_bytes for r in rows),
|
|
"external_pct": (external / total * 100.0) if total else 0.0,
|
|
"top_ips": sorted(({"ip": k, **v} for k, v in ip_counts.items()),
|
|
key=lambda x: x["count"], reverse=True)[:top],
|
|
"top_countries": ranked(country_counts, "country"),
|
|
"top_routers": ranked(router_counts, "router"),
|
|
}
|
|
|
|
|
|
@traefik_bp.get("/traffic")
|
|
@require_role(UserRole.viewer)
|
|
async def traffic():
|
|
"""HTMX fragment: traffic origin summary scoped to selected time range."""
|
|
since, range_key = parse_range(request.args.get("range"))
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(
|
|
select(TraefikAccessSummary)
|
|
.where(TraefikAccessSummary.period_end >= since)
|
|
.order_by(TraefikAccessSummary.period_end.asc())
|
|
# No LIMIT — all summaries in range needed for accurate totals;
|
|
# sparkline subsamples to 80 points afterwards
|
|
)
|
|
rows = result.scalars().all()
|
|
|
|
if not rows:
|
|
al = _access_log()
|
|
return await render_template(
|
|
"traefik/traffic.html", summary=None, history=[], range_key=range_key,
|
|
access_log_enabled=bool(al.get("enabled", False)), access_log_path=al.get("path", ""),
|
|
)
|
|
|
|
summary = summarize_access(rows)
|
|
|
|
history = [
|
|
{
|
|
"period_end": r.period_end,
|
|
"external_pct": (r.external_requests / r.total_requests * 100.0)
|
|
if r.total_requests else 0.0,
|
|
}
|
|
for r in rows
|
|
]
|
|
|
|
summary["sparkline_external"] = _sparkline([h["external_pct"] for h in subsample(history, 80)])
|
|
|
|
return await render_template(
|
|
"traefik/traffic.html", summary=summary, history=history, range_key=range_key
|
|
)
|