bvandeusenandClaude Opus 5.5
9f14620271
#5741 "Traefik routers are graded against their own normal", milestone 513. - A router's 4xx share and p95 learn their normal (learn_normal in the plugin's setup). Once a router has 5 days of history, a reading over the fixed cutoff but inside its normal is healthy, and the healthy table says it is usual for that router. One out of its normal is a warning that says what is usual, and a metric anomaly. - The 5xx share is a failure: fixed cutoff always, whatever the history or traffic (System 182's ruling). - A 4xx share over fewer requests than Settings -> Thresholds' new "Fewest requests to judge 4xx" (30 per 15 minutes) isn't graded on the page, and isn't judged for anomalies. - Core: learn_normal(gate=(metric, least)) judges a key only while its resource's gate metric averages at least `least`; `least` may be a function, so it follows a setting. learn_normal(since=...) learns only from history after a moment: Traefik latency learns from #5755's per-scrape figures only. - The Traefik page lists the routers' metric anomalies of the last 30 days (_anomaly_card.html). The dashboard widget keeps its why to problems. - Thresholds and docs say when the cutoffs give way to a normal. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
750 lines
32 KiB
Python
750 lines
32 KiB
Python
# plugins/traefik/routes.py
|
|
from __future__ import annotations
|
|
from collections import defaultdict
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace
|
|
from quart import Blueprint, current_app, render_template, request
|
|
from sqlalchemy import func, select
|
|
import json
|
|
import time
|
|
|
|
from steward.auth.middleware import require_role
|
|
from steward.models.users import UserRole
|
|
from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample, time_bucket
|
|
from steward.core import anomalies as anom
|
|
from steward.core.formatting import cert_left, status_class
|
|
from steward.core.norms import norms_at
|
|
from steward.models.metrics import PluginMetricLatest
|
|
from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
|
|
from .scheduler import router_domains
|
|
from .scraper import cert_names, covers
|
|
|
|
traefik_bp = Blueprint("traefik", __name__, template_folder="templates")
|
|
|
|
|
|
def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
|
|
"""Generate an inline SVG sparkline from a list of float values."""
|
|
if len(values) < 2:
|
|
return f'<svg width="{width}" height="{height}"></svg>'
|
|
mn, mx = min(values), max(values)
|
|
if mx == mn:
|
|
mx = mn + 1.0
|
|
step = width / (len(values) - 1)
|
|
pts = []
|
|
for i, v in enumerate(values):
|
|
x = i * step
|
|
y = height - (v - mn) / (mx - mn) * (height - 2) - 1
|
|
pts.append(f"{x:.1f},{y:.1f}")
|
|
poly = " ".join(pts)
|
|
return (
|
|
f'<svg width="{width}" height="{height}" viewBox="0 0 {width} {height}" '
|
|
f'style="vertical-align:middle;">'
|
|
f'<polyline points="{poly}" fill="none" stroke="#6060c0" stroke-width="1.5"/>'
|
|
f'</svg>'
|
|
)
|
|
|
|
|
|
_GRADE_ORDER = {"crit": 0, "warn": 1, "ok": 2}
|
|
|
|
|
|
def error_cutoffs(thresholds: dict) -> tuple[float, float]:
|
|
"""(5xx, 4xx): the error shares, in percent, past which a router is failing
|
|
or worth a look. Settings → Thresholds, Traefik routers (#5380)."""
|
|
return (float(thresholds.get("error_5xx", {}).get("crit", 0.1)),
|
|
float(thresholds.get("error_4xx", {}).get("warn", 1.0)))
|
|
|
|
|
|
# Each router's 4xx share and p95 are judged against its own normal once it
|
|
# has one (#5741, System 182): an error-pages service answers 4xx all day and
|
|
# that is its normal, not a problem. Departing by less than the floor is never
|
|
# news. A share over fewer than min_requests() in the window is not graded: one
|
|
# 404 of three requests is 33%. The 5xx share is a failure and keeps its
|
|
# fixed cutoff whatever the history or traffic (the operator's ruling).
|
|
FLOOR_4XX = 2.0 # percentage points
|
|
FLOOR_P95 = 50.0 # ms
|
|
NORMED = ("error_rate_4xx_pct", "latency_p95_ms")
|
|
|
|
|
|
def min_requests(thresholds: dict) -> float:
|
|
"""Fewest requests over the STATE_WINDOW for a 4xx share to be graded
|
|
(Settings → Thresholds, traefik.threshold.min_requests)."""
|
|
return float(thresholds.get("router_requests", {}).get("warn", 30))
|
|
|
|
|
|
def _against(value: float, norm, floor: float, unit: str) -> tuple[bool, str]:
|
|
"""(departs, what is usual) for a reading over its fixed cutoff."""
|
|
if norm is None:
|
|
return True, ""
|
|
usual = f"usual {norm.usual_hold:.1f}{unit}" if unit == "%" else f"usual {norm.usual_hold:.0f}{unit}"
|
|
return anom.departs(value, norm, floor), f"{usual} at this hour"
|
|
|
|
|
|
def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = None) -> tuple[str, str | None]:
|
|
"""(grade, why) for a router's reading (recent_window's, or one scrape's):
|
|
crit while it serves 5xx past its cutoff, warn on a p95 past the latency
|
|
critical cutoff or a 4xx share past its cutoff (all Settings → Thresholds),
|
|
otherwise ok. `norms` ({metric: Norm}) are the router's normals for this
|
|
hour: a p95 or 4xx share past its cutoff but within its normal is ok, and
|
|
`why` then says it is usual. `over` names the span the reading covers."""
|
|
norms = norms or {}
|
|
error_5xx, error_4xx = error_cutoffs(thresholds)
|
|
if (reading.error_rate_5xx_pct or 0) > error_5xx:
|
|
return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}"
|
|
usual = []
|
|
p95 = reading.latency_p95_ms
|
|
if p95 is not None and status_class(p95, "latency", thresholds) == "crit":
|
|
out, said = _against(p95, norms.get("latency_p95_ms"), FLOOR_P95, " ms")
|
|
if out:
|
|
return "warn", f"p95 latency is {p95:.0f} ms{over}" + (f", {said}" if said else "")
|
|
usual.append(f"p95 {p95:.0f} ms is {said}")
|
|
requests = getattr(reading, "requests", None)
|
|
e4 = reading.error_rate_4xx_pct or 0
|
|
if e4 > error_4xx and (requests is None or requests >= min_requests(thresholds)):
|
|
out, said = _against(e4, norms.get("error_rate_4xx_pct"), FLOOR_4XX, "%")
|
|
if out:
|
|
return "warn", f"{e4:.1f}% of responses are 4xx{over}" + (f", {said}" if said else "")
|
|
usual.append(f"4xx {e4:.1f}% is {said}")
|
|
return "ok", ("; ".join(usual) or None)
|
|
|
|
|
|
async def router_norms(db, names: list[str], now: datetime) -> dict[str, dict]:
|
|
"""{router: {metric: Norm}}: each router's learned normals for this hour.
|
|
A router without enough history yet is absent, and graded on the cutoffs."""
|
|
got = await norms_at(db, [("traefik", n, m) for n in names for m in NORMED], now)
|
|
out: dict[str, dict] = {}
|
|
for (_s, router, metric), norm in got.items():
|
|
out.setdefault(router, {})[metric] = norm
|
|
return out
|
|
|
|
|
|
# A router is graded on its last STATE_WINDOW of traffic, not on one scrape:
|
|
# a share near a cutoff crossed it on one scrape and fell back on the next,
|
|
# so problem cards came and went every poll (#5214). Over the window a real
|
|
# problem stays listed until it has been clean a while, and a one-scrape
|
|
# blip is diluted rather than shown.
|
|
STATE_WINDOW = timedelta(minutes=15)
|
|
STATE_WINDOW_WORDS = " over the last 15 min"
|
|
|
|
|
|
async def recent_window(db, names: list[str], now: datetime) -> dict[str, SimpleNamespace]:
|
|
"""{router: one reading of its last STATE_WINDOW}: the 4xx and 5xx shares
|
|
weighted by each scrape's traffic (an idle scrape's 100% counts for
|
|
little), p95 averaged, and about how many requests it answered. A router
|
|
with no rows in the window is absent."""
|
|
if not names:
|
|
return {}
|
|
rate = TraefikMetric.request_rate
|
|
result = await db.execute(
|
|
select(
|
|
TraefikMetric.router_name,
|
|
func.sum(rate).label("req"),
|
|
func.sum(rate * TraefikMetric.error_rate_4xx_pct).label("e4w"),
|
|
func.sum(rate * TraefikMetric.error_rate_5xx_pct).label("e5w"),
|
|
func.avg(TraefikMetric.error_rate_4xx_pct).label("e4"),
|
|
func.avg(TraefikMetric.error_rate_5xx_pct).label("e5"),
|
|
func.avg(TraefikMetric.latency_p95_ms).label("p95"),
|
|
func.avg(rate).label("rate"),
|
|
)
|
|
.where(TraefikMetric.router_name.in_(names))
|
|
.where(TraefikMetric.scraped_at >= now - STATE_WINDOW)
|
|
.group_by(TraefikMetric.router_name)
|
|
)
|
|
out = {}
|
|
for r in result.all():
|
|
req = r.req or 0.0
|
|
out[r.router_name] = SimpleNamespace(
|
|
error_rate_4xx_pct=r.e4w / req if req > 0 else r.e4,
|
|
error_rate_5xx_pct=r.e5w / req if req > 0 else r.e5,
|
|
latency_p95_ms=r.p95,
|
|
requests=(r.rate or 0.0) * STATE_WINDOW.total_seconds(),
|
|
)
|
|
return out
|
|
|
|
|
|
async def status_codes(db, names: list[str], since: datetime) -> dict[str, list[tuple[str, int]]]:
|
|
"""{router: [(code, responses), …]} for the 4xx and 5xx codes each router
|
|
answered since `since`, most first (#5711). A router with none is absent."""
|
|
if not names:
|
|
return {}
|
|
total = func.sum(TraefikStatusCode.count)
|
|
rows = (await db.execute(
|
|
select(TraefikStatusCode.router_name, TraefikStatusCode.code, total.label("n"))
|
|
.where(TraefikStatusCode.router_name.in_(names))
|
|
.where(TraefikStatusCode.scraped_at >= since)
|
|
.group_by(TraefikStatusCode.router_name, TraefikStatusCode.code)
|
|
.order_by(TraefikStatusCode.router_name, total.desc(), TraefikStatusCode.code)
|
|
)).all()
|
|
out: dict[str, list[tuple[str, int]]] = {}
|
|
for r in rows:
|
|
out.setdefault(r.router_name, []).append((r.code, int(r.n)))
|
|
return out
|
|
|
|
|
|
def grade_routers(routers: list[dict], window: dict, thresholds: dict, norms: dict | None = None) -> None:
|
|
"""Set each router's state and why from its recent window, or from its
|
|
latest scrape when the window holds nothing, against its normals
|
|
(router_norms') where it has them."""
|
|
norms = norms or {}
|
|
for r in routers:
|
|
reading, own = window.get(r["name"]), norms.get(r["name"])
|
|
r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS, own) if reading
|
|
else router_state(r["latest"], thresholds, norms=own))
|
|
|
|
|
|
# A router missing from this many scrapes in a row is no longer reported:
|
|
# Traefik dropped it (a removed service), so its last state is history.
|
|
GONE_AFTER_SCRAPES = 3
|
|
|
|
|
|
def gone_routers(routers: list[dict], scrape_seconds: int) -> tuple[list[dict], list[dict]]:
|
|
"""(current, gone): a router whose newest row trails the newest scrape of
|
|
any router by GONE_AFTER_SCRAPES intervals is gone, newest-seen first."""
|
|
if not routers:
|
|
return [], []
|
|
newest = max(r["latest"].scraped_at for r in routers)
|
|
cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
|
|
current = [r for r in routers if r["latest"].scraped_at >= cutoff]
|
|
gone = sorted((r for r in routers if r["latest"].scraped_at < cutoff),
|
|
key=lambda r: r["latest"].scraped_at, reverse=True)
|
|
return current, gone
|
|
|
|
|
|
def problem_since(history: list, thresholds: dict) -> tuple[datetime | None, bool]:
|
|
"""(start, before_range) of a router's current problem, read from its
|
|
history buckets (router_history's, oldest first): the start of the
|
|
trailing run of buckets that grade as a problem. (None, False) when the
|
|
newest bucket reads ok, as for a problem too recent or too brief for a
|
|
bucket to show. `before_range` says the run reaches back to the first
|
|
bucket, so the problem began before the range shown."""
|
|
start = None
|
|
for point in reversed(history):
|
|
if router_state(point, thresholds)[0] == "ok":
|
|
break
|
|
start = point
|
|
if start is None:
|
|
return None, False
|
|
return start.bucket, start is history[0]
|
|
|
|
|
|
def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]:
|
|
"""(problems, healthy): failing routers before slow or noisy ones, each
|
|
group by name; healthy routers by name."""
|
|
problems = sorted((r for r in routers if r["state"] != "ok"),
|
|
key=lambda r: (_GRADE_ORDER[r["state"]], r["name"].lower()))
|
|
healthy = sorted((r for r in routers if r["state"] == "ok"), key=lambda r: r["name"].lower())
|
|
return problems, healthy
|
|
|
|
|
|
# The figures the scraper records for every router (scheduler._do_scrape).
|
|
ROUTER_METRICS = ("request_rate", "error_rate_4xx_pct", "error_rate_5xx_pct", "latency_p50_ms",
|
|
"latency_p95_ms", "latency_p99_ms", "response_bytes_rate")
|
|
|
|
|
|
async def latest_per_router(db) -> list[SimpleNamespace]:
|
|
"""Each router's newest scrape, read from plugin_metric_latest: the scraper
|
|
records every figure through MetricBatch, which keeps one row per router
|
|
and figure there (latest_by_resource's table, #5208). A walk of
|
|
traefik_metrics' index, one step per router, took 7 s on the live
|
|
instance (#5355); each step lands on the router's oldest entries, which
|
|
retention deletes every minute, so it likely waded through dead ones.
|
|
|
|
Each router reads as one scrape, with the attributes a TraefikMetric row
|
|
has; certificates, recorded under the same source, are left out."""
|
|
rows = (await db.execute(select(PluginMetricLatest).where(
|
|
PluginMetricLatest.source_module == "traefik",
|
|
PluginMetricLatest.metric_name.in_(ROUTER_METRICS)))).scalars()
|
|
by_router: dict[str, dict] = {}
|
|
for r in rows:
|
|
by_router.setdefault(r.resource_name, {})[r.metric_name] = r
|
|
return [SimpleNamespace(router_name=name, scraped_at=max(m.recorded_at for m in figures.values()),
|
|
**{k: figures[k].value if k in figures else None for k in ROUTER_METRICS})
|
|
for name, figures in by_router.items() if "request_rate" in figures]
|
|
|
|
|
|
def current_certs(certs, scrape_seconds: int) -> tuple[list, list]:
|
|
"""(served, old): a certificate is served while Traefik still reports it.
|
|
Rows are upserted by serial and never deleted, so a renewed or retired
|
|
certificate lingers; one missing from the last GONE_AFTER_SCRAPES
|
|
scrapes is old (#5254)."""
|
|
certs = list(certs)
|
|
if not certs:
|
|
return [], []
|
|
newest = max(c.scraped_at for c in certs)
|
|
cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
|
|
return ([c for c in certs if c.scraped_at >= cutoff],
|
|
sorted((c for c in certs if c.scraped_at < cutoff), key=lambda c: c.scraped_at, reverse=True))
|
|
|
|
|
|
def router_hosts(name: str, domains: dict[str, list[str]]) -> list[str]:
|
|
"""The hosts a metrics router name serves. The API names a router
|
|
"app@docker"; metrics may carry it as "websecure-app@docker", one per
|
|
entrypoint, so a name not found as-is is tried without that prefix."""
|
|
if name in domains:
|
|
return domains[name]
|
|
for api_name, hosts in domains.items():
|
|
if name.endswith("-" + api_name):
|
|
return hosts
|
|
return []
|
|
|
|
|
|
def certs_by_router(routers: list[str], domains: dict[str, list[str]], certs, now) -> dict[str, dict]:
|
|
"""{router: {"cn", "days"}}: the soonest-expiring served certificate valid
|
|
for one of the router's hosts. A router with no hosts, or none covered,
|
|
is absent."""
|
|
out: dict[str, dict] = {}
|
|
names = [(c, cert_names(c.cn, c.sans)) for c in certs]
|
|
for router in routers:
|
|
for host in router_hosts(router, domains):
|
|
for c, cn in names:
|
|
if covers(cn, host):
|
|
days = (c.not_after - now).total_seconds() / 86400.0
|
|
if router not in out or days < out[router]["days"]:
|
|
out[router] = {"cn": c.cn or host, "days": days}
|
|
return out
|
|
|
|
|
|
def _access_log() -> dict:
|
|
"""The plugin's access_log settings ({} when unset or malformed)."""
|
|
cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {})
|
|
return cfg if isinstance(cfg, dict) else {}
|
|
|
|
|
|
@traefik_bp.get("/")
|
|
@require_role(UserRole.viewer)
|
|
async def index():
|
|
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
|
|
access_log_enabled = bool(_access_log().get("enabled", False))
|
|
current_range = request.args.get("range", DEFAULT_RANGE)
|
|
return await render_template(
|
|
"traefik/index.html",
|
|
poll_interval=poll_interval,
|
|
access_log_enabled=access_log_enabled,
|
|
current_range=current_range,
|
|
)
|
|
|
|
|
|
@traefik_bp.get("/rows")
|
|
@require_role(UserRole.viewer)
|
|
async def rows():
|
|
"""HTMX fragment: service cards with sparklines scoped to selected time range."""
|
|
since, range_key = parse_range(request.args.get("range"))
|
|
now = datetime.now(timezone.utc)
|
|
thresholds = current_app.config.get("THRESHOLDS", {})
|
|
|
|
scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
|
|
phases = _Phases()
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
# Every known router, with its most recent raw row; a router Traefik
|
|
# stopped reporting gets no sparklines, so its history is not read.
|
|
newest = await latest_per_router(db)
|
|
current, gone = gone_routers([{"name": m.router_name, "latest": m} for m in newest], scrape)
|
|
phases.mark("latest")
|
|
|
|
names = [r["name"] for r in current]
|
|
history = await router_history(db, names, since, bucket_seconds(since))
|
|
phases.mark("history")
|
|
grade_routers(current, await recent_window(db, names, now), thresholds,
|
|
await router_norms(db, names, now))
|
|
phases.mark("window")
|
|
# The routers' metric anomalies, last 30 days (#5741).
|
|
anomalies = await anom.views(db, await anom.list_anomalies(
|
|
db, source="traefik", since=now - timedelta(days=30), limit=20), links=False)
|
|
for r in current:
|
|
points = history[r["name"]]
|
|
r["since"], r["since_before"] = (problem_since(points, thresholds) if r["state"] != "ok"
|
|
else (None, False))
|
|
r["sparkline_req"] = _sparkline([p.request_rate for p in points])
|
|
r["sparkline_bytes"] = _sparkline([p.response_bytes_rate for p in points])
|
|
r["sparkline_p95"] = _sparkline([p.latency_p95_ms for p in points if p.latency_p95_ms is not None])
|
|
r["sparkline_4xx"] = _sparkline([p.error_rate_4xx_pct for p in points])
|
|
r["sparkline_5xx"] = _sparkline([p.error_rate_5xx_pct for p in points])
|
|
phases.mark("sparklines")
|
|
# The codes behind each problem, over the window it is graded on.
|
|
codes = await status_codes(db, [r["name"] for r in current if r["state"] != "ok"], now - STATE_WINDOW)
|
|
for r in current:
|
|
r["codes"] = codes.get(r["name"], [])
|
|
# Codes are kept from #5711 on: until the first of them is a window
|
|
# old, a router with none listed may simply predate the recording.
|
|
first_code = await db.scalar(select(func.min(TraefikStatusCode.scraped_at)))
|
|
codes_known = first_code is not None and first_code <= now - STATE_WINDOW
|
|
phases.mark("codes")
|
|
|
|
# Latest global stat (always most recent, not range-filtered)
|
|
result = await db.execute(
|
|
select(TraefikGlobalStat)
|
|
.order_by(TraefikGlobalStat.scraped_at.desc())
|
|
.limit(1)
|
|
)
|
|
global_stat = result.scalar_one_or_none()
|
|
|
|
# All certs, sorted by soonest expiry
|
|
result = await db.execute(
|
|
select(TraefikCert).order_by(TraefikCert.not_after)
|
|
)
|
|
raw_certs = result.scalars().all()
|
|
phases.mark("stats_certs")
|
|
|
|
served, old = current_certs(raw_certs, scrape)
|
|
domains = router_domains()
|
|
|
|
def cert_row(c):
|
|
names = cert_names(c.cn, c.sans)
|
|
return {
|
|
"serial": c.serial,
|
|
"cn": c.cn or c.serial,
|
|
"sans": [n for n in names if n != c.cn],
|
|
"not_after": c.not_after,
|
|
"days_remaining": (c.not_after - now).total_seconds() / 86400.0,
|
|
# The routers whose hosts it covers; None when Traefik's API
|
|
# couldn't say which hosts each router serves.
|
|
"routers": None if domains is None else sorted(
|
|
{r["name"].split("@")[0] for r in current
|
|
for h in router_hosts(r["name"], domains) if covers(names, h)}),
|
|
}
|
|
|
|
certs = [cert_row(c) for c in served]
|
|
old_certs = [cert_row(c) for c in old]
|
|
problems, healthy = split_routers(current)
|
|
phases.mark("cert_rows")
|
|
error_5xx, error_4xx = error_cutoffs(thresholds)
|
|
page = await render_template(
|
|
"traefik/rows.html",
|
|
problems=problems,
|
|
healthy=healthy,
|
|
gone=gone,
|
|
current_range=range_key,
|
|
codes_known=codes_known,
|
|
error_4xx=error_4xx,
|
|
error_5xx=error_5xx,
|
|
global_stat=global_stat,
|
|
certs=certs,
|
|
old_certs=old_certs,
|
|
range_key=range_key,
|
|
anomalies=anomalies,
|
|
back="/plugins/traefik/",
|
|
base_resource=None,
|
|
)
|
|
phases.mark("render")
|
|
phases.report("rows", f"routers={len(current)} range={range_key}")
|
|
return page
|
|
|
|
|
|
@traefik_bp.get("/widget")
|
|
@require_role(UserRole.viewer)
|
|
async def widget():
|
|
"""The dashboard Routers widget (#5254), in the widget-shape order: a
|
|
verdict, routers in trouble with why, then the busiest healthy ones under
|
|
one header. Only routers and certificates Traefik still reports. A
|
|
certificate's expiry shows on the router it serves, and makes that router
|
|
a problem once it is inside the Settings → Thresholds TLS window."""
|
|
now = datetime.now(timezone.utc)
|
|
thresholds = current_app.config.get("THRESHOLDS", {})
|
|
scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
|
|
|
|
phases = _Phases()
|
|
async with current_app.db_sessionmaker() as db:
|
|
latest = await latest_per_router(db)
|
|
phases.mark("latest")
|
|
current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
|
|
window = await recent_window(db, [r["name"] for r in current], now)
|
|
norms = await router_norms(db, [r["name"] for r in current], now)
|
|
phases.mark("window")
|
|
certs = (await db.execute(select(TraefikCert))).scalars().all()
|
|
phases.mark("certs")
|
|
phases.report("widget", f"routers={len(current)}")
|
|
|
|
served, _old = current_certs(certs, scrape)
|
|
domains = router_domains()
|
|
by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
|
|
grade_routers(current, window, thresholds, norms)
|
|
for r in current:
|
|
r["cert"] = by_router.get(r["name"])
|
|
if r["cert"]:
|
|
grade = status_class(r["cert"]["days"], "tls_days", thresholds)
|
|
r["cert"]["grade"] = grade
|
|
if _GRADE_ORDER.get(grade, 2) < _GRADE_ORDER[r["state"]]:
|
|
left = cert_left(r["cert"]["days"]) # "12 days", "expires today", "expired …"
|
|
r["state"], r["why"] = grade, (
|
|
f"certificate expires in {left}" if left[0].isdigit() else f"certificate {left}")
|
|
problems, healthy = split_routers(current)
|
|
healthy.sort(key=lambda r: -(r["latest"].request_rate or 0))
|
|
# With no router domains to go on, certificates can't sit on their
|
|
# routers: the expiring ones are listed after the routers instead.
|
|
loose = [] if domains is not None else sorted(
|
|
({"cn": c.cn or c.serial, "days": (c.not_after - now).total_seconds() / 86400.0} for c in served),
|
|
key=lambda c: c["days"])
|
|
loose = [c for c in loose if status_class(c["days"], "tls_days", thresholds) in ("warn", "crit")]
|
|
return await render_template(
|
|
"traefik/widget.html",
|
|
problems=problems, healthy=healthy[:ROUTERS_SHOWN],
|
|
healthy_more=max(0, len(healthy) - ROUTERS_SHOWN),
|
|
total=len(current), loose_certs=loose, domains_known=domains is not None,
|
|
)
|
|
|
|
|
|
# Healthy routers on the dashboard widget (the busiest); the page lists all.
|
|
ROUTERS_SHOWN = 6
|
|
|
|
|
|
@traefik_bp.get("/widget/access_log")
|
|
@require_role(UserRole.viewer)
|
|
async def widget_access_log():
|
|
"""HTMX dashboard widget: traffic origin summary with IP filter."""
|
|
from collections import defaultdict
|
|
ip_filter = request.args.get("ip_filter", "all")
|
|
limit = max(1, min(50, int(request.args.get("limit", 10) or 10)))
|
|
widget_id = request.args.get("wid", "0")
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(
|
|
select(TraefikAccessSummary)
|
|
.order_by(TraefikAccessSummary.period_end.desc())
|
|
.limit(12) # last ~1 hour of 5-min windows
|
|
)
|
|
rows = result.scalars().all()
|
|
|
|
if not rows:
|
|
return await render_template("traefik/widget_access_log.html",
|
|
summary=None, widget_id=widget_id)
|
|
|
|
total = sum(r.total_requests for r in rows)
|
|
internal = sum(r.internal_requests for r in rows)
|
|
external = sum(r.external_requests for r in rows)
|
|
|
|
ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True})
|
|
for row in rows:
|
|
for entry in json.loads(row.top_ips_json):
|
|
k = entry["ip"]
|
|
ip_counts[k]["count"] += entry["count"]
|
|
ip_counts[k]["internal"] = entry.get("internal", True)
|
|
|
|
all_ips = sorted(
|
|
[{"ip": k, **v} for k, v in ip_counts.items()],
|
|
key=lambda x: x["count"], reverse=True,
|
|
)
|
|
if ip_filter == "internal":
|
|
all_ips = [e for e in all_ips if e["internal"]]
|
|
elif ip_filter == "external":
|
|
all_ips = [e for e in all_ips if not e["internal"]]
|
|
|
|
summary = {
|
|
"total": total,
|
|
"internal": internal,
|
|
"external": external,
|
|
"external_pct": (external / total * 100.0) if total else 0.0,
|
|
"top_ips": all_ips[:limit],
|
|
"ip_filter": ip_filter,
|
|
}
|
|
return await render_template("traefik/widget_access_log.html",
|
|
summary=summary, widget_id=widget_id)
|
|
|
|
|
|
# A rows or widget request past this logs where its time went, phase by
|
|
# phase, so a slow query names itself (#5355: 8 s on every poll, warm).
|
|
SLOW_PAGE_SECONDS = 1.0
|
|
|
|
|
|
class _Phases:
|
|
"""Wall-clock per phase of one request; report() logs them when slow."""
|
|
|
|
def __init__(self):
|
|
self.start = self.last = time.monotonic()
|
|
self.took: dict[str, float] = {}
|
|
|
|
def mark(self, name: str) -> None:
|
|
now = time.monotonic()
|
|
self.took[name] = now - self.last
|
|
self.last = now
|
|
|
|
def report(self, what: str, detail: str = "") -> None:
|
|
total = time.monotonic() - self.start
|
|
if total >= SLOW_PAGE_SECONDS:
|
|
current_app.logger.warning("traefik %s: slow %.1fs (%s) %s", what, total,
|
|
" ".join(f"{k}={v:.2f}s" for k, v in self.took.items()), detail)
|
|
|
|
|
|
async def router_history(db, names: list[str], since, bucket_secs: int) -> dict[str, list]:
|
|
"""{router: [rows of request_rate, latency_p95_ms, error_rate_5xx_pct,
|
|
error_rate_4xx_pct, response_bytes_rate]}
|
|
per `bucket_secs` window since `since`, oldest first: one query for every
|
|
router, answered from the covering (router_name, scraped_at) index
|
|
(traefik_004). It was a query per router, each row a heap read, which
|
|
took the rows page to 12-28 s."""
|
|
history: dict[str, list] = {n: [] for n in names}
|
|
if not history:
|
|
return history
|
|
bucket_col = time_bucket(TraefikMetric.scraped_at, bucket_secs)
|
|
result = await db.execute(
|
|
select(
|
|
TraefikMetric.router_name,
|
|
func.avg(TraefikMetric.request_rate).label("request_rate"),
|
|
func.avg(TraefikMetric.latency_p95_ms).label("latency_p95_ms"),
|
|
func.avg(TraefikMetric.error_rate_5xx_pct).label("error_rate_5xx_pct"),
|
|
func.avg(TraefikMetric.error_rate_4xx_pct).label("error_rate_4xx_pct"),
|
|
func.avg(TraefikMetric.response_bytes_rate).label("response_bytes_rate"),
|
|
bucket_col,
|
|
)
|
|
.where(TraefikMetric.router_name.in_(names))
|
|
.where(TraefikMetric.scraped_at >= since)
|
|
.group_by(TraefikMetric.router_name, bucket_col)
|
|
.order_by(TraefikMetric.router_name, bucket_col)
|
|
)
|
|
for row in result.all():
|
|
history[row.router_name].append(row)
|
|
return history
|
|
|
|
|
|
async def request_history(db, since, bucket_seconds: int) -> list[dict]:
|
|
"""[{"at", "req_rate", "err_pct"}] per time bucket since `since`: each
|
|
scrape summed over routers (the total req/s), averaged over the bucket;
|
|
the 5xx share weighted by each router's traffic, so a quiet router's
|
|
errors don't count as much as a busy one's."""
|
|
per_scrape = (
|
|
select(
|
|
TraefikMetric.scraped_at.label("at"),
|
|
func.sum(TraefikMetric.request_rate).label("req"),
|
|
func.sum(TraefikMetric.request_rate * TraefikMetric.error_rate_5xx_pct).label("err_weighted"),
|
|
)
|
|
.where(TraefikMetric.scraped_at >= since)
|
|
.group_by(TraefikMetric.scraped_at)
|
|
.subquery()
|
|
)
|
|
bucket_col = time_bucket(per_scrape.c.at, bucket_seconds)
|
|
rows = (await db.execute(
|
|
select(
|
|
func.avg(per_scrape.c.req).label("req_rate"),
|
|
func.sum(per_scrape.c.err_weighted).label("err_weighted"),
|
|
func.sum(per_scrape.c.req).label("req_sum"),
|
|
func.min(per_scrape.c.at).label("at"),
|
|
bucket_col,
|
|
)
|
|
.group_by(bucket_col)
|
|
.order_by(bucket_col)
|
|
)).all()
|
|
return [{"at": r.at, "req_rate": round(r.req_rate or 0, 3),
|
|
"err_pct": round((r.err_weighted or 0) / r.req_sum, 2) if r.req_sum else 0.0}
|
|
for r in rows]
|
|
|
|
|
|
@traefik_bp.get("/widget/request_chart")
|
|
@require_role(UserRole.viewer)
|
|
async def widget_request_chart():
|
|
"""HTMX dashboard widget: requests per second across all routers, and the
|
|
share of them answered 5xx, over the last few hours (#5256). Each scrape
|
|
is summed over routers first (the total), then averaged per bucket; the
|
|
5xx share is weighted by each router's traffic, not averaged per router."""
|
|
widget_id = request.args.get("wid", "0")
|
|
if request.args.get("range"):
|
|
# The Traefik page: the page's own time range (#5710).
|
|
since, span = parse_range(request.args.get("range"))
|
|
b_secs = bucket_seconds(since)
|
|
else:
|
|
hours = max(1, min(24, int(request.args.get("hours", 6) or 6)))
|
|
since, span = datetime.now(timezone.utc) - timedelta(hours=hours), f"{hours}h"
|
|
b_secs = max(60, (hours * 3600) // 80) # ~80 buckets
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
history = await request_history(db, since, b_secs)
|
|
|
|
labels = [int(r["at"].timestamp() * 1000) for r in history] # epoch ms; the browser shows local time
|
|
req_rates = [r["req_rate"] for r in history]
|
|
err_pcts = [r["err_pct"] for r in history]
|
|
|
|
return await render_template(
|
|
"traefik/widget_request_chart.html",
|
|
labels_json=json.dumps(labels),
|
|
req_rate_json=json.dumps(req_rates),
|
|
error_rate_json=json.dumps(err_pcts),
|
|
widget_id=widget_id,
|
|
span=span,
|
|
days=(datetime.now(timezone.utc) - since) > timedelta(hours=24),
|
|
now_req=req_rates[-1] if req_rates else None,
|
|
peak_req=max(req_rates) if req_rates else None,
|
|
peak_err=max(err_pcts) if err_pcts else None,
|
|
)
|
|
|
|
|
|
@traefik_bp.get("/traffic")
|
|
@require_role(UserRole.viewer)
|
|
async def traffic():
|
|
"""HTMX fragment: traffic origin summary scoped to selected time range."""
|
|
since, range_key = parse_range(request.args.get("range"))
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(
|
|
select(TraefikAccessSummary)
|
|
.where(TraefikAccessSummary.period_end >= since)
|
|
.order_by(TraefikAccessSummary.period_end.asc())
|
|
# No LIMIT — all summaries in range needed for accurate totals;
|
|
# sparkline subsamples to 80 points afterwards
|
|
)
|
|
rows = result.scalars().all()
|
|
|
|
if not rows:
|
|
al = _access_log()
|
|
return await render_template(
|
|
"traefik/traffic.html", summary=None, history=[], range_key=range_key,
|
|
access_log_enabled=bool(al.get("enabled", False)), access_log_path=al.get("path", ""),
|
|
)
|
|
|
|
total = sum(r.total_requests for r in rows)
|
|
internal = sum(r.internal_requests for r in rows)
|
|
external = sum(r.external_requests for r in rows)
|
|
total_bytes = sum(r.total_bytes for r in rows)
|
|
|
|
ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True, "country": None})
|
|
country_counts: dict[str, int] = defaultdict(int)
|
|
router_counts: dict[str, int] = defaultdict(int)
|
|
|
|
for row in rows:
|
|
for entry in json.loads(row.top_ips_json):
|
|
k = entry["ip"]
|
|
ip_counts[k]["count"] += entry["count"]
|
|
ip_counts[k]["internal"] = entry["internal"]
|
|
ip_counts[k]["country"] = entry.get("country")
|
|
for entry in json.loads(row.top_countries_json):
|
|
country_counts[entry["country"]] += entry["count"]
|
|
for entry in json.loads(row.top_routers_json):
|
|
router_counts[entry["router"]] += entry["count"]
|
|
|
|
top_ips = sorted(
|
|
[{"ip": k, **v} for k, v in ip_counts.items()],
|
|
key=lambda x: x["count"], reverse=True
|
|
)[:10]
|
|
top_countries = sorted(
|
|
[{"country": k, "count": v} for k, v in country_counts.items()],
|
|
key=lambda x: x["count"], reverse=True
|
|
)[:10]
|
|
top_routers = sorted(
|
|
[{"router": k, "count": v} for k, v in router_counts.items()],
|
|
key=lambda x: x["count"], reverse=True
|
|
)[:10]
|
|
|
|
history = [
|
|
{
|
|
"period_end": r.period_end,
|
|
"external_pct": (r.external_requests / r.total_requests * 100.0)
|
|
if r.total_requests else 0.0,
|
|
}
|
|
for r in rows
|
|
]
|
|
|
|
summary = {
|
|
"total": total,
|
|
"internal": internal,
|
|
"external": external,
|
|
"total_bytes": total_bytes,
|
|
"external_pct": (external / total * 100.0) if total else 0.0,
|
|
"top_ips": top_ips,
|
|
"top_countries": top_countries,
|
|
"top_routers": top_routers,
|
|
"sparkline_external": _sparkline(
|
|
[h["external_pct"] for h in subsample(history, 80)]
|
|
),
|
|
}
|
|
|
|
return await render_template(
|
|
"traefik/traffic.html", summary=summary, history=history, range_key=range_key
|
|
)
|