feat(traefik): Routers widget shows current routers only, one header row, certs on the routers they serve (#5254)
#5254 "Traefik — Routers: current routers only, header row, certs on the router they serve". - Router hosts come from Traefik's API (/api/http|tcp/routers), read best-effort on each scrape. An optional api_url overrides it; the default is /api on the metrics host. A certificate's expiry then shows in a TLS column on the router whose host it covers. An expiring one makes that router a problem, graded by Settings → Thresholds. - Only current: routers and certificates must have appeared in the last three scrapes. Certificates are upserted by serial and never deleted, so renewed or retired ones lingered on the widget. - Widget order: verdict, problem routers with the reason, then the busiest healthy routers under one header (Router / Req/s / Bandwidth / p95 / TLS). A missing value shows "—". - If the API is unreachable, expiring certificates are listed after the routers, not above them. - The page splits certificates into served (with the routers using each, plus SANs) and "No longer served". - Latest row per router: an index walk (recursive CTE + LATERAL) replaces DISTINCT plus a query per router. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -14,6 +14,9 @@ tags:
|
||||
|
||||
config:
|
||||
metrics_url: "http://localhost:8080/metrics"
|
||||
# Traefik's API, for which hosts each router serves (so a certificate's
|
||||
# expiry shows on its router). Empty: /api on the metrics URL's host.
|
||||
api_url: ""
|
||||
scrape_interval_seconds: 60
|
||||
access_log:
|
||||
enabled: false
|
||||
|
||||
+130
-66
@@ -3,14 +3,16 @@ from __future__ import annotations
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from quart import Blueprint, current_app, render_template, request
|
||||
from sqlalchemy import Integer, cast, func, select
|
||||
from sqlalchemy import Integer, cast, func, select, text
|
||||
import json
|
||||
|
||||
from steward.auth.middleware import require_role
|
||||
from steward.models.users import UserRole
|
||||
from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample
|
||||
from steward.core.formatting import status_class
|
||||
from steward.core.formatting import cert_left, status_class
|
||||
from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric
|
||||
from .scheduler import router_domains
|
||||
from .scraper import cert_names, covers
|
||||
|
||||
traefik_bp = Blueprint("traefik", __name__, template_folder="templates")
|
||||
|
||||
@@ -85,6 +87,71 @@ def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]:
|
||||
return problems, healthy
|
||||
|
||||
|
||||
# Newest row per router. Postgres has no skip scan, so DISTINCT or max() GROUP
|
||||
# BY reads every row in retention; this walks the (router_name, scraped_at)
|
||||
# index one router at a time instead: a step per router, not per row.
|
||||
_LATEST_PER_ROUTER = text("""
|
||||
WITH RECURSIVE names AS (
|
||||
SELECT min(router_name) AS name FROM traefik_metrics
|
||||
UNION ALL
|
||||
SELECT (SELECT min(router_name) FROM traefik_metrics WHERE router_name > names.name)
|
||||
FROM names WHERE names.name IS NOT NULL
|
||||
)
|
||||
SELECT m.* FROM names
|
||||
JOIN LATERAL (
|
||||
SELECT * FROM traefik_metrics WHERE router_name = names.name
|
||||
ORDER BY scraped_at DESC LIMIT 1
|
||||
) m ON true
|
||||
""")
|
||||
|
||||
|
||||
async def latest_per_router(db) -> list:
|
||||
return list((await db.execute(
|
||||
select(TraefikMetric).from_statement(_LATEST_PER_ROUTER))).scalars().all())
|
||||
|
||||
|
||||
def current_certs(certs, scrape_seconds: int) -> tuple[list, list]:
|
||||
"""(served, old): a certificate is served while Traefik still reports it.
|
||||
Rows are upserted by serial and never deleted, so a renewed or retired
|
||||
certificate lingers; one missing from the last GONE_AFTER_SCRAPES
|
||||
scrapes is old (#5254)."""
|
||||
certs = list(certs)
|
||||
if not certs:
|
||||
return [], []
|
||||
newest = max(c.scraped_at for c in certs)
|
||||
cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
|
||||
return ([c for c in certs if c.scraped_at >= cutoff],
|
||||
sorted((c for c in certs if c.scraped_at < cutoff), key=lambda c: c.scraped_at, reverse=True))
|
||||
|
||||
|
||||
def router_hosts(name: str, domains: dict[str, list[str]]) -> list[str]:
|
||||
"""The hosts a metrics router name serves. The API names a router
|
||||
"app@docker"; metrics may carry it as "websecure-app@docker", one per
|
||||
entrypoint, so a name not found as-is is tried without that prefix."""
|
||||
if name in domains:
|
||||
return domains[name]
|
||||
for api_name, hosts in domains.items():
|
||||
if name.endswith("-" + api_name):
|
||||
return hosts
|
||||
return []
|
||||
|
||||
|
||||
def certs_by_router(routers: list[str], domains: dict[str, list[str]], certs, now) -> dict[str, dict]:
|
||||
"""{router: {"cn", "days"}}: the soonest-expiring served certificate valid
|
||||
for one of the router's hosts. A router with no hosts, or none covered,
|
||||
is absent."""
|
||||
out: dict[str, dict] = {}
|
||||
names = [(c, cert_names(c.cn, c.sans)) for c in certs]
|
||||
for router in routers:
|
||||
for host in router_hosts(router, domains):
|
||||
for c, cn in names:
|
||||
if covers(cn, host):
|
||||
days = (c.not_after - now).total_seconds() / 86400.0
|
||||
if router not in out or days < out[router]["days"]:
|
||||
out[router] = {"cn": c.cn or host, "days": days}
|
||||
return out
|
||||
|
||||
|
||||
def _access_log() -> dict:
|
||||
"""The plugin's access_log settings ({} when unset or malformed)."""
|
||||
cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {})
|
||||
@@ -114,13 +181,9 @@ async def rows():
|
||||
thresholds = current_app.config.get("THRESHOLDS", {})
|
||||
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
# All known routers
|
||||
result = await db.execute(
|
||||
select(TraefikMetric.router_name)
|
||||
.distinct()
|
||||
.order_by(TraefikMetric.router_name)
|
||||
)
|
||||
routers = [row[0] for row in result.all()]
|
||||
# Every known router, with its most recent raw row
|
||||
newest = {m.router_name: m for m in await latest_per_router(db)}
|
||||
routers = sorted(newest)
|
||||
|
||||
b_secs = bucket_seconds(since)
|
||||
bucket_col = (
|
||||
@@ -145,16 +208,7 @@ async def rows():
|
||||
)
|
||||
history = result.all()
|
||||
|
||||
# Latest value — always the most recent raw row
|
||||
result = await db.execute(
|
||||
select(TraefikMetric)
|
||||
.where(TraefikMetric.router_name == router)
|
||||
.order_by(TraefikMetric.scraped_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
latest = result.scalar_one_or_none()
|
||||
if not latest:
|
||||
continue
|
||||
latest = newest[router]
|
||||
|
||||
state, why = router_state(latest, thresholds)
|
||||
router_data.append({
|
||||
@@ -181,19 +235,28 @@ async def rows():
|
||||
)
|
||||
raw_certs = result.scalars().all()
|
||||
|
||||
certs = []
|
||||
for c in raw_certs:
|
||||
days = (c.not_after - now).total_seconds() / 86400.0
|
||||
certs.append({
|
||||
scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
|
||||
current, gone = gone_routers(router_data, scrape)
|
||||
served, old = current_certs(raw_certs, scrape)
|
||||
domains = router_domains()
|
||||
|
||||
def cert_row(c):
|
||||
names = cert_names(c.cn, c.sans)
|
||||
return {
|
||||
"serial": c.serial,
|
||||
"cn": c.cn or c.serial,
|
||||
"sans": c.sans,
|
||||
"sans": [n for n in names if n != c.cn],
|
||||
"not_after": c.not_after,
|
||||
"days_remaining": days,
|
||||
})
|
||||
"days_remaining": (c.not_after - now).total_seconds() / 86400.0,
|
||||
# The routers whose hosts it covers; None when Traefik's API
|
||||
# couldn't say which hosts each router serves.
|
||||
"routers": None if domains is None else sorted(
|
||||
{r["name"].split("@")[0] for r in current
|
||||
for h in router_hosts(r["name"], domains) if covers(names, h)}),
|
||||
}
|
||||
|
||||
current, gone = gone_routers(
|
||||
router_data, current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"])
|
||||
certs = [cert_row(c) for c in served]
|
||||
old_certs = [cert_row(c) for c in old]
|
||||
problems, healthy = split_routers(current)
|
||||
return await render_template(
|
||||
"traefik/rows.html",
|
||||
@@ -204,6 +267,7 @@ async def rows():
|
||||
error_5xx=ERROR_5XX_PCT,
|
||||
global_stat=global_stat,
|
||||
certs=certs,
|
||||
old_certs=old_certs,
|
||||
range_key=range_key,
|
||||
)
|
||||
|
||||
@@ -211,53 +275,53 @@ async def rows():
|
||||
@traefik_bp.get("/widget")
|
||||
@require_role(UserRole.viewer)
|
||||
async def widget():
|
||||
"""HTMX dashboard widget fragment."""
|
||||
"""The dashboard Routers widget (#5254), in the widget-shape order: a
|
||||
verdict, routers in trouble with why, then the busiest healthy ones under
|
||||
one header. Only routers and certificates Traefik still reports. A
|
||||
certificate's expiry shows on the router it serves, and makes that router
|
||||
a problem once it is inside the Settings → Thresholds TLS window."""
|
||||
now = datetime.now(timezone.utc)
|
||||
thresholds = current_app.config.get("THRESHOLDS", {})
|
||||
scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
|
||||
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
result = await db.execute(
|
||||
select(TraefikMetric.router_name).distinct().order_by(TraefikMetric.router_name)
|
||||
)
|
||||
routers = [row[0] for row in result.all()]
|
||||
router_data = []
|
||||
for router in routers:
|
||||
result = await db.execute(
|
||||
select(TraefikMetric)
|
||||
.where(TraefikMetric.router_name == router)
|
||||
.order_by(TraefikMetric.scraped_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
latest = result.scalar_one_or_none()
|
||||
if latest:
|
||||
router_data.append({"name": router, "latest": latest})
|
||||
|
||||
router_data.sort(key=lambda r: (
|
||||
-(1 if r["latest"].error_rate_5xx_pct > 0.1 else 0),
|
||||
-(1 if r["latest"].error_rate_4xx_pct > 5.0 else 0),
|
||||
-r["latest"].request_rate,
|
||||
))
|
||||
total_routers = len(router_data)
|
||||
router_data = router_data[:5]
|
||||
|
||||
result = await db.execute(
|
||||
select(TraefikCert).order_by(TraefikCert.not_after)
|
||||
)
|
||||
raw_certs = result.scalars().all()
|
||||
|
||||
expiring_certs = []
|
||||
for c in raw_certs:
|
||||
days = (c.not_after - now).total_seconds() / 86400.0
|
||||
if days < 30:
|
||||
expiring_certs.append({"cn": c.cn or c.serial, "days_remaining": days})
|
||||
latest = await latest_per_router(db)
|
||||
certs = (await db.execute(select(TraefikCert))).scalars().all()
|
||||
|
||||
current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
|
||||
served, _old = current_certs(certs, scrape)
|
||||
domains = router_domains()
|
||||
by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
|
||||
for r in current:
|
||||
r["state"], r["why"] = router_state(r["latest"], thresholds)
|
||||
r["cert"] = by_router.get(r["name"])
|
||||
if r["cert"]:
|
||||
grade = status_class(r["cert"]["days"], "tls_days", thresholds)
|
||||
r["cert"]["grade"] = grade
|
||||
if _GRADE_ORDER.get(grade, 2) < _GRADE_ORDER[r["state"]]:
|
||||
left = cert_left(r["cert"]["days"]) # "12 days", "expires today", "expired …"
|
||||
r["state"], r["why"] = grade, (
|
||||
f"certificate expires in {left}" if left[0].isdigit() else f"certificate {left}")
|
||||
problems, healthy = split_routers(current)
|
||||
healthy.sort(key=lambda r: -(r["latest"].request_rate or 0))
|
||||
# With no router domains to go on, certificates can't sit on their
|
||||
# routers: the expiring ones are listed after the routers instead.
|
||||
loose = [] if domains is not None else sorted(
|
||||
({"cn": c.cn or c.serial, "days": (c.not_after - now).total_seconds() / 86400.0} for c in served),
|
||||
key=lambda c: c["days"])
|
||||
loose = [c for c in loose if status_class(c["days"], "tls_days", thresholds) in ("warn", "crit")]
|
||||
return await render_template(
|
||||
"traefik/widget.html",
|
||||
router_data=router_data,
|
||||
expiring_certs=expiring_certs,
|
||||
total_routers=total_routers,
|
||||
problems=problems, healthy=healthy[:ROUTERS_SHOWN],
|
||||
healthy_more=max(0, len(healthy) - ROUTERS_SHOWN),
|
||||
total=len(current), loose_certs=loose, domains_known=domains is not None,
|
||||
)
|
||||
|
||||
|
||||
# Healthy routers on the dashboard widget (the busiest); the page lists all.
|
||||
ROUTERS_SHOWN = 6
|
||||
|
||||
|
||||
@traefik_bp.get("/widget/access_log")
|
||||
@require_role(UserRole.viewer)
|
||||
async def widget_access_log():
|
||||
|
||||
@@ -16,6 +16,16 @@ logger = logging.getLogger(__name__)
|
||||
_prev_metrics: dict = {}
|
||||
_prev_time: float = 0.0
|
||||
|
||||
# Router name -> host names from Traefik's API, refreshed each scrape (#5254).
|
||||
# None until the API has answered once, and again whenever it stops answering,
|
||||
# so readers can tell "no domains" from "don't know".
|
||||
_router_domains: dict[str, list[str]] | None = None
|
||||
|
||||
|
||||
def router_domains() -> dict[str, list[str]] | None:
|
||||
return _router_domains
|
||||
|
||||
|
||||
# Access log state
|
||||
_geoip_db = None
|
||||
_access_log_last_tick: "datetime | None" = None
|
||||
@@ -38,15 +48,17 @@ def make_scrape_task(app: "Quart") -> ScheduledTask:
|
||||
|
||||
async def _do_scrape(app: "Quart") -> None:
|
||||
"""Fetch Traefik metrics, write to DB, and emit plugin_metrics."""
|
||||
global _prev_metrics, _prev_time
|
||||
global _prev_metrics, _prev_time, _router_domains
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from .models import TraefikCert, TraefikGlobalStat, TraefikMetric
|
||||
from .scraper import (
|
||||
api_base,
|
||||
compute_global_stats,
|
||||
compute_router_metrics,
|
||||
extract_certs,
|
||||
fetch_metrics,
|
||||
fetch_router_domains,
|
||||
)
|
||||
from steward.core.alerts import MetricBatch
|
||||
|
||||
@@ -57,6 +69,13 @@ async def _do_scrape(app: "Quart") -> None:
|
||||
except Exception:
|
||||
logger.exception("Traefik scrape failed (url=%s)", metrics_url)
|
||||
return
|
||||
cfg = app.config["PLUGINS"]["traefik"]
|
||||
try:
|
||||
_router_domains = await fetch_router_domains(api_base(metrics_url, cfg.get("api_url") or ""))
|
||||
except Exception as exc:
|
||||
if _router_domains is not None:
|
||||
logger.warning("Traefik API stopped answering (%s); certificates shown without routers", exc)
|
||||
_router_domains = None
|
||||
|
||||
now = time.monotonic()
|
||||
elapsed = now - _prev_time if _prev_time else 60.0
|
||||
|
||||
@@ -274,3 +274,69 @@ async def fetch_metrics(metrics_url: str) -> ParsedMetrics:
|
||||
response = await client.get(metrics_url)
|
||||
response.raise_for_status()
|
||||
return parse_prometheus(response.text)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Router domains (Traefik API)
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# The metrics name each router but not the hosts it answers for, so which
|
||||
# certificate serves which router is only knowable from Traefik's API. Read
|
||||
# once per scrape; when the API isn't reachable, routers simply have no
|
||||
# domains and certificates are shown on their own (#5254).
|
||||
|
||||
_HOST_MATCHER = re.compile(r"Host(?:SNI)?\(([^)]*)\)")
|
||||
_QUOTED = re.compile(r"[`\"']([^`\"']+)[`\"']")
|
||||
|
||||
|
||||
def rule_hosts(rule: str) -> list[str]:
|
||||
"""The host names a router rule matches: Host(`a`, `b`) and HostSNI(`c`)
|
||||
give ["a", "b", "c"]; HostRegexp and path-only rules give nothing."""
|
||||
hosts: list[str] = []
|
||||
for args in _HOST_MATCHER.findall(rule or ""):
|
||||
hosts.extend(h.strip().lower() for h in _QUOTED.findall(args) if h.strip() != "*")
|
||||
return hosts
|
||||
|
||||
|
||||
def api_base(metrics_url: str, api_url: str = "") -> str:
|
||||
"""The Traefik API's base: the configured api_url, else /api on the
|
||||
metrics URL's own host (the default "traefik" entrypoint serves both)."""
|
||||
if api_url:
|
||||
return api_url.rstrip("/")
|
||||
parts = httpx.URL(metrics_url)
|
||||
return str(parts.copy_with(path="/api", query=None, fragment=None)).rstrip("/")
|
||||
|
||||
|
||||
async def fetch_router_domains(base: str) -> dict[str, list[str]]:
|
||||
"""{router name: host names} for every enabled HTTP and TCP router."""
|
||||
out: dict[str, list[str]] = {}
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
for kind in ("http", "tcp"):
|
||||
response = await client.get(f"{base}/{kind}/routers", params={"per_page": 1000})
|
||||
if kind == "tcp" and response.status_code == 404:
|
||||
continue
|
||||
response.raise_for_status()
|
||||
for r in response.json():
|
||||
if r.get("status", "enabled") != "enabled" or not r.get("name"):
|
||||
continue
|
||||
hosts = rule_hosts(r.get("rule", ""))
|
||||
if hosts:
|
||||
out.setdefault(r["name"], []).extend(hosts)
|
||||
return out
|
||||
|
||||
|
||||
def cert_names(cn: str | None, sans: str | None) -> set[str]:
|
||||
"""Every name a certificate covers: its CN and SANs, lower-cased."""
|
||||
names = {n.strip().lower() for n in (sans or "").split(",") if n.strip()}
|
||||
if cn:
|
||||
names.add(cn.strip().lower())
|
||||
return names
|
||||
|
||||
|
||||
def covers(names: set[str], host: str) -> bool:
|
||||
"""Whether a certificate with these names is valid for host, a wildcard
|
||||
"*.example.com" covering one label: "a.example.com", not "a.b.example.com"."""
|
||||
if host in names:
|
||||
return True
|
||||
head, _, rest = host.partition(".")
|
||||
return bool(rest) and f"*.{rest}" in names
|
||||
|
||||
@@ -37,7 +37,7 @@
|
||||
</div>
|
||||
{% endmacro %}
|
||||
|
||||
{% if problems or healthy or gone or certs %}
|
||||
{% if problems or healthy or gone or certs or old_certs %}
|
||||
{# ── Routers that need attention ─────────────────────────────────────────── #}
|
||||
{% if problems %}
|
||||
<div style="column-width:340px;column-gap:1rem;">
|
||||
@@ -94,22 +94,43 @@
|
||||
</script>
|
||||
{% endif %}
|
||||
|
||||
{# ── Certificates, soonest expiry first ──────────────────────────────────── #}
|
||||
{% if certs %}
|
||||
{# ── Certificates Traefik still serves, soonest expiry first; then the ones
|
||||
it no longer reports (renewed or retired), kept here but off the
|
||||
dashboard (#5254). #}
|
||||
{% if certs or old_certs %}
|
||||
<div class="card">
|
||||
<div class="section-title" style="margin-bottom:0.75rem;">TLS certificates</div>
|
||||
{% if certs %}
|
||||
<table class="table" style="font-size:0.85rem;">
|
||||
<thead><tr><th>Name</th><th>Expires</th><th style="text-align:right;">Left</th></tr></thead>
|
||||
<thead><tr><th>Name</th><th>Routers</th><th>Expires</th><th style="text-align:right;">Left</th></tr></thead>
|
||||
<tbody>
|
||||
{% for cert in certs %}
|
||||
<tr>
|
||||
<td style="word-break:break-all;">{{ cert.cn }}</td>
|
||||
<td style="word-break:break-all;">{{ cert.cn }}{% if cert.sans %}<div class="field-hint">also {{ cert.sans|join(", ") }}</div>{% endif %}</td>
|
||||
<td class="field-hint">{% if cert.routers is none %}—{% elif cert.routers %}{{ cert.routers|join(", ") }}{% else %}not in use{% endif %}</td>
|
||||
<td class="field-hint">{{ cert.not_after.strftime("%Y-%m-%d") }}</td>
|
||||
<td style="text-align:right;white-space:nowrap;" class="tone-{{ status_info(status_class(cert.days_remaining, 'tls_days'))[0] }}">{{ cert_left(cert.days_remaining) }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% endif %}
|
||||
{% if old_certs %}
|
||||
<details style="margin-top:0.75rem;">
|
||||
<summary class="field-hint">No longer served ({{ old_certs|length }}) — renewed or retired; Traefik stopped reporting them</summary>
|
||||
<table class="table" style="font-size:0.85rem;margin-top:0.5rem;">
|
||||
<thead><tr><th>Name</th><th>Expires</th></tr></thead>
|
||||
<tbody>
|
||||
{% for cert in old_certs %}
|
||||
<tr style="color:var(--text-muted);">
|
||||
<td style="word-break:break-all;">{{ cert.cn }}</td>
|
||||
<td class="field-hint">{{ cert.not_after.strftime("%Y-%m-%d") }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</details>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
|
||||
@@ -1,54 +1,55 @@
|
||||
{# plugins/traefik/templates/traefik/widget.html #}
|
||||
{% if expiring_certs %}
|
||||
<div style="margin-bottom:0.6rem;">
|
||||
{% for cert in expiring_certs %}
|
||||
<div style="display:flex;justify-content:space-between;font-size:0.8rem;padding:0.15rem 0;
|
||||
{% if cert.days_remaining < 7 %}color:var(--red);{% else %}color:var(--yellow);{% endif %}">
|
||||
<span>⚠ {{ cert.cn }}</span>
|
||||
<span>{{ cert_left(cert.days_remaining) }}</span>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if router_data %}
|
||||
{% for r in router_data %}
|
||||
<div class="ping-row">
|
||||
<div class="ping-meta">
|
||||
<span class="ping-name" style="font-size:0.8rem;word-break:break-all;">{{ r.name }}</span>
|
||||
</div>
|
||||
<div style="flex:1;display:flex;gap:1.5rem;font-size:0.82rem;font-variant-numeric:tabular-nums;">
|
||||
<span title="Requests/s">
|
||||
<span style="color:var(--text-muted);">req/s</span>
|
||||
<span style="margin-left:0.3rem;">{{ "%.2f"|format(r.latest.request_rate) }}</span>
|
||||
</span>
|
||||
<span title="Bandwidth">
|
||||
<span style="color:var(--text-muted);">bw</span>
|
||||
<span style="margin-left:0.3rem;">
|
||||
{% set bps = r.latest.response_bytes_rate %}
|
||||
{{ bps|rate }}
|
||||
</span>
|
||||
</span>
|
||||
<span title="p95 latency">
|
||||
<span style="color:var(--text-muted);">p95</span>
|
||||
<span style="margin-left:0.3rem;
|
||||
{% if r.latest.latency_p95_ms > 500 %}color:var(--red){% elif r.latest.latency_p95_ms > 200 %}color:var(--yellow){% else %}color:var(--green){% endif %};">
|
||||
{{ "%.0f"|format(r.latest.latency_p95_ms) }}ms
|
||||
</span>
|
||||
</span>
|
||||
{% if r.latest.error_rate_5xx_pct > 0 %}
|
||||
<span title="5xx error rate">
|
||||
<span style="color:var(--red);">{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}% 5xx</span>
|
||||
</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% if total_routers > 5 %}
|
||||
<div style="color:var(--text-dim);font-size:0.75rem;padding:0.25rem 0;">
|
||||
+{{ total_routers - 5 }} more — <a href="/plugins/traefik/">view all →</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
{# plugins/traefik/templates/traefik/widget.html — the dashboard Routers widget
|
||||
(#5254): a verdict, routers in trouble with why, then the busiest healthy
|
||||
ones, under one header. Only routers Traefik still reports. A certificate
|
||||
shows on the router it serves (TLS column, days left); when Traefik's API
|
||||
can't say which hosts a router serves, the expiring certificates are listed
|
||||
after the routers instead. #}
|
||||
{% from "_macros.html" import status_dot %}
|
||||
{% if not total %}
|
||||
<div class="w-verdict" style="color:var(--text-muted);">No Traefik data yet.</div>
|
||||
{% else %}
|
||||
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No Traefik data yet.</p>
|
||||
{% set crit = problems | selectattr("state", "equalto", "crit") | list %}
|
||||
<div class="w-verdict {{ 'tone-crit' if crit else ('tone-warn' if problems or loose_certs else 'tone-ok') }}">
|
||||
{%- if problems|length == 1 %}{{ problems[0].name.split('@')[0] }}: {{ problems[0].why }}.
|
||||
{%- elif problems %}{{ problems|length }} of {{ total }} routers need a look.
|
||||
{%- else %}All {{ total }} router{{ 's' if total != 1 }} healthy{% if loose_certs %}; {{ loose_certs|length }} certificate{{ 's' if loose_certs|length != 1 }} expiring{% endif %}.{% endif -%}
|
||||
</div>
|
||||
<table class="w-table">
|
||||
<colgroup><col><col style="width:4rem"><col class="w-wide-only" style="width:5.5rem"><col style="width:4.2rem">{% if domains_known %}<col style="width:4.5rem">{% endif %}</colgroup>
|
||||
<thead><tr><th>Router</th><th>Req/s</th><th class="w-wide-only">Bandwidth</th><th>p95</th>{% if domains_known %}<th>TLS</th>{% endif %}</tr></thead>
|
||||
<tbody>
|
||||
{% for r in problems + healthy %}
|
||||
{% set m = r.latest %}
|
||||
<tr>
|
||||
<td class="w-name" title="{{ r.name }}">
|
||||
{{- status_dot(r.state) -}}
|
||||
{{ r.name.split('@')[0] }}
|
||||
{%- if r.why %}<span class="w-sub tone-{{ r.state }}">{{ r.why }}</span>{% endif %}
|
||||
</td>
|
||||
<td>{% if m.request_rate is none %}—{% else %}{{ "%.2f"|format(m.request_rate) }}{% endif %}</td>
|
||||
<td class="w-wide-only">{% if m.response_bytes_rate is none %}—{% else %}{{ m.response_bytes_rate|rate }}{% endif %}</td>
|
||||
<td style="{{ threshold_style(m.latency_p95_ms, 'latency') if m.latency_p95_ms is not none }}">
|
||||
{%- if m.latency_p95_ms is none %}—{% else %}{{ "%.0f"|format(m.latency_p95_ms) }} ms{% endif -%}
|
||||
</td>
|
||||
{% if domains_known %}
|
||||
<td {% if r.cert %}style="{{ threshold_style(r.cert.days, 'tls_days') }}" title="Certificate {{ r.cert.cn }}"{% endif %}>
|
||||
{%- if r.cert %}{{ cert_left(r.cert.days) }}{% else %}—{% endif -%}
|
||||
</td>
|
||||
{% endif %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% if healthy_more %}<div class="w-more">+{{ healthy_more }} more healthy{% if session.user_id %} · <a href="/plugins/traefik/">all routers</a>{% endif %}</div>{% endif %}
|
||||
{% if loose_certs %}
|
||||
<div class="w-section">Certificates expiring</div>
|
||||
{% for c in loose_certs %}
|
||||
<div class="w-item">
|
||||
{{- status_dot(status_class(c.days, 'tls_days')) -}}
|
||||
<span class="w-label" title="{{ c.cn }}">{{ c.cn }}</span>
|
||||
<span class="w-meta" style="{{ threshold_style(c.days, 'tls_days') }}">{{ cert_left(c.days) }}</span>
|
||||
</div>
|
||||
{% endfor %}
|
||||
<div class="w-note">Traefik's API didn't say which hosts each router serves, so these can't be shown on their routers.</div>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Integration: the newest Traefik row per router, read by walking the
|
||||
(router_name, scraped_at) index rather than grouping every row (#5254).
|
||||
Requires STEWARD_DATABASE_URL."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
if not os.environ.get("STEWARD_DATABASE_URL"):
|
||||
pytest.skip("needs Postgres")
|
||||
from steward.app import create_app
|
||||
return create_app(testing=False)
|
||||
|
||||
|
||||
def test_each_router_reads_as_its_newest_row(app):
|
||||
from plugins.traefik.models import TraefikMetric
|
||||
from plugins.traefik.routes import latest_per_router
|
||||
|
||||
now = datetime.now(timezone.utc).replace(microsecond=0)
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
a, b = f"a-{tag}@docker", f"b-{tag}@docker"
|
||||
|
||||
def row(name, ago, rate):
|
||||
return TraefikMetric(router_name=name, scraped_at=now - timedelta(minutes=ago), request_rate=rate,
|
||||
error_rate_4xx_pct=0.0, error_rate_5xx_pct=0.0, latency_p50_ms=1.0,
|
||||
latency_p95_ms=2.0, latency_p99_ms=3.0, response_bytes_rate=0.0)
|
||||
|
||||
async def go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
s.add_all([row(a, 3, 1.0), row(a, 1, 2.0), row(a, 2, 3.0), row(b, 5, 4.0)])
|
||||
async with s.begin():
|
||||
return {m.router_name: m for m in await latest_per_router(s)}
|
||||
|
||||
got = asyncio.run(go())
|
||||
assert got[a].request_rate == 2.0 and got[a].scraped_at == now - timedelta(minutes=1)
|
||||
assert got[b].request_rate == 4.0
|
||||
@@ -48,10 +48,10 @@ async def test_the_page_leads_with_problems_and_folds_the_healthy(app):
|
||||
problems, healthy = split_routers([
|
||||
_router("api", _latest(e5=3.0)), _router("web", _latest()), _router("docs", _latest())])
|
||||
now = datetime.now(timezone.utc)
|
||||
certs = [{"serial": "1", "cn": "soon.example", "sans": [], "not_after": now + timedelta(days=2),
|
||||
"days_remaining": 2.0},
|
||||
{"serial": "2", "cn": "fine.example", "sans": [], "not_after": now + timedelta(days=60),
|
||||
"days_remaining": 60.0}]
|
||||
certs = [{"serial": "1", "cn": "soon.example", "sans": [], "routers": None,
|
||||
"not_after": now + timedelta(days=2), "days_remaining": 2.0},
|
||||
{"serial": "2", "cn": "fine.example", "sans": [], "routers": None,
|
||||
"not_after": now + timedelta(days=60), "days_remaining": 60.0}]
|
||||
async with app.test_request_context("/plugins/traefik/rows"):
|
||||
page = await render_plugin_template(app, "traefik", "traefik/rows.html", problems=problems,
|
||||
healthy=healthy, global_stat=None, certs=certs, range_key="24h",
|
||||
@@ -110,3 +110,123 @@ async def test_the_traffic_empty_state_knows_whether_the_log_is_on(app, enabled,
|
||||
range_key="24h", access_log_enabled=enabled,
|
||||
access_log_path="/var/log/traefik/access.log")
|
||||
assert says in page and not_says not in page
|
||||
|
||||
|
||||
# ── Routers widget: current routers, certs on the router they serve (#5254) ──
|
||||
|
||||
@pytest.mark.parametrize("rule, hosts", [
|
||||
("Host(`app.example.com`)", ["app.example.com"]),
|
||||
("Host(`a.example.com`, `B.example.com`) && PathPrefix(`/api`)", ["a.example.com", "b.example.com"]),
|
||||
("Host(`a.example.com`) || Host(`b.example.com`)", ["a.example.com", "b.example.com"]),
|
||||
("HostSNI(`db.example.com`)", ["db.example.com"]),
|
||||
("HostSNI(`*`)", []),
|
||||
("PathPrefix(`/`)", []),
|
||||
("", []),
|
||||
])
|
||||
def test_a_router_rule_names_its_hosts(rule, hosts):
|
||||
from plugins.traefik.scraper import rule_hosts
|
||||
assert rule_hosts(rule) == hosts
|
||||
|
||||
|
||||
def test_a_certificate_covers_its_names_and_one_wildcard_label():
|
||||
from plugins.traefik.scraper import cert_names, covers
|
||||
names = cert_names("Example.com", "*.example.com, www.other.org")
|
||||
assert names == {"example.com", "*.example.com", "www.other.org"}
|
||||
assert covers(names, "example.com") and covers(names, "app.example.com")
|
||||
assert covers(names, "www.other.org")
|
||||
assert not covers(names, "a.b.example.com") and not covers(names, "other.org")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metrics_url, api_url, base", [
|
||||
("http://traefik:8080/metrics", "", "http://traefik:8080/api"),
|
||||
("http://traefik:8080/metrics?x=1", "", "http://traefik:8080/api"),
|
||||
("http://traefik:8080/metrics", "http://other:9000/api/", "http://other:9000/api"),
|
||||
])
|
||||
def test_the_api_sits_beside_the_metrics_unless_configured(metrics_url, api_url, base):
|
||||
from plugins.traefik.scraper import api_base
|
||||
assert api_base(metrics_url, api_url) == base
|
||||
|
||||
|
||||
def test_a_metrics_router_finds_its_api_name_with_or_without_the_entrypoint():
|
||||
from plugins.traefik.routes import router_hosts
|
||||
domains = {"app@docker": ["app.example.com"]}
|
||||
assert router_hosts("app@docker", domains) == ["app.example.com"]
|
||||
assert router_hosts("websecure-app@docker", domains) == ["app.example.com"]
|
||||
assert router_hosts("myapp@docker", domains) == []
|
||||
|
||||
|
||||
def _cert(serial, cn, days, seen_ago=30, sans=None):
|
||||
now = datetime.now(timezone.utc)
|
||||
return NS(serial=serial, cn=cn, sans=sans, not_after=now + timedelta(days=days),
|
||||
scraped_at=now - timedelta(seconds=seen_ago))
|
||||
|
||||
|
||||
def test_a_certificate_traefik_stopped_reporting_is_old_not_current():
|
||||
from plugins.traefik.routes import current_certs
|
||||
live, renewed = _cert("1", "a.example", 80), _cert("2", "a.example", 5, seen_ago=3600)
|
||||
served, old = current_certs([live, renewed], 60)
|
||||
assert served == [live] and old == [renewed]
|
||||
assert current_certs([], 60) == ([], [])
|
||||
|
||||
|
||||
def test_each_router_gets_the_soonest_certificate_for_its_hosts():
|
||||
from plugins.traefik.routes import certs_by_router
|
||||
now = datetime.now(timezone.utc)
|
||||
certs = [_cert("1", "*.example.com", 40), _cert("2", "app.example.com", 10), _cert("3", "x.org", 2)]
|
||||
got = certs_by_router(["app@docker", "docs@docker", "plain@docker"],
|
||||
{"app@docker": ["app.example.com"], "docs@docker": ["docs.example.com"]},
|
||||
certs, now)
|
||||
assert got["app@docker"]["cn"] == "app.example.com" and int(got["app@docker"]["days"]) == 9
|
||||
assert got["docs@docker"]["cn"] == "*.example.com"
|
||||
assert "plain@docker" not in got
|
||||
|
||||
|
||||
async def test_the_widget_gives_a_verdict_then_routers_under_one_header(app):
|
||||
slow = _router("api@docker", _latest(p95=450.0))
|
||||
expiring = _router("shop@docker", _latest(req=3.0))
|
||||
expiring.update(state="crit", why="certificate expires in 2 days",
|
||||
cert={"cn": "shop.example", "days": 2.0, "grade": "crit"})
|
||||
fine = _router("web@docker", _latest(req=9.0))
|
||||
fine["cert"] = {"cn": "web.example", "days": 60.0, "grade": "ok"}
|
||||
async with app.test_request_context("/plugins/traefik/widget"):
|
||||
page = await render_plugin_template(
|
||||
app, "traefik", "traefik/widget.html", problems=[expiring, slow], healthy=[fine],
|
||||
healthy_more=4, total=7, loose_certs=[], domains_known=True)
|
||||
page = " ".join(page.split())
|
||||
assert "2 of 7 routers need a look." in page
|
||||
assert page.count("<th>Router</th>") == 1 and "<th>TLS</th>" in page
|
||||
# Problems first, each with why; the cert sits on its router, not above.
|
||||
assert page.index(">shop") < page.index(">api") < page.index(">web")
|
||||
assert "certificate expires in 2 days" in page and "p95 latency is 450 ms" in page
|
||||
assert "60 days" in page and "Certificates expiring" not in page
|
||||
assert '</span>shop<span class="w-sub' in page and ">shop@docker" not in page # provider off the label
|
||||
assert "+4 more healthy" in page
|
||||
|
||||
|
||||
async def test_without_router_domains_expiring_certs_follow_the_routers(app):
|
||||
fine = _router("web@docker", _latest())
|
||||
async with app.test_request_context("/plugins/traefik/widget"):
|
||||
page = await render_plugin_template(
|
||||
app, "traefik", "traefik/widget.html", problems=[], healthy=[fine], healthy_more=0, total=1,
|
||||
loose_certs=[{"cn": "old.example", "days": 5.0}], domains_known=False)
|
||||
page = " ".join(page.split())
|
||||
assert "All 1 router healthy; 1 certificate expiring." in page
|
||||
assert "<th>TLS</th>" not in page
|
||||
assert page.index("<th>Router</th>") < page.index("Certificates expiring") < page.index("old.example")
|
||||
|
||||
|
||||
async def test_the_page_sets_certificates_traefik_no_longer_serves_apart(app):
|
||||
now = datetime.now(timezone.utc)
|
||||
certs = [{"serial": "1", "cn": "app.example", "sans": ["www.app.example"], "routers": ["app"],
|
||||
"not_after": now + timedelta(days=40), "days_remaining": 40.0},
|
||||
{"serial": "2", "cn": "spare.example", "sans": [], "routers": [],
|
||||
"not_after": now + timedelta(days=40), "days_remaining": 40.0}]
|
||||
old = [{"serial": "3", "cn": "gone.example", "sans": [], "routers": [],
|
||||
"not_after": now + timedelta(days=5), "days_remaining": 5.0}]
|
||||
async with app.test_request_context("/plugins/traefik/rows"):
|
||||
page = await render_plugin_template(app, "traefik", "traefik/rows.html", problems=[], healthy=[],
|
||||
gone=[], global_stat=None, certs=certs, old_certs=old,
|
||||
range_key="24h", error_4xx=1.0, error_5xx=0.1)
|
||||
page = " ".join(page.split())
|
||||
assert "also www.app.example" in page and ">app</td>" in page and ">not in use</td>" in page
|
||||
assert page.index("spare.example") < page.index("No longer served (1)") < page.index("gone.example")
|
||||
|
||||
Reference in New Issue
Block a user