feat(traefik): Routers widget shows current routers only, one header row, certs on the routers they serve (#5254)
CI / lint (push) Successful in 2s
CI / integration (push) Successful in 52s
CI / unit (push) Failing after 1m17s
CI / publish (push) Skipped

#5254 "Traefik — Routers: current routers only, header row, certs on the router they serve".

- Router hosts come from Traefik's API (/api/http|tcp/routers), read
  best-effort on each scrape. An optional api_url overrides it; the default
  is /api on the metrics host. A certificate's expiry then shows in a TLS
  column on the router whose host it covers. An expiring one makes that
  router a problem, graded by Settings → Thresholds.
- Only current: routers and certificates must have appeared in the last
  three scrapes. Certificates are upserted by serial and never deleted, so
  renewed or retired ones lingered on the widget.
- Widget order: verdict, problem routers with the reason, then the busiest
  healthy routers under one header (Router / Req/s / Bandwidth / p95 / TLS).
  A missing value shows "—".
- If the API is unreachable, expiring certificates are listed after the
  routers, not above them.
- The page splits certificates into served (with the routers using each,
  plus SANs) and "No longer served".
- Latest row per router: an index walk (recursive CTE + LATERAL) replaces
  DISTINCT plus a query per router.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-07 13:25:11 -04:00
co-authored by Claude Opus 5.5
parent d72a1991a7
commit ea5531ac8f
8 changed files with 468 additions and 128 deletions
+3
View File
@@ -14,6 +14,9 @@ tags:
config:
metrics_url: "http://localhost:8080/metrics"
# Traefik's API, for which hosts each router serves (so a certificate's
# expiry shows on its router). Empty: /api on the metrics URL's host.
api_url: ""
scrape_interval_seconds: 60
access_log:
enabled: false
+130 -66
View File
@@ -3,14 +3,16 @@ from __future__ import annotations
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from quart import Blueprint, current_app, render_template, request
from sqlalchemy import Integer, cast, func, select
from sqlalchemy import Integer, cast, func, select, text
import json
from steward.auth.middleware import require_role
from steward.models.users import UserRole
from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample
from steward.core.formatting import status_class
from steward.core.formatting import cert_left, status_class
from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric
from .scheduler import router_domains
from .scraper import cert_names, covers
traefik_bp = Blueprint("traefik", __name__, template_folder="templates")
@@ -85,6 +87,71 @@ def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]:
return problems, healthy
# Newest row per router. Postgres has no skip scan, so DISTINCT or max() GROUP
# BY reads every row in retention; this walks the (router_name, scraped_at)
# index one router at a time instead: a step per router, not per row.
_LATEST_PER_ROUTER = text("""
WITH RECURSIVE names AS (
SELECT min(router_name) AS name FROM traefik_metrics
UNION ALL
SELECT (SELECT min(router_name) FROM traefik_metrics WHERE router_name > names.name)
FROM names WHERE names.name IS NOT NULL
)
SELECT m.* FROM names
JOIN LATERAL (
SELECT * FROM traefik_metrics WHERE router_name = names.name
ORDER BY scraped_at DESC LIMIT 1
) m ON true
""")
async def latest_per_router(db) -> list:
return list((await db.execute(
select(TraefikMetric).from_statement(_LATEST_PER_ROUTER))).scalars().all())
def current_certs(certs, scrape_seconds: int) -> tuple[list, list]:
"""(served, old): a certificate is served while Traefik still reports it.
Rows are upserted by serial and never deleted, so a renewed or retired
certificate lingers; one missing from the last GONE_AFTER_SCRAPES
scrapes is old (#5254)."""
certs = list(certs)
if not certs:
return [], []
newest = max(c.scraped_at for c in certs)
cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
return ([c for c in certs if c.scraped_at >= cutoff],
sorted((c for c in certs if c.scraped_at < cutoff), key=lambda c: c.scraped_at, reverse=True))
def router_hosts(name: str, domains: dict[str, list[str]]) -> list[str]:
"""The hosts a metrics router name serves. The API names a router
"app@docker"; metrics may carry it as "websecure-app@docker", one per
entrypoint, so a name not found as-is is tried without that prefix."""
if name in domains:
return domains[name]
for api_name, hosts in domains.items():
if name.endswith("-" + api_name):
return hosts
return []
def certs_by_router(routers: list[str], domains: dict[str, list[str]], certs, now) -> dict[str, dict]:
"""{router: {"cn", "days"}}: the soonest-expiring served certificate valid
for one of the router's hosts. A router with no hosts, or none covered,
is absent."""
out: dict[str, dict] = {}
names = [(c, cert_names(c.cn, c.sans)) for c in certs]
for router in routers:
for host in router_hosts(router, domains):
for c, cn in names:
if covers(cn, host):
days = (c.not_after - now).total_seconds() / 86400.0
if router not in out or days < out[router]["days"]:
out[router] = {"cn": c.cn or host, "days": days}
return out
def _access_log() -> dict:
"""The plugin's access_log settings ({} when unset or malformed)."""
cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {})
@@ -114,13 +181,9 @@ async def rows():
thresholds = current_app.config.get("THRESHOLDS", {})
async with current_app.db_sessionmaker() as db:
# All known routers
result = await db.execute(
select(TraefikMetric.router_name)
.distinct()
.order_by(TraefikMetric.router_name)
)
routers = [row[0] for row in result.all()]
# Every known router, with its most recent raw row
newest = {m.router_name: m for m in await latest_per_router(db)}
routers = sorted(newest)
b_secs = bucket_seconds(since)
bucket_col = (
@@ -145,16 +208,7 @@ async def rows():
)
history = result.all()
# Latest value — always the most recent raw row
result = await db.execute(
select(TraefikMetric)
.where(TraefikMetric.router_name == router)
.order_by(TraefikMetric.scraped_at.desc())
.limit(1)
)
latest = result.scalar_one_or_none()
if not latest:
continue
latest = newest[router]
state, why = router_state(latest, thresholds)
router_data.append({
@@ -181,19 +235,28 @@ async def rows():
)
raw_certs = result.scalars().all()
certs = []
for c in raw_certs:
days = (c.not_after - now).total_seconds() / 86400.0
certs.append({
scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
current, gone = gone_routers(router_data, scrape)
served, old = current_certs(raw_certs, scrape)
domains = router_domains()
def cert_row(c):
names = cert_names(c.cn, c.sans)
return {
"serial": c.serial,
"cn": c.cn or c.serial,
"sans": c.sans,
"sans": [n for n in names if n != c.cn],
"not_after": c.not_after,
"days_remaining": days,
})
"days_remaining": (c.not_after - now).total_seconds() / 86400.0,
# The routers whose hosts it covers; None when Traefik's API
# couldn't say which hosts each router serves.
"routers": None if domains is None else sorted(
{r["name"].split("@")[0] for r in current
for h in router_hosts(r["name"], domains) if covers(names, h)}),
}
current, gone = gone_routers(
router_data, current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"])
certs = [cert_row(c) for c in served]
old_certs = [cert_row(c) for c in old]
problems, healthy = split_routers(current)
return await render_template(
"traefik/rows.html",
@@ -204,6 +267,7 @@ async def rows():
error_5xx=ERROR_5XX_PCT,
global_stat=global_stat,
certs=certs,
old_certs=old_certs,
range_key=range_key,
)
@@ -211,53 +275,53 @@ async def rows():
@traefik_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
"""HTMX dashboard widget fragment."""
"""The dashboard Routers widget (#5254), in the widget-shape order: a
verdict, routers in trouble with why, then the busiest healthy ones under
one header. Only routers and certificates Traefik still reports. A
certificate's expiry shows on the router it serves, and makes that router
a problem once it is inside the Settings → Thresholds TLS window."""
now = datetime.now(timezone.utc)
thresholds = current_app.config.get("THRESHOLDS", {})
scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(TraefikMetric.router_name).distinct().order_by(TraefikMetric.router_name)
)
routers = [row[0] for row in result.all()]
router_data = []
for router in routers:
result = await db.execute(
select(TraefikMetric)
.where(TraefikMetric.router_name == router)
.order_by(TraefikMetric.scraped_at.desc())
.limit(1)
)
latest = result.scalar_one_or_none()
if latest:
router_data.append({"name": router, "latest": latest})
router_data.sort(key=lambda r: (
-(1 if r["latest"].error_rate_5xx_pct > 0.1 else 0),
-(1 if r["latest"].error_rate_4xx_pct > 5.0 else 0),
-r["latest"].request_rate,
))
total_routers = len(router_data)
router_data = router_data[:5]
result = await db.execute(
select(TraefikCert).order_by(TraefikCert.not_after)
)
raw_certs = result.scalars().all()
expiring_certs = []
for c in raw_certs:
days = (c.not_after - now).total_seconds() / 86400.0
if days < 30:
expiring_certs.append({"cn": c.cn or c.serial, "days_remaining": days})
latest = await latest_per_router(db)
certs = (await db.execute(select(TraefikCert))).scalars().all()
current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
served, _old = current_certs(certs, scrape)
domains = router_domains()
by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
for r in current:
r["state"], r["why"] = router_state(r["latest"], thresholds)
r["cert"] = by_router.get(r["name"])
if r["cert"]:
grade = status_class(r["cert"]["days"], "tls_days", thresholds)
r["cert"]["grade"] = grade
if _GRADE_ORDER.get(grade, 2) < _GRADE_ORDER[r["state"]]:
left = cert_left(r["cert"]["days"]) # "12 days", "expires today", "expired …"
r["state"], r["why"] = grade, (
f"certificate expires in {left}" if left[0].isdigit() else f"certificate {left}")
problems, healthy = split_routers(current)
healthy.sort(key=lambda r: -(r["latest"].request_rate or 0))
# With no router domains to go on, certificates can't sit on their
# routers: the expiring ones are listed after the routers instead.
loose = [] if domains is not None else sorted(
({"cn": c.cn or c.serial, "days": (c.not_after - now).total_seconds() / 86400.0} for c in served),
key=lambda c: c["days"])
loose = [c for c in loose if status_class(c["days"], "tls_days", thresholds) in ("warn", "crit")]
return await render_template(
"traefik/widget.html",
router_data=router_data,
expiring_certs=expiring_certs,
total_routers=total_routers,
problems=problems, healthy=healthy[:ROUTERS_SHOWN],
healthy_more=max(0, len(healthy) - ROUTERS_SHOWN),
total=len(current), loose_certs=loose, domains_known=domains is not None,
)
# Healthy routers on the dashboard widget (the busiest); the page lists all.
ROUTERS_SHOWN = 6
@traefik_bp.get("/widget/access_log")
@require_role(UserRole.viewer)
async def widget_access_log():
+20 -1
View File
@@ -16,6 +16,16 @@ logger = logging.getLogger(__name__)
_prev_metrics: dict = {}
_prev_time: float = 0.0
# Router name -> host names from Traefik's API, refreshed each scrape (#5254).
# None until the API has answered once, and again whenever it stops answering,
# so readers can tell "no domains" from "don't know".
_router_domains: dict[str, list[str]] | None = None
def router_domains() -> dict[str, list[str]] | None:
return _router_domains
# Access log state
_geoip_db = None
_access_log_last_tick: "datetime | None" = None
@@ -38,15 +48,17 @@ def make_scrape_task(app: "Quart") -> ScheduledTask:
async def _do_scrape(app: "Quart") -> None:
"""Fetch Traefik metrics, write to DB, and emit plugin_metrics."""
global _prev_metrics, _prev_time
global _prev_metrics, _prev_time, _router_domains
from datetime import datetime, timezone
from .models import TraefikCert, TraefikGlobalStat, TraefikMetric
from .scraper import (
api_base,
compute_global_stats,
compute_router_metrics,
extract_certs,
fetch_metrics,
fetch_router_domains,
)
from steward.core.alerts import MetricBatch
@@ -57,6 +69,13 @@ async def _do_scrape(app: "Quart") -> None:
except Exception:
logger.exception("Traefik scrape failed (url=%s)", metrics_url)
return
cfg = app.config["PLUGINS"]["traefik"]
try:
_router_domains = await fetch_router_domains(api_base(metrics_url, cfg.get("api_url") or ""))
except Exception as exc:
if _router_domains is not None:
logger.warning("Traefik API stopped answering (%s); certificates shown without routers", exc)
_router_domains = None
now = time.monotonic()
elapsed = now - _prev_time if _prev_time else 60.0
+66
View File
@@ -274,3 +274,69 @@ async def fetch_metrics(metrics_url: str) -> ParsedMetrics:
response = await client.get(metrics_url)
response.raise_for_status()
return parse_prometheus(response.text)
# ──────────────────────────────────────────────────────────────────────────────
# Router domains (Traefik API)
# ──────────────────────────────────────────────────────────────────────────────
#
# The metrics name each router but not the hosts it answers for, so which
# certificate serves which router is only knowable from Traefik's API. Read
# once per scrape; when the API isn't reachable, routers simply have no
# domains and certificates are shown on their own (#5254).
_HOST_MATCHER = re.compile(r"Host(?:SNI)?\(([^)]*)\)")
_QUOTED = re.compile(r"[`\"']([^`\"']+)[`\"']")
def rule_hosts(rule: str) -> list[str]:
"""The host names a router rule matches: Host(`a`, `b`) and HostSNI(`c`)
give ["a", "b", "c"]; HostRegexp and path-only rules give nothing."""
hosts: list[str] = []
for args in _HOST_MATCHER.findall(rule or ""):
hosts.extend(h.strip().lower() for h in _QUOTED.findall(args) if h.strip() != "*")
return hosts
def api_base(metrics_url: str, api_url: str = "") -> str:
"""The Traefik API's base: the configured api_url, else /api on the
metrics URL's own host (the default "traefik" entrypoint serves both)."""
if api_url:
return api_url.rstrip("/")
parts = httpx.URL(metrics_url)
return str(parts.copy_with(path="/api", query=None, fragment=None)).rstrip("/")
async def fetch_router_domains(base: str) -> dict[str, list[str]]:
"""{router name: host names} for every enabled HTTP and TCP router."""
out: dict[str, list[str]] = {}
async with httpx.AsyncClient(timeout=10.0) as client:
for kind in ("http", "tcp"):
response = await client.get(f"{base}/{kind}/routers", params={"per_page": 1000})
if kind == "tcp" and response.status_code == 404:
continue
response.raise_for_status()
for r in response.json():
if r.get("status", "enabled") != "enabled" or not r.get("name"):
continue
hosts = rule_hosts(r.get("rule", ""))
if hosts:
out.setdefault(r["name"], []).extend(hosts)
return out
def cert_names(cn: str | None, sans: str | None) -> set[str]:
"""Every name a certificate covers: its CN and SANs, lower-cased."""
names = {n.strip().lower() for n in (sans or "").split(",") if n.strip()}
if cn:
names.add(cn.strip().lower())
return names
def covers(names: set[str], host: str) -> bool:
"""Whether a certificate with these names is valid for host, a wildcard
"*.example.com" covering one label: "a.example.com", not "a.b.example.com"."""
if host in names:
return True
head, _, rest = host.partition(".")
return bool(rest) and f"*.{rest}" in names
+26 -5
View File
@@ -37,7 +37,7 @@
</div>
{% endmacro %}
{% if problems or healthy or gone or certs %}
{% if problems or healthy or gone or certs or old_certs %}
{# ── Routers that need attention ─────────────────────────────────────────── #}
{% if problems %}
<div style="column-width:340px;column-gap:1rem;">
@@ -94,22 +94,43 @@
</script>
{% endif %}
{# ── Certificates, soonest expiry first ──────────────────────────────────── #}
{% if certs %}
{# ── Certificates Traefik still serves, soonest expiry first; then the ones
it no longer reports (renewed or retired), kept here but off the
dashboard (#5254). #}
{% if certs or old_certs %}
<div class="card">
<div class="section-title" style="margin-bottom:0.75rem;">TLS certificates</div>
{% if certs %}
<table class="table" style="font-size:0.85rem;">
<thead><tr><th>Name</th><th>Expires</th><th style="text-align:right;">Left</th></tr></thead>
<thead><tr><th>Name</th><th>Routers</th><th>Expires</th><th style="text-align:right;">Left</th></tr></thead>
<tbody>
{% for cert in certs %}
<tr>
<td style="word-break:break-all;">{{ cert.cn }}</td>
<td style="word-break:break-all;">{{ cert.cn }}{% if cert.sans %}<div class="field-hint">also {{ cert.sans|join(", ") }}</div>{% endif %}</td>
<td class="field-hint">{% if cert.routers is none %}—{% elif cert.routers %}{{ cert.routers|join(", ") }}{% else %}not in use{% endif %}</td>
<td class="field-hint">{{ cert.not_after.strftime("%Y-%m-%d") }}</td>
<td style="text-align:right;white-space:nowrap;" class="tone-{{ status_info(status_class(cert.days_remaining, 'tls_days'))[0] }}">{{ cert_left(cert.days_remaining) }}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}
{% if old_certs %}
<details style="margin-top:0.75rem;">
<summary class="field-hint">No longer served ({{ old_certs|length }}) — renewed or retired; Traefik stopped reporting them</summary>
<table class="table" style="font-size:0.85rem;margin-top:0.5rem;">
<thead><tr><th>Name</th><th>Expires</th></tr></thead>
<tbody>
{% for cert in old_certs %}
<tr style="color:var(--text-muted);">
<td style="word-break:break-all;">{{ cert.cn }}</td>
<td class="field-hint">{{ cert.not_after.strftime("%Y-%m-%d") }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</details>
{% endif %}
</div>
{% endif %}
+53 -52
View File
@@ -1,54 +1,55 @@
{# plugins/traefik/templates/traefik/widget.html #}
{% if expiring_certs %}
<div style="margin-bottom:0.6rem;">
{% for cert in expiring_certs %}
<div style="display:flex;justify-content:space-between;font-size:0.8rem;padding:0.15rem 0;
{% if cert.days_remaining < 7 %}color:var(--red);{% else %}color:var(--yellow);{% endif %}">
<span>&#9888; {{ cert.cn }}</span>
<span>{{ cert_left(cert.days_remaining) }}</span>
</div>
{% endfor %}
</div>
{% endif %}
{% if router_data %}
{% for r in router_data %}
<div class="ping-row">
<div class="ping-meta">
<span class="ping-name" style="font-size:0.8rem;word-break:break-all;">{{ r.name }}</span>
</div>
<div style="flex:1;display:flex;gap:1.5rem;font-size:0.82rem;font-variant-numeric:tabular-nums;">
<span title="Requests/s">
<span style="color:var(--text-muted);">req/s</span>
<span style="margin-left:0.3rem;">{{ "%.2f"|format(r.latest.request_rate) }}</span>
</span>
<span title="Bandwidth">
<span style="color:var(--text-muted);">bw</span>
<span style="margin-left:0.3rem;">
{% set bps = r.latest.response_bytes_rate %}
{{ bps|rate }}
</span>
</span>
<span title="p95 latency">
<span style="color:var(--text-muted);">p95</span>
<span style="margin-left:0.3rem;
{% if r.latest.latency_p95_ms > 500 %}color:var(--red){% elif r.latest.latency_p95_ms > 200 %}color:var(--yellow){% else %}color:var(--green){% endif %};">
{{ "%.0f"|format(r.latest.latency_p95_ms) }}ms
</span>
</span>
{% if r.latest.error_rate_5xx_pct > 0 %}
<span title="5xx error rate">
<span style="color:var(--red);">{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}% 5xx</span>
</span>
{% endif %}
</div>
</div>
{% endfor %}
{% if total_routers > 5 %}
<div style="color:var(--text-dim);font-size:0.75rem;padding:0.25rem 0;">
+{{ total_routers - 5 }} more — <a href="/plugins/traefik/">view all →</a>
</div>
{% endif %}
{# plugins/traefik/templates/traefik/widget.html — the dashboard Routers widget
(#5254): a verdict, routers in trouble with why, then the busiest healthy
ones, under one header. Only routers Traefik still reports. A certificate
shows on the router it serves (TLS column, days left); when Traefik's API
can't say which hosts a router serves, the expiring certificates are listed
after the routers instead. #}
{% from "_macros.html" import status_dot %}
{% if not total %}
<div class="w-verdict" style="color:var(--text-muted);">No Traefik data yet.</div>
{% else %}
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No Traefik data yet.</p>
{% set crit = problems | selectattr("state", "equalto", "crit") | list %}
<div class="w-verdict {{ 'tone-crit' if crit else ('tone-warn' if problems or loose_certs else 'tone-ok') }}">
{%- if problems|length == 1 %}{{ problems[0].name.split('@')[0] }}: {{ problems[0].why }}.
{%- elif problems %}{{ problems|length }} of {{ total }} routers need a look.
{%- else %}All {{ total }} router{{ 's' if total != 1 }} healthy{% if loose_certs %}; {{ loose_certs|length }} certificate{{ 's' if loose_certs|length != 1 }} expiring{% endif %}.{% endif -%}
</div>
<table class="w-table">
<colgroup><col><col style="width:4rem"><col class="w-wide-only" style="width:5.5rem"><col style="width:4.2rem">{% if domains_known %}<col style="width:4.5rem">{% endif %}</colgroup>
<thead><tr><th>Router</th><th>Req/s</th><th class="w-wide-only">Bandwidth</th><th>p95</th>{% if domains_known %}<th>TLS</th>{% endif %}</tr></thead>
<tbody>
{% for r in problems + healthy %}
{% set m = r.latest %}
<tr>
<td class="w-name" title="{{ r.name }}">
{{- status_dot(r.state) -}}
{{ r.name.split('@')[0] }}
{%- if r.why %}<span class="w-sub tone-{{ r.state }}">{{ r.why }}</span>{% endif %}
</td>
<td>{% if m.request_rate is none %}—{% else %}{{ "%.2f"|format(m.request_rate) }}{% endif %}</td>
<td class="w-wide-only">{% if m.response_bytes_rate is none %}—{% else %}{{ m.response_bytes_rate|rate }}{% endif %}</td>
<td style="{{ threshold_style(m.latency_p95_ms, 'latency') if m.latency_p95_ms is not none }}">
{%- if m.latency_p95_ms is none %}—{% else %}{{ "%.0f"|format(m.latency_p95_ms) }} ms{% endif -%}
</td>
{% if domains_known %}
<td {% if r.cert %}style="{{ threshold_style(r.cert.days, 'tls_days') }}" title="Certificate {{ r.cert.cn }}"{% endif %}>
{%- if r.cert %}{{ cert_left(r.cert.days) }}{% else %}—{% endif -%}
</td>
{% endif %}
</tr>
{% endfor %}
</tbody>
</table>
{% if healthy_more %}<div class="w-more">+{{ healthy_more }} more healthy{% if session.user_id %} · <a href="/plugins/traefik/">all routers</a>{% endif %}</div>{% endif %}
{% if loose_certs %}
<div class="w-section">Certificates expiring</div>
{% for c in loose_certs %}
<div class="w-item">
{{- status_dot(status_class(c.days, 'tls_days')) -}}
<span class="w-label" title="{{ c.cn }}">{{ c.cn }}</span>
<span class="w-meta" style="{{ threshold_style(c.days, 'tls_days') }}">{{ cert_left(c.days) }}</span>
</div>
{% endfor %}
<div class="w-note">Traefik's API didn't say which hosts each router serves, so these can't be shown on their routers.</div>
{% endif %}
{% endif %}
+46
View File
@@ -0,0 +1,46 @@
"""Integration: the newest Traefik row per router, read by walking the
(router_name, scraped_at) index rather than grouping every row (#5254).
Requires STEWARD_DATABASE_URL."""
from __future__ import annotations
import asyncio
import os
import uuid
from datetime import datetime, timedelta, timezone
import pytest
pytestmark = pytest.mark.integration
@pytest.fixture
def app():
if not os.environ.get("STEWARD_DATABASE_URL"):
pytest.skip("needs Postgres")
from steward.app import create_app
return create_app(testing=False)
def test_each_router_reads_as_its_newest_row(app):
from plugins.traefik.models import TraefikMetric
from plugins.traefik.routes import latest_per_router
now = datetime.now(timezone.utc).replace(microsecond=0)
tag = uuid.uuid4().hex[:8]
a, b = f"a-{tag}@docker", f"b-{tag}@docker"
def row(name, ago, rate):
return TraefikMetric(router_name=name, scraped_at=now - timedelta(minutes=ago), request_rate=rate,
error_rate_4xx_pct=0.0, error_rate_5xx_pct=0.0, latency_p50_ms=1.0,
latency_p95_ms=2.0, latency_p99_ms=3.0, response_bytes_rate=0.0)
async def go():
async with app.db_sessionmaker() as s:
async with s.begin():
s.add_all([row(a, 3, 1.0), row(a, 1, 2.0), row(a, 2, 3.0), row(b, 5, 4.0)])
async with s.begin():
return {m.router_name: m for m in await latest_per_router(s)}
got = asyncio.run(go())
assert got[a].request_rate == 2.0 and got[a].scraped_at == now - timedelta(minutes=1)
assert got[b].request_rate == 4.0
+124 -4
View File
@@ -48,10 +48,10 @@ async def test_the_page_leads_with_problems_and_folds_the_healthy(app):
problems, healthy = split_routers([
_router("api", _latest(e5=3.0)), _router("web", _latest()), _router("docs", _latest())])
now = datetime.now(timezone.utc)
certs = [{"serial": "1", "cn": "soon.example", "sans": [], "not_after": now + timedelta(days=2),
"days_remaining": 2.0},
{"serial": "2", "cn": "fine.example", "sans": [], "not_after": now + timedelta(days=60),
"days_remaining": 60.0}]
certs = [{"serial": "1", "cn": "soon.example", "sans": [], "routers": None,
"not_after": now + timedelta(days=2), "days_remaining": 2.0},
{"serial": "2", "cn": "fine.example", "sans": [], "routers": None,
"not_after": now + timedelta(days=60), "days_remaining": 60.0}]
async with app.test_request_context("/plugins/traefik/rows"):
page = await render_plugin_template(app, "traefik", "traefik/rows.html", problems=problems,
healthy=healthy, global_stat=None, certs=certs, range_key="24h",
@@ -110,3 +110,123 @@ async def test_the_traffic_empty_state_knows_whether_the_log_is_on(app, enabled,
range_key="24h", access_log_enabled=enabled,
access_log_path="/var/log/traefik/access.log")
assert says in page and not_says not in page
# ── Routers widget: current routers, certs on the router they serve (#5254) ──
@pytest.mark.parametrize("rule, hosts", [
("Host(`app.example.com`)", ["app.example.com"]),
("Host(`a.example.com`, `B.example.com`) && PathPrefix(`/api`)", ["a.example.com", "b.example.com"]),
("Host(`a.example.com`) || Host(`b.example.com`)", ["a.example.com", "b.example.com"]),
("HostSNI(`db.example.com`)", ["db.example.com"]),
("HostSNI(`*`)", []),
("PathPrefix(`/`)", []),
("", []),
])
def test_a_router_rule_names_its_hosts(rule, hosts):
from plugins.traefik.scraper import rule_hosts
assert rule_hosts(rule) == hosts
def test_a_certificate_covers_its_names_and_one_wildcard_label():
from plugins.traefik.scraper import cert_names, covers
names = cert_names("Example.com", "*.example.com, www.other.org")
assert names == {"example.com", "*.example.com", "www.other.org"}
assert covers(names, "example.com") and covers(names, "app.example.com")
assert covers(names, "www.other.org")
assert not covers(names, "a.b.example.com") and not covers(names, "other.org")
@pytest.mark.parametrize("metrics_url, api_url, base", [
("http://traefik:8080/metrics", "", "http://traefik:8080/api"),
("http://traefik:8080/metrics?x=1", "", "http://traefik:8080/api"),
("http://traefik:8080/metrics", "http://other:9000/api/", "http://other:9000/api"),
])
def test_the_api_sits_beside_the_metrics_unless_configured(metrics_url, api_url, base):
from plugins.traefik.scraper import api_base
assert api_base(metrics_url, api_url) == base
def test_a_metrics_router_finds_its_api_name_with_or_without_the_entrypoint():
from plugins.traefik.routes import router_hosts
domains = {"app@docker": ["app.example.com"]}
assert router_hosts("app@docker", domains) == ["app.example.com"]
assert router_hosts("websecure-app@docker", domains) == ["app.example.com"]
assert router_hosts("myapp@docker", domains) == []
def _cert(serial, cn, days, seen_ago=30, sans=None):
now = datetime.now(timezone.utc)
return NS(serial=serial, cn=cn, sans=sans, not_after=now + timedelta(days=days),
scraped_at=now - timedelta(seconds=seen_ago))
def test_a_certificate_traefik_stopped_reporting_is_old_not_current():
from plugins.traefik.routes import current_certs
live, renewed = _cert("1", "a.example", 80), _cert("2", "a.example", 5, seen_ago=3600)
served, old = current_certs([live, renewed], 60)
assert served == [live] and old == [renewed]
assert current_certs([], 60) == ([], [])
def test_each_router_gets_the_soonest_certificate_for_its_hosts():
from plugins.traefik.routes import certs_by_router
now = datetime.now(timezone.utc)
certs = [_cert("1", "*.example.com", 40), _cert("2", "app.example.com", 10), _cert("3", "x.org", 2)]
got = certs_by_router(["app@docker", "docs@docker", "plain@docker"],
{"app@docker": ["app.example.com"], "docs@docker": ["docs.example.com"]},
certs, now)
assert got["app@docker"]["cn"] == "app.example.com" and int(got["app@docker"]["days"]) == 9
assert got["docs@docker"]["cn"] == "*.example.com"
assert "plain@docker" not in got
async def test_the_widget_gives_a_verdict_then_routers_under_one_header(app):
slow = _router("api@docker", _latest(p95=450.0))
expiring = _router("shop@docker", _latest(req=3.0))
expiring.update(state="crit", why="certificate expires in 2 days",
cert={"cn": "shop.example", "days": 2.0, "grade": "crit"})
fine = _router("web@docker", _latest(req=9.0))
fine["cert"] = {"cn": "web.example", "days": 60.0, "grade": "ok"}
async with app.test_request_context("/plugins/traefik/widget"):
page = await render_plugin_template(
app, "traefik", "traefik/widget.html", problems=[expiring, slow], healthy=[fine],
healthy_more=4, total=7, loose_certs=[], domains_known=True)
page = " ".join(page.split())
assert "2 of 7 routers need a look." in page
assert page.count("<th>Router</th>") == 1 and "<th>TLS</th>" in page
# Problems first, each with why; the cert sits on its router, not above.
assert page.index(">shop") < page.index(">api") < page.index(">web")
assert "certificate expires in 2 days" in page and "p95 latency is 450 ms" in page
assert "60 days" in page and "Certificates expiring" not in page
assert '</span>shop<span class="w-sub' in page and ">shop@docker" not in page # provider off the label
assert "+4 more healthy" in page
async def test_without_router_domains_expiring_certs_follow_the_routers(app):
fine = _router("web@docker", _latest())
async with app.test_request_context("/plugins/traefik/widget"):
page = await render_plugin_template(
app, "traefik", "traefik/widget.html", problems=[], healthy=[fine], healthy_more=0, total=1,
loose_certs=[{"cn": "old.example", "days": 5.0}], domains_known=False)
page = " ".join(page.split())
assert "All 1 router healthy; 1 certificate expiring." in page
assert "<th>TLS</th>" not in page
assert page.index("<th>Router</th>") < page.index("Certificates expiring") < page.index("old.example")
async def test_the_page_sets_certificates_traefik_no_longer_serves_apart(app):
now = datetime.now(timezone.utc)
certs = [{"serial": "1", "cn": "app.example", "sans": ["www.app.example"], "routers": ["app"],
"not_after": now + timedelta(days=40), "days_remaining": 40.0},
{"serial": "2", "cn": "spare.example", "sans": [], "routers": [],
"not_after": now + timedelta(days=40), "days_remaining": 40.0}]
old = [{"serial": "3", "cn": "gone.example", "sans": [], "routers": [],
"not_after": now + timedelta(days=5), "days_remaining": 5.0}]
async with app.test_request_context("/plugins/traefik/rows"):
page = await render_plugin_template(app, "traefik", "traefik/rows.html", problems=[], healthy=[],
gone=[], global_stat=None, certs=certs, old_certs=old,
range_key="24h", error_4xx=1.0, error_5xx=0.1)
page = " ".join(page.split())
assert "also www.app.example" in page and ">app</td>" in page and ">not in use</td>" in page
assert page.index("spare.example") < page.index("No longer served (1)") < page.index("gone.example")