bvandeusenandClaude Opus 5.5
39b6066f3b
#5256 "Traefik — Request chart: blank on the live dashboard; time axis and honest labels". - "Req/s" was the average of the routers, not the total. Each scrape is now summed over routers, then averaged per bucket (request_history). - The 5xx share is now weighted by each router's traffic. The axis is no longer pinned at 0–100 (suggestedMax 5). - The x axis shows times, in the viewer's local time. The tooltip shows both series at that time. - A verdict line comes first: now / peak req/s and the peak 5xx share. The figures read even if the chart doesn't. - The blank live chart's cause is still unknown. Chart.js and chart-colors.js both load on the dashboard. A missing library or a thrown error is now written into the widget instead of leaving it empty, so the cause will show on the page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
249 lines
13 KiB
Python
249 lines
13 KiB
Python
"""The Traefik page puts failing and slow routers first and folds the healthy
|
|
ones into one table; certificates are graded by Settings → Thresholds (#5126)."""
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace as NS
|
|
|
|
import pytest
|
|
|
|
from plugins.traefik.routes import gone_routers, router_state, split_routers
|
|
from steward.core.settings import to_thresholds_cfg
|
|
from tests._templates import render_plugin_template
|
|
from tests.test_style_tokens import hex_colours
|
|
|
|
TH = to_thresholds_cfg({}) # latency crit 200 ms
|
|
|
|
|
|
def _latest(req=1.0, p95=20.0, e4=0.0, e5=0.0):
|
|
return NS(request_rate=req, response_bytes_rate=1000.0, latency_p95_ms=p95, latency_p99_ms=p95 * 1.5,
|
|
error_rate_4xx_pct=e4, error_rate_5xx_pct=e5,
|
|
scraped_at=datetime.now(timezone.utc) - timedelta(seconds=30))
|
|
|
|
|
|
@pytest.mark.parametrize("latest, grade", [
|
|
(_latest(), "ok"),
|
|
(_latest(e5=2.5), "crit"),
|
|
(_latest(p95=450.0), "warn"),
|
|
(_latest(e4=5.0), "warn"),
|
|
(_latest(p95=120.0, e4=0.5), "ok"), # slow-ish but under the critical cutoff
|
|
])
|
|
def test_router_state(latest, grade):
|
|
assert router_state(latest, TH)[0] == grade
|
|
|
|
|
|
def _router(name, latest):
|
|
state, why = router_state(latest, TH)
|
|
return {"name": name, "latest": latest, "state": state, "why": why,
|
|
"sparkline_req": "", "sparkline_p95": "", "sparkline_5xx": ""}
|
|
|
|
|
|
def test_failing_routers_come_before_slow_ones_and_healthy_ones_split_off():
|
|
problems, healthy = split_routers([
|
|
_router("b-ok", _latest()), _router("slow", _latest(p95=450.0)),
|
|
_router("broken", _latest(e5=3.0)), _router("a-ok", _latest())])
|
|
assert [r["name"] for r in problems] == ["broken", "slow"]
|
|
assert [r["name"] for r in healthy] == ["a-ok", "b-ok"]
|
|
|
|
|
|
async def test_the_page_leads_with_problems_and_folds_the_healthy(app):
|
|
problems, healthy = split_routers([
|
|
_router("api", _latest(e5=3.0)), _router("web", _latest()), _router("docs", _latest())])
|
|
now = datetime.now(timezone.utc)
|
|
certs = [{"serial": "1", "cn": "soon.example", "sans": [], "routers": None,
|
|
"not_after": now + timedelta(days=2), "days_remaining": 2.0},
|
|
{"serial": "2", "cn": "fine.example", "sans": [], "routers": None,
|
|
"not_after": now + timedelta(days=60), "days_remaining": 60.0}]
|
|
async with app.test_request_context("/plugins/traefik/rows"):
|
|
page = await render_plugin_template(app, "traefik", "traefik/rows.html", problems=problems,
|
|
healthy=healthy, global_stat=None, certs=certs, range_key="24h",
|
|
error_4xx=1.0, error_5xx=0.1)
|
|
|
|
assert page.index(">api</h3>") < page.index('id="traefik-healthy"')
|
|
assert "3.0% of responses are 5xx" in page
|
|
assert ">2 healthy routers</summary>" in page
|
|
assert page.index(">docs</td>") < page.index(">web</td>")
|
|
assert 'class="tone-crit">2 days</td>' in page and 'class="tone-ok">60 days</td>' in page
|
|
assert "font-weight:600" not in page and not hex_colours(page), hex_colours(page)
|
|
|
|
|
|
def test_a_router_traefik_stopped_reporting_is_set_apart_not_failing():
|
|
live = _router("api", _latest())
|
|
removed = _router("old", _latest(e5=9.0))
|
|
removed["latest"].scraped_at = live["latest"].scraped_at - timedelta(minutes=10)
|
|
recent = _router("blip", _latest())
|
|
recent["latest"].scraped_at = live["latest"].scraped_at - timedelta(seconds=60)
|
|
current, gone = gone_routers([live, removed, recent], 30) # 3 scrapes = 90 s
|
|
assert [r["name"] for r in current] == ["api", "blip"]
|
|
assert [r["name"] for r in gone] == ["old"]
|
|
assert gone_routers([], 30) == ([], [])
|
|
|
|
|
|
async def test_gone_routers_are_listed_by_when_they_were_last_seen(app):
|
|
removed = _router("old", _latest(e5=9.0))
|
|
removed["latest"].scraped_at = datetime.now(timezone.utc) - timedelta(hours=2)
|
|
async with app.test_request_context("/plugins/traefik/rows"):
|
|
page = await render_plugin_template(app, "traefik", "traefik/rows.html", problems=[], healthy=[],
|
|
gone=[removed], global_stat=None, certs=[], range_key="24h",
|
|
error_4xx=1.0, error_5xx=0.1)
|
|
page = " ".join(page.split())
|
|
assert "1 router no longer reported</summary>" in page
|
|
assert "old</span> <span class=\"problem-detail\">last seen <time" in page and ">2h ago</time>" in page
|
|
assert "Failing" not in page
|
|
|
|
|
|
@pytest.mark.parametrize("days, says", [
|
|
(12.7, "12 days"), (1.2, "1 day"), (0.4, "expires today"),
|
|
(-0.3, "expired today"), (-1.5, "expired 1 day ago"), (-12.0, "expired 12 days ago"), (None, "—"),
|
|
])
|
|
def test_a_certificate_past_its_date_says_expired(days, says):
|
|
"""#5212: "expired 12 days ago", never "-12 days"."""
|
|
from steward.core.formatting import cert_left
|
|
assert cert_left(days) == says
|
|
|
|
|
|
@pytest.mark.parametrize("enabled, says, not_says", [
|
|
(False, "The access log is off", "No requests in the last"),
|
|
(True, "No requests in the last 24h", "The access log is off"),
|
|
])
|
|
async def test_the_traffic_empty_state_knows_whether_the_log_is_on(app, enabled, says, not_says):
|
|
async with app.test_request_context("/plugins/traefik/traffic"):
|
|
page = await render_plugin_template(app, "traefik", "traefik/traffic.html", summary=None, history=[],
|
|
range_key="24h", access_log_enabled=enabled,
|
|
access_log_path="/var/log/traefik/access.log")
|
|
assert says in page and not_says not in page
|
|
|
|
|
|
# ── Routers widget: current routers, certs on the router they serve (#5254) ──
|
|
|
|
@pytest.mark.parametrize("rule, hosts", [
|
|
("Host(`app.example.com`)", ["app.example.com"]),
|
|
("Host(`a.example.com`, `B.example.com`) && PathPrefix(`/api`)", ["a.example.com", "b.example.com"]),
|
|
("Host(`a.example.com`) || Host(`b.example.com`)", ["a.example.com", "b.example.com"]),
|
|
("HostSNI(`db.example.com`)", ["db.example.com"]),
|
|
("HostSNI(`*`)", []),
|
|
("PathPrefix(`/`)", []),
|
|
("", []),
|
|
])
|
|
def test_a_router_rule_names_its_hosts(rule, hosts):
|
|
from plugins.traefik.scraper import rule_hosts
|
|
assert rule_hosts(rule) == hosts
|
|
|
|
|
|
def test_a_certificate_covers_its_names_and_one_wildcard_label():
|
|
from plugins.traefik.scraper import cert_names, covers
|
|
names = cert_names("Example.com", "*.example.com, www.other.org")
|
|
assert names == {"example.com", "*.example.com", "www.other.org"}
|
|
assert covers(names, "example.com") and covers(names, "app.example.com")
|
|
assert covers(names, "www.other.org")
|
|
assert not covers(names, "a.b.example.com") and not covers(names, "other.org")
|
|
|
|
|
|
@pytest.mark.parametrize("metrics_url, api_url, base", [
|
|
("http://traefik:8080/metrics", "", "http://traefik:8080/api"),
|
|
("http://traefik:8080/metrics?x=1", "", "http://traefik:8080/api"),
|
|
("http://traefik:8080/metrics", "http://other:9000/api/", "http://other:9000/api"),
|
|
])
|
|
def test_the_api_sits_beside_the_metrics_unless_configured(metrics_url, api_url, base):
|
|
from plugins.traefik.scraper import api_base
|
|
assert api_base(metrics_url, api_url) == base
|
|
|
|
|
|
def test_a_metrics_router_finds_its_api_name_with_or_without_the_entrypoint():
|
|
from plugins.traefik.routes import router_hosts
|
|
domains = {"app@docker": ["app.example.com"]}
|
|
assert router_hosts("app@docker", domains) == ["app.example.com"]
|
|
assert router_hosts("websecure-app@docker", domains) == ["app.example.com"]
|
|
assert router_hosts("myapp@docker", domains) == []
|
|
|
|
|
|
def _cert(serial, cn, days, seen_ago=30, sans=None):
|
|
now = datetime.now(timezone.utc)
|
|
return NS(serial=serial, cn=cn, sans=sans, not_after=now + timedelta(days=days),
|
|
scraped_at=now - timedelta(seconds=seen_ago))
|
|
|
|
|
|
def test_a_certificate_traefik_stopped_reporting_is_old_not_current():
|
|
from plugins.traefik.routes import current_certs
|
|
live, renewed = _cert("1", "a.example", 80), _cert("2", "a.example", 5, seen_ago=3600)
|
|
served, old = current_certs([live, renewed], 60)
|
|
assert served == [live] and old == [renewed]
|
|
assert current_certs([], 60) == ([], [])
|
|
|
|
|
|
def test_each_router_gets_the_soonest_certificate_for_its_hosts():
|
|
from plugins.traefik.routes import certs_by_router
|
|
now = datetime.now(timezone.utc)
|
|
certs = [_cert("1", "*.example.com", 40), _cert("2", "app.example.com", 10), _cert("3", "x.org", 2)]
|
|
got = certs_by_router(["app@docker", "docs@docker", "plain@docker"],
|
|
{"app@docker": ["app.example.com"], "docs@docker": ["docs.example.com"]},
|
|
certs, now)
|
|
assert got["app@docker"]["cn"] == "app.example.com" and round(got["app@docker"]["days"]) == 10
|
|
assert got["docs@docker"]["cn"] == "*.example.com"
|
|
assert "plain@docker" not in got
|
|
|
|
|
|
async def test_the_widget_gives_a_verdict_then_routers_under_one_header(app):
|
|
slow = _router("api@docker", _latest(p95=450.0))
|
|
expiring = _router("shop@docker", _latest(req=3.0))
|
|
expiring.update(state="crit", why="certificate expires in 2 days",
|
|
cert={"cn": "shop.example", "days": 2.0, "grade": "crit"})
|
|
fine = _router("web@docker", _latest(req=9.0))
|
|
fine["cert"] = {"cn": "web.example", "days": 60.0, "grade": "ok"}
|
|
async with app.test_request_context("/plugins/traefik/widget"):
|
|
page = await render_plugin_template(
|
|
app, "traefik", "traefik/widget.html", problems=[expiring, slow], healthy=[fine],
|
|
healthy_more=4, total=7, loose_certs=[], domains_known=True)
|
|
page = " ".join(page.split())
|
|
assert "2 of 7 routers need a look." in page
|
|
assert page.count("<th>Router</th>") == 1 and "<th>TLS</th>" in page
|
|
# Problems first, each with why; the cert sits on its router, not above.
|
|
assert page.index(">shop") < page.index(">api") < page.index(">web")
|
|
assert "certificate expires in 2 days" in page and "p95 latency is 450 ms" in page
|
|
assert "60 days" in page and "Certificates expiring" not in page
|
|
assert '</span>shop<span class="w-sub' in page and ">shop@docker" not in page # provider off the label
|
|
assert "+4 more healthy" in page
|
|
|
|
|
|
async def test_without_router_domains_expiring_certs_follow_the_routers(app):
|
|
fine = _router("web@docker", _latest())
|
|
async with app.test_request_context("/plugins/traefik/widget"):
|
|
page = await render_plugin_template(
|
|
app, "traefik", "traefik/widget.html", problems=[], healthy=[fine], healthy_more=0, total=1,
|
|
loose_certs=[{"cn": "old.example", "days": 5.0}], domains_known=False)
|
|
page = " ".join(page.split())
|
|
assert "All 1 router healthy; 1 certificate expiring." in page
|
|
assert "<th>TLS</th>" not in page
|
|
assert page.index("<th>Router</th>") < page.index("Certificates expiring") < page.index("old.example")
|
|
|
|
|
|
async def test_the_page_sets_certificates_traefik_no_longer_serves_apart(app):
|
|
now = datetime.now(timezone.utc)
|
|
certs = [{"serial": "1", "cn": "app.example", "sans": ["www.app.example"], "routers": ["app"],
|
|
"not_after": now + timedelta(days=40), "days_remaining": 40.0},
|
|
{"serial": "2", "cn": "spare.example", "sans": [], "routers": [],
|
|
"not_after": now + timedelta(days=40), "days_remaining": 40.0}]
|
|
old = [{"serial": "3", "cn": "gone.example", "sans": [], "routers": [],
|
|
"not_after": now + timedelta(days=5), "days_remaining": 5.0}]
|
|
async with app.test_request_context("/plugins/traefik/rows"):
|
|
page = await render_plugin_template(app, "traefik", "traefik/rows.html", problems=[], healthy=[],
|
|
gone=[], global_stat=None, certs=certs, old_certs=old,
|
|
range_key="24h", error_4xx=1.0, error_5xx=0.1)
|
|
page = " ".join(page.split())
|
|
assert "also www.app.example" in page and ">app</td>" in page and ">not in use</td>" in page
|
|
assert page.index("spare.example") < page.index("No longer served (1)") < page.index("gone.example")
|
|
|
|
|
|
async def test_the_request_chart_says_its_figures_before_the_chart(app):
|
|
async with app.test_request_context("/plugins/traefik/widget/request_chart"):
|
|
page = await render_plugin_template(
|
|
app, "traefik", "traefik/widget_request_chart.html", labels_json="[1, 2]",
|
|
req_rate_json="[1.5, 4.25]", error_rate_json="[0, 0.5]", widget_id="7", hours=6,
|
|
now_req=4.25, peak_req=4.25, peak_err=0.5)
|
|
empty = await render_plugin_template(
|
|
app, "traefik", "traefik/widget_request_chart.html", labels_json="[]", hours=6)
|
|
page = " ".join(page.split())
|
|
assert "Now 4.25 req/s across all routers, peak 4.25. 5xx peaked at 0.50% of responses." in page
|
|
assert page.index("w-verdict") < page.index("<canvas")
|
|
assert "Requests/s (all routers)" in page and "max: 100" not in page
|
|
assert "couldn't be drawn" in page # a failure says why in place of the chart
|
|
assert "No Traefik requests recorded in the last 6h." in empty
|