2 Commits
Author SHA1 Message Date
bvandeusenandClaude Opus 5.5 ea5531ac8f feat(traefik): Routers widget shows current routers only, one header row, certs on the routers they serve (#5254)
CI / lint (push) Successful in 2s
CI / publish (push) Skipped
CI / integration (push) Successful in 52s
CI / unit (push) Failing after 1m17s
#5254 "Traefik — Routers: current routers only, header row, certs on the router they serve".

- Router hosts come from Traefik's API (/api/http|tcp/routers), read
  best-effort on each scrape. An optional api_url overrides it; the default
  is /api on the metrics host. A certificate's expiry then shows in a TLS
  column on the router whose host it covers. An expiring one makes that
  router a problem, graded by Settings → Thresholds.
- Only current: routers and certificates must have appeared in the last
  three scrapes. Certificates are upserted by serial and never deleted, so
  renewed or retired ones lingered on the widget.
- Widget order: verdict, problem routers with the reason, then the busiest
  healthy routers under one header (Router / Req/s / Bandwidth / p95 / TLS).
  A missing value shows "—".
- If the API is unreachable, expiring certificates are listed after the
  routers, not above them.
- The page splits certificates into served (with the routers using each,
  plus SANs) and "No longer served".
- Latest row per router: an index walk (recursive CTE + LATERAL) replaces
  DISTINCT plus a query per router.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-07 13:25:11 -04:00
bvandeusenandClaude Opus 5.5 d72a1991a7 feat(docker): Busiest Containers shows each figure with its scale; readable names (#5253)
The widget names the busiest container in a verdict ("Busiest: tdarr on
VDNT-Docker03, 1.6 of 8 cores."), then a fixed-layout table (container,
host, CPU in cores, memory in bytes) with each figure's share drawn as
shading behind it: of the host's cores, and of the memory limit or the
host's memory.

The numbers were already rendered, but each row grew as wide as its longest
name. A grid item won't shrink below its content, so Gitea CI container
names pushed the figures out of view and the cell scrolled sideways. Names
now come from display_name(): a swarm task by its service (no slot number),
an Actions job container as "CI: <workflow> · <job>". They cut off with …
and show whole on hover.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-07 13:18:44 -04:00
13 changed files with 555 additions and 165 deletions
+1 -4
View File
@@ -284,7 +284,6 @@ async def widget_resources():
.where(DockerMetric.scraped_at >= since)
.group_by(DockerMetric.host_id, DockerMetric.container_name)
)).all()}
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
hosts = await docker_svc.host_map(db, {c.host_id for c in running})
capacity = await docker_svc.host_capacity(db, {c.host_id for c in running})
@@ -293,10 +292,9 @@ async def widget_resources():
return c.cpu_pct if v is None else float(v)
busiest = sorted(running, key=lambda c: (-(cpu_of(c) or 0), c.name))[:limit]
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
rows_data = [{
"c": c,
"label": docker_svc.task_label(c, node_hostname),
"label": docker_svc.display_name(c),
"host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id,
"usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c)),
} for c in busiest]
@@ -304,7 +302,6 @@ async def widget_resources():
"docker/widget_resources.html",
rows=rows_data,
minutes=CPU_AVERAGE_MINUTES,
multi_host=len({c.host_id for c in busiest}) > 1,
widget_id=widget_id,
)
+19
View File
@@ -6,6 +6,7 @@ Quart's pipeline) call the same code.
"""
from __future__ import annotations
import re
from datetime import datetime, timedelta, timezone
from sqlalchemy import func, select
@@ -126,6 +127,24 @@ def task_label(c, node_hostname: dict | None = None) -> str:
return f"{svc} · {host}" if host else svc
_CI_JOB = re.compile(
r"^(?:GITEA|FORGEJO)-ACTIONS-TASK-\d+-WORKFLOW-(?P<workflow>.+?)-JOB-(?P<job>.+?)(?:-[0-9a-f]{16,})?$")
def display_name(c) -> str:
"""A container's name as a list can show it in one short line (#5253): a
swarm task by its service (the host says where it runs), a Gitea or
Forgejo Actions job container as "CI: <workflow> · <job>", anything else
by its own name."""
svc = getattr(c, "service_name", None)
if svc:
return svc
m = _CI_JOB.match(c.name or "")
if m:
return f"CI: {m['workflow']} · {m['job']}"
return c.name
def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = None) -> dict:
"""A container's CPU and memory in terms a reader can weigh.
@@ -1,42 +1,41 @@
{# docker/widget_resources.html — dashboard widget: the containers using the
most CPU (averaged over the last few minutes, so the order holds still),
each with its CPU in cores and memory in bytes. The CPU bar is a share of
the host's CPUs; the memory bar a share of the container's limit, or of
the host's memory when no limit is set. #}
{% from "docker/_usage.html" import cpu, mem %}
{# docker/widget_resources.html — the dashboard Busiest Containers widget
(#5253): a verdict naming the busiest, then a table of the top containers
by CPU (averaged over the last few minutes, so the order holds still). Each
figure carries its unit; the shaded bar behind it is its share of the
host's cores, or of the memory limit (the host's memory when none is set). #}
{% if not rows %}
<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No running containers.</div>
<div class="w-verdict" style="color:var(--text-muted);">No running containers.</div>
{% else %}
{% macro bar(label, pct, tone, text, title) %}
<div style="display:flex;align-items:center;gap:0.45rem;margin-bottom:2px;" title="{{ title }}">
<span style="font-size:0.62rem;color:var(--text-dim);width:1.9rem;flex-shrink:0;letter-spacing:0.03em;">{{ label }}</span>
<div style="flex:1;height:6px;background:var(--bg-elevated);border-radius:3px;overflow:hidden;">
{% if pct is not none %}
<div style="height:100%;border-radius:3px;width:{{ [[pct, 0]|max, 100]|min }}%;
background:{% if tone == 'crit' %}var(--red){% elif tone == 'warn' %}var(--orange){% else %}var(--accent){% endif %};
transition:width 0.4s;"></div>
{% endif %}
</div>
<span style="font-size:0.7rem;color:var(--text-muted);font-family:ui-monospace,monospace;min-width:4.5rem;text-align:right;flex-shrink:0;white-space:nowrap;">{{ text }}</span>
{% set top = rows[0] %}
<div class="w-verdict">
Busiest: <strong>{{ top.label }}</strong> on {{ top.host_name }},
{% if top.usage.cores is not none %}{{ "%.2f"|format(top.usage.cores) if top.usage.cores < 1 else "%.1f"|format(top.usage.cores) }}
{%- if top.usage.cpus %} of {{ top.usage.cpus }} cores{% else %} cores{% endif %}{% else %}no CPU reading yet{% endif %}.
</div>
{% endmacro %}
<div style="font-size:0.7rem;color:var(--text-dim);margin-bottom:0.45rem;">Busiest by CPU, {{ minutes }}-minute average</div>
<div style="display:grid;gap:0.6rem;">
<table class="w-table">
<colgroup><col><col class="w-wide-only" style="width:7.5rem"><col style="width:5.5rem"><col style="width:6.5rem"></colgroup>
<thead><tr><th>Container</th><th class="w-wide-only">Host</th><th>CPU</th><th>Memory</th></tr></thead>
<tbody>
{% for r in rows %}
{% set u = r.usage %}
<div>
<div style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-bottom:0.25rem;" title="{{ r.c.name }}">
<a href="/plugins/docker/container/{{ r.c.host_id }}/{{ r.c.name }}" style="color:inherit;text-decoration:none;">{{ r.label }}</a>{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
</div>
{% if u.cores is not none %}
{{ bar("CPU", u.cpu_share, status_class(u.cpu_share, 'cpu'), cpu(u),
("%.1f%% of the host's %d core%s" % (u.cpu_share, u.cpus, "" if u.cpus == 1 else "s")) if u.cpus else "Host core count unknown") }}
<tr>
<td class="w-name" title="{{ r.c.name }}">
{%- if session.user_id %}<a href="/plugins/docker/container/{{ r.c.host_id }}/{{ r.c.name }}">{{ r.label }}</a>{% else %}{{ r.label }}{% endif -%}
</td>
<td class="w-wide-only" title="{{ r.host_name }}">{{ r.host_name }}</td>
{% if u.cores is none %}<td>—</td>{% else %}
<td class="w-meter tone-{{ status_class(u.cpu_share, 'cpu') if u.cpus else 'dim' }}" style="--pct:{{ [[u.cpu_share or 0, 0]|max, 100]|min }}%"
title="{% if u.cpus %}{{ '%.1f'|format(u.cpu_share) }}% of {{ r.host_name }}'s {{ u.cpus }} cores{% else %}The host's core count isn't known{% endif %}">
{{- "%.2f"|format(u.cores) if u.cores < 1 else "%.1f"|format(u.cores) }} cores</td>
{% endif %}
{% if u.mem_bytes is not none %}
{{ bar("MEM", u.mem_pct if u.mem_of else none, status_class(u.mem_pct, 'mem') if u.mem_of else 'unknown', mem(u),
("%.1f%% of %s" % (u.mem_pct, u.mem_of)) if u.mem_of and u.mem_pct is not none else "") }}
{% if u.mem_bytes is none %}<td>—</td>{% else %}
<td class="w-meter tone-{{ status_class(u.mem_pct, 'mem') if u.mem_of else 'dim' }}" style="--pct:{{ [[u.mem_pct or 0, 0]|max, 100]|min }}%"
title="{% if u.mem_pct is not none and u.mem_of %}{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% else %}No memory limit set{% endif %}">
{{- u.mem_bytes|bytes }}</td>
{% endif %}
</div>
</tr>
{% endfor %}
</div>
</tbody>
</table>
<div class="w-note">CPU over the last {{ minutes }} minutes. Shading: share of the host's cores, and of the memory limit (or the host's memory when no limit is set).</div>
{% endif %}
+3
View File
@@ -14,6 +14,9 @@ tags:
config:
metrics_url: "http://localhost:8080/metrics"
# Traefik's API, for which hosts each router serves (so a certificate's
# expiry shows on its router). Empty: /api on the metrics URL's host.
api_url: ""
scrape_interval_seconds: 60
access_log:
enabled: false
+130 -66
View File
@@ -3,14 +3,16 @@ from __future__ import annotations
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from quart import Blueprint, current_app, render_template, request
from sqlalchemy import Integer, cast, func, select
from sqlalchemy import Integer, cast, func, select, text
import json
from steward.auth.middleware import require_role
from steward.models.users import UserRole
from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample
from steward.core.formatting import status_class
from steward.core.formatting import cert_left, status_class
from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric
from .scheduler import router_domains
from .scraper import cert_names, covers
traefik_bp = Blueprint("traefik", __name__, template_folder="templates")
@@ -85,6 +87,71 @@ def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]:
return problems, healthy
# Newest row per router. Postgres has no skip scan, so DISTINCT or max() GROUP
# BY reads every row in retention; this walks the (router_name, scraped_at)
# index one router at a time instead: a step per router, not per row.
_LATEST_PER_ROUTER = text("""
WITH RECURSIVE names AS (
SELECT min(router_name) AS name FROM traefik_metrics
UNION ALL
SELECT (SELECT min(router_name) FROM traefik_metrics WHERE router_name > names.name)
FROM names WHERE names.name IS NOT NULL
)
SELECT m.* FROM names
JOIN LATERAL (
SELECT * FROM traefik_metrics WHERE router_name = names.name
ORDER BY scraped_at DESC LIMIT 1
) m ON true
""")
async def latest_per_router(db) -> list:
return list((await db.execute(
select(TraefikMetric).from_statement(_LATEST_PER_ROUTER))).scalars().all())
def current_certs(certs, scrape_seconds: int) -> tuple[list, list]:
"""(served, old): a certificate is served while Traefik still reports it.
Rows are upserted by serial and never deleted, so a renewed or retired
certificate lingers; one missing from the last GONE_AFTER_SCRAPES
scrapes is old (#5254)."""
certs = list(certs)
if not certs:
return [], []
newest = max(c.scraped_at for c in certs)
cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
return ([c for c in certs if c.scraped_at >= cutoff],
sorted((c for c in certs if c.scraped_at < cutoff), key=lambda c: c.scraped_at, reverse=True))
def router_hosts(name: str, domains: dict[str, list[str]]) -> list[str]:
"""The hosts a metrics router name serves. The API names a router
"app@docker"; metrics may carry it as "websecure-app@docker", one per
entrypoint, so a name not found as-is is tried without that prefix."""
if name in domains:
return domains[name]
for api_name, hosts in domains.items():
if name.endswith("-" + api_name):
return hosts
return []
def certs_by_router(routers: list[str], domains: dict[str, list[str]], certs, now) -> dict[str, dict]:
"""{router: {"cn", "days"}}: the soonest-expiring served certificate valid
for one of the router's hosts. A router with no hosts, or none covered,
is absent."""
out: dict[str, dict] = {}
names = [(c, cert_names(c.cn, c.sans)) for c in certs]
for router in routers:
for host in router_hosts(router, domains):
for c, cn in names:
if covers(cn, host):
days = (c.not_after - now).total_seconds() / 86400.0
if router not in out or days < out[router]["days"]:
out[router] = {"cn": c.cn or host, "days": days}
return out
def _access_log() -> dict:
"""The plugin's access_log settings ({} when unset or malformed)."""
cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {})
@@ -114,13 +181,9 @@ async def rows():
thresholds = current_app.config.get("THRESHOLDS", {})
async with current_app.db_sessionmaker() as db:
# All known routers
result = await db.execute(
select(TraefikMetric.router_name)
.distinct()
.order_by(TraefikMetric.router_name)
)
routers = [row[0] for row in result.all()]
# Every known router, with its most recent raw row
newest = {m.router_name: m for m in await latest_per_router(db)}
routers = sorted(newest)
b_secs = bucket_seconds(since)
bucket_col = (
@@ -145,16 +208,7 @@ async def rows():
)
history = result.all()
# Latest value — always the most recent raw row
result = await db.execute(
select(TraefikMetric)
.where(TraefikMetric.router_name == router)
.order_by(TraefikMetric.scraped_at.desc())
.limit(1)
)
latest = result.scalar_one_or_none()
if not latest:
continue
latest = newest[router]
state, why = router_state(latest, thresholds)
router_data.append({
@@ -181,19 +235,28 @@ async def rows():
)
raw_certs = result.scalars().all()
certs = []
for c in raw_certs:
days = (c.not_after - now).total_seconds() / 86400.0
certs.append({
scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
current, gone = gone_routers(router_data, scrape)
served, old = current_certs(raw_certs, scrape)
domains = router_domains()
def cert_row(c):
names = cert_names(c.cn, c.sans)
return {
"serial": c.serial,
"cn": c.cn or c.serial,
"sans": c.sans,
"sans": [n for n in names if n != c.cn],
"not_after": c.not_after,
"days_remaining": days,
})
"days_remaining": (c.not_after - now).total_seconds() / 86400.0,
# The routers whose hosts it covers; None when Traefik's API
# couldn't say which hosts each router serves.
"routers": None if domains is None else sorted(
{r["name"].split("@")[0] for r in current
for h in router_hosts(r["name"], domains) if covers(names, h)}),
}
current, gone = gone_routers(
router_data, current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"])
certs = [cert_row(c) for c in served]
old_certs = [cert_row(c) for c in old]
problems, healthy = split_routers(current)
return await render_template(
"traefik/rows.html",
@@ -204,6 +267,7 @@ async def rows():
error_5xx=ERROR_5XX_PCT,
global_stat=global_stat,
certs=certs,
old_certs=old_certs,
range_key=range_key,
)
@@ -211,53 +275,53 @@ async def rows():
@traefik_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
"""HTMX dashboard widget fragment."""
"""The dashboard Routers widget (#5254), in the widget-shape order: a
verdict, routers in trouble with why, then the busiest healthy ones under
one header. Only routers and certificates Traefik still reports. A
certificate's expiry shows on the router it serves, and makes that router
a problem once it is inside the Settings → Thresholds TLS window."""
now = datetime.now(timezone.utc)
thresholds = current_app.config.get("THRESHOLDS", {})
scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(TraefikMetric.router_name).distinct().order_by(TraefikMetric.router_name)
)
routers = [row[0] for row in result.all()]
router_data = []
for router in routers:
result = await db.execute(
select(TraefikMetric)
.where(TraefikMetric.router_name == router)
.order_by(TraefikMetric.scraped_at.desc())
.limit(1)
)
latest = result.scalar_one_or_none()
if latest:
router_data.append({"name": router, "latest": latest})
router_data.sort(key=lambda r: (
-(1 if r["latest"].error_rate_5xx_pct > 0.1 else 0),
-(1 if r["latest"].error_rate_4xx_pct > 5.0 else 0),
-r["latest"].request_rate,
))
total_routers = len(router_data)
router_data = router_data[:5]
result = await db.execute(
select(TraefikCert).order_by(TraefikCert.not_after)
)
raw_certs = result.scalars().all()
expiring_certs = []
for c in raw_certs:
days = (c.not_after - now).total_seconds() / 86400.0
if days < 30:
expiring_certs.append({"cn": c.cn or c.serial, "days_remaining": days})
latest = await latest_per_router(db)
certs = (await db.execute(select(TraefikCert))).scalars().all()
current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
served, _old = current_certs(certs, scrape)
domains = router_domains()
by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
for r in current:
r["state"], r["why"] = router_state(r["latest"], thresholds)
r["cert"] = by_router.get(r["name"])
if r["cert"]:
grade = status_class(r["cert"]["days"], "tls_days", thresholds)
r["cert"]["grade"] = grade
if _GRADE_ORDER.get(grade, 2) < _GRADE_ORDER[r["state"]]:
left = cert_left(r["cert"]["days"]) # "12 days", "expires today", "expired …"
r["state"], r["why"] = grade, (
f"certificate expires in {left}" if left[0].isdigit() else f"certificate {left}")
problems, healthy = split_routers(current)
healthy.sort(key=lambda r: -(r["latest"].request_rate or 0))
# With no router domains to go on, certificates can't sit on their
# routers: the expiring ones are listed after the routers instead.
loose = [] if domains is not None else sorted(
({"cn": c.cn or c.serial, "days": (c.not_after - now).total_seconds() / 86400.0} for c in served),
key=lambda c: c["days"])
loose = [c for c in loose if status_class(c["days"], "tls_days", thresholds) in ("warn", "crit")]
return await render_template(
"traefik/widget.html",
router_data=router_data,
expiring_certs=expiring_certs,
total_routers=total_routers,
problems=problems, healthy=healthy[:ROUTERS_SHOWN],
healthy_more=max(0, len(healthy) - ROUTERS_SHOWN),
total=len(current), loose_certs=loose, domains_known=domains is not None,
)
# Healthy routers on the dashboard widget (the busiest); the page lists all.
ROUTERS_SHOWN = 6
@traefik_bp.get("/widget/access_log")
@require_role(UserRole.viewer)
async def widget_access_log():
+20 -1
View File
@@ -16,6 +16,16 @@ logger = logging.getLogger(__name__)
_prev_metrics: dict = {}
_prev_time: float = 0.0
# Router name -> host names from Traefik's API, refreshed each scrape (#5254).
# None until the API has answered once, and again whenever it stops answering,
# so readers can tell "no domains" from "don't know".
_router_domains: dict[str, list[str]] | None = None
def router_domains() -> dict[str, list[str]] | None:
return _router_domains
# Access log state
_geoip_db = None
_access_log_last_tick: "datetime | None" = None
@@ -38,15 +48,17 @@ def make_scrape_task(app: "Quart") -> ScheduledTask:
async def _do_scrape(app: "Quart") -> None:
"""Fetch Traefik metrics, write to DB, and emit plugin_metrics."""
global _prev_metrics, _prev_time
global _prev_metrics, _prev_time, _router_domains
from datetime import datetime, timezone
from .models import TraefikCert, TraefikGlobalStat, TraefikMetric
from .scraper import (
api_base,
compute_global_stats,
compute_router_metrics,
extract_certs,
fetch_metrics,
fetch_router_domains,
)
from steward.core.alerts import MetricBatch
@@ -57,6 +69,13 @@ async def _do_scrape(app: "Quart") -> None:
except Exception:
logger.exception("Traefik scrape failed (url=%s)", metrics_url)
return
cfg = app.config["PLUGINS"]["traefik"]
try:
_router_domains = await fetch_router_domains(api_base(metrics_url, cfg.get("api_url") or ""))
except Exception as exc:
if _router_domains is not None:
logger.warning("Traefik API stopped answering (%s); certificates shown without routers", exc)
_router_domains = None
now = time.monotonic()
elapsed = now - _prev_time if _prev_time else 60.0
+66
View File
@@ -274,3 +274,69 @@ async def fetch_metrics(metrics_url: str) -> ParsedMetrics:
response = await client.get(metrics_url)
response.raise_for_status()
return parse_prometheus(response.text)
# ──────────────────────────────────────────────────────────────────────────────
# Router domains (Traefik API)
# ──────────────────────────────────────────────────────────────────────────────
#
# The metrics name each router but not the hosts it answers for, so which
# certificate serves which router is only knowable from Traefik's API. Read
# once per scrape; when the API isn't reachable, routers simply have no
# domains and certificates are shown on their own (#5254).
_HOST_MATCHER = re.compile(r"Host(?:SNI)?\(([^)]*)\)")
_QUOTED = re.compile(r"[`\"']([^`\"']+)[`\"']")
def rule_hosts(rule: str) -> list[str]:
"""The host names a router rule matches: Host(`a`, `b`) and HostSNI(`c`)
give ["a", "b", "c"]; HostRegexp and path-only rules give nothing."""
hosts: list[str] = []
for args in _HOST_MATCHER.findall(rule or ""):
hosts.extend(h.strip().lower() for h in _QUOTED.findall(args) if h.strip() != "*")
return hosts
def api_base(metrics_url: str, api_url: str = "") -> str:
"""The Traefik API's base: the configured api_url, else /api on the
metrics URL's own host (the default "traefik" entrypoint serves both)."""
if api_url:
return api_url.rstrip("/")
parts = httpx.URL(metrics_url)
return str(parts.copy_with(path="/api", query=None, fragment=None)).rstrip("/")
async def fetch_router_domains(base: str) -> dict[str, list[str]]:
"""{router name: host names} for every enabled HTTP and TCP router."""
out: dict[str, list[str]] = {}
async with httpx.AsyncClient(timeout=10.0) as client:
for kind in ("http", "tcp"):
response = await client.get(f"{base}/{kind}/routers", params={"per_page": 1000})
if kind == "tcp" and response.status_code == 404:
continue
response.raise_for_status()
for r in response.json():
if r.get("status", "enabled") != "enabled" or not r.get("name"):
continue
hosts = rule_hosts(r.get("rule", ""))
if hosts:
out.setdefault(r["name"], []).extend(hosts)
return out
def cert_names(cn: str | None, sans: str | None) -> set[str]:
"""Every name a certificate covers: its CN and SANs, lower-cased."""
names = {n.strip().lower() for n in (sans or "").split(",") if n.strip()}
if cn:
names.add(cn.strip().lower())
return names
def covers(names: set[str], host: str) -> bool:
"""Whether a certificate with these names is valid for host, a wildcard
"*.example.com" covering one label: "a.example.com", not "a.b.example.com"."""
if host in names:
return True
head, _, rest = host.partition(".")
return bool(rest) and f"*.{rest}" in names
+26 -5
View File
@@ -37,7 +37,7 @@
</div>
{% endmacro %}
{% if problems or healthy or gone or certs %}
{% if problems or healthy or gone or certs or old_certs %}
{# ── Routers that need attention ─────────────────────────────────────────── #}
{% if problems %}
<div style="column-width:340px;column-gap:1rem;">
@@ -94,22 +94,43 @@
</script>
{% endif %}
{# ── Certificates, soonest expiry first ──────────────────────────────────── #}
{% if certs %}
{# ── Certificates Traefik still serves, soonest expiry first; then the ones
it no longer reports (renewed or retired), kept here but off the
dashboard (#5254). #}
{% if certs or old_certs %}
<div class="card">
<div class="section-title" style="margin-bottom:0.75rem;">TLS certificates</div>
{% if certs %}
<table class="table" style="font-size:0.85rem;">
<thead><tr><th>Name</th><th>Expires</th><th style="text-align:right;">Left</th></tr></thead>
<thead><tr><th>Name</th><th>Routers</th><th>Expires</th><th style="text-align:right;">Left</th></tr></thead>
<tbody>
{% for cert in certs %}
<tr>
<td style="word-break:break-all;">{{ cert.cn }}</td>
<td style="word-break:break-all;">{{ cert.cn }}{% if cert.sans %}<div class="field-hint">also {{ cert.sans|join(", ") }}</div>{% endif %}</td>
<td class="field-hint">{% if cert.routers is none %}—{% elif cert.routers %}{{ cert.routers|join(", ") }}{% else %}not in use{% endif %}</td>
<td class="field-hint">{{ cert.not_after.strftime("%Y-%m-%d") }}</td>
<td style="text-align:right;white-space:nowrap;" class="tone-{{ status_info(status_class(cert.days_remaining, 'tls_days'))[0] }}">{{ cert_left(cert.days_remaining) }}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}
{% if old_certs %}
<details style="margin-top:0.75rem;">
<summary class="field-hint">No longer served ({{ old_certs|length }}) — renewed or retired; Traefik stopped reporting them</summary>
<table class="table" style="font-size:0.85rem;margin-top:0.5rem;">
<thead><tr><th>Name</th><th>Expires</th></tr></thead>
<tbody>
{% for cert in old_certs %}
<tr style="color:var(--text-muted);">
<td style="word-break:break-all;">{{ cert.cn }}</td>
<td class="field-hint">{{ cert.not_after.strftime("%Y-%m-%d") }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</details>
{% endif %}
</div>
{% endif %}
+53 -52
View File
@@ -1,54 +1,55 @@
{# plugins/traefik/templates/traefik/widget.html #}
{% if expiring_certs %}
<div style="margin-bottom:0.6rem;">
{% for cert in expiring_certs %}
<div style="display:flex;justify-content:space-between;font-size:0.8rem;padding:0.15rem 0;
{% if cert.days_remaining < 7 %}color:var(--red);{% else %}color:var(--yellow);{% endif %}">
<span>&#9888; {{ cert.cn }}</span>
<span>{{ cert_left(cert.days_remaining) }}</span>
</div>
{% endfor %}
</div>
{% endif %}
{% if router_data %}
{% for r in router_data %}
<div class="ping-row">
<div class="ping-meta">
<span class="ping-name" style="font-size:0.8rem;word-break:break-all;">{{ r.name }}</span>
</div>
<div style="flex:1;display:flex;gap:1.5rem;font-size:0.82rem;font-variant-numeric:tabular-nums;">
<span title="Requests/s">
<span style="color:var(--text-muted);">req/s</span>
<span style="margin-left:0.3rem;">{{ "%.2f"|format(r.latest.request_rate) }}</span>
</span>
<span title="Bandwidth">
<span style="color:var(--text-muted);">bw</span>
<span style="margin-left:0.3rem;">
{% set bps = r.latest.response_bytes_rate %}
{{ bps|rate }}
</span>
</span>
<span title="p95 latency">
<span style="color:var(--text-muted);">p95</span>
<span style="margin-left:0.3rem;
{% if r.latest.latency_p95_ms > 500 %}color:var(--red){% elif r.latest.latency_p95_ms > 200 %}color:var(--yellow){% else %}color:var(--green){% endif %};">
{{ "%.0f"|format(r.latest.latency_p95_ms) }}ms
</span>
</span>
{% if r.latest.error_rate_5xx_pct > 0 %}
<span title="5xx error rate">
<span style="color:var(--red);">{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}% 5xx</span>
</span>
{% endif %}
</div>
</div>
{% endfor %}
{% if total_routers > 5 %}
<div style="color:var(--text-dim);font-size:0.75rem;padding:0.25rem 0;">
+{{ total_routers - 5 }} more — <a href="/plugins/traefik/">view all →</a>
</div>
{% endif %}
{# plugins/traefik/templates/traefik/widget.html — the dashboard Routers widget
(#5254): a verdict, routers in trouble with why, then the busiest healthy
ones, under one header. Only routers Traefik still reports. A certificate
shows on the router it serves (TLS column, days left); when Traefik's API
can't say which hosts a router serves, the expiring certificates are listed
after the routers instead. #}
{% from "_macros.html" import status_dot %}
{% if not total %}
<div class="w-verdict" style="color:var(--text-muted);">No Traefik data yet.</div>
{% else %}
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No Traefik data yet.</p>
{% set crit = problems | selectattr("state", "equalto", "crit") | list %}
<div class="w-verdict {{ 'tone-crit' if crit else ('tone-warn' if problems or loose_certs else 'tone-ok') }}">
{%- if problems|length == 1 %}{{ problems[0].name.split('@')[0] }}: {{ problems[0].why }}.
{%- elif problems %}{{ problems|length }} of {{ total }} routers need a look.
{%- else %}All {{ total }} router{{ 's' if total != 1 }} healthy{% if loose_certs %}; {{ loose_certs|length }} certificate{{ 's' if loose_certs|length != 1 }} expiring{% endif %}.{% endif -%}
</div>
<table class="w-table">
<colgroup><col><col style="width:4rem"><col class="w-wide-only" style="width:5.5rem"><col style="width:4.2rem">{% if domains_known %}<col style="width:4.5rem">{% endif %}</colgroup>
<thead><tr><th>Router</th><th>Req/s</th><th class="w-wide-only">Bandwidth</th><th>p95</th>{% if domains_known %}<th>TLS</th>{% endif %}</tr></thead>
<tbody>
{% for r in problems + healthy %}
{% set m = r.latest %}
<tr>
<td class="w-name" title="{{ r.name }}">
{{- status_dot(r.state) -}}
{{ r.name.split('@')[0] }}
{%- if r.why %}<span class="w-sub tone-{{ r.state }}">{{ r.why }}</span>{% endif %}
</td>
<td>{% if m.request_rate is none %}—{% else %}{{ "%.2f"|format(m.request_rate) }}{% endif %}</td>
<td class="w-wide-only">{% if m.response_bytes_rate is none %}—{% else %}{{ m.response_bytes_rate|rate }}{% endif %}</td>
<td style="{{ threshold_style(m.latency_p95_ms, 'latency') if m.latency_p95_ms is not none }}">
{%- if m.latency_p95_ms is none %}—{% else %}{{ "%.0f"|format(m.latency_p95_ms) }} ms{% endif -%}
</td>
{% if domains_known %}
<td {% if r.cert %}style="{{ threshold_style(r.cert.days, 'tls_days') }}" title="Certificate {{ r.cert.cn }}"{% endif %}>
{%- if r.cert %}{{ cert_left(r.cert.days) }}{% else %}—{% endif -%}
</td>
{% endif %}
</tr>
{% endfor %}
</tbody>
</table>
{% if healthy_more %}<div class="w-more">+{{ healthy_more }} more healthy{% if session.user_id %} · <a href="/plugins/traefik/">all routers</a>{% endif %}</div>{% endif %}
{% if loose_certs %}
<div class="w-section">Certificates expiring</div>
{% for c in loose_certs %}
<div class="w-item">
{{- status_dot(status_class(c.days, 'tls_days')) -}}
<span class="w-label" title="{{ c.cn }}">{{ c.cn }}</span>
<span class="w-meta" style="{{ threshold_style(c.days, 'tls_days') }}">{{ cert_left(c.days) }}</span>
</div>
{% endfor %}
<div class="w-note">Traefik's API didn't say which hosts each router serves, so these can't be shown on their routers.</div>
{% endif %}
{% endif %}
+5
View File
@@ -404,6 +404,11 @@ body.dash-editing .widget-drawer { transform:translateY(0); }
.w-item .w-label { flex:1; min-width:0; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; }
.w-item .w-label a { color:inherit; text-decoration:none; }
.w-item .w-meta { flex-shrink:0; font-size:0.72rem; color:var(--text-muted); font-variant-numeric:tabular-nums; }
/* A number with its share drawn behind it (--pct, 0–100), in the number's own
tone: the bar never stands without the figure it measures. */
.w-meter { background:linear-gradient(90deg, color-mix(in srgb, currentColor 20%, transparent) var(--pct, 0%), transparent var(--pct, 0%)); border-radius:3px; }
.w-note { font-size:0.7rem; color:var(--text-dim); margin-top:0.3rem; }
@container (max-width: 26rem) { .w-wide-only { display:none; } }
/* Time range selector */
+46
View File
@@ -0,0 +1,46 @@
"""Integration: the newest Traefik row per router, read by walking the
(router_name, scraped_at) index rather than grouping every row (#5254).
Requires STEWARD_DATABASE_URL."""
from __future__ import annotations
import asyncio
import os
import uuid
from datetime import datetime, timedelta, timezone
import pytest
pytestmark = pytest.mark.integration
@pytest.fixture
def app():
if not os.environ.get("STEWARD_DATABASE_URL"):
pytest.skip("needs Postgres")
from steward.app import create_app
return create_app(testing=False)
def test_each_router_reads_as_its_newest_row(app):
from plugins.traefik.models import TraefikMetric
from plugins.traefik.routes import latest_per_router
now = datetime.now(timezone.utc).replace(microsecond=0)
tag = uuid.uuid4().hex[:8]
a, b = f"a-{tag}@docker", f"b-{tag}@docker"
def row(name, ago, rate):
return TraefikMetric(router_name=name, scraped_at=now - timedelta(minutes=ago), request_rate=rate,
error_rate_4xx_pct=0.0, error_rate_5xx_pct=0.0, latency_p50_ms=1.0,
latency_p95_ms=2.0, latency_p99_ms=3.0, response_bytes_rate=0.0)
async def go():
async with app.db_sessionmaker() as s:
async with s.begin():
s.add_all([row(a, 3, 1.0), row(a, 1, 2.0), row(a, 2, 3.0), row(b, 5, 4.0)])
async with s.begin():
return {m.router_name: m for m in await latest_per_router(s)}
got = asyncio.run(go())
assert got[a].request_rate == 2.0 and got[a].scraped_at == now - timedelta(minutes=1)
assert got[b].request_rate == 4.0
+30
View File
@@ -208,3 +208,33 @@ async def test_the_services_widget_says_when_all_is_well(app):
healthy=[{"kind": "service", "s": _widget_svc("a")}])
assert "2/2 services healthy" in page and "nothing failed in 24h" in page and "tone-ok" in page
assert "Needs a look" not in page and "Failed in the last 24h" not in page
def test_a_container_reads_by_its_service_or_ci_job():
"""#5253: no task id, no slot number, no forty-character CI container name."""
assert svc.display_name(_c("scribe_app.1.x7q", service_name="scribe_app")) == "scribe_app"
ci = "GITEA-ACTIONS-TASK-32792-WORKFLOW-Desktop-Tauri-JOB-verify-Web-d718b95dad0b02ad3cc1"
assert svc.display_name(_c(ci)) == "CI: Desktop-Tauri · verify-Web"
assert svc.display_name(_c("portainer_agent")) == "portainer_agent"
async def test_busiest_containers_show_the_number_with_its_scale(app):
"""#5253: every bar has its figure; the widget names the busiest."""
rows = [
{"c": _c("GITEA-ACTIONS-TASK-1-WORKFLOW-Android-JOB-build-0123456789abcdef0"),
"label": "CI: Android · build", "host_name": "VDNT-Docker03",
"usage": {"cores": 1.62, "cpus": 8, "cpu_share": 20.25, "mem_bytes": 820 * 2**20,
"mem_pct": 5.0, "mem_of": "the host's memory", "mem_limit": None}},
{"c": _c("tdarr"), "label": "tdarr", "host_name": "VDNT-Docker03",
"usage": {"cores": 0.25, "cpus": 8, "cpu_share": 3.1, "mem_bytes": None,
"mem_pct": None, "mem_of": None, "mem_limit": None}},
]
async with app.test_request_context("/plugins/docker/widget/resources"):
page = await render_plugin_template(app, "docker", "docker/widget_resources.html",
rows=rows, minutes=5, widget_id="1")
assert "Busiest: <strong>CI: Android · build</strong> on VDNT-Docker03" in page and "1.6 of 8 cores" in page
assert ">1.6 cores<" in page and ">0.25 cores<" in page and ">820.0 MiB<" in page
assert "--pct:20.25%" in page and ">—<" in page
assert page.count("<th") == 4
assert ">CI: Android · build<" in page # the long CI name only as the hover title
assert not hex_colours(page), hex_colours(page)
+124 -4
View File
@@ -48,10 +48,10 @@ async def test_the_page_leads_with_problems_and_folds_the_healthy(app):
problems, healthy = split_routers([
_router("api", _latest(e5=3.0)), _router("web", _latest()), _router("docs", _latest())])
now = datetime.now(timezone.utc)
certs = [{"serial": "1", "cn": "soon.example", "sans": [], "not_after": now + timedelta(days=2),
"days_remaining": 2.0},
{"serial": "2", "cn": "fine.example", "sans": [], "not_after": now + timedelta(days=60),
"days_remaining": 60.0}]
certs = [{"serial": "1", "cn": "soon.example", "sans": [], "routers": None,
"not_after": now + timedelta(days=2), "days_remaining": 2.0},
{"serial": "2", "cn": "fine.example", "sans": [], "routers": None,
"not_after": now + timedelta(days=60), "days_remaining": 60.0}]
async with app.test_request_context("/plugins/traefik/rows"):
page = await render_plugin_template(app, "traefik", "traefik/rows.html", problems=problems,
healthy=healthy, global_stat=None, certs=certs, range_key="24h",
@@ -110,3 +110,123 @@ async def test_the_traffic_empty_state_knows_whether_the_log_is_on(app, enabled,
range_key="24h", access_log_enabled=enabled,
access_log_path="/var/log/traefik/access.log")
assert says in page and not_says not in page
# ── Routers widget: current routers, certs on the router they serve (#5254) ──
@pytest.mark.parametrize("rule, hosts", [
("Host(`app.example.com`)", ["app.example.com"]),
("Host(`a.example.com`, `B.example.com`) && PathPrefix(`/api`)", ["a.example.com", "b.example.com"]),
("Host(`a.example.com`) || Host(`b.example.com`)", ["a.example.com", "b.example.com"]),
("HostSNI(`db.example.com`)", ["db.example.com"]),
("HostSNI(`*`)", []),
("PathPrefix(`/`)", []),
("", []),
])
def test_a_router_rule_names_its_hosts(rule, hosts):
from plugins.traefik.scraper import rule_hosts
assert rule_hosts(rule) == hosts
def test_a_certificate_covers_its_names_and_one_wildcard_label():
from plugins.traefik.scraper import cert_names, covers
names = cert_names("Example.com", "*.example.com, www.other.org")
assert names == {"example.com", "*.example.com", "www.other.org"}
assert covers(names, "example.com") and covers(names, "app.example.com")
assert covers(names, "www.other.org")
assert not covers(names, "a.b.example.com") and not covers(names, "other.org")
@pytest.mark.parametrize("metrics_url, api_url, base", [
("http://traefik:8080/metrics", "", "http://traefik:8080/api"),
("http://traefik:8080/metrics?x=1", "", "http://traefik:8080/api"),
("http://traefik:8080/metrics", "http://other:9000/api/", "http://other:9000/api"),
])
def test_the_api_sits_beside_the_metrics_unless_configured(metrics_url, api_url, base):
from plugins.traefik.scraper import api_base
assert api_base(metrics_url, api_url) == base
def test_a_metrics_router_finds_its_api_name_with_or_without_the_entrypoint():
from plugins.traefik.routes import router_hosts
domains = {"app@docker": ["app.example.com"]}
assert router_hosts("app@docker", domains) == ["app.example.com"]
assert router_hosts("websecure-app@docker", domains) == ["app.example.com"]
assert router_hosts("myapp@docker", domains) == []
def _cert(serial, cn, days, seen_ago=30, sans=None):
now = datetime.now(timezone.utc)
return NS(serial=serial, cn=cn, sans=sans, not_after=now + timedelta(days=days),
scraped_at=now - timedelta(seconds=seen_ago))
def test_a_certificate_traefik_stopped_reporting_is_old_not_current():
from plugins.traefik.routes import current_certs
live, renewed = _cert("1", "a.example", 80), _cert("2", "a.example", 5, seen_ago=3600)
served, old = current_certs([live, renewed], 60)
assert served == [live] and old == [renewed]
assert current_certs([], 60) == ([], [])
def test_each_router_gets_the_soonest_certificate_for_its_hosts():
from plugins.traefik.routes import certs_by_router
now = datetime.now(timezone.utc)
certs = [_cert("1", "*.example.com", 40), _cert("2", "app.example.com", 10), _cert("3", "x.org", 2)]
got = certs_by_router(["app@docker", "docs@docker", "plain@docker"],
{"app@docker": ["app.example.com"], "docs@docker": ["docs.example.com"]},
certs, now)
assert got["app@docker"]["cn"] == "app.example.com" and int(got["app@docker"]["days"]) == 9
assert got["docs@docker"]["cn"] == "*.example.com"
assert "plain@docker" not in got
async def test_the_widget_gives_a_verdict_then_routers_under_one_header(app):
slow = _router("api@docker", _latest(p95=450.0))
expiring = _router("shop@docker", _latest(req=3.0))
expiring.update(state="crit", why="certificate expires in 2 days",
cert={"cn": "shop.example", "days": 2.0, "grade": "crit"})
fine = _router("web@docker", _latest(req=9.0))
fine["cert"] = {"cn": "web.example", "days": 60.0, "grade": "ok"}
async with app.test_request_context("/plugins/traefik/widget"):
page = await render_plugin_template(
app, "traefik", "traefik/widget.html", problems=[expiring, slow], healthy=[fine],
healthy_more=4, total=7, loose_certs=[], domains_known=True)
page = " ".join(page.split())
assert "2 of 7 routers need a look." in page
assert page.count("<th>Router</th>") == 1 and "<th>TLS</th>" in page
# Problems first, each with why; the cert sits on its router, not above.
assert page.index(">shop") < page.index(">api") < page.index(">web")
assert "certificate expires in 2 days" in page and "p95 latency is 450 ms" in page
assert "60 days" in page and "Certificates expiring" not in page
assert '</span>shop<span class="w-sub' in page and ">shop@docker" not in page # provider off the label
assert "+4 more healthy" in page
async def test_without_router_domains_expiring_certs_follow_the_routers(app):
fine = _router("web@docker", _latest())
async with app.test_request_context("/plugins/traefik/widget"):
page = await render_plugin_template(
app, "traefik", "traefik/widget.html", problems=[], healthy=[fine], healthy_more=0, total=1,
loose_certs=[{"cn": "old.example", "days": 5.0}], domains_known=False)
page = " ".join(page.split())
assert "All 1 router healthy; 1 certificate expiring." in page
assert "<th>TLS</th>" not in page
assert page.index("<th>Router</th>") < page.index("Certificates expiring") < page.index("old.example")
async def test_the_page_sets_certificates_traefik_no_longer_serves_apart(app):
now = datetime.now(timezone.utc)
certs = [{"serial": "1", "cn": "app.example", "sans": ["www.app.example"], "routers": ["app"],
"not_after": now + timedelta(days=40), "days_remaining": 40.0},
{"serial": "2", "cn": "spare.example", "sans": [], "routers": [],
"not_after": now + timedelta(days=40), "days_remaining": 40.0}]
old = [{"serial": "3", "cn": "gone.example", "sans": [], "routers": [],
"not_after": now + timedelta(days=5), "days_remaining": 5.0}]
async with app.test_request_context("/plugins/traefik/rows"):
page = await render_plugin_template(app, "traefik", "traefik/rows.html", problems=[], healthy=[],
gone=[], global_stat=None, certs=certs, old_certs=old,
range_key="24h", error_4xx=1.0, error_5xx=0.1)
page = " ".join(page.split())
assert "also www.app.example" in page and ">app</td>" in page and ">not in use</td>" in page
assert page.index("spare.example") < page.index("No longer served (1)") < page.index("gone.example")