feat(plugins): Traefik problems first, UniFi/SNMP/fleet on shared shapes, audit names its entities, templates on tokens (#5126)
Traefik: routers are graded from their latest scrape (router_state: crit on any real 5xx share, warn on p95 past the latency critical cutoff or a high 4xx share). Failing and slow routers lead as cards that say why; healthy ones fold into one table that stays open or shut across polls. Proxy counts are stat tiles; certificates are graded by Settings -> Thresholds (tls_days). UniFi: WAN health is a stat strip (latency graded by the thresholds), devices carry status dots, event categories and firewall actions are badges, the alarm count reuses .count-badge (renamed from .nav-count). SNMP: each device wears a status badge and its readings are stat tiles. Agent fleet: the table comes first, with a Status column (Reporting / Silent / Never reported, by the host page's stale_after_seconds), silent agents first, hosts linked, last seen as |ago. Audit: entries name their entity (one query per type, steward/audit/ service.py) and link it; something deleted since keeps the name its entry recorded and says "(deleted)". Tokens: charts read colours through chart-colors.js (stewardColor resolves a token to the rgb the browser computed; Chart.js defaults take the page's text and border colours), so host, history, Traefik and SNMP charts drop their hex. Template hex colours reach 0 (HEX_CEILING = 0), the last rgba pills go, and heavy weights (600/700/bold, 60 of them) become 500, the house pair being 400/500; a guard keeps templates there. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -9,6 +9,7 @@ import json
|
||||
from steward.auth.middleware import require_role
|
||||
from steward.models.users import UserRole
|
||||
from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample
|
||||
from steward.core.formatting import status_class
|
||||
from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric
|
||||
|
||||
traefik_bp = Blueprint("traefik", __name__, template_folder="templates")
|
||||
@@ -36,6 +37,36 @@ def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
|
||||
)
|
||||
|
||||
|
||||
# A router serving any real share of 5xx is failing; a 4xx share above this
|
||||
# is worth a look (clients hitting missing routes, auth failures).
|
||||
ERROR_5XX_PCT = 0.1
|
||||
ERROR_4XX_PCT = 1.0
|
||||
_GRADE_ORDER = {"crit": 0, "warn": 1, "ok": 2}
|
||||
|
||||
|
||||
def router_state(latest, thresholds: dict) -> tuple[str, str | None]:
|
||||
"""(grade, why) for a router's latest scrape: crit while it serves 5xx,
|
||||
warn on a p95 past the latency critical cutoff (Settings → Thresholds)
|
||||
or a high 4xx share, otherwise ok with nothing to say."""
|
||||
if (latest.error_rate_5xx_pct or 0) > ERROR_5XX_PCT:
|
||||
return "crit", f"{latest.error_rate_5xx_pct:.1f}% of responses are 5xx"
|
||||
p95 = latest.latency_p95_ms
|
||||
if p95 is not None and status_class(p95, "latency", thresholds) == "crit":
|
||||
return "warn", f"p95 latency is {p95:.0f} ms"
|
||||
if (latest.error_rate_4xx_pct or 0) > ERROR_4XX_PCT:
|
||||
return "warn", f"{latest.error_rate_4xx_pct:.1f}% of responses are 4xx"
|
||||
return "ok", None
|
||||
|
||||
|
||||
def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]:
|
||||
"""(problems, healthy): failing routers before slow or noisy ones, each
|
||||
group by name; healthy routers by name."""
|
||||
problems = sorted((r for r in routers if r["state"] != "ok"),
|
||||
key=lambda r: (_GRADE_ORDER[r["state"]], r["name"].lower()))
|
||||
healthy = sorted((r for r in routers if r["state"] == "ok"), key=lambda r: r["name"].lower())
|
||||
return problems, healthy
|
||||
|
||||
|
||||
@traefik_bp.get("/")
|
||||
@require_role(UserRole.viewer)
|
||||
async def index():
|
||||
@@ -59,6 +90,7 @@ async def rows():
|
||||
"""HTMX fragment: service cards with sparklines scoped to selected time range."""
|
||||
since, range_key = parse_range(request.args.get("range"))
|
||||
now = datetime.now(timezone.utc)
|
||||
thresholds = current_app.config.get("THRESHOLDS", {})
|
||||
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
# All known routers
|
||||
@@ -103,9 +135,12 @@ async def rows():
|
||||
if not latest:
|
||||
continue
|
||||
|
||||
state, why = router_state(latest, thresholds)
|
||||
router_data.append({
|
||||
"name": router,
|
||||
"latest": latest,
|
||||
"state": state,
|
||||
"why": why,
|
||||
"sparkline_req": _sparkline([r.request_rate for r in history]),
|
||||
"sparkline_p95": _sparkline([r.latency_p95_ms for r in history]),
|
||||
"sparkline_5xx": _sparkline([r.error_rate_5xx_pct for r in history]),
|
||||
@@ -136,9 +171,13 @@ async def rows():
|
||||
"days_remaining": days,
|
||||
})
|
||||
|
||||
problems, healthy = split_routers(router_data)
|
||||
return await render_template(
|
||||
"traefik/rows.html",
|
||||
router_data=router_data,
|
||||
problems=problems,
|
||||
healthy=healthy,
|
||||
error_4xx=ERROR_4XX_PCT,
|
||||
error_5xx=ERROR_5XX_PCT,
|
||||
global_stat=global_stat,
|
||||
certs=certs,
|
||||
range_key=range_key,
|
||||
|
||||
@@ -1,124 +1,104 @@
|
||||
{# plugins/traefik/templates/traefik/rows.html #}
|
||||
|
||||
{# ── Global stats bar ──────────────────────────────────────────────────────── #}
|
||||
{% if global_stat %}
|
||||
<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--bg-card);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;">
|
||||
|
||||
{% if global_stat.open_conns_total is not none %}
|
||||
<span>
|
||||
<span style="color:var(--text-muted);">Open conns</span>
|
||||
<span style="margin-left:0.4rem;font-weight:600;">{{ global_stat.open_conns_total | int }}</span>
|
||||
</span>
|
||||
{% endif %}
|
||||
{# traefik/rows.html — HTMX fragment for the Traefik page: routers that need
|
||||
attention first (routes.router_state), the proxy's own counts, the healthy
|
||||
routers folded into one table, then certificates by soonest expiry. #}
|
||||
{% from "_macros.html" import stat_tile, status_badge %}
|
||||
|
||||
{# ── Counts ───────────────────────────────────────────────────────────────── #}
|
||||
<div class="stat-strip">
|
||||
{{ stat_tile("Needs attention", problems|length, state="crit" if problems else "inactive") }}
|
||||
{{ stat_tile("Healthy routers", healthy|length, state="up" if healthy else "inactive") }}
|
||||
{% if global_stat %}
|
||||
{% if global_stat.config_last_reload_success is not none %}
|
||||
<span>
|
||||
<span style="color:var(--text-muted);">Config</span>
|
||||
{% if global_stat.config_last_reload_success == 1.0 %}
|
||||
<span style="margin-left:0.4rem;color:var(--green);">✓ OK</span>
|
||||
{% else %}
|
||||
<span style="margin-left:0.4rem;color:var(--red);">✗ failed</span>
|
||||
{% endif %}
|
||||
{% if global_stat.config_reloads_total is not none %}
|
||||
<span style="color:var(--text-dim);margin-left:0.3rem;">({{ global_stat.config_reloads_total | int }} reloads)</span>
|
||||
{% endif %}
|
||||
</span>
|
||||
{{ stat_tile("Config", "OK" if global_stat.config_last_reload_success == 1.0 else "Reload failed",
|
||||
state="up" if global_stat.config_last_reload_success == 1.0 else "crit",
|
||||
hint=(global_stat.config_reloads_total|int ~ " reloads") if global_stat.config_reloads_total is not none else none) }}
|
||||
{% endif %}
|
||||
|
||||
{% if global_stat.process_memory_bytes is not none %}
|
||||
<span>
|
||||
<span style="color:var(--text-muted);">Memory</span>
|
||||
<span style="margin-left:0.4rem;">
|
||||
{{ global_stat.process_memory_bytes|bytes }}
|
||||
</span>
|
||||
</span>
|
||||
{% if global_stat.open_conns_total is not none %}{{ stat_tile("Open connections", global_stat.open_conns_total|int) }}{% endif %}
|
||||
{% if global_stat.process_memory_bytes is not none %}{{ stat_tile("Memory", global_stat.process_memory_bytes|bytes) }}{% endif %}
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<span style="margin-left:auto;color:var(--text-dim);font-size:0.78rem;">
|
||||
{{ global_stat.scraped_at.strftime("%H:%M:%S") }} UTC
|
||||
</span>
|
||||
{% macro router_card(r) %}
|
||||
<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
|
||||
<div class="host-health-head">
|
||||
{{ status_badge(r.state, "Failing" if r.state == 'crit' else "Warning") }}
|
||||
<h3 class="section-title" style="margin:0;word-break:break-all;">{{ r.name }}</h3>
|
||||
</div>
|
||||
<p class="problem-detail" style="margin-bottom:0.5rem;">{{ r.why }}</p>
|
||||
<table class="table" style="font-size:0.85rem;">
|
||||
<tr><td class="field-hint">Req/s</td><td>{{ "%.2f"|format(r.latest.request_rate) }}</td><td>{{ r.sparkline_req|safe }}</td></tr>
|
||||
<tr><td class="field-hint">Bandwidth</td><td>{{ r.latest.response_bytes_rate|rate }}</td><td></td></tr>
|
||||
<tr><td class="field-hint">p95</td><td class="tone-{{ status_info(status_class(r.latest.latency_p95_ms, 'latency'))[0] }}">{{ "%.0f"|format(r.latest.latency_p95_ms) }} ms</td><td>{{ r.sparkline_p95|safe }}</td></tr>
|
||||
<tr><td class="field-hint">p99</td><td>{{ "%.0f"|format(r.latest.latency_p99_ms) }} ms</td><td></td></tr>
|
||||
<tr><td class="field-hint">4xx</td><td class="{{ 'tone-warn' if r.latest.error_rate_4xx_pct > error_4xx else '' }}">{{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%</td><td></td></tr>
|
||||
<tr><td class="field-hint">5xx</td><td class="{{ 'tone-crit' if r.latest.error_rate_5xx_pct > error_5xx else '' }}">{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%</td><td>{{ r.sparkline_5xx|safe }}</td></tr>
|
||||
</table>
|
||||
<div class="field-hint" style="margin-top:0.5rem;">Last scraped {{ r.latest.scraped_at|ago }}</div>
|
||||
</div>
|
||||
{% endmacro %}
|
||||
|
||||
{% if problems or healthy or certs %}
|
||||
{# ── Routers that need attention ─────────────────────────────────────────── #}
|
||||
{% if problems %}
|
||||
<div style="column-width:340px;column-gap:1rem;">
|
||||
{% for r in problems %}{{ router_card(r) }}{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{# ── Per-service cards + cert card in shared columns layout ───────────────── #}
|
||||
{% if router_data or certs %}
|
||||
<div style="column-width:340px;column-count:3;column-gap:1rem;">
|
||||
|
||||
{% for r in router_data %}
|
||||
<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
|
||||
<h3 class="section-title" style="margin-bottom:0.75rem;word-break:break-all;">{{ r.name }}</h3>
|
||||
<table class="table" style="font-size:0.85rem;">
|
||||
<tr>
|
||||
<td style="color:var(--text-muted);padding:0.2rem 0;">Req/s</td>
|
||||
<td style="padding:0.2rem 0.5rem;">{{ "%.2f"|format(r.latest.request_rate) }}</td>
|
||||
<td style="padding:0.2rem 0;">{{ r.sparkline_req | safe }}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="color:var(--text-muted);padding:0.2rem 0;">Bandwidth</td>
|
||||
<td style="padding:0.2rem 0.5rem;">
|
||||
{% set bps = r.latest.response_bytes_rate %}
|
||||
{{ bps|rate }}
|
||||
</td>
|
||||
<td style="padding:0.2rem 0;"></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="color:var(--text-muted);padding:0.2rem 0;">p95 ms</td>
|
||||
<td style="padding:0.2rem 0.5rem;">{{ "%.1f"|format(r.latest.latency_p95_ms) }}</td>
|
||||
<td style="padding:0.2rem 0;">{{ r.sparkline_p95 | safe }}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="color:var(--text-muted);padding:0.2rem 0;">p99 ms</td>
|
||||
<td style="padding:0.2rem 0.5rem;">{{ "%.1f"|format(r.latest.latency_p99_ms) }}</td>
|
||||
<td style="padding:0.2rem 0;"></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="color:var(--text-muted);padding:0.2rem 0;">4xx %</td>
|
||||
<td style="color:{% if r.latest.error_rate_4xx_pct > 1 %}var(--yellow){% else %}var(--green){% endif %};padding:0.2rem 0.5rem;">
|
||||
{{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%
|
||||
</td>
|
||||
<td style="padding:0.2rem 0;"></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="color:var(--text-muted);padding:0.2rem 0;">5xx %</td>
|
||||
<td style="color:{% if r.latest.error_rate_5xx_pct > 0.1 %}var(--red){% else %}var(--green){% endif %};padding:0.2rem 0.5rem;">
|
||||
{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%
|
||||
</td>
|
||||
<td style="padding:0.2rem 0;">{{ r.sparkline_5xx | safe }}</td>
|
||||
</tr>
|
||||
</table>
|
||||
<div style="color:var(--text-dim);font-size:0.75rem;margin-top:0.5rem;">
|
||||
Last scraped {{ r.latest.scraped_at.strftime("%H:%M:%S") }} UTC
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
{% if certs %}
|
||||
<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
|
||||
<div class="section-title" style="margin-bottom:0.75rem;">TLS Certificates</div>
|
||||
<table class="table" style="font-size:0.83rem;width:100%;">
|
||||
{% for cert in certs %}
|
||||
{# ── Healthy routers, folded ─────────────────────────────────────────────── #}
|
||||
{% if healthy %}
|
||||
<details class="card" id="traefik-healthy" style="margin-bottom:1rem;">
|
||||
<summary class="section-title" style="cursor:pointer;">{{ healthy|length }} healthy router{{ 's' if healthy|length != 1 }}</summary>
|
||||
<div class="table-wrap" style="margin-top:0.75rem;">
|
||||
<table class="table" style="font-size:0.85rem;">
|
||||
<thead><tr><th>Router</th><th>Req/s</th><th>Bandwidth</th><th>p95</th><th>4xx</th><th>5xx</th></tr></thead>
|
||||
<tbody>
|
||||
{% for r in healthy %}
|
||||
<tr>
|
||||
<td style="padding:0.2rem 0 0.2rem 0;word-break:break-all;">{{ cert.cn }}</td>
|
||||
<td style="text-align:right;padding:0.2rem 0;white-space:nowrap;font-weight:600;
|
||||
{% if cert.days_remaining < 7 %}color:var(--red);
|
||||
{% elif cert.days_remaining < 30 %}color:var(--yellow);
|
||||
{% else %}color:var(--green);{% endif %}">
|
||||
{{ cert.days_remaining | int }}d
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td colspan="2" style="padding:0 0 0.4rem 0;color:var(--text-dim);font-size:0.75rem;">
|
||||
{{ cert.not_after.strftime("%Y-%m-%d") }}
|
||||
</td>
|
||||
<td style="word-break:break-all;">{{ r.name }}</td>
|
||||
<td>{{ "%.2f"|format(r.latest.request_rate) }}</td>
|
||||
<td>{{ r.latest.response_bytes_rate|rate }}</td>
|
||||
<td class="tone-{{ status_info(status_class(r.latest.latency_p95_ms, 'latency'))[0] }}">{{ "%.0f"|format(r.latest.latency_p95_ms) }} ms</td>
|
||||
<td>{{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%</td>
|
||||
<td>{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</details>
|
||||
<script>
|
||||
// The fragment is re-polled; keep the healthy list as the reader left it.
|
||||
(function () {
|
||||
var el = document.getElementById("traefik-healthy");
|
||||
if (!el) return;
|
||||
el.open = !!window.__traefikHealthyOpen;
|
||||
el.addEventListener("toggle", function () { window.__traefikHealthyOpen = el.open; });
|
||||
})();
|
||||
</script>
|
||||
{% endif %}
|
||||
|
||||
{# ── Certificates, soonest expiry first ──────────────────────────────────── #}
|
||||
{% if certs %}
|
||||
<div class="card">
|
||||
<div class="section-title" style="margin-bottom:0.75rem;">TLS certificates</div>
|
||||
<table class="table" style="font-size:0.85rem;">
|
||||
<thead><tr><th>Name</th><th>Expires</th><th style="text-align:right;">Left</th></tr></thead>
|
||||
<tbody>
|
||||
{% for cert in certs %}
|
||||
<tr>
|
||||
<td style="word-break:break-all;">{{ cert.cn }}</td>
|
||||
<td class="field-hint">{{ cert.not_after.strftime("%Y-%m-%d") }}</td>
|
||||
<td style="text-align:right;white-space:nowrap;" class="tone-{{ status_info(status_class(cert.days_remaining, 'tls_days'))[0] }}">{{ cert.days_remaining|int }} days</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="card">
|
||||
<p style="color:var(--text-dim);">No Traefik metrics yet. Configure <code>metrics_url</code> in <code>config.yaml</code> under <code>plugins.traefik</code>.</p>
|
||||
<p class="field-hint">No Traefik metrics yet. Configure <code>metrics_url</code> in <code>config.yaml</code> under <code>plugins.traefik</code>.</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
@@ -5,15 +5,15 @@
|
||||
<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--bg-card);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;align-items:center;">
|
||||
<span>
|
||||
<span style="color:var(--text-muted);">Total</span>
|
||||
<span style="margin-left:0.4rem;font-weight:600;">{{ summary.total }}</span>
|
||||
<span style="margin-left:0.4rem;font-weight:500;">{{ summary.total }}</span>
|
||||
</span>
|
||||
<span>
|
||||
<span style="color:var(--text-muted);">Internal</span>
|
||||
<span style="margin-left:0.4rem;color:var(--green);font-weight:600;">{{ summary.internal }}</span>
|
||||
<span style="margin-left:0.4rem;color:var(--green);font-weight:500;">{{ summary.internal }}</span>
|
||||
</span>
|
||||
<span>
|
||||
<span style="color:var(--text-muted);">External</span>
|
||||
<span style="margin-left:0.4rem;color:{% if summary.external_pct > 50 %}var(--yellow){% else %}var(--green){% endif %};font-weight:600;">
|
||||
<span style="margin-left:0.4rem;color:{% if summary.external_pct > 50 %}var(--yellow){% else %}var(--green){% endif %};font-weight:500;">
|
||||
{{ summary.external }} ({{ "%.1f"|format(summary.external_pct) }}%)
|
||||
</span>
|
||||
</span>
|
||||
|
||||
@@ -5,11 +5,11 @@
|
||||
|
||||
<div style="display:flex;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
|
||||
<div>
|
||||
<span style="font-size:1.4rem;font-weight:700;line-height:1;">{{ summary.total | int }}</span>
|
||||
<span style="font-size:1.4rem;font-weight:500;line-height:1;">{{ summary.total | int }}</span>
|
||||
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">total req</span>
|
||||
</div>
|
||||
<div>
|
||||
<span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--orange);">
|
||||
<span style="font-size:1.4rem;font-weight:500;line-height:1;color:var(--orange);">
|
||||
{{ "%.1f" | format(summary.external_pct) }}%
|
||||
</span>
|
||||
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">external</span>
|
||||
|
||||
@@ -19,8 +19,8 @@
|
||||
{
|
||||
label: "Req/s",
|
||||
data: {{ req_rate_json | safe }},
|
||||
borderColor: "#7c5ef4",
|
||||
backgroundColor: "rgba(124,94,244,0.08)",
|
||||
borderColor: stewardColor("--blue"),
|
||||
backgroundColor: stewardColor("--blue-dim"),
|
||||
fill: true,
|
||||
tension: 0.3,
|
||||
pointRadius: 0,
|
||||
@@ -30,7 +30,7 @@
|
||||
{
|
||||
label: "5xx %",
|
||||
data: {{ error_rate_json | safe }},
|
||||
borderColor: "#e83848",
|
||||
borderColor: stewardColor("--red"),
|
||||
fill: false,
|
||||
tension: 0.3,
|
||||
pointRadius: 0,
|
||||
@@ -45,24 +45,23 @@
|
||||
animation: false,
|
||||
plugins: {
|
||||
legend: {
|
||||
labels: { color: "#6858a0", boxWidth: 10, font: { size: 10 } }
|
||||
labels: { boxWidth: 10, font: { size: 10 } }
|
||||
}
|
||||
},
|
||||
scales: {
|
||||
x: { display: false },
|
||||
y: {
|
||||
min: 0,
|
||||
grid: { color: "rgba(30,30,62,0.8)" },
|
||||
ticks: { color: "#6858a0", font: { size: 10 } },
|
||||
title: { display: true, text: "req/s", color: "#6858a0", font: { size: 9 } },
|
||||
ticks: { font: { size: 10 } },
|
||||
title: { display: true, text: "req/s", font: { size: 9 } },
|
||||
},
|
||||
y2: {
|
||||
position: "right",
|
||||
min: 0,
|
||||
max: 100,
|
||||
grid: { drawOnChartArea: false },
|
||||
ticks: { color: "#e83848", font: { size: 10 } },
|
||||
title: { display: true, text: "5xx%", color: "#e83848", font: { size: 9 } },
|
||||
ticks: { color: stewardColor("--red"), font: { size: 10 } },
|
||||
title: { display: true, text: "5xx%", color: stewardColor("--red"), font: { size: 9 } },
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user