FabledSteward/plugins/traefik/routes.py

# plugins/traefik/routes.py
from __future__ import annotations
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from quart import Blueprint, current_app, render_template, request
from sqlalchemy import func, select
import json
import time

from steward.auth.middleware import require_role
from steward.models.users import UserRole
from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample, time_bucket
from steward.core import anomalies as anom
from steward.core.formatting import cert_left, status_class
from steward.core.norms import norms_at
from steward.core.settings import floor_of
from steward.models.metrics import PluginMetricLatest
from . import meaning
from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode
from .scheduler import error_page_services, router_domains
from .scraper import cert_names, covers

traefik_bp = Blueprint("traefik", __name__, template_folder="templates")


def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
    """Generate an inline SVG sparkline from a list of float values."""
    if len(values) < 2:
        return f'<svg width="{width}" height="{height}"></svg>'
    mn, mx = min(values), max(values)
    if mx == mn:
        mx = mn + 1.0
    step = width / (len(values) - 1)
    pts = []
    for i, v in enumerate(values):
        x = i * step
        y = height - (v - mn) / (mx - mn) * (height - 2) - 1
        pts.append(f"{x:.1f},{y:.1f}")
    poly = " ".join(pts)
    return (
        f'<svg width="{width}" height="{height}" viewBox="0 0 {width} {height}" '
        f'style="vertical-align:middle;">'
        f'<polyline points="{poly}" fill="none" stroke="#6060c0" stroke-width="1.5"/>'
        f'</svg>'
    )


_GRADE_ORDER = {"crit": 0, "warn": 1, "ok": 2}


def error_cutoffs(thresholds: dict) -> tuple[float, float]:
    """(5xx, 4xx): the error shares, in percent, past which a router is failing
    or worth a look. Settings → Thresholds, Traefik routers (#5380)."""
    return (float(thresholds.get("error_5xx", {}).get("crit", 0.1)),
            float(thresholds.get("error_4xx", {}).get("warn", 1.0)))


# Each router's 4xx share and p95 are judged against its own normal once it
# has one (#5741, System 182): an error-pages service answers 4xx all day and
# that is its normal, not a problem. Departing by less than the floor
# (Settings → Thresholds, norms.floor.router_*) is never news. A share over
# fewer than min_requests() in the window is not graded: one 404 of three
# requests is 33%. The 5xx share is a failure and keeps its fixed cutoff
# whatever the history or traffic (the operator's ruling).
NORMED = ("error_rate_4xx_pct", "latency_p95_ms")


def min_requests(thresholds: dict) -> float:
    """Fewest requests over the STATE_WINDOW for a 4xx share to be graded
    (Settings → Thresholds, traefik.threshold.min_requests)."""
    return float(thresholds.get("router_requests", {}).get("warn", 30))


def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = None) -> tuple[str, str | None]:
    """(grade, why) for a router's reading (recent_window's, or one scrape's):
    crit while it serves 5xx past its cutoff, warn on a p95 past the latency
    critical cutoff or a 4xx share past its cutoff (all Settings → Thresholds),
    otherwise ok. `norms` ({metric: Norm}) are the router's normals for this
    hour: a p95 or 4xx share past its cutoff but within its normal is ok, and
    `why` then says it is usual. `over` names the span the reading covers."""
    norms = norms or {}
    error_5xx, error_4xx = error_cutoffs(thresholds)
    if (reading.error_rate_5xx_pct or 0) > error_5xx:
        return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}"
    usual = []
    p95 = reading.latency_p95_ms
    if p95 is not None and status_class(p95, "latency", thresholds) == "crit":
        out, said = anom.against(p95, norms.get("latency_p95_ms"),
                                  floor_of(thresholds, "router_p95_ms"), " ms")
        if out:
            return "warn", f"p95 latency is {p95:.0f} ms{over}" + (f", {said}" if said else "")
        usual.append(f"p95 {p95:.0f} ms is {said}")
    requests = getattr(reading, "requests", None)
    e4 = reading.error_rate_4xx_pct or 0
    if e4 > error_4xx and (requests is None or requests >= min_requests(thresholds)):
        out, said = anom.against(e4, norms.get("error_rate_4xx_pct"),
                                  floor_of(thresholds, "router_4xx_pct"), "%")
        if out:
            return "warn", f"{e4:.1f}% of responses are 4xx{over}" + (f", {said}" if said else "")
        usual.append(f"4xx {e4:.1f}% is {said}")
    return "ok", ("; ".join(usual) or None)


async def router_norms(db, names: list[str], now: datetime) -> dict[str, dict]:
    """{router: {metric: Norm}}: each router's learned normals for this hour.
    A router without enough history yet is absent, and graded on the cutoffs."""
    got = await norms_at(db, [("traefik", n, m) for n in names for m in NORMED], now)
    out: dict[str, dict] = {}
    for (_s, router, metric), norm in got.items():
        out.setdefault(router, {})[metric] = norm
    return out


# A router is graded on its last STATE_WINDOW of traffic, not on one scrape:
# a share near a cutoff crossed it on one scrape and fell back on the next,
# so problem cards came and went every poll (#5214). Over the window a real
# problem stays listed until it has been clean a while, and a one-scrape
# blip is diluted rather than shown.
STATE_WINDOW = timedelta(minutes=15)
STATE_WINDOW_WORDS = " over the last 15 min"


async def recent_window(db, names: list[str], now: datetime) -> dict[str, SimpleNamespace]:
    """{router: one reading of its last STATE_WINDOW}: the 4xx and 5xx shares
    weighted by each scrape's traffic (an idle scrape's 100% counts for
    little), p95 averaged, and about how many requests it answered. A router
    with no rows in the window is absent."""
    if not names:
        return {}
    rate = TraefikMetric.request_rate
    result = await db.execute(
        select(
            TraefikMetric.router_name,
            func.sum(rate).label("req"),
            func.sum(rate * TraefikMetric.error_rate_4xx_pct).label("e4w"),
            func.sum(rate * TraefikMetric.error_rate_5xx_pct).label("e5w"),
            func.avg(TraefikMetric.error_rate_4xx_pct).label("e4"),
            func.avg(TraefikMetric.error_rate_5xx_pct).label("e5"),
            func.avg(TraefikMetric.latency_p95_ms).label("p95"),
            func.avg(rate).label("rate"),
        )
        .where(TraefikMetric.router_name.in_(names))
        .where(TraefikMetric.scraped_at >= now - STATE_WINDOW)
        .group_by(TraefikMetric.router_name)
    )
    out = {}
    for r in result.all():
        req = r.req or 0.0
        out[r.router_name] = SimpleNamespace(
            error_rate_4xx_pct=r.e4w / req if req > 0 else r.e4,
            error_rate_5xx_pct=r.e5w / req if req > 0 else r.e5,
            latency_p95_ms=r.p95,
            requests=(r.rate or 0.0) * STATE_WINDOW.total_seconds(),
        )
    return out


async def status_codes(db, names: list[str], since: datetime) -> dict[str, list[tuple[str, int]]]:
    """{router: [(code, responses), …]} for the 4xx and 5xx codes each router
    answered since `since`, most first (#5711). A router with none is absent."""
    if not names:
        return {}
    total = func.sum(TraefikStatusCode.count)
    rows = (await db.execute(
        select(TraefikStatusCode.router_name, TraefikStatusCode.code, total.label("n"))
        .where(TraefikStatusCode.router_name.in_(names))
        .where(TraefikStatusCode.scraped_at >= since)
        .group_by(TraefikStatusCode.router_name, TraefikStatusCode.code)
        .order_by(TraefikStatusCode.router_name, total.desc(), TraefikStatusCode.code)
    )).all()
    out: dict[str, list[tuple[str, int]]] = {}
    for r in rows:
        out.setdefault(r.router_name, []).append((r.code, int(r.n)))
    return out


ERROR_PAGES_WHY = "serves other routers' error pages: its 4xx and 5xx are theirs, graded on them"


def grade_routers(routers: list[dict], window: dict, thresholds: dict, norms: dict | None = None,
                  error_pages: frozenset[str] = frozenset()) -> None:
    """Set each router's state and why from its recent window, or from its
    latest scrape when the window holds nothing, against its normals
    (router_norms') where it has them. A service Traefik's errors middleware
    sends error pages to (`error_pages`) is healthy whatever its codes: it
    answers with the code of the error it covers for (#5780)."""
    norms = norms or {}
    for r in routers:
        if r["name"] in error_pages:
            r["state"], r["why"], r["kind"], r["normed"] = "ok", ERROR_PAGES_WHY, None, False
            continue
        reading, own = window.get(r["name"]), norms.get(r["name"])
        r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS, own) if reading
                                else router_state(r["latest"], thresholds, norms=own))
        r["kind"] = meaning.problem_kind(r["state"], r["why"])
        r["normed"] = bool(own) and {"slow": "latency_p95_ms", "4xx": "error_rate_4xx_pct"}.get(
            r["kind"], "") in own


async def swarm_service_names(db) -> list[str]:
    """Every swarm service the Docker plugin has stored; [] without it."""
    from steward.core.plugin_manager import loaded_plugin_submodule
    models = loaded_plugin_submodule("docker", "models")
    if models is None:
        return []
    return list((await db.execute(select(models.DockerSwarmService.service_name).distinct())).scalars())


def explain_problems(problems: list[dict], services: list[str]) -> None:
    """Give each problem card what it means and its one next step (#5712):
    `meaning` (meaning.explain's) and `service`, the swarm service behind
    the router when Steward can tell which."""
    for r in problems:
        kind = r.get("kind") or meaning.problem_kind(r["state"], r["why"])
        r["meaning"] = meaning.explain(kind, r.get("codes", []),
                                       getattr(r["latest"], "latency_p50_ms", None),
                                       getattr(r["latest"], "latency_p95_ms", None),
                                       r.get("normed", False))
        r["service"] = meaning.swarm_service(r["name"], services)


# A router missing from this many scrapes in a row is no longer reported:
# Traefik dropped it (a removed service), so its last state is history.
GONE_AFTER_SCRAPES = 3


def gone_routers(routers: list[dict], scrape_seconds: int) -> tuple[list[dict], list[dict]]:
    """(current, gone): a router whose newest row trails the newest scrape of
    any router by GONE_AFTER_SCRAPES intervals is gone, newest-seen first."""
    if not routers:
        return [], []
    newest = max(r["latest"].scraped_at for r in routers)
    cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
    current = [r for r in routers if r["latest"].scraped_at >= cutoff]
    gone = sorted((r for r in routers if r["latest"].scraped_at < cutoff),
                  key=lambda r: r["latest"].scraped_at, reverse=True)
    return current, gone


def problem_since(history: list, thresholds: dict) -> tuple[datetime | None, bool]:
    """(start, before_range) of a router's current problem, read from its
    history buckets (router_history's, oldest first): the start of the
    trailing run of buckets that grade as a problem. (None, False) when the
    newest bucket reads ok, as for a problem too recent or too brief for a
    bucket to show. `before_range` says the run reaches back to the first
    bucket, so the problem began before the range shown."""
    start = None
    for point in reversed(history):
        if router_state(point, thresholds)[0] == "ok":
            break
        start = point
    if start is None:
        return None, False
    return start.bucket, start is history[0]


def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]:
    """(problems, healthy): failing routers before slow or noisy ones, each
    group by name; healthy routers by name."""
    problems = sorted((r for r in routers if r["state"] != "ok"),
                      key=lambda r: (_GRADE_ORDER[r["state"]], r["name"].lower()))
    healthy = sorted((r for r in routers if r["state"] == "ok"), key=lambda r: r["name"].lower())
    return problems, healthy


# The figures the scraper records for every router (scheduler._do_scrape).
ROUTER_METRICS = ("request_rate", "error_rate_4xx_pct", "error_rate_5xx_pct", "latency_p50_ms",
                  "latency_p95_ms", "latency_p99_ms", "response_bytes_rate")


async def latest_per_router(db) -> list[SimpleNamespace]:
    """Each router's newest scrape, read from plugin_metric_latest: the scraper
    records every figure through MetricBatch, which keeps one row per router
    and figure there (latest_by_resource's table, #5208). A walk of
    traefik_metrics' index, one step per router, took 7 s on the live
    instance (#5355); each step lands on the router's oldest entries, which
    retention deletes every minute, so it likely waded through dead ones.

    Each router reads as one scrape, with the attributes a TraefikMetric row
    has; certificates, recorded under the same source, are left out."""
    rows = (await db.execute(select(PluginMetricLatest).where(
        PluginMetricLatest.source_module == "traefik",
        PluginMetricLatest.metric_name.in_(ROUTER_METRICS)))).scalars()
    by_router: dict[str, dict] = {}
    for r in rows:
        by_router.setdefault(r.resource_name, {})[r.metric_name] = r
    return [SimpleNamespace(router_name=name, scraped_at=max(m.recorded_at for m in figures.values()),
                            **{k: figures[k].value if k in figures else None for k in ROUTER_METRICS})
            for name, figures in by_router.items() if "request_rate" in figures]


def current_certs(certs, scrape_seconds: int) -> tuple[list, list]:
    """(served, old): a certificate is served while Traefik still reports it.
    Rows are upserted by serial and never deleted, so a renewed or retired
    certificate lingers; one missing from the last GONE_AFTER_SCRAPES
    scrapes is old (#5254)."""
    certs = list(certs)
    if not certs:
        return [], []
    newest = max(c.scraped_at for c in certs)
    cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
    return ([c for c in certs if c.scraped_at >= cutoff],
            sorted((c for c in certs if c.scraped_at < cutoff), key=lambda c: c.scraped_at, reverse=True))


def router_hosts(name: str, domains: dict[str, list[str]]) -> list[str]:
    """The hosts a metrics router name serves. The API names a router
    "app@docker"; metrics may carry it as "websecure-app@docker", one per
    entrypoint, so a name not found as-is is tried without that prefix."""
    if name in domains:
        return domains[name]
    for api_name, hosts in domains.items():
        if name.endswith("-" + api_name):
            return hosts
    return []


def certs_by_router(routers: list[str], domains: dict[str, list[str]], certs, now) -> dict[str, dict]:
    """{router: {"cn", "days"}}: the soonest-expiring served certificate valid
    for one of the router's hosts. A router with no hosts, or none covered,
    is absent."""
    out: dict[str, dict] = {}
    names = [(c, cert_names(c.cn, c.sans)) for c in certs]
    for router in routers:
        for host in router_hosts(router, domains):
            for c, cn in names:
                if covers(cn, host):
                    days = (c.not_after - now).total_seconds() / 86400.0
                    if router not in out or days < out[router]["days"]:
                        out[router] = {"cn": c.cn or host, "days": days}
    return out


def _access_log() -> dict:
    """The plugin's access_log settings ({} when unset or malformed)."""
    cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {})
    return cfg if isinstance(cfg, dict) else {}


@traefik_bp.get("/")
@require_role(UserRole.viewer)
async def index():
    poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
    access_log_enabled = bool(_access_log().get("enabled", False))
    current_range = request.args.get("range", DEFAULT_RANGE)
    return await render_template(
        "traefik/index.html",
        poll_interval=poll_interval,
        access_log_enabled=access_log_enabled,
        current_range=current_range,
    )


@traefik_bp.get("/rows")
@require_role(UserRole.viewer)
async def rows():
    """HTMX fragment: service cards with sparklines scoped to selected time range."""
    since, range_key = parse_range(request.args.get("range"))
    now = datetime.now(timezone.utc)
    thresholds = current_app.config.get("THRESHOLDS", {})

    scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
    phases = _Phases()

    async with current_app.db_sessionmaker() as db:
        # Every known router, with its most recent raw row; a router Traefik
        # stopped reporting gets no sparklines, so its history is not read.
        newest = await latest_per_router(db)
        current, gone = gone_routers([{"name": m.router_name, "latest": m} for m in newest], scrape)
        phases.mark("latest")

        names = [r["name"] for r in current]
        history = await router_history(db, names, since, bucket_seconds(since))
        phases.mark("history")
        grade_routers(current, await recent_window(db, names, now), thresholds,
                      await router_norms(db, names, now), error_page_services())
        phases.mark("window")
        # The routers' metric anomalies, last 30 days (#5741).
        anomalies = await anom.views(db, await anom.list_anomalies(
            db, source="traefik", since=now - timedelta(days=30), limit=20), links=False)
        for r in current:
            points = history[r["name"]]
            r["since"], r["since_before"] = (problem_since(points, thresholds) if r["state"] != "ok"
                                             else (None, False))
            r["sparkline_req"] = _sparkline([p.request_rate for p in points])
            r["sparkline_bytes"] = _sparkline([p.response_bytes_rate for p in points])
            r["sparkline_p95"] = _sparkline([p.latency_p95_ms for p in points if p.latency_p95_ms is not None])
            r["sparkline_4xx"] = _sparkline([p.error_rate_4xx_pct for p in points])
            r["sparkline_5xx"] = _sparkline([p.error_rate_5xx_pct for p in points])
        phases.mark("sparklines")
        # The codes behind each problem, over the window it is graded on.
        codes = await status_codes(db, [r["name"] for r in current if r["state"] != "ok"], now - STATE_WINDOW)
        for r in current:
            r["codes"] = codes.get(r["name"], [])
        # Codes are kept from #5711 on: until the first of them is a window
        # old, a router with none listed may simply predate the recording.
        first_code = await db.scalar(select(func.min(TraefikStatusCode.scraped_at)))
        codes_known = first_code is not None and first_code <= now - STATE_WINDOW
        services = await swarm_service_names(db) if any(r["state"] != "ok" for r in current) else []
        phases.mark("codes")

        # Latest global stat (always most recent, not range-filtered)
        result = await db.execute(
            select(TraefikGlobalStat)
            .order_by(TraefikGlobalStat.scraped_at.desc())
            .limit(1)
        )
        global_stat = result.scalar_one_or_none()

        # All certs, sorted by soonest expiry
        result = await db.execute(
            select(TraefikCert).order_by(TraefikCert.not_after)
        )
        raw_certs = result.scalars().all()
        phases.mark("stats_certs")

    served, old = current_certs(raw_certs, scrape)
    domains = router_domains()

    def cert_row(c):
        names = cert_names(c.cn, c.sans)
        return {
            "serial": c.serial,
            "cn": c.cn or c.serial,
            "sans": [n for n in names if n != c.cn],
            "not_after": c.not_after,
            "days_remaining": (c.not_after - now).total_seconds() / 86400.0,
            # The routers whose hosts it covers; None when Traefik's API
            # couldn't say which hosts each router serves.
            "routers": None if domains is None else sorted(
                {r["name"].split("@")[0] for r in current
                 for h in router_hosts(r["name"], domains) if covers(names, h)}),
        }

    certs = [cert_row(c) for c in served]
    old_certs = [cert_row(c) for c in old]
    problems, healthy = split_routers(current)
    explain_problems(problems, services)
    phases.mark("cert_rows")
    error_5xx, error_4xx = error_cutoffs(thresholds)
    page = await render_template(
        "traefik/rows.html",
        problems=problems,
        healthy=healthy,
        gone=gone,
        current_range=range_key,
        codes_known=codes_known,
        error_4xx=error_4xx,
        error_5xx=error_5xx,
        global_stat=global_stat,
        certs=certs,
        old_certs=old_certs,
        range_key=range_key,
        anomalies=anomalies,
        back="/plugins/traefik/",
        base_resource=None,
    )
    phases.mark("render")
    phases.report("rows", f"routers={len(current)} range={range_key}")
    return page


@traefik_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
    """The dashboard Routers widget (#5254), in the widget-shape order: a
    verdict, routers in trouble with why, then the busiest healthy ones under
    one header. Only routers and certificates Traefik still reports. A
    certificate's expiry shows on the router it serves, and makes that router
    a problem once it is inside the Settings → Thresholds TLS window."""
    now = datetime.now(timezone.utc)
    thresholds = current_app.config.get("THRESHOLDS", {})
    scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]

    phases = _Phases()
    async with current_app.db_sessionmaker() as db:
        latest = await latest_per_router(db)
        phases.mark("latest")
        current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
        window = await recent_window(db, [r["name"] for r in current], now)
        norms = await router_norms(db, [r["name"] for r in current], now)
        phases.mark("window")
        certs = (await db.execute(select(TraefikCert))).scalars().all()
        phases.mark("certs")
    phases.report("widget", f"routers={len(current)}")

    served, _old = current_certs(certs, scrape)
    domains = router_domains()
    by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
    grade_routers(current, window, thresholds, norms, error_page_services())
    for r in current:
        r["cert"] = by_router.get(r["name"])
        if r["cert"]:
            grade = status_class(r["cert"]["days"], "tls_days", thresholds)
            r["cert"]["grade"] = grade
            if _GRADE_ORDER.get(grade, 2) < _GRADE_ORDER[r["state"]]:
                left = cert_left(r["cert"]["days"])     # "12 days", "expires today", "expired …"
                r["state"], r["why"] = grade, (
                    f"certificate expires in {left}" if left[0].isdigit() else f"certificate {left}")
    problems, healthy = split_routers(current)
    healthy.sort(key=lambda r: -(r["latest"].request_rate or 0))
    # With no router domains to go on, certificates can't sit on their
    # routers: the expiring ones are listed after the routers instead.
    loose = [] if domains is not None else sorted(
        ({"cn": c.cn or c.serial, "days": (c.not_after - now).total_seconds() / 86400.0} for c in served),
        key=lambda c: c["days"])
    loose = [c for c in loose if status_class(c["days"], "tls_days", thresholds) in ("warn", "crit")]
    return await render_template(
        "traefik/widget.html",
        problems=problems, healthy=healthy[:ROUTERS_SHOWN],
        healthy_more=max(0, len(healthy) - ROUTERS_SHOWN),
        total=len(current), loose_certs=loose, domains_known=domains is not None,
    )


# Healthy routers on the dashboard widget (the busiest); the page lists all.
ROUTERS_SHOWN = 6


@traefik_bp.get("/widget/access_log")
@require_role(UserRole.viewer)
async def widget_access_log():
    """HTMX dashboard widget: traffic origin summary with IP filter."""
    from collections import defaultdict
    ip_filter = request.args.get("ip_filter", "all")
    limit = max(1, min(50, int(request.args.get("limit", 10) or 10)))
    widget_id = request.args.get("wid", "0")

    async with current_app.db_sessionmaker() as db:
        result = await db.execute(
            select(TraefikAccessSummary)
            .order_by(TraefikAccessSummary.period_end.desc())
            .limit(12)  # last ~1 hour of 5-min windows
        )
        rows = result.scalars().all()

    if not rows:
        return await render_template("traefik/widget_access_log.html",
                                     summary=None, widget_id=widget_id)

    total = sum(r.total_requests for r in rows)
    internal = sum(r.internal_requests for r in rows)
    external = sum(r.external_requests for r in rows)

    ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True})
    for row in rows:
        for entry in json.loads(row.top_ips_json):
            k = entry["ip"]
            ip_counts[k]["count"] += entry["count"]
            ip_counts[k]["internal"] = entry.get("internal", True)

    all_ips = sorted(
        [{"ip": k, **v} for k, v in ip_counts.items()],
        key=lambda x: x["count"], reverse=True,
    )
    if ip_filter == "internal":
        all_ips = [e for e in all_ips if e["internal"]]
    elif ip_filter == "external":
        all_ips = [e for e in all_ips if not e["internal"]]

    summary = {
        "total": total,
        "internal": internal,
        "external": external,
        "external_pct": (external / total * 100.0) if total else 0.0,
        "top_ips": all_ips[:limit],
        "ip_filter": ip_filter,
    }
    return await render_template("traefik/widget_access_log.html",
                                 summary=summary, widget_id=widget_id)


# A rows or widget request past this logs where its time went, phase by
# phase, so a slow query names itself (#5355: 8 s on every poll, warm).
SLOW_PAGE_SECONDS = 1.0


class _Phases:
    """Wall-clock per phase of one request; report() logs them when slow."""

    def __init__(self):
        self.start = self.last = time.monotonic()
        self.took: dict[str, float] = {}

    def mark(self, name: str) -> None:
        now = time.monotonic()
        self.took[name] = now - self.last
        self.last = now

    def report(self, what: str, detail: str = "") -> None:
        total = time.monotonic() - self.start
        if total >= SLOW_PAGE_SECONDS:
            current_app.logger.warning("traefik %s: slow %.1fs (%s) %s", what, total,
                                       " ".join(f"{k}={v:.2f}s" for k, v in self.took.items()), detail)


async def router_history(db, names: list[str], since, bucket_secs: int) -> dict[str, list]:
    """{router: [rows of request_rate, latency_p95_ms, error_rate_5xx_pct,
    error_rate_4xx_pct, response_bytes_rate]}
    per `bucket_secs` window since `since`, oldest first: one query for every
    router, answered from the covering (router_name, scraped_at) index
    (traefik_004). It was a query per router, each row a heap read, which
    took the rows page to 12-28 s."""
    history: dict[str, list] = {n: [] for n in names}
    if not history:
        return history
    bucket_col = time_bucket(TraefikMetric.scraped_at, bucket_secs)
    result = await db.execute(
        select(
            TraefikMetric.router_name,
            func.avg(TraefikMetric.request_rate).label("request_rate"),
            func.avg(TraefikMetric.latency_p95_ms).label("latency_p95_ms"),
            func.avg(TraefikMetric.error_rate_5xx_pct).label("error_rate_5xx_pct"),
            func.avg(TraefikMetric.error_rate_4xx_pct).label("error_rate_4xx_pct"),
            func.avg(TraefikMetric.response_bytes_rate).label("response_bytes_rate"),
            bucket_col,
        )
        .where(TraefikMetric.router_name.in_(names))
        .where(TraefikMetric.scraped_at >= since)
        .group_by(TraefikMetric.router_name, bucket_col)
        .order_by(TraefikMetric.router_name, bucket_col)
    )
    for row in result.all():
        history[row.router_name].append(row)
    return history


async def request_history(db, since, bucket_seconds: int) -> list[dict]:
    """[{"at", "req_rate", "err_pct"}] per time bucket since `since`: each
    scrape summed over routers (the total req/s), averaged over the bucket;
    the 5xx share weighted by each router's traffic, so a quiet router's
    errors don't count as much as a busy one's."""
    per_scrape = (
        select(
            TraefikMetric.scraped_at.label("at"),
            func.sum(TraefikMetric.request_rate).label("req"),
            func.sum(TraefikMetric.request_rate * TraefikMetric.error_rate_5xx_pct).label("err_weighted"),
        )
        .where(TraefikMetric.scraped_at >= since)
        .group_by(TraefikMetric.scraped_at)
        .subquery()
    )
    bucket_col = time_bucket(per_scrape.c.at, bucket_seconds)
    rows = (await db.execute(
        select(
            func.avg(per_scrape.c.req).label("req_rate"),
            func.sum(per_scrape.c.err_weighted).label("err_weighted"),
            func.sum(per_scrape.c.req).label("req_sum"),
            func.min(per_scrape.c.at).label("at"),
            bucket_col,
        )
        .group_by(bucket_col)
        .order_by(bucket_col)
    )).all()
    return [{"at": r.at, "req_rate": round(r.req_rate or 0, 3),
             "err_pct": round((r.err_weighted or 0) / r.req_sum, 2) if r.req_sum else 0.0}
            for r in rows]


@traefik_bp.get("/widget/request_chart")
@require_role(UserRole.viewer)
async def widget_request_chart():
    """HTMX dashboard widget: requests per second across all routers, and the
    share of them answered 5xx, over the last few hours (#5256). Each scrape
    is summed over routers first (the total), then averaged per bucket; the
    5xx share is weighted by each router's traffic, not averaged per router."""
    widget_id = request.args.get("wid", "0")
    if request.args.get("range"):
        # The Traefik page: the page's own time range (#5710).
        since, span = parse_range(request.args.get("range"))
        b_secs = bucket_seconds(since)
    else:
        hours = max(1, min(24, int(request.args.get("hours", 6) or 6)))
        since, span = datetime.now(timezone.utc) - timedelta(hours=hours), f"{hours}h"
        b_secs = max(60, (hours * 3600) // 80)  # ~80 buckets

    async with current_app.db_sessionmaker() as db:
        history = await request_history(db, since, b_secs)

    labels = [int(r["at"].timestamp() * 1000) for r in history]   # epoch ms; the browser shows local time
    req_rates = [r["req_rate"] for r in history]
    err_pcts = [r["err_pct"] for r in history]

    return await render_template(
        "traefik/widget_request_chart.html",
        labels_json=json.dumps(labels),
        req_rate_json=json.dumps(req_rates),
        error_rate_json=json.dumps(err_pcts),
        widget_id=widget_id,
        span=span,
        days=(datetime.now(timezone.utc) - since) > timedelta(hours=24),
        now_req=req_rates[-1] if req_rates else None,
        peak_req=max(req_rates) if req_rates else None,
        peak_err=max(err_pcts) if err_pcts else None,
    )


@traefik_bp.get("/traffic")
@require_role(UserRole.viewer)
async def traffic():
    """HTMX fragment: traffic origin summary scoped to selected time range."""
    since, range_key = parse_range(request.args.get("range"))

    async with current_app.db_sessionmaker() as db:
        result = await db.execute(
            select(TraefikAccessSummary)
            .where(TraefikAccessSummary.period_end >= since)
            .order_by(TraefikAccessSummary.period_end.asc())
            # No LIMIT — all summaries in range needed for accurate totals;
            # sparkline subsamples to 80 points afterwards
        )
        rows = result.scalars().all()

    if not rows:
        al = _access_log()
        return await render_template(
            "traefik/traffic.html", summary=None, history=[], range_key=range_key,
            access_log_enabled=bool(al.get("enabled", False)), access_log_path=al.get("path", ""),
        )

    total = sum(r.total_requests for r in rows)
    internal = sum(r.internal_requests for r in rows)
    external = sum(r.external_requests for r in rows)
    total_bytes = sum(r.total_bytes for r in rows)

    ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True, "country": None})
    country_counts: dict[str, int] = defaultdict(int)
    router_counts: dict[str, int] = defaultdict(int)

    for row in rows:
        for entry in json.loads(row.top_ips_json):
            k = entry["ip"]
            ip_counts[k]["count"] += entry["count"]
            ip_counts[k]["internal"] = entry["internal"]
            ip_counts[k]["country"] = entry.get("country")
        for entry in json.loads(row.top_countries_json):
            country_counts[entry["country"]] += entry["count"]
        for entry in json.loads(row.top_routers_json):
            router_counts[entry["router"]] += entry["count"]

    top_ips = sorted(
        [{"ip": k, **v} for k, v in ip_counts.items()],
        key=lambda x: x["count"], reverse=True
    )[:10]
    top_countries = sorted(
        [{"country": k, "count": v} for k, v in country_counts.items()],
        key=lambda x: x["count"], reverse=True
    )[:10]
    top_routers = sorted(
        [{"router": k, "count": v} for k, v in router_counts.items()],
        key=lambda x: x["count"], reverse=True
    )[:10]

    history = [
        {
            "period_end": r.period_end,
            "external_pct": (r.external_requests / r.total_requests * 100.0)
            if r.total_requests else 0.0,
        }
        for r in rows
    ]

    summary = {
        "total": total,
        "internal": internal,
        "external": external,
        "total_bytes": total_bytes,
        "external_pct": (external / total * 100.0) if total else 0.0,
        "top_ips": top_ips,
        "top_countries": top_countries,
        "top_routers": top_routers,
        "sparkline_external": _sparkline(
            [h["external_pct"] for h in subsample(history, 80)]
        ),
    }

    return await render_template(
        "traefik/traffic.html", summary=summary, history=history, range_key=range_key
    )