FabledSteward/plugins/traefik/routes.py

# plugins/traefik/routes.py
from __future__ import annotations
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from quart import Blueprint, current_app, render_template, request
from sqlalchemy import Integer, cast, func, select
import json

from steward.auth.middleware import require_role
from steward.models.users import UserRole
from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample
from steward.core.formatting import status_class
from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric

traefik_bp = Blueprint("traefik", __name__, template_folder="templates")


def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
    """Generate an inline SVG sparkline from a list of float values."""
    if len(values) < 2:
        return f'<svg width="{width}" height="{height}"></svg>'
    mn, mx = min(values), max(values)
    if mx == mn:
        mx = mn + 1.0
    step = width / (len(values) - 1)
    pts = []
    for i, v in enumerate(values):
        x = i * step
        y = height - (v - mn) / (mx - mn) * (height - 2) - 1
        pts.append(f"{x:.1f},{y:.1f}")
    poly = " ".join(pts)
    return (
        f'<svg width="{width}" height="{height}" viewBox="0 0 {width} {height}" '
        f'style="vertical-align:middle;">'
        f'<polyline points="{poly}" fill="none" stroke="#6060c0" stroke-width="1.5"/>'
        f'</svg>'
    )


# A router serving any real share of 5xx is failing; a 4xx share above this
# is worth a look (clients hitting missing routes, auth failures).
ERROR_5XX_PCT = 0.1
ERROR_4XX_PCT = 1.0
_GRADE_ORDER = {"crit": 0, "warn": 1, "ok": 2}


def router_state(latest, thresholds: dict) -> tuple[str, str | None]:
    """(grade, why) for a router's latest scrape: crit while it serves 5xx,
    warn on a p95 past the latency critical cutoff (Settings → Thresholds)
    or a high 4xx share, otherwise ok with nothing to say."""
    if (latest.error_rate_5xx_pct or 0) > ERROR_5XX_PCT:
        return "crit", f"{latest.error_rate_5xx_pct:.1f}% of responses are 5xx"
    p95 = latest.latency_p95_ms
    if p95 is not None and status_class(p95, "latency", thresholds) == "crit":
        return "warn", f"p95 latency is {p95:.0f} ms"
    if (latest.error_rate_4xx_pct or 0) > ERROR_4XX_PCT:
        return "warn", f"{latest.error_rate_4xx_pct:.1f}% of responses are 4xx"
    return "ok", None


# A router missing from this many scrapes in a row is no longer reported:
# Traefik dropped it (a removed service), so its last state is history.
GONE_AFTER_SCRAPES = 3


def gone_routers(routers: list[dict], scrape_seconds: int) -> tuple[list[dict], list[dict]]:
    """(current, gone): a router whose newest row trails the newest scrape of
    any router by GONE_AFTER_SCRAPES intervals is gone, newest-seen first."""
    if not routers:
        return [], []
    newest = max(r["latest"].scraped_at for r in routers)
    cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
    current = [r for r in routers if r["latest"].scraped_at >= cutoff]
    gone = sorted((r for r in routers if r["latest"].scraped_at < cutoff),
                  key=lambda r: r["latest"].scraped_at, reverse=True)
    return current, gone


def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]:
    """(problems, healthy): failing routers before slow or noisy ones, each
    group by name; healthy routers by name."""
    problems = sorted((r for r in routers if r["state"] != "ok"),
                      key=lambda r: (_GRADE_ORDER[r["state"]], r["name"].lower()))
    healthy = sorted((r for r in routers if r["state"] == "ok"), key=lambda r: r["name"].lower())
    return problems, healthy


def _access_log() -> dict:
    """The plugin's access_log settings ({} when unset or malformed)."""
    cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {})
    return cfg if isinstance(cfg, dict) else {}


@traefik_bp.get("/")
@require_role(UserRole.viewer)
async def index():
    poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
    access_log_enabled = bool(_access_log().get("enabled", False))
    current_range = request.args.get("range", DEFAULT_RANGE)
    return await render_template(
        "traefik/index.html",
        poll_interval=poll_interval,
        access_log_enabled=access_log_enabled,
        current_range=current_range,
    )


@traefik_bp.get("/rows")
@require_role(UserRole.viewer)
async def rows():
    """HTMX fragment: service cards with sparklines scoped to selected time range."""
    since, range_key = parse_range(request.args.get("range"))
    now = datetime.now(timezone.utc)
    thresholds = current_app.config.get("THRESHOLDS", {})

    async with current_app.db_sessionmaker() as db:
        # All known routers
        result = await db.execute(
            select(TraefikMetric.router_name)
            .distinct()
            .order_by(TraefikMetric.router_name)
        )
        routers = [row[0] for row in result.all()]

        b_secs = bucket_seconds(since)
        bucket_col = (
            cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs
        ).label("bucket")

        router_data = []
        for router in routers:
            # Bucketed history for sparklines — always ~80 points regardless of range
            result = await db.execute(
                select(
                    func.avg(TraefikMetric.request_rate).label("request_rate"),
                    func.avg(TraefikMetric.latency_p95_ms).label("latency_p95_ms"),
                    func.avg(TraefikMetric.error_rate_5xx_pct).label("error_rate_5xx_pct"),
                    func.min(TraefikMetric.scraped_at).label("scraped_at"),
                    bucket_col,
                )
                .where(TraefikMetric.router_name == router)
                .where(TraefikMetric.scraped_at >= since)
                .group_by(bucket_col)
                .order_by(bucket_col)
            )
            history = result.all()

            # Latest value — always the most recent raw row
            result = await db.execute(
                select(TraefikMetric)
                .where(TraefikMetric.router_name == router)
                .order_by(TraefikMetric.scraped_at.desc())
                .limit(1)
            )
            latest = result.scalar_one_or_none()
            if not latest:
                continue

            state, why = router_state(latest, thresholds)
            router_data.append({
                "name": router,
                "latest": latest,
                "state": state,
                "why": why,
                "sparkline_req": _sparkline([r.request_rate for r in history]),
                "sparkline_p95": _sparkline([r.latency_p95_ms for r in history]),
                "sparkline_5xx": _sparkline([r.error_rate_5xx_pct for r in history]),
            })

        # Latest global stat (always most recent, not range-filtered)
        result = await db.execute(
            select(TraefikGlobalStat)
            .order_by(TraefikGlobalStat.scraped_at.desc())
            .limit(1)
        )
        global_stat = result.scalar_one_or_none()

        # All certs, sorted by soonest expiry
        result = await db.execute(
            select(TraefikCert).order_by(TraefikCert.not_after)
        )
        raw_certs = result.scalars().all()

    certs = []
    for c in raw_certs:
        days = (c.not_after - now).total_seconds() / 86400.0
        certs.append({
            "serial": c.serial,
            "cn": c.cn or c.serial,
            "sans": c.sans,
            "not_after": c.not_after,
            "days_remaining": days,
        })

    current, gone = gone_routers(
        router_data, current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"])
    problems, healthy = split_routers(current)
    return await render_template(
        "traefik/rows.html",
        problems=problems,
        healthy=healthy,
        gone=gone,
        error_4xx=ERROR_4XX_PCT,
        error_5xx=ERROR_5XX_PCT,
        global_stat=global_stat,
        certs=certs,
        range_key=range_key,
    )


@traefik_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
    """HTMX dashboard widget fragment."""
    now = datetime.now(timezone.utc)

    async with current_app.db_sessionmaker() as db:
        result = await db.execute(
            select(TraefikMetric.router_name).distinct().order_by(TraefikMetric.router_name)
        )
        routers = [row[0] for row in result.all()]
        router_data = []
        for router in routers:
            result = await db.execute(
                select(TraefikMetric)
                .where(TraefikMetric.router_name == router)
                .order_by(TraefikMetric.scraped_at.desc())
                .limit(1)
            )
            latest = result.scalar_one_or_none()
            if latest:
                router_data.append({"name": router, "latest": latest})

        router_data.sort(key=lambda r: (
            -(1 if r["latest"].error_rate_5xx_pct > 0.1 else 0),
            -(1 if r["latest"].error_rate_4xx_pct > 5.0 else 0),
            -r["latest"].request_rate,
        ))
        total_routers = len(router_data)
        router_data = router_data[:5]

        result = await db.execute(
            select(TraefikCert).order_by(TraefikCert.not_after)
        )
        raw_certs = result.scalars().all()

    expiring_certs = []
    for c in raw_certs:
        days = (c.not_after - now).total_seconds() / 86400.0
        if days < 30:
            expiring_certs.append({"cn": c.cn or c.serial, "days_remaining": days})

    return await render_template(
        "traefik/widget.html",
        router_data=router_data,
        expiring_certs=expiring_certs,
        total_routers=total_routers,
    )


@traefik_bp.get("/widget/access_log")
@require_role(UserRole.viewer)
async def widget_access_log():
    """HTMX dashboard widget: traffic origin summary with IP filter."""
    from collections import defaultdict
    ip_filter = request.args.get("ip_filter", "all")
    limit = max(1, min(50, int(request.args.get("limit", 10) or 10)))
    widget_id = request.args.get("wid", "0")

    async with current_app.db_sessionmaker() as db:
        result = await db.execute(
            select(TraefikAccessSummary)
            .order_by(TraefikAccessSummary.period_end.desc())
            .limit(12)  # last ~1 hour of 5-min windows
        )
        rows = result.scalars().all()

    if not rows:
        return await render_template("traefik/widget_access_log.html",
                                     summary=None, widget_id=widget_id)

    total = sum(r.total_requests for r in rows)
    internal = sum(r.internal_requests for r in rows)
    external = sum(r.external_requests for r in rows)

    ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True})
    for row in rows:
        for entry in json.loads(row.top_ips_json):
            k = entry["ip"]
            ip_counts[k]["count"] += entry["count"]
            ip_counts[k]["internal"] = entry.get("internal", True)

    all_ips = sorted(
        [{"ip": k, **v} for k, v in ip_counts.items()],
        key=lambda x: x["count"], reverse=True,
    )
    if ip_filter == "internal":
        all_ips = [e for e in all_ips if e["internal"]]
    elif ip_filter == "external":
        all_ips = [e for e in all_ips if not e["internal"]]

    summary = {
        "total": total,
        "internal": internal,
        "external": external,
        "external_pct": (external / total * 100.0) if total else 0.0,
        "top_ips": all_ips[:limit],
        "ip_filter": ip_filter,
    }
    return await render_template("traefik/widget_access_log.html",
                                 summary=summary, widget_id=widget_id)


@traefik_bp.get("/widget/request_chart")
@require_role(UserRole.viewer)
async def widget_request_chart():
    """HTMX dashboard widget: Chart.js request rate + error % over time."""
    from datetime import timedelta
    hours = max(1, min(24, int(request.args.get("hours", 6) or 6)))
    widget_id = request.args.get("wid", "0")

    since = datetime.now(timezone.utc) - timedelta(hours=hours)
    b_secs = max(60, (hours * 3600) // 80)  # ~80 buckets

    bucket_col = (
        cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs
    ).label("bucket")

    async with current_app.db_sessionmaker() as db:
        result = await db.execute(
            select(
                func.avg(TraefikMetric.request_rate).label("req_rate"),
                func.avg(TraefikMetric.error_rate_5xx_pct).label("err_pct"),
                func.min(TraefikMetric.scraped_at).label("scraped_at"),
                bucket_col,
            )
            .where(TraefikMetric.scraped_at >= since)
            .group_by(bucket_col)
            .order_by(bucket_col)
        )
        history = result.all()

    labels = list(range(len(history)))
    req_rates = [round(r.req_rate or 0, 3) for r in history]
    err_pcts = [round(r.err_pct or 0, 2) for r in history]

    return await render_template(
        "traefik/widget_request_chart.html",
        labels_json=json.dumps(labels),
        req_rate_json=json.dumps(req_rates),
        error_rate_json=json.dumps(err_pcts),
        widget_id=widget_id,
        hours=hours,
    )


@traefik_bp.get("/traffic")
@require_role(UserRole.viewer)
async def traffic():
    """HTMX fragment: traffic origin summary scoped to selected time range."""
    since, range_key = parse_range(request.args.get("range"))

    async with current_app.db_sessionmaker() as db:
        result = await db.execute(
            select(TraefikAccessSummary)
            .where(TraefikAccessSummary.period_end >= since)
            .order_by(TraefikAccessSummary.period_end.asc())
            # No LIMIT — all summaries in range needed for accurate totals;
            # sparkline subsamples to 80 points afterwards
        )
        rows = result.scalars().all()

    if not rows:
        al = _access_log()
        return await render_template(
            "traefik/traffic.html", summary=None, history=[], range_key=range_key,
            access_log_enabled=bool(al.get("enabled", False)), access_log_path=al.get("path", ""),
        )

    total = sum(r.total_requests for r in rows)
    internal = sum(r.internal_requests for r in rows)
    external = sum(r.external_requests for r in rows)
    total_bytes = sum(r.total_bytes for r in rows)

    ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True, "country": None})
    country_counts: dict[str, int] = defaultdict(int)
    router_counts: dict[str, int] = defaultdict(int)

    for row in rows:
        for entry in json.loads(row.top_ips_json):
            k = entry["ip"]
            ip_counts[k]["count"] += entry["count"]
            ip_counts[k]["internal"] = entry["internal"]
            ip_counts[k]["country"] = entry.get("country")
        for entry in json.loads(row.top_countries_json):
            country_counts[entry["country"]] += entry["count"]
        for entry in json.loads(row.top_routers_json):
            router_counts[entry["router"]] += entry["count"]

    top_ips = sorted(
        [{"ip": k, **v} for k, v in ip_counts.items()],
        key=lambda x: x["count"], reverse=True
    )[:10]
    top_countries = sorted(
        [{"country": k, "count": v} for k, v in country_counts.items()],
        key=lambda x: x["count"], reverse=True
    )[:10]
    top_routers = sorted(
        [{"router": k, "count": v} for k, v in router_counts.items()],
        key=lambda x: x["count"], reverse=True
    )[:10]

    history = [
        {
            "period_end": r.period_end,
            "external_pct": (r.external_requests / r.total_requests * 100.0)
            if r.total_requests else 0.0,
        }
        for r in rows
    ]

    summary = {
        "total": total,
        "internal": internal,
        "external": external,
        "total_bytes": total_bytes,
        "external_pct": (external / total * 100.0) if total else 0.0,
        "top_ips": top_ips,
        "top_countries": top_countries,
        "top_routers": top_routers,
        "sparkline_external": _sparkline(
            [h["external_pct"] for h in subsample(history, 80)]
        ),
    }

    return await render_template(
        "traefik/traffic.html", summary=summary, history=history, range_key=range_key
    )