Roundtable-plugins/traefik/access_log.py

# plugins/traefik/access_log.py
"""Traefik JSON access log tailer and traffic-origin aggregator.

Reads new lines from Traefik's JSON access log on each scheduler tick,
classifies source IPs as internal (RFC1918) or external, optionally
looks up country codes via MaxMind GeoLite2, and returns an aggregated
summary ready to persist in TraefikAccessSummary.
"""
from __future__ import annotations

import ipaddress
import json
from collections import defaultdict
from pathlib import Path

# ── Module-level file-position state ─────────────────────────────────────────
_log_position: int = 0   # byte offset of last read
_log_inode: int = 0      # detect rotation by inode change

# ── Private IP ranges (RFC1918 + loopback + link-local) ──────────────────────
_PRIVATE_NETS = [
    ipaddress.ip_network("10.0.0.0/8"),
    ipaddress.ip_network("172.16.0.0/12"),
    ipaddress.ip_network("192.168.0.0/16"),
    ipaddress.ip_network("127.0.0.0/8"),
    ipaddress.ip_network("169.254.0.0/16"),
    ipaddress.ip_network("::1/128"),
    ipaddress.ip_network("fc00::/7"),
    ipaddress.ip_network("fe80::/10"),
]


def _strip_port(raw: str) -> str:
    """Strip trailing :port from an IPv4 address string. Leaves IPv6 untouched."""
    if not raw:
        return raw
    # Bracketed IPv6 with port: [::1]:12345
    if raw.startswith("["):
        return raw[1:raw.index("]")] if "]" in raw else raw
    # IPv4:port has exactly one colon
    if raw.count(":") == 1:
        return raw.rsplit(":", 1)[0]
    return raw


def is_internal(raw_ip: str) -> bool:
    """Return True if the address falls within a private/loopback range."""
    ip_str = _strip_port(raw_ip)
    try:
        addr = ipaddress.ip_address(ip_str)
        return any(addr in net for net in _PRIVATE_NETS)
    except ValueError:
        return False


def _country(ip_str: str, geoip_db) -> str | None:
    """Return ISO country code via an open maxminddb reader, or None."""
    if geoip_db is None:
        return None
    clean = _strip_port(ip_str)
    try:
        record = geoip_db.get(clean)
        if record:
            return record.get("country", {}).get("iso_code")
    except Exception:
        pass
    return None


def read_new_lines(log_path: str) -> list[dict]:
    """Return all new JSON-parsed access log entries since the last call.

    Handles log rotation: if the file shrinks or its inode changes, we
    reset to the beginning of the new file.
    """
    global _log_position, _log_inode

    path = Path(log_path)
    if not path.exists():
        return []

    try:
        stat = path.stat()
    except OSError:
        return []

    current_inode = stat.st_ino
    current_size = stat.st_size

    # Rotation detected
    if current_inode != _log_inode or current_size < _log_position:
        _log_position = 0
        _log_inode = current_inode

    if current_size == _log_position:
        return []

    entries: list[dict] = []
    try:
        with open(log_path, "rb") as f:
            f.seek(_log_position)
            raw = f.read(current_size - _log_position)
            _log_position = current_size
    except OSError:
        return []

    for raw_line in raw.split(b"\n"):
        raw_line = raw_line.strip()
        if not raw_line:
            continue
        try:
            entries.append(json.loads(raw_line))
        except json.JSONDecodeError:
            continue

    return entries


def aggregate(entries: list[dict], geoip_db=None, top_n: int = 10) -> dict | None:
    """Aggregate a batch of access log entries into a summary dict.

    Returns None if there are no entries.
    Result keys map directly to TraefikAccessSummary columns (excluding
    id and period_start/period_end, which the caller provides).
    """
    if not entries:
        return None

    total = 0
    internal = 0
    external = 0
    total_bytes = 0.0

    by_router: dict[str, int] = defaultdict(int)
    ip_counts: dict[str, int] = defaultdict(int)
    ip_internal: dict[str, bool] = {}
    ip_country: dict[str, str | None] = {}
    country_counts: dict[str, int] = defaultdict(int)

    for entry in entries:
        total += 1
        raw_ip = entry.get("ClientHost", "")
        router = entry.get("RouterName") or entry.get("ServiceName") or "unknown"
        total_bytes += entry.get("DownstreamContentSize") or 0

        by_router[router] += 1

        internal_flag = is_internal(raw_ip)
        if internal_flag:
            internal += 1
        else:
            external += 1
            if raw_ip not in ip_country:
                ip_country[raw_ip] = _country(raw_ip, geoip_db)
            c = ip_country[raw_ip]
            if c:
                country_counts[c] += 1

        ip_counts[raw_ip] += 1
        ip_internal[raw_ip] = internal_flag

    top_ips = [
        {
            "ip": ip,
            "count": cnt,
            "internal": ip_internal[ip],
            "country": ip_country.get(ip),
        }
        for ip, cnt in sorted(ip_counts.items(), key=lambda x: x[1], reverse=True)[:top_n]
    ]
    top_countries = [
        {"country": c, "count": n}
        for c, n in sorted(country_counts.items(), key=lambda x: x[1], reverse=True)[:top_n]
    ]
    top_routers = [
        {"router": r, "count": n}
        for r, n in sorted(by_router.items(), key=lambda x: x[1], reverse=True)[:top_n]
    ]

    return {
        "total_requests": total,
        "internal_requests": internal,
        "external_requests": external,
        "total_bytes": total_bytes,
        "top_ips_json": json.dumps(top_ips),
        "top_countries_json": json.dumps(top_countries),
        "top_routers_json": json.dumps(top_routers),
    }


def open_geoip(db_path: str | None):
    """Open a MaxMind mmdb file if available. Returns None if not configured or missing."""
    if not db_path:
        return None
    try:
        import maxminddb  # type: ignore[import]
        return maxminddb.open_database(db_path)
    except Exception:
        return None