# plugins/traefik/access_log.py """Traefik JSON access log tailer and traffic-origin aggregator. Reads new lines from Traefik's JSON access log on each scheduler tick, classifies source IPs as internal (RFC1918) or external, optionally looks up country codes via MaxMind GeoLite2, and returns an aggregated summary ready to persist in TraefikAccessSummary. """ from __future__ import annotations import ipaddress import json from collections import defaultdict from pathlib import Path # ── Module-level file-position state ───────────────────────────────────────── _log_position: int = 0 # byte offset of last read _log_inode: int = 0 # detect rotation by inode change # ── Private IP ranges (RFC1918 + loopback + link-local) ────────────────────── _PRIVATE_NETS = [ ipaddress.ip_network("10.0.0.0/8"), ipaddress.ip_network("172.16.0.0/12"), ipaddress.ip_network("192.168.0.0/16"), ipaddress.ip_network("127.0.0.0/8"), ipaddress.ip_network("169.254.0.0/16"), ipaddress.ip_network("::1/128"), ipaddress.ip_network("fc00::/7"), ipaddress.ip_network("fe80::/10"), ] def _strip_port(raw: str) -> str: """Strip trailing :port from an IPv4 address string. Leaves IPv6 untouched.""" if not raw: return raw # Bracketed IPv6 with port: [::1]:12345 if raw.startswith("["): return raw[1:raw.index("]")] if "]" in raw else raw # IPv4:port has exactly one colon if raw.count(":") == 1: return raw.rsplit(":", 1)[0] return raw def is_internal(raw_ip: str) -> bool: """Return True if the address falls within a private/loopback range.""" ip_str = _strip_port(raw_ip) try: addr = ipaddress.ip_address(ip_str) return any(addr in net for net in _PRIVATE_NETS) except ValueError: return False def _country(ip_str: str, geoip_db) -> str | None: """Return ISO country code via an open maxminddb reader, or None.""" if geoip_db is None: return None clean = _strip_port(ip_str) try: record = geoip_db.get(clean) if record: return record.get("country", {}).get("iso_code") except Exception: pass return None def read_new_lines(log_path: str) -> list[dict]: """Return all new JSON-parsed access log entries since the last call. Handles log rotation: if the file shrinks or its inode changes, we reset to the beginning of the new file. """ global _log_position, _log_inode path = Path(log_path) if not path.exists(): return [] try: stat = path.stat() except OSError: return [] current_inode = stat.st_ino current_size = stat.st_size # Rotation detected if current_inode != _log_inode or current_size < _log_position: _log_position = 0 _log_inode = current_inode if current_size == _log_position: return [] entries: list[dict] = [] try: with open(log_path, "rb") as f: f.seek(_log_position) raw = f.read(current_size - _log_position) _log_position = current_size except OSError: return [] for raw_line in raw.split(b"\n"): raw_line = raw_line.strip() if not raw_line: continue try: entries.append(json.loads(raw_line)) except json.JSONDecodeError: continue return entries def aggregate(entries: list[dict], geoip_db=None, top_n: int = 10) -> dict | None: """Aggregate a batch of access log entries into a summary dict. Returns None if there are no entries. Result keys map directly to TraefikAccessSummary columns (excluding id and period_start/period_end, which the caller provides). """ if not entries: return None total = 0 internal = 0 external = 0 total_bytes = 0.0 by_router: dict[str, int] = defaultdict(int) ip_counts: dict[str, int] = defaultdict(int) ip_internal: dict[str, bool] = {} ip_country: dict[str, str | None] = {} country_counts: dict[str, int] = defaultdict(int) for entry in entries: total += 1 raw_ip = entry.get("ClientHost", "") router = entry.get("RouterName") or entry.get("ServiceName") or "unknown" total_bytes += entry.get("DownstreamContentSize") or 0 by_router[router] += 1 internal_flag = is_internal(raw_ip) if internal_flag: internal += 1 else: external += 1 if raw_ip not in ip_country: ip_country[raw_ip] = _country(raw_ip, geoip_db) c = ip_country[raw_ip] if c: country_counts[c] += 1 ip_counts[raw_ip] += 1 ip_internal[raw_ip] = internal_flag top_ips = [ { "ip": ip, "count": cnt, "internal": ip_internal[ip], "country": ip_country.get(ip), } for ip, cnt in sorted(ip_counts.items(), key=lambda x: x[1], reverse=True)[:top_n] ] top_countries = [ {"country": c, "count": n} for c, n in sorted(country_counts.items(), key=lambda x: x[1], reverse=True)[:top_n] ] top_routers = [ {"router": r, "count": n} for r, n in sorted(by_router.items(), key=lambda x: x[1], reverse=True)[:top_n] ] return { "total_requests": total, "internal_requests": internal, "external_requests": external, "total_bytes": total_bytes, "top_ips_json": json.dumps(top_ips), "top_countries_json": json.dumps(top_countries), "top_routers_json": json.dumps(top_routers), } def open_geoip(db_path: str | None): """Open a MaxMind mmdb file if available. Returns None if not configured or missing.""" if not db_path: return None try: import maxminddb # type: ignore[import] return maxminddb.open_database(db_path) except Exception: return None