feat(plugins): fold first-party plugins in-tree; bundled + external roots

First-party plugins (host_agent, http, snmp, traefik, unifi, docker) are now tracked under plugins/ and baked into the image, so they version atomically with core — ending the cross-repo import drift the roundtable->steward rename exposed. History for these files is preserved in the archived Roundtable-plugins repo. Plugin discovery becomes multi-root: PLUGIN_DIR (single) -> PLUGIN_DIRS (bundled first, then external) + PLUGIN_INSTALL_DIR. Bundled ships in the image; third-party plugins still mount at runtime into the external root (STEWARD_PLUGIN_DIR, default /data/plugins) and downloads/installs land there. Bundled shadows external on a name collision. - config.py: load_bootstrap returns plugin_dirs + plugin_install_dir - app.py: iterate PLUGIN_DIRS at the migration + load sites - migration_runner.py: discover_all_in() unions every plugin root - plugin_manager.py: resolve_plugin_path() (pure, first-root-wins); load / install / hot-reload span all roots; installs target the external root - settings/routes.py: _discover_plugins scans all roots, dedup bundled-first - Dockerfile: COPY plugins/ ; docker-compose: drop host bind, document external - tests/test_plugin_dirs.py: resolution, multi-root discovery, bootstrap split Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-01 08:37:24 -04:00
parent 88ab5b917e
commit a7a281cb11
99 changed files with 7931 additions and 52 deletions
@@ -0,0 +1,199 @@
+# plugins/traefik/access_log.py
+"""Traefik JSON access log tailer and traffic-origin aggregator.
+
+Reads new lines from Traefik's JSON access log on each scheduler tick,
+classifies source IPs as internal (RFC1918) or external, optionally
+looks up country codes via MaxMind GeoLite2, and returns an aggregated
+summary ready to persist in TraefikAccessSummary.
+"""
+from __future__ import annotations
+
+import ipaddress
+import json
+from collections import defaultdict
+from pathlib import Path
+
+# ── Module-level file-position state ─────────────────────────────────────────
+_log_position: int = 0   # byte offset of last read
+_log_inode: int = 0      # detect rotation by inode change
+
+# ── Private IP ranges (RFC1918 + loopback + link-local) ──────────────────────
+_PRIVATE_NETS = [
+    ipaddress.ip_network("10.0.0.0/8"),
+    ipaddress.ip_network("172.16.0.0/12"),
+    ipaddress.ip_network("192.168.0.0/16"),
+    ipaddress.ip_network("127.0.0.0/8"),
+    ipaddress.ip_network("169.254.0.0/16"),
+    ipaddress.ip_network("::1/128"),
+    ipaddress.ip_network("fc00::/7"),
+    ipaddress.ip_network("fe80::/10"),
+]
+
+
+def _strip_port(raw: str) -> str:
+    """Strip trailing :port from an IPv4 address string. Leaves IPv6 untouched."""
+    if not raw:
+        return raw
+    # Bracketed IPv6 with port: [::1]:12345
+    if raw.startswith("["):
+        return raw[1:raw.index("]")] if "]" in raw else raw
+    # IPv4:port has exactly one colon
+    if raw.count(":") == 1:
+        return raw.rsplit(":", 1)[0]
+    return raw
+
+
+def is_internal(raw_ip: str) -> bool:
+    """Return True if the address falls within a private/loopback range."""
+    ip_str = _strip_port(raw_ip)
+    try:
+        addr = ipaddress.ip_address(ip_str)
+        return any(addr in net for net in _PRIVATE_NETS)
+    except ValueError:
+        return False
+
+
+def _country(ip_str: str, geoip_db) -> str | None:
+    """Return ISO country code via an open maxminddb reader, or None."""
+    if geoip_db is None:
+        return None
+    clean = _strip_port(ip_str)
+    try:
+        record = geoip_db.get(clean)
+        if record:
+            return record.get("country", {}).get("iso_code")
+    except Exception:
+        pass
+    return None
+
+
+def read_new_lines(log_path: str) -> list[dict]:
+    """Return all new JSON-parsed access log entries since the last call.
+
+    Handles log rotation: if the file shrinks or its inode changes, we
+    reset to the beginning of the new file.
+    """
+    global _log_position, _log_inode
+
+    path = Path(log_path)
+    if not path.exists():
+        return []
+
+    try:
+        stat = path.stat()
+    except OSError:
+        return []
+
+    current_inode = stat.st_ino
+    current_size = stat.st_size
+
+    # Rotation detected
+    if current_inode != _log_inode or current_size < _log_position:
+        _log_position = 0
+        _log_inode = current_inode
+
+    if current_size == _log_position:
+        return []
+
+    entries: list[dict] = []
+    try:
+        with open(log_path, "rb") as f:
+            f.seek(_log_position)
+            raw = f.read(current_size - _log_position)
+            _log_position = current_size
+    except OSError:
+        return []
+
+    for raw_line in raw.split(b"\n"):
+        raw_line = raw_line.strip()
+        if not raw_line:
+            continue
+        try:
+            entries.append(json.loads(raw_line))
+        except json.JSONDecodeError:
+            continue
+
+    return entries
+
+
+def aggregate(entries: list[dict], geoip_db=None, top_n: int = 10) -> dict | None:
+    """Aggregate a batch of access log entries into a summary dict.
+
+    Returns None if there are no entries.
+    Result keys map directly to TraefikAccessSummary columns (excluding
+    id and period_start/period_end, which the caller provides).
+    """
+    if not entries:
+        return None
+
+    total = 0
+    internal = 0
+    external = 0
+    total_bytes = 0.0
+
+    by_router: dict[str, int] = defaultdict(int)
+    ip_counts: dict[str, int] = defaultdict(int)
+    ip_internal: dict[str, bool] = {}
+    ip_country: dict[str, str | None] = {}
+    country_counts: dict[str, int] = defaultdict(int)
+
+    for entry in entries:
+        total += 1
+        raw_ip = entry.get("ClientHost", "")
+        router = entry.get("RouterName") or entry.get("ServiceName") or "unknown"
+        total_bytes += entry.get("DownstreamContentSize") or 0
+
+        by_router[router] += 1
+
+        internal_flag = is_internal(raw_ip)
+        if internal_flag:
+            internal += 1
+        else:
+            external += 1
+            if raw_ip not in ip_country:
+                ip_country[raw_ip] = _country(raw_ip, geoip_db)
+            c = ip_country[raw_ip]
+            if c:
+                country_counts[c] += 1
+
+        ip_counts[raw_ip] += 1
+        ip_internal[raw_ip] = internal_flag
+
+    top_ips = [
+        {
+            "ip": ip,
+            "count": cnt,
+            "internal": ip_internal[ip],
+            "country": ip_country.get(ip),
+        }
+        for ip, cnt in sorted(ip_counts.items(), key=lambda x: x[1], reverse=True)[:top_n]
+    ]
+    top_countries = [
+        {"country": c, "count": n}
+        for c, n in sorted(country_counts.items(), key=lambda x: x[1], reverse=True)[:top_n]
+    ]
+    top_routers = [
+        {"router": r, "count": n}
+        for r, n in sorted(by_router.items(), key=lambda x: x[1], reverse=True)[:top_n]
+    ]
+
+    return {
+        "total_requests": total,
+        "internal_requests": internal,
+        "external_requests": external,
+        "total_bytes": total_bytes,
+        "top_ips_json": json.dumps(top_ips),
+        "top_countries_json": json.dumps(top_countries),
+        "top_routers_json": json.dumps(top_routers),
+    }
+
+
+def open_geoip(db_path: str | None):
+    """Open a MaxMind mmdb file if available. Returns None if not configured or missing."""
+    if not db_path:
+        return None
+    try:
+        import maxminddb  # type: ignore[import]
+        return maxminddb.open_database(db_path)
+    except Exception:
+        return None