diff --git a/.gitignore b/.gitignore index fb2b9af..b4b5ccd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,10 @@ # Planning files docs/superpowers/ -# Plugin directory — managed as a separate repo (bvandeusen/steward-plugins) -# PLUGIN_DIR in config.yaml points here at runtime -/plugins/ +# First-party plugins are tracked in-tree under plugins/ and ship in the image. +# Third-party plugins are mounted at runtime into the external dir +# (STEWARD_PLUGIN_DIR, default /data/plugins) and are not tracked here. +/plugins/**/*.zip # Python __pycache__/ @@ -50,3 +51,6 @@ Thumbs.db # Playbook cache (runtime data) playbook_cache/ + +# Superpowers brainstorm scratch (visual companion mockups, state) +.superpowers/ diff --git a/Dockerfile b/Dockerfile index 938501c..4172e1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,10 @@ COPY steward/ steward/ RUN pip install --no-cache-dir . COPY alembic.ini . +# First-party plugins ship inside the image (bundled root at /app/plugins). +# Third-party plugins are mounted at runtime into the external root +# (STEWARD_PLUGIN_DIR, default /data/plugins). +COPY plugins/ plugins/ COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh diff --git a/docker-compose.yml b/docker-compose.yml index faedd9f..3e1ce35 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,10 @@ services: ports: - "5000:5000" volumes: + # First-party plugins are baked into the image at /app/plugins. + # Third-party plugins persist in the external root /data/plugins + # (inside app_data); STEWARD_PLUGIN_DIR defaults there, no extra mount needed. - app_data:/data - - ./plugins:/app/plugins:ro - /mnt/Data/traefik/log:/var/log/traefik:ro - /var/run/docker.sock:/var/run/docker.sock:ro environment: diff --git a/plugins/__init__.py b/plugins/__init__.py new file mode 100644 index 0000000..1648f8e --- /dev/null +++ b/plugins/__init__.py @@ -0,0 +1 @@ +# plugins/__init__.py diff --git a/plugins/docker/__init__.py b/plugins/docker/__init__.py new file mode 100644 index 0000000..d569556 --- /dev/null +++ b/plugins/docker/__init__.py @@ -0,0 +1,24 @@ +# plugins/docker/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + from .models import DockerContainer, DockerMetric # noqa: F401 — register with Base + + +def get_scheduled_tasks() -> list: + from .scheduler import make_task + return [make_task(_app)] + + +def get_blueprint(): + from .routes import docker_bp + return docker_bp diff --git a/plugins/docker/migrations/__init__.py b/plugins/docker/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/docker/migrations/env.py b/plugins/docker/migrations/env.py new file mode 100644 index 0000000..5c20755 --- /dev/null +++ b/plugins/docker/migrations/env.py @@ -0,0 +1,70 @@ +# plugins/docker/migrations/env.py +"""Alembic env.py for the Docker plugin.""" +import asyncio +import os +import sys +from pathlib import Path +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config +from alembic import context + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from fabledscryer.models.base import Base +import fabledscryer.models # noqa: F401 +from plugins.docker.models import DockerContainer, DockerMetric # noqa: F401 + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _get_url() -> str: + import yaml + cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml") + try: + with open(cfg_path) as f: + cfg = yaml.safe_load(f) or {} + url = cfg.get("database", {}).get("url", "") + except FileNotFoundError: + url = "" + return os.environ.get("FABLEDSCRYER_DATABASE__URL", url) + + +def run_migrations_offline() -> None: + context.configure( + url=_get_url(), target_metadata=target_metadata, + literal_binds=True, dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + cfg = config.get_section(config.config_ini_section, {}) + cfg["sqlalchemy.url"] = _get_url() + connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/plugins/docker/migrations/versions/__init__.py b/plugins/docker/migrations/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/docker/migrations/versions/docker_001_initial.py b/plugins/docker/migrations/versions/docker_001_initial.py new file mode 100644 index 0000000..90e5298 --- /dev/null +++ b/plugins/docker/migrations/versions/docker_001_initial.py @@ -0,0 +1,47 @@ +"""Docker plugin initial tables + +Revision ID: docker_001_initial +Revises: (none — branch off core via depends_on) +Create Date: 2026-03-22 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = "docker" +depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",) + + +def upgrade() -> None: + op.create_table( + "docker_containers", + sa.Column("name", sa.String(255), primary_key=True), + sa.Column("container_id", sa.String(64), nullable=False, server_default=""), + sa.Column("image", sa.String(512), nullable=False, server_default=""), + sa.Column("status", sa.String(32), nullable=False, server_default="unknown"), + sa.Column("cpu_pct", sa.Float, nullable=True), + sa.Column("mem_usage_bytes", sa.Integer, nullable=True), + sa.Column("mem_limit_bytes", sa.Integer, nullable=True), + sa.Column("mem_pct", sa.Float, nullable=True), + sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"), + sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "docker_metrics", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("container_name", sa.String(255), nullable=False, index=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False, index=True), + sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"), + sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"), + sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"), + ) + + +def downgrade() -> None: + op.drop_table("docker_metrics") + op.drop_table("docker_containers") diff --git a/plugins/docker/models.py b/plugins/docker/models.py new file mode 100644 index 0000000..09c1cf3 --- /dev/null +++ b/plugins/docker/models.py @@ -0,0 +1,49 @@ +# plugins/docker/models.py +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import DateTime, Float, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from fabledscryer.models.base import Base + + +class DockerContainer(Base): + """Latest known state per container — upserted by name on each scrape.""" + __tablename__ = "docker_containers" + + # Container name is the stable natural key; container_id changes on recreation + name: Mapped[str] = mapped_column(String(255), primary_key=True) + container_id: Mapped[str] = mapped_column(String(64), nullable=False, default="") + image: Mapped[str] = mapped_column(String(512), nullable=False, default="") + status: Mapped[str] = mapped_column(String(32), nullable=False, default="unknown") + # running | stopped | paused | exited | dead + cpu_pct: Mapped[float | None] = mapped_column(Float, nullable=True) + mem_usage_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) + mem_limit_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) + mem_pct: Mapped[float | None] = mapped_column(Float, nullable=True) + restart_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + ports_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + # JSON: [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}] + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + scraped_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + +class DockerMetric(Base): + """Time-series CPU/memory per container — one row per scrape per running container.""" + __tablename__ = "docker_metrics" + + id: Mapped[str] = mapped_column( + String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + ) + container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + scraped_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + index=True, + ) + cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + mem_usage_bytes: Mapped[int] = mapped_column(Integer, nullable=False, default=0) diff --git a/plugins/docker/plugin.yaml b/plugins/docker/plugin.yaml new file mode 100644 index 0000000..8745302 --- /dev/null +++ b/plugins/docker/plugin.yaml @@ -0,0 +1,17 @@ +name: docker +version: "1.0.0" +description: "Docker container status, resource usage, restart tracking via Docker socket" +author: "FabledScryer" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/docker" +tags: + - containers + - docker + - infrastructure + +config: + socket_path: /var/run/docker.sock + scrape_interval_seconds: 60 + include_stopped: false diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py new file mode 100644 index 0000000..12d0383 --- /dev/null +++ b/plugins/docker/routes.py @@ -0,0 +1,161 @@ +# plugins/docker/routes.py +from __future__ import annotations +import json +from datetime import datetime, timezone +from quart import Blueprint, current_app, render_template, request +from sqlalchemy import Integer, cast, func, select + +from fabledscryer.auth.middleware import require_role +from fabledscryer.models.users import UserRole +from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds +from .models import DockerContainer, DockerMetric + +docker_bp = Blueprint("docker", __name__, template_folder="templates") + + +def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: + if len(values) < 2: + return f'' + mn, mx = min(values), max(values) + if mx == mn: + mx = mn + 1.0 + step = width / (len(values) - 1) + pts = [] + for i, v in enumerate(values): + x = i * step + y = height - (v - mn) / (mx - mn) * (height - 2) - 1 + pts.append(f"{x:.1f},{y:.1f}") + poly = " ".join(pts) + return ( + f'' + f'' + f'' + ) + + +@docker_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + current_range = request.args.get("range", DEFAULT_RANGE) + return await render_template( + "docker/index.html", + poll_interval=poll_interval, + current_range=current_range, + ) + + +@docker_bp.get("/rows") +@require_role(UserRole.viewer) +async def rows(): + """HTMX fragment: container list with status and resource sparklines.""" + since, range_key = parse_range(request.args.get("range")) + b_secs = bucket_seconds(since) + + bucket_col = ( + cast(func.strftime('%s', DockerMetric.scraped_at), Integer) / b_secs + ).label("bucket") + + async with current_app.db_sessionmaker() as db: + # All known containers ordered by running first, then name + result = await db.execute( + select(DockerContainer) + .order_by( + # running first + (DockerContainer.status == "running").desc(), + DockerContainer.name, + ) + ) + containers = list(result.scalars()) + + # Build per-container sparkline histories + histories: dict[str, list] = {} + for c in containers: + result = await db.execute( + select( + func.avg(DockerMetric.cpu_pct).label("cpu_pct"), + func.avg(DockerMetric.mem_pct).label("mem_pct"), + bucket_col, + ) + .where(DockerMetric.container_name == c.name) + .where(DockerMetric.scraped_at >= since) + .group_by(bucket_col) + .order_by(bucket_col) + ) + histories[c.name] = result.all() + + running = sum(1 for c in containers if c.status == "running") + stopped = len(containers) - running + + container_data = [] + for c in containers: + hist = histories.get(c.name, []) + ports = json.loads(c.ports_json) if c.ports_json else [] + container_data.append({ + "container": c, + "ports": ports, + "sparkline_cpu": _sparkline([r.cpu_pct or 0 for r in hist]), + "sparkline_mem": _sparkline([r.mem_pct or 0 for r in hist]), + }) + + return await render_template( + "docker/rows.html", + container_data=container_data, + running=running, + stopped=stopped, + range_key=range_key, + ) + + +@docker_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + """HTMX dashboard widget: container status overview.""" + show_stopped = request.args.get("show_stopped", "no") == "yes" + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(DockerContainer) + .order_by( + (DockerContainer.status == "running").desc(), + DockerContainer.name, + ) + ) + all_containers = list(result.scalars()) + + running = [c for c in all_containers if c.status == "running"] + stopped = [c for c in all_containers if c.status != "running"] + display = all_containers if show_stopped else running + + return await render_template( + "docker/widget.html", + containers=display, + running_count=len(running), + stopped_count=len(stopped), + show_stopped=show_stopped, + widget_id=widget_id, + ) + + +@docker_bp.get("/widget/resources") +@require_role(UserRole.viewer) +async def widget_resources(): + """HTMX dashboard widget: CPU + memory usage for running containers.""" + limit = max(1, min(20, int(request.args.get("limit", 10) or 10))) + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(DockerContainer) + .where(DockerContainer.status == "running") + .order_by(DockerContainer.cpu_pct.desc().nullslast()) + ) + containers = list(result.scalars())[:limit] + + return await render_template( + "docker/widget_resources.html", + containers=containers, + widget_id=widget_id, + ) diff --git a/plugins/docker/scheduler.py b/plugins/docker/scheduler.py new file mode 100644 index 0000000..eef8c61 --- /dev/null +++ b/plugins/docker/scheduler.py @@ -0,0 +1,93 @@ +# plugins/docker/scheduler.py +from __future__ import annotations +import json +import logging +from datetime import datetime, timezone + +from fabledscryer.core.scheduler import ScheduledTask +from fabledscryer.core.alerts import record_metric + +logger = logging.getLogger(__name__) + + +def make_task(app) -> ScheduledTask: + interval = int( + app.config["PLUGINS"]["docker"].get("scrape_interval_seconds", 60) + ) + + async def scrape(): + await _do_scrape(app) + + return ScheduledTask( + name="docker_scrape", + coro_factory=scrape, + interval_seconds=interval, + run_on_startup=True, + ) + + +async def _do_scrape(app) -> None: + from .scraper import scrape_docker + from .models import DockerContainer, DockerMetric + + cfg = app.config["PLUGINS"]["docker"] + socket_path = cfg.get("socket_path", "/var/run/docker.sock") + include_stopped = bool(cfg.get("include_stopped", False)) + + try: + containers = await scrape_docker(socket_path, include_stopped) + except ConnectionError as exc: + logger.error("Docker scrape failed: %s", exc) + return + except Exception: + logger.exception("Docker scrape error") + return + + now = datetime.now(timezone.utc) + + async with app.db_sessionmaker() as session: + async with session.begin(): + for c in containers: + # Upsert container state + existing = await session.get(DockerContainer, c["name"]) + if existing is None: + existing = DockerContainer(name=c["name"]) + session.add(existing) + existing.container_id = c["container_id"] + existing.image = c["image"] + existing.status = c["status"] + existing.cpu_pct = c["cpu_pct"] + existing.mem_usage_bytes = c["mem_usage_bytes"] + existing.mem_limit_bytes = c["mem_limit_bytes"] + existing.mem_pct = c["mem_pct"] + existing.restart_count = c["restart_count"] + existing.ports_json = json.dumps(c["ports"]) + existing.started_at = c["started_at"] + existing.scraped_at = now + + # Time-series metric (running containers only) + if c["status"] == "running" and c["cpu_pct"] is not None: + session.add(DockerMetric( + container_name=c["name"], + scraped_at=now, + cpu_pct=c["cpu_pct"], + mem_pct=c["mem_pct"] or 0.0, + mem_usage_bytes=c["mem_usage_bytes"] or 0, + )) + + # Feed alert pipeline + await record_metric( + session=session, + source_module="docker", + resource_name=c["name"], + metric_name="cpu_pct", + value=c["cpu_pct"], + ) + if c["mem_pct"] is not None: + await record_metric( + session=session, + source_module="docker", + resource_name=c["name"], + metric_name="mem_pct", + value=c["mem_pct"], + ) diff --git a/plugins/docker/scraper.py b/plugins/docker/scraper.py new file mode 100644 index 0000000..96f601b --- /dev/null +++ b/plugins/docker/scraper.py @@ -0,0 +1,140 @@ +# plugins/docker/scraper.py +"""Docker API client via Unix socket.""" +from __future__ import annotations +import asyncio +import logging +from datetime import datetime, timezone + +import httpx + +logger = logging.getLogger(__name__) + + +def _short_id(container_id: str) -> str: + return container_id[:12] if container_id else "" + + +def _calc_cpu_pct(stats: dict) -> float: + """Calculate CPU % from a Docker stats snapshot (one-shot).""" + try: + cpu = stats.get("cpu_stats", {}) + precpu = stats.get("precpu_stats", {}) + cpu_delta = ( + cpu["cpu_usage"]["total_usage"] - precpu["cpu_usage"]["total_usage"] + ) + sys_delta = cpu.get("system_cpu_usage", 0) - precpu.get("system_cpu_usage", 0) + num_cpus = cpu.get("online_cpus") or len( + cpu["cpu_usage"].get("percpu_usage") or [None] + ) + if sys_delta <= 0 or cpu_delta < 0: + return 0.0 + return round((cpu_delta / sys_delta) * num_cpus * 100.0, 2) + except (KeyError, TypeError, ZeroDivisionError): + return 0.0 + + +def _calc_mem(stats: dict) -> tuple[int, int, float]: + """Return (usage_bytes, limit_bytes, mem_pct) from Docker stats.""" + try: + mem = stats.get("memory_stats", {}) + usage = mem.get("usage", 0) + limit = mem.get("limit", 0) + # Subtract page cache for working set (cgroup v1: "cache", v2: "inactive_file") + cache = mem.get("stats", {}).get("cache", 0) or \ + mem.get("stats", {}).get("inactive_file", 0) + actual = max(0, usage - cache) + pct = round(actual / limit * 100.0, 2) if limit > 0 else 0.0 + return actual, limit, pct + except (KeyError, TypeError): + return 0, 0, 0.0 + + +def _parse_ports(ports: list) -> list[dict]: + """Normalise Docker port bindings to a compact list.""" + result = [] + for p in (ports or []): + if p.get("PublicPort"): + result.append({ + "host_port": p["PublicPort"], + "container_port": p["PrivatePort"], + "protocol": p.get("Type", "tcp"), + }) + return result + + +async def scrape_docker(socket_path: str, include_stopped: bool) -> list[dict]: + """ + Fetch all containers + resource stats from the Docker daemon. + Returns a list of normalised dicts ready for the scheduler to persist. + Raises ConnectionError if the socket is unreachable. + """ + transport = httpx.AsyncHTTPTransport(uds=socket_path) + try: + async with httpx.AsyncClient( + transport=transport, + base_url="http://docker", + timeout=10.0, + ) as client: + resp = await client.get( + "/containers/json", + params={"all": "true" if include_stopped else "false"}, + ) + resp.raise_for_status() + containers = resp.json() + + async def _get_stats(c: dict) -> tuple[dict, dict | None]: + if c.get("State") != "running": + return c, None + try: + r = await client.get( + f"/containers/{c['Id']}/stats", + params={"stream": "false", "one-shot": "true"}, + timeout=5.0, + ) + return c, r.json() + except Exception as exc: + logger.debug( + "Stats unavailable for %s: %s", _short_id(c.get("Id", "")), exc + ) + return c, None + + pairs = await asyncio.gather(*[_get_stats(c) for c in containers]) + + except httpx.ConnectError as exc: + raise ConnectionError( + f"Cannot connect to Docker socket at {socket_path}: {exc}" + ) from exc + + results = [] + now = datetime.now(timezone.utc) + for container, stats in pairs: + names = container.get("Names") or [] + name = names[0].lstrip("/") if names else _short_id(container.get("Id", "")) + + cpu_pct = mem_usage = mem_limit = mem_pct = None + if stats is not None: + cpu_pct = _calc_cpu_pct(stats) + mem_usage, mem_limit, mem_pct = _calc_mem(stats) + + # Created is a Unix epoch int from /containers/json + created_ts = container.get("Created") + started_at = ( + datetime.fromtimestamp(created_ts, tz=timezone.utc) + if isinstance(created_ts, (int, float)) else None + ) + + results.append({ + "name": name, + "container_id": _short_id(container.get("Id", "")), + "image": container.get("Image", ""), + "status": container.get("State", "unknown"), + "cpu_pct": cpu_pct, + "mem_usage_bytes": mem_usage, + "mem_limit_bytes": mem_limit, + "mem_pct": mem_pct, + "restart_count": 0, # would require /inspect; left as 0 for now + "ports": _parse_ports(container.get("Ports", [])), + "started_at": started_at, + }) + + return results diff --git a/plugins/docker/templates/docker/index.html b/plugins/docker/templates/docker/index.html new file mode 100644 index 0000000..42c9bb5 --- /dev/null +++ b/plugins/docker/templates/docker/index.html @@ -0,0 +1,16 @@ +{% extends "base.html" %} +{% block title %}Docker — Fabled Scryer{% endblock %} +{% block content %} +
+

Docker

+ {% include "_time_range.html" %} +
+ +
+
Loading...
+
+{% endblock %} diff --git a/plugins/docker/templates/docker/rows.html b/plugins/docker/templates/docker/rows.html new file mode 100644 index 0000000..3b6c84e --- /dev/null +++ b/plugins/docker/templates/docker/rows.html @@ -0,0 +1,86 @@ +{# docker/rows.html — HTMX fragment for the Docker main page #} + +{# ── Summary strip ─────────────────────────────────────────────────────────── #} +
+
+
Running
+ {{ running }} +
+
+
Stopped
+ {{ stopped }} +
+
+
Total
+ {{ container_data | length }} +
+
+ +{# ── Container table ─────────────────────────────────────────────────────── #} +{% if container_data %} +
+ + + + + + + + + + + + + + {% for item in container_data %} + {% set c = item.container %} + + + + + + + + + + {% endfor %} + +
ContainerImagePortsCPU %CPU historyMem %Mem history
+
+ +
+
{{ c.name }}
+
{{ c.status }}
+
+
+
+ {{ c.image }} + + {% for p in item.ports %} +
{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}
+ {% endfor %} +
+ {% if c.cpu_pct is not none %} + + {{ "%.1f" | format(c.cpu_pct) }}% + + {% else %} + + {% endif %} + {{ item.sparkline_cpu | safe }} + {% if c.mem_pct is not none %} + + {{ "%.1f" | format(c.mem_pct) }}% + + {% else %} + + {% endif %} + {{ item.sparkline_mem | safe }}
+
+{% else %} +
+
+ No containers found. Make sure the Docker socket is accessible and the plugin is configured correctly. +
+
+{% endif %} diff --git a/plugins/docker/templates/docker/widget.html b/plugins/docker/templates/docker/widget.html new file mode 100644 index 0000000..427018f --- /dev/null +++ b/plugins/docker/templates/docker/widget.html @@ -0,0 +1,35 @@ +{# docker/widget.html — dashboard widget: container status overview #} +{% if not containers and running_count == 0 and stopped_count == 0 %} +
No containers found.
+{% else %} + +
+
+ {{ running_count }} + running +
+ {% if stopped_count %} +
+ {{ stopped_count }} + stopped +
+ {% endif %} +
+ +
+ {% for c in containers %} +
+ + + {{ c.name }} + + {% if c.cpu_pct is not none %} + + {{ "%.1f" | format(c.cpu_pct) }}% + + {% endif %} +
+ {% endfor %} +
+ +{% endif %} diff --git a/plugins/docker/templates/docker/widget_resources.html b/plugins/docker/templates/docker/widget_resources.html new file mode 100644 index 0000000..39c90dc --- /dev/null +++ b/plugins/docker/templates/docker/widget_resources.html @@ -0,0 +1,36 @@ +{# docker/widget_resources.html — dashboard widget: CPU + memory bars for running containers #} +{% if not containers %} +
No running containers.
+{% else %} +
+ {% for c in containers %} +
+
+ + {{ c.name }} + + + {% if c.cpu_pct is not none %}CPU {{ "%.1f" | format(c.cpu_pct) }}%{% endif %} + {% if c.mem_pct is not none %} · Mem {{ "%.1f" | format(c.mem_pct) }}%{% endif %} + +
+ {% if c.cpu_pct is not none %} +
+
+
+
+ {% endif %} + {% if c.mem_pct is not none %} +
+
+
+
+ {% endif %} +
+ {% endfor %} +
+{% endif %} diff --git a/plugins/host_agent/__init__.py b/plugins/host_agent/__init__.py new file mode 100644 index 0000000..6a7b229 --- /dev/null +++ b/plugins/host_agent/__init__.py @@ -0,0 +1,23 @@ +# plugins/host_agent/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + + +def get_scheduled_tasks() -> list: + from .scheduler import make_task + return [make_task(_app)] + + +def get_blueprint(): + from .routes import host_agent_bp + return host_agent_bp diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py new file mode 100644 index 0000000..34f52d6 --- /dev/null +++ b/plugins/host_agent/agent.py @@ -0,0 +1,370 @@ +# plugins/host_agent/agent.py +"""Steward host agent — pushes resource metrics to a Steward instance. + +Python 3.8+ stdlib only. Target ~300 lines. Served to targets at +GET /plugins/host_agent/agent.py. +""" +from __future__ import annotations + +import json +import os +import shutil +import signal +import socket +import sys +import time +import urllib.error +import urllib.request +from collections import deque +from datetime import datetime, timezone + +AGENT_VERSION = "1.0.0" + + +class ConfigError(Exception): + pass + + +REQUIRED_KEYS = ("url", "token") +INT_KEYS = ("interval_seconds",) +LIST_KEYS = ("mounts",) + + +def read_config(path: str) -> dict: + """Parse a flat `key = value` config file.""" + cfg: dict = {} + try: + with open(path, "r", encoding="utf-8") as f: + for lineno, raw in enumerate(f, 1): + line = raw.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + raise ConfigError(f"{path}:{lineno}: expected 'key = value'") + key, _, value = line.partition("=") + key = key.strip() + value = value.strip() + if key in INT_KEYS: + try: + cfg[key] = int(value) + except ValueError: + raise ConfigError(f"{path}:{lineno}: {key} must be int") + elif key in LIST_KEYS: + cfg[key] = [v.strip() for v in value.split(",") if v.strip()] + else: + cfg[key] = value + except FileNotFoundError: + raise ConfigError(f"{path}: not found") + + missing = [k for k in REQUIRED_KEYS if k not in cfg] + if missing: + raise ConfigError(f"{path}: missing required key(s): {', '.join(missing)}") + + cfg.setdefault("interval_seconds", 30) + return cfg + + +# ─── collectors ────────────────────────────────────────────────────────────── + +STAT_PATH = "/proc/stat" +MEMINFO_PATH = "/proc/meminfo" +LOADAVG_PATH = "/proc/loadavg" +UPTIME_PATH = "/proc/uptime" +OS_RELEASE_PATH = "/etc/os-release" + + +def _read_file(path: str) -> str: + with open(path, "r", encoding="utf-8") as f: + return f.read() + + +def _parse_cpu_line(stat_text: str) -> tuple[int, int]: + """Return (total_jiffies, idle_jiffies) for the aggregate cpu line.""" + for line in stat_text.splitlines(): + if line.startswith("cpu "): + parts = line.split() + fields = [int(x) for x in parts[1:]] + idle = fields[3] + (fields[4] if len(fields) > 4 else 0) # idle + iowait + total = sum(fields) + return total, idle + raise RuntimeError("no aggregate cpu line in /proc/stat") + + +def collect_cpu(sample_window: float = 0.2) -> float: + """Return CPU utilization % over sample_window seconds.""" + t1, i1 = _parse_cpu_line(_read_file(STAT_PATH)) + time.sleep(sample_window) + t2, i2 = _parse_cpu_line(_read_file(STAT_PATH)) + dt = t2 - t1 + di = i2 - i1 + if dt <= 0: + return 0.0 + return round(100.0 * (dt - di) / dt, 2) + + +def collect_memory() -> dict: + info: dict[str, int] = {} + for line in _read_file(MEMINFO_PATH).splitlines(): + if ":" not in line: + continue + key, _, rest = line.partition(":") + parts = rest.strip().split() + if not parts: + continue + try: + value_kb = int(parts[0]) + except ValueError: + continue + info[key] = value_kb * 1024 # bytes + total = info.get("MemTotal", 0) + available = info.get("MemAvailable", 0) + swap_total = info.get("SwapTotal", 0) + swap_free = info.get("SwapFree", 0) + return { + "total_bytes": total, + "used_bytes": max(total - available, 0), + "available_bytes": available, + "swap_used_bytes": max(swap_total - swap_free, 0), + } + + +def collect_storage(mounts: list[str]) -> list[dict]: + out: list[dict] = [] + for m in mounts: + try: + u = shutil.disk_usage(m) + except (FileNotFoundError, PermissionError): + continue + out.append({"mount": m, "total_bytes": u.total, "used_bytes": u.used}) + return out + + +def collect_load() -> dict: + parts = _read_file(LOADAVG_PATH).split() + return {"1m": float(parts[0]), "5m": float(parts[1]), "15m": float(parts[2])} + + +def collect_uptime() -> int: + return int(float(_read_file(UPTIME_PATH).split()[0])) + + +def collect_metadata() -> dict: + u = os.uname() + distro = "unknown" + try: + for line in _read_file(OS_RELEASE_PATH).splitlines(): + if line.startswith("PRETTY_NAME="): + distro = line.split("=", 1)[1].strip().strip('"') + break + except (OSError, FileNotFoundError): + pass + return {"kernel": u.release, "distro": distro, "arch": u.machine} + + +def default_mounts() -> list[str]: + """Return real mount points from /proc/mounts, skipping pseudo filesystems.""" + skip_types = {"tmpfs", "devtmpfs", "proc", "sysfs", "cgroup", "cgroup2", + "overlay", "squashfs", "ramfs", "devpts", "mqueue", "pstore", + "securityfs", "debugfs", "tracefs", "hugetlbfs", "fusectl", + "configfs", "bpf", "autofs", "efivarfs", "binfmt_misc"} + mounts: list[str] = [] + try: + with open("/proc/mounts", "r") as f: + for line in f: + parts = line.split() + if len(parts) < 3: + continue + if parts[2] in skip_types: + continue + mounts.append(parts[1]) + except OSError: + return ["/"] + return mounts or ["/"] + + +# ─── ring buffer ───────────────────────────────────────────────────────────── + + +class RingBuffer: + def __init__(self, maxlen: int = 20) -> None: + self._dq: deque = deque(maxlen=maxlen) + + def push(self, item) -> None: + self._dq.append(item) + + def drain(self) -> list: + out = list(self._dq) + self._dq.clear() + return out + + def __len__(self) -> int: + return len(self._dq) + + +# ─── payload ───────────────────────────────────────────────────────────────── + + +def build_sample(mounts: list[str]) -> dict: + """Collect one full sample. Partial samples allowed if a collector fails.""" + sample: dict = {"ts": datetime.now(timezone.utc).isoformat()} + try: + sample["cpu_pct"] = collect_cpu() + except Exception: + sample["cpu_pct"] = None + try: + sample["mem"] = collect_memory() + except Exception: + sample["mem"] = None + try: + sample["load"] = collect_load() + except Exception: + sample["load"] = None + try: + sample["uptime_secs"] = collect_uptime() + except Exception: + sample["uptime_secs"] = None + try: + sample["storage"] = collect_storage(mounts) + except Exception: + sample["storage"] = [] + return sample + + +def build_payload(samples: list[dict], hostname: str, metadata: dict) -> dict: + return { + "agent_version": AGENT_VERSION, + "hostname": hostname, + "metadata": metadata, + "samples": samples, + } + + +# ─── POST + backoff ────────────────────────────────────────────────────────── + + +BACKOFF_CAP = 300 + + +def next_backoff(current: int) -> int: + if current <= 0: + return 30 + return min(current * 2, BACKOFF_CAP) + + +def post_payload(url: str, token: str, payload: dict) -> tuple[bool, int | None]: + body = json.dumps(payload).encode("utf-8") + req = urllib.request.Request( + url.rstrip("/") + "/plugins/host_agent/ingest", + data=body, + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + "User-Agent": f"steward-host-agent/{AGENT_VERSION}", + }, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=10) as resp: + return (200 <= resp.status < 300, resp.status) + except urllib.error.HTTPError as e: + return (False, e.code) + except (urllib.error.URLError, TimeoutError, socket.timeout, OSError): + return (False, None) + + +# ─── main loop ─────────────────────────────────────────────────────────────── + + +_reload_requested = False +_shutdown_requested = False + + +def _handle_hup(_signum, _frame): + global _reload_requested + _reload_requested = True + + +def _handle_term(_signum, _frame): + global _shutdown_requested + _shutdown_requested = True + + +def _log(level: str, msg: str) -> None: + sys.stderr.write(f"[{level}] {msg}\n") + sys.stderr.flush() + + +def main_loop(conf_path: str) -> int: + global _reload_requested, _shutdown_requested + signal.signal(signal.SIGHUP, _handle_hup) + signal.signal(signal.SIGTERM, _handle_term) + + try: + cfg = read_config(conf_path) + except ConfigError as e: + _log("ERROR", str(e)) + return 2 + + metadata = collect_metadata() + hostname = cfg.get("hostname") or socket.gethostname() + mounts = cfg.get("mounts") or default_mounts() + buffer = RingBuffer(maxlen=20) + backoff = 0 + + _log("INFO", f"steward-host-agent {AGENT_VERSION} starting " + f"(url={cfg['url']}, interval={cfg['interval_seconds']}s)") + + while not _shutdown_requested: + if _reload_requested: + try: + cfg = read_config(conf_path) + mounts = cfg.get("mounts") or default_mounts() + _log("INFO", "config reloaded") + except ConfigError as e: + _log("ERROR", f"reload failed: {e}") + _reload_requested = False + + sample = build_sample(mounts) + buffered = buffer.drain() + payload = build_payload( + samples=buffered + [sample], + hostname=hostname, + metadata=metadata, + ) + ok, status = post_payload(cfg["url"], cfg["token"], payload) + + if ok: + if buffered: + _log("INFO", f"flushed {len(buffered)} buffered samples") + backoff = 0 + sleep_for = cfg["interval_seconds"] + else: + if status == 400: + _log("ERROR", "server rejected payload (400) — dropping sample") + elif status == 401: + _log("ERROR", "token rejected (401) — check config + UI") + buffer.push(sample) + else: + _log("WARN", f"POST failed (status={status}); buffering") + for s in buffered: + buffer.push(s) + buffer.push(sample) + backoff = next_backoff(backoff) + sleep_for = backoff + + slept = 0.0 + while slept < sleep_for and not _shutdown_requested and not _reload_requested: + time.sleep(min(1.0, sleep_for - slept)) + slept += 1.0 + + _log("INFO", "SIGTERM — flushing and exiting") + final = buffer.drain() + if final: + post_payload(cfg["url"], cfg["token"], + build_payload(final, hostname, metadata)) + return 0 + + +if __name__ == "__main__": + conf = os.environ.get("STEWARD_AGENT_CONFIG", "/etc/steward-agent.conf") + sys.exit(main_loop(conf)) diff --git a/plugins/host_agent/migrations/__init__.py b/plugins/host_agent/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/host_agent/migrations/env.py b/plugins/host_agent/migrations/env.py new file mode 100644 index 0000000..6545c0e --- /dev/null +++ b/plugins/host_agent/migrations/env.py @@ -0,0 +1,70 @@ +# plugins/host_agent/migrations/env.py +"""Alembic env.py for the host_agent plugin.""" +import asyncio +import os +import sys +from pathlib import Path +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config +from alembic import context + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from steward.models.base import Base +import steward.models # noqa: F401 +from plugins.host_agent.models import HostAgentRegistration # noqa: F401 + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _get_url() -> str: + import yaml + cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml") + try: + with open(cfg_path) as f: + cfg = yaml.safe_load(f) or {} + url = cfg.get("database", {}).get("url", "") + except FileNotFoundError: + url = "" + return os.environ.get("STEWARD_DATABASE__URL", url) + + +def run_migrations_offline() -> None: + context.configure( + url=_get_url(), target_metadata=target_metadata, + literal_binds=True, dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + cfg = config.get_section(config.config_ini_section, {}) + cfg["sqlalchemy.url"] = _get_url() + connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/plugins/host_agent/migrations/versions/__init__.py b/plugins/host_agent/migrations/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/host_agent/migrations/versions/host_agent_001_initial.py b/plugins/host_agent/migrations/versions/host_agent_001_initial.py new file mode 100644 index 0000000..d241257 --- /dev/null +++ b/plugins/host_agent/migrations/versions/host_agent_001_initial.py @@ -0,0 +1,36 @@ +# plugins/host_agent/migrations/versions/host_agent_001_initial.py +"""host_agent plugin initial tables + +Revision ID: host_agent_001_initial +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "host_agent_001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = "host_agent" +depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",) + + +def upgrade() -> None: + op.create_table( + "host_agent_registrations", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("host_id", sa.String(36), + sa.ForeignKey("hosts.id", ondelete="CASCADE"), + nullable=False, unique=True), + sa.Column("token_hash", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("token_created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("agent_version", sa.String(32), nullable=True), + sa.Column("kernel", sa.String(128), nullable=True), + sa.Column("distro", sa.String(128), nullable=True), + sa.Column("arch", sa.String(32), nullable=True), + sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("host_agent_registrations") diff --git a/plugins/host_agent/models.py b/plugins/host_agent/models.py new file mode 100644 index 0000000..b93d725 --- /dev/null +++ b/plugins/host_agent/models.py @@ -0,0 +1,32 @@ +# plugins/host_agent/models.py +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import Column, String, DateTime, ForeignKey +from steward.models.base import Base + + +def _uuid() -> str: + return str(uuid.uuid4()) + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +class HostAgentRegistration(Base): + __tablename__ = "host_agent_registrations" + + id = Column(String(36), primary_key=True, default=_uuid) + host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"), + unique=True, nullable=False) + token_hash = Column(String(64), nullable=False, unique=True, index=True) + token_created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow) + agent_version = Column(String(32), nullable=True) + kernel = Column(String(128), nullable=True) + distro = Column(String(128), nullable=True) + arch = Column(String(32), nullable=True) + last_seen_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow) + updated_at = Column(DateTime(timezone=True), nullable=False, + default=_utcnow, onupdate=_utcnow) diff --git a/plugins/host_agent/plugin.yaml b/plugins/host_agent/plugin.yaml new file mode 100644 index 0000000..09b9022 --- /dev/null +++ b/plugins/host_agent/plugin.yaml @@ -0,0 +1,19 @@ +# plugins/host_agent/plugin.yaml +name: host_agent +version: "1.0.0" +description: "Remote Linux host resource monitoring via a lightweight Python push agent (CPU, memory, storage, load, uptime)" +author: "Steward" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/host_agent" +tags: + - host + - monitoring + - cpu + - memory + - storage + +config: + stale_after_seconds: 180 + default_interval_seconds: 30 diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py new file mode 100644 index 0000000..f023811 --- /dev/null +++ b/plugins/host_agent/routes.py @@ -0,0 +1,398 @@ +# plugins/host_agent/routes.py +from __future__ import annotations + +import hashlib +from datetime import datetime, timezone +from typing import Any + +from pathlib import Path + +import secrets +from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for +from steward.auth.middleware import require_role +from steward.models.users import UserRole +from sqlalchemy import select, func +from datetime import timedelta + +from steward.core.settings import public_base_url +from steward.models.hosts import Host +from steward.models.metrics import PluginMetric +from .models import HostAgentRegistration + +host_agent_bp = Blueprint("host_agent", __name__, template_folder="templates") + +SOURCE_MODULE = "host_agent" + + +def _hash_token(raw: str) -> str: + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def _parse_ts(ts: str) -> datetime: + if ts.endswith("Z"): + ts = ts[:-1] + "+00:00" + return datetime.fromisoformat(ts) + + +def _expand_sample_to_metrics( + sample: dict, host_name: str, recorded_at: datetime +) -> list[PluginMetric]: + rows: list[PluginMetric] = [] + + def row(metric: str, resource: str, value: float) -> None: + rows.append(PluginMetric( + source_module=SOURCE_MODULE, + resource_name=resource, + metric_name=metric, + value=float(value), + recorded_at=recorded_at, + )) + + if sample.get("cpu_pct") is not None: + row("cpu_pct", host_name, sample["cpu_pct"]) + + mem = sample.get("mem") or {} + if mem.get("total_bytes"): + total = mem["total_bytes"] + available = mem.get("available_bytes", 0) + used_pct = 100.0 * (total - available) / total if total else 0.0 + row("mem_used_pct", host_name, used_pct) + row("mem_available_bytes", host_name, available) + row("swap_used_bytes", host_name, mem.get("swap_used_bytes", 0)) + + load = sample.get("load") or {} + for key in ("1m", "5m", "15m"): + if key in load: + row(f"load_{key}", host_name, load[key]) + + if sample.get("uptime_secs") is not None: + row("uptime_secs", host_name, sample["uptime_secs"]) + + worst_pct = 0.0 + for disk in sample.get("storage") or []: + total = disk.get("total_bytes", 0) + used = disk.get("used_bytes", 0) + pct = 100.0 * used / total if total else 0.0 + mount_res = f"{host_name}:{disk['mount']}" + row("disk_used_pct", mount_res, pct) + row("disk_used_bytes", mount_res, used) + row("disk_total_bytes", mount_res, total) + if pct > worst_pct: + worst_pct = pct + if sample.get("storage"): + row("disk_used_pct_worst", host_name, worst_pct) + + return rows + + +def _error(status: int, code: str, detail: str | None = None) -> tuple[Any, int]: + body: dict = {"ok": False, "error": code} + if detail: + body["detail"] = detail + return jsonify(body), status + + +@host_agent_bp.post("/ingest") +async def ingest(): + auth = request.headers.get("Authorization", "") + if not auth.startswith("Bearer "): + return _error(401, "invalid_token") + raw = auth[len("Bearer "):].strip() + if not raw: + return _error(401, "invalid_token") + + try: + payload = await request.get_json(force=True) + except Exception: + return _error(400, "malformed_payload", "invalid JSON") + if not isinstance(payload, dict) or "samples" not in payload: + return _error(400, "malformed_payload", "missing 'samples'") + + samples = payload.get("samples") or [] + if not isinstance(samples, list) or not samples: + return _error(400, "malformed_payload", "'samples' must be a non-empty list") + + token_hash = _hash_token(raw) + async with current_app.db_sessionmaker() as session: + reg = (await session.execute( + select(HostAgentRegistration).where( + HostAgentRegistration.token_hash == token_hash) + )).scalar_one_or_none() + if reg is None: + return _error(401, "invalid_token") + host = (await session.execute( + select(Host).where(Host.id == reg.host_id) + )).scalar_one_or_none() + if host is None: + return _error(401, "invalid_token") + + accepted = 0 + latest_ts: datetime | None = None + for sample in samples: + try: + recorded_at = _parse_ts(sample["ts"]) + except (KeyError, ValueError): + continue + metrics = _expand_sample_to_metrics(sample, host.name, recorded_at) + for m in metrics: + session.add(m) + accepted += 1 + if latest_ts is None or recorded_at > latest_ts: + latest_ts = recorded_at + + if accepted == 0: + await session.rollback() + return _error(400, "malformed_payload", "no valid samples") + + md = payload.get("metadata") or {} + changed = False + if latest_ts and (reg.last_seen_at is None or latest_ts > reg.last_seen_at): + reg.last_seen_at = latest_ts + changed = True + version = payload.get("agent_version") + if version and reg.agent_version != version: + reg.agent_version = version + changed = True + for field in ("kernel", "distro", "arch"): + if field in md and getattr(reg, field) != md[field]: + setattr(reg, field, md[field]) + changed = True + if changed: + reg.updated_at = datetime.now(timezone.utc) + + if latest_ts: + skew = abs((datetime.now(timezone.utc) - latest_ts).total_seconds()) + if skew > 300: + current_app.logger.warning( + "host_agent ingest: clock skew %.0fs for host=%s", skew, host.name) + + await session.commit() + + return jsonify({"ok": True, "samples_accepted": accepted}), 200 + + +AGENT_SOURCE_PATH = Path(__file__).parent / "agent.py" + + +def _agent_version() -> str: + try: + for line in AGENT_SOURCE_PATH.read_text().splitlines(): + if line.startswith("AGENT_VERSION"): + return line.split("=", 1)[1].strip().strip('"').strip("'") + except OSError: + pass + return "unknown" + + +@host_agent_bp.get("/install.sh") +async def install_script(): + token = request.args.get("token", "") + if not token: + return _error(401, "invalid_token") + async with current_app.db_sessionmaker() as session: + reg = (await session.execute( + select(HostAgentRegistration).where( + HostAgentRegistration.token_hash == _hash_token(token)) + )).scalar_one_or_none() + if reg is None: + return _error(401, "invalid_token") + host = (await session.execute( + select(Host).where(Host.id == reg.host_id) + )).scalar_one_or_none() + if host is None: + return _error(401, "invalid_token") + + url = public_base_url(request) + + rendered = await render_template( + "install.sh.j2", + url=url, + token=token, + agent_version=_agent_version(), + host_name=host.name, + host_address=host.address, + ) + return Response(rendered, mimetype="text/plain") + + +@host_agent_bp.get("/agent.py") +async def agent_source(): + try: + body = AGENT_SOURCE_PATH.read_text(encoding="utf-8") + except OSError: + return _error(500, "agent_missing") + return Response(body, mimetype="text/x-python") + + +@host_agent_bp.get("/widget") +async def widget_table(): + """Fleet-glance table: one row per monitored host, latest metrics.""" + async with current_app.db_sessionmaker() as session: + regs = (await session.execute(select(HostAgentRegistration))).scalars().all() + host_ids = [r.host_id for r in regs] + hosts = { + h.id: h for h in (await session.execute( + select(Host).where(Host.id.in_(host_ids)) + )).scalars().all() + } if host_ids else {} + + subq = ( + select( + PluginMetric.resource_name, + PluginMetric.metric_name, + func.max(PluginMetric.recorded_at).label("max_ts"), + ) + .where(PluginMetric.source_module == SOURCE_MODULE) + .group_by(PluginMetric.resource_name, PluginMetric.metric_name) + ).subquery() + latest_rows = (await session.execute( + select(PluginMetric).join( + subq, + (PluginMetric.resource_name == subq.c.resource_name) & + (PluginMetric.metric_name == subq.c.metric_name) & + (PluginMetric.recorded_at == subq.c.max_ts), + ).where(PluginMetric.source_module == SOURCE_MODULE) + )).scalars().all() + + latest: dict[str, dict[str, float]] = {} + for row in latest_rows: + if ":" in row.resource_name: + continue + latest.setdefault(row.resource_name, {})[row.metric_name] = row.value + + rows = [] + for reg in regs: + host = hosts.get(reg.host_id) + if host is None: + continue + m = latest.get(host.name, {}) + rows.append({ + "host": host, + "reg": reg, + "cpu_pct": m.get("cpu_pct"), + "mem_used_pct": m.get("mem_used_pct"), + "disk_worst": m.get("disk_used_pct_worst"), + "load_1m": m.get("load_1m"), + }) + + return await render_template("widget_table.html", rows=rows) + + +@host_agent_bp.get("/widget/history") +async def widget_history(): + host_id = request.args.get("host_id", "") + hours = int(request.args.get("hours", "6")) + cutoff = datetime.now(timezone.utc) - timedelta(hours=hours) + + async with current_app.db_sessionmaker() as session: + host = (await session.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return _error(404, "not_found") + points = (await session.execute( + select(PluginMetric).where( + PluginMetric.source_module == SOURCE_MODULE, + PluginMetric.resource_name == host.name, + PluginMetric.metric_name.in_( + ("cpu_pct", "mem_used_pct", "disk_used_pct_worst")), + PluginMetric.recorded_at >= cutoff, + ).order_by(PluginMetric.recorded_at) + )).scalars().all() + + series: dict[str, list[dict]] = {"cpu_pct": [], "mem_used_pct": [], "disk_used_pct_worst": []} + for p in points: + series[p.metric_name].append({"t": p.recorded_at.isoformat(), "v": p.value}) + + return await render_template("widget_history.html", host=host, series=series, hours=hours) + + +def _new_token_pair() -> tuple[str, str]: + raw = secrets.token_urlsafe(32) + return raw, _hash_token(raw) + + +@host_agent_bp.get("/settings/") +@require_role(UserRole.admin) +async def settings_list(): + async with current_app.db_sessionmaker() as session: + regs = (await session.execute(select(HostAgentRegistration))).scalars().all() + hosts_by_id = { + h.id: h for h in (await session.execute( + select(Host).where(Host.id.in_([r.host_id for r in regs])))).scalars().all() + } if regs else {} + + new_token = request.args.get("new_token") + new_host_id = request.args.get("host_id") + install_url = None + if new_token and new_host_id: + install_url = f"{public_base_url(request)}/plugins/host_agent/install.sh?token={new_token}" + + return await render_template( + "settings_list.html", + registrations=[ + {"reg": r, "host": hosts_by_id.get(r.host_id)} for r in regs + ], + new_token=new_token, + install_url=install_url, + ) + + +@host_agent_bp.post("/settings/add-host") +@require_role(UserRole.admin) +async def add_host(): + form = await request.form + name = (form.get("name") or "").strip() + address = (form.get("address") or "").strip() + if not name: + return _error(400, "missing_name") + + async with current_app.db_sessionmaker() as session: + host = (await session.execute( + select(Host).where(Host.name == name))).scalar_one_or_none() + if host is None: + host = Host(name=name, address=address) + session.add(host) + await session.flush() + existing = (await session.execute( + select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host.id))).scalar_one_or_none() + if existing is not None: + await session.rollback() + return _error(400, "already_registered") + + raw, hashed = _new_token_pair() + reg = HostAgentRegistration(host_id=host.id, token_hash=hashed) + session.add(reg) + await session.commit() + host_id = host.id + + return redirect(url_for("host_agent.settings_list") + f"?new_token={raw}&host_id={host_id}") + + +@host_agent_bp.post("/settings//rotate-token") +@require_role(UserRole.admin) +async def rotate_token(host_id: str): + async with current_app.db_sessionmaker() as session: + reg = (await session.execute( + select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host_id))).scalar_one_or_none() + if reg is None: + return _error(404, "not_found") + raw, hashed = _new_token_pair() + reg.token_hash = hashed + reg.token_created_at = datetime.now(timezone.utc) + await session.commit() + return redirect(url_for("host_agent.settings_list") + f"?new_token={raw}&host_id={host_id}") + + +@host_agent_bp.post("/settings//delete") +@require_role(UserRole.admin) +async def delete_registration(host_id: str): + async with current_app.db_sessionmaker() as session: + reg = (await session.execute( + select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host_id))).scalar_one_or_none() + if reg is not None: + await session.delete(reg) + await session.commit() + return redirect(url_for("host_agent.settings_list")) diff --git a/plugins/host_agent/scheduler.py b/plugins/host_agent/scheduler.py new file mode 100644 index 0000000..85056c1 --- /dev/null +++ b/plugins/host_agent/scheduler.py @@ -0,0 +1,78 @@ +# plugins/host_agent/scheduler.py +from __future__ import annotations +import logging +from datetime import datetime, timedelta, timezone +from typing import Iterable + +from sqlalchemy import select + +from steward.core.scheduler import ScheduledTask +from steward.models.hosts import Host +from .models import HostAgentRegistration + +logger = logging.getLogger(__name__) + + +def _filter_stale( + regs: Iterable, + *, + now: datetime, + stale_after_seconds: int, +) -> list: + """Pure staleness filter: returns the subset with last_seen_at strictly + older than (now - stale_after_seconds). Rows with last_seen_at=None are + never stale (they are unregistered-in-practice).""" + cutoff = now - timedelta(seconds=stale_after_seconds) + return [ + r for r in regs + if r.last_seen_at is not None and r.last_seen_at < cutoff + ] + + +async def find_stale_registrations(app, stale_after_seconds: int = 180) -> list[dict]: + async with app.db_sessionmaker() as session: + all_regs = (await session.execute( + select(HostAgentRegistration) + )).scalars().all() + stale_rows = _filter_stale( + all_regs, + now=datetime.now(timezone.utc), + stale_after_seconds=stale_after_seconds, + ) + if not stale_rows: + return [] + hosts = { + h.id: h for h in (await session.execute( + select(Host).where(Host.id.in_([r.host_id for r in stale_rows])) + )).scalars().all() + } + return [ + { + "host_id": r.host_id, + "host_name": hosts[r.host_id].name if r.host_id in hosts else "?", + "last_seen_at": r.last_seen_at, + } + for r in stale_rows + ] + + +def make_task(app) -> ScheduledTask: + async def _check_stale(): + try: + stale = await find_stale_registrations(app) + except Exception: + logger.exception("host_agent stale check failed") + return + if stale: + logger.info( + "host_agent: %d stale agent(s): %s", + len(stale), + [s["host_name"] for s in stale], + ) + + return ScheduledTask( + name="host_agent_stale_check", + coro_factory=_check_stale, + interval_seconds=60, + run_on_startup=False, + ) diff --git a/plugins/host_agent/templates/install.sh.j2 b/plugins/host_agent/templates/install.sh.j2 new file mode 100644 index 0000000..02d55be --- /dev/null +++ b/plugins/host_agent/templates/install.sh.j2 @@ -0,0 +1,81 @@ +#!/bin/sh +# Steward host agent installer +# Generated for: {{ host_name }} ({{ host_address }}) +# Steward URL: {{ url }} +set -e + +STEWARD_URL="{{ url }}" +AGENT_TOKEN="{{ token }}" +AGENT_VERSION="{{ agent_version }}" + +AGENT_USER="steward-agent" +AGENT_DIR="/usr/local/lib/steward-agent" +CONF_FILE="/etc/steward-agent.conf" +UNIT_FILE="/etc/systemd/system/steward-agent.service" + +# ── preflight ──────────────────────────────────────────────────────────────── +[ "$(id -u)" = "0" ] || { echo "must run as root (use sudo)"; exit 1; } +command -v systemctl >/dev/null 2>&1 || { echo "systemd not found — unsupported init system"; exit 1; } +command -v python3 >/dev/null 2>&1 || { echo "python3 not found — install python3 first"; exit 1; } + +# Handle --uninstall +if [ "${1:-}" = "--uninstall" ]; then + systemctl disable --now steward-agent.service 2>/dev/null || true + rm -f "$UNIT_FILE" "$CONF_FILE" + rm -rf "$AGENT_DIR" + systemctl daemon-reload + # keep the system user — removing it risks orphaned files elsewhere + echo "uninstalled" + exit 0 +fi + +# ── create system user ─────────────────────────────────────────────────────── +if ! id "$AGENT_USER" >/dev/null 2>&1; then + useradd --system --no-create-home --shell /usr/sbin/nologin "$AGENT_USER" +fi + +# ── drop the agent file ────────────────────────────────────────────────────── +mkdir -p "$AGENT_DIR" +curl -sSL "${STEWARD_URL}/plugins/host_agent/agent.py" -o "$AGENT_DIR/agent.py" +chmod 0755 "$AGENT_DIR/agent.py" +chown root:root "$AGENT_DIR/agent.py" + +# ── write config ───────────────────────────────────────────────────────────── +cat > "$CONF_FILE" < "$UNIT_FILE" <Host Agent — Registered Hosts + +{% if new_token %} +
+

New token — copy this install command now

+

+ This is the only time the raw token is shown. Rotate if you lose it. +

+
curl -sSL '{{ install_url }}' | sudo sh
+
+{% endif %} + +
+
+
+ + +
+
+ + +
+ +
+
+ +
+ + + + + + + + + {% for item in registrations %} + + + + + + + + {% else %} + + {% endfor %} + +
HostAgent versionDistroLast seenActions
{{ item.host.name if item.host else item.reg.host_id }}{{ item.reg.agent_version or "—" }}{{ item.reg.distro or "—" }}{{ item.reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") if item.reg.last_seen_at else "never" }} +
+ +
+
+ +
+
No hosts registered. Add one above.
+
+{% endblock %} diff --git a/plugins/host_agent/templates/widget_history.html b/plugins/host_agent/templates/widget_history.html new file mode 100644 index 0000000..35e363f --- /dev/null +++ b/plugins/host_agent/templates/widget_history.html @@ -0,0 +1,24 @@ +
+

{{ host.name }} — last {{ hours }}h

+ + +
diff --git a/plugins/host_agent/templates/widget_table.html b/plugins/host_agent/templates/widget_table.html new file mode 100644 index 0000000..a646fad --- /dev/null +++ b/plugins/host_agent/templates/widget_table.html @@ -0,0 +1,34 @@ +{# host_agent widget — fleet glance, flex rows (no header wrap) #} +{% if rows %} +
+ {% for r in rows %} + {% set stale = not r.reg.last_seen_at %} +
+ + + {{ r.host.name }} + + + {% if r.cpu_pct is not none %} + cpu {{ "%.0f"|format(r.cpu_pct) }}% + {% endif %} + {% if r.mem_used_pct is not none %} + mem {{ "%.0f"|format(r.mem_used_pct) }}% + {% endif %} + {% if r.disk_worst is not none %} + disk {{ "%.0f"|format(r.disk_worst) }}% + {% endif %} + {% if r.load_1m is not none %} + load {{ "%.2f"|format(r.load_1m) }} + {% endif %} + + + {{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }} + +
+ {% endfor %} +
+{% else %} +

No hosts with agent data yet.

+{% endif %} diff --git a/plugins/http/__init__.py b/plugins/http/__init__.py new file mode 100644 index 0000000..cb79685 --- /dev/null +++ b/plugins/http/__init__.py @@ -0,0 +1,24 @@ +# plugins/http/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + from .models import HttpMonitor, HttpResult # noqa: F401 + + +def get_scheduled_tasks() -> list: + from .scheduler import make_task + return [make_task(_app)] + + +def get_blueprint(): + from .routes import http_bp + return http_bp diff --git a/plugins/http/checker.py b/plugins/http/checker.py new file mode 100644 index 0000000..7eec743 --- /dev/null +++ b/plugins/http/checker.py @@ -0,0 +1,105 @@ +# plugins/http/checker.py +"""Core HTTP check logic — runs a single monitor check and returns a result dict.""" +from __future__ import annotations +import asyncio +import json +import logging +import ssl +import time +from datetime import datetime, timezone +from urllib.parse import urlparse + +import httpx + +logger = logging.getLogger(__name__) + + +async def _get_tls_expiry(hostname: str, port: int) -> datetime | None: + """Attempt a bare TLS handshake to extract the certificate expiry date.""" + ctx = ssl.create_default_context() + try: + reader, writer = await asyncio.wait_for( + asyncio.open_connection(hostname, port, ssl=ctx), + timeout=5.0, + ) + cert = writer.get_extra_info("ssl_object").getpeercert() + writer.close() + try: + await writer.wait_closed() + except Exception: + pass + exp_str = cert.get("notAfter", "") + if not exp_str: + return None + # Format: "Mar 22 12:00:00 2027 GMT" + return datetime.strptime(exp_str, "%b %d %H:%M:%S %Y %Z").replace( + tzinfo=timezone.utc + ) + except Exception as exc: + logger.debug("TLS check failed for %s:%d — %s", hostname, port, exc) + return None + + +async def run_check( + url: str, + method: str = "GET", + expected_status: int = 200, + content_match: str = "", + headers: dict | None = None, + timeout_seconds: int = 10, + follow_redirects: bool = True, + verify_ssl: bool = True, +) -> dict: + """ + Perform one HTTP check. Returns a dict with: + is_up, status_code, response_ms, content_matched, error_msg, tls_expires_at + """ + result: dict = { + "is_up": False, + "status_code": None, + "response_ms": None, + "content_matched": None, + "error_msg": None, + "tls_expires_at": None, + } + + parsed = urlparse(url) + is_https = parsed.scheme.lower() == "https" + + t0 = time.monotonic() + try: + async with httpx.AsyncClient( + follow_redirects=follow_redirects, + verify=verify_ssl, + timeout=timeout_seconds, + ) as client: + response = await client.request( + method, + url, + headers=headers or {}, + ) + result["response_ms"] = round((time.monotonic() - t0) * 1000, 1) + result["status_code"] = response.status_code + + status_ok = response.status_code == expected_status + if content_match: + matched = content_match in response.text + result["content_matched"] = matched + result["is_up"] = status_ok and matched + else: + result["is_up"] = status_ok + + except httpx.TimeoutException: + result["response_ms"] = round((time.monotonic() - t0) * 1000, 1) + result["error_msg"] = "Timeout" + except httpx.ConnectError as exc: + result["error_msg"] = f"Connection error: {exc}" + except Exception as exc: + result["error_msg"] = str(exc)[:512] + + # TLS expiry — only for HTTPS, and only when the check succeeded or we got a response + if is_https and result["status_code"] is not None: + port = parsed.port or 443 + result["tls_expires_at"] = await _get_tls_expiry(parsed.hostname, port) + + return result diff --git a/plugins/http/migrations/__init__.py b/plugins/http/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/http/migrations/env.py b/plugins/http/migrations/env.py new file mode 100644 index 0000000..fd308a2 --- /dev/null +++ b/plugins/http/migrations/env.py @@ -0,0 +1,70 @@ +# plugins/http/migrations/env.py +"""Alembic env.py for the HTTP monitoring plugin.""" +import asyncio +import os +import sys +from pathlib import Path +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config +from alembic import context + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from fabledscryer.models.base import Base +import fabledscryer.models # noqa: F401 +from plugins.http.models import HttpMonitor, HttpResult # noqa: F401 + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _get_url() -> str: + import yaml + cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml") + try: + with open(cfg_path) as f: + cfg = yaml.safe_load(f) or {} + url = cfg.get("database", {}).get("url", "") + except FileNotFoundError: + url = "" + return os.environ.get("FABLEDSCRYER_DATABASE__URL", url) + + +def run_migrations_offline() -> None: + context.configure( + url=_get_url(), target_metadata=target_metadata, + literal_binds=True, dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + cfg = config.get_section(config.config_ini_section, {}) + cfg["sqlalchemy.url"] = _get_url() + connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/plugins/http/migrations/versions/__init__.py b/plugins/http/migrations/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/http/migrations/versions/http_001_initial.py b/plugins/http/migrations/versions/http_001_initial.py new file mode 100644 index 0000000..c546023 --- /dev/null +++ b/plugins/http/migrations/versions/http_001_initial.py @@ -0,0 +1,52 @@ +"""HTTP monitoring plugin initial tables + +Revision ID: http_001_initial +Revises: (none — branch off core via depends_on) +Create Date: 2026-03-22 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "http_001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = "http" +depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",) + + +def upgrade() -> None: + op.create_table( + "http_monitors", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("name", sa.String(128), nullable=False), + sa.Column("url", sa.String(2048), nullable=False), + sa.Column("method", sa.String(8), nullable=False, server_default="GET"), + sa.Column("expected_status", sa.Integer, nullable=False, server_default="200"), + sa.Column("content_match", sa.String(512), nullable=False, server_default=""), + sa.Column("headers_json", sa.Text, nullable=False, server_default="{}"), + sa.Column("timeout_seconds", sa.Integer, nullable=False, server_default="10"), + sa.Column("check_interval_seconds", sa.Integer, nullable=False, server_default="0"), + sa.Column("follow_redirects", sa.Boolean, nullable=False, server_default="1"), + sa.Column("verify_ssl", sa.Boolean, nullable=False, server_default="1"), + sa.Column("enabled", sa.Boolean, nullable=False, server_default="1"), + sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "http_results", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("monitor_id", sa.String(36), nullable=False, index=True), + sa.Column("checked_at", sa.DateTime(timezone=True), nullable=False, index=True), + sa.Column("status_code", sa.Integer, nullable=True), + sa.Column("response_ms", sa.Float, nullable=True), + sa.Column("is_up", sa.Boolean, nullable=False, server_default="0"), + sa.Column("content_matched", sa.Boolean, nullable=True), + sa.Column("error_msg", sa.String(512), nullable=True), + sa.Column("tls_expires_at", sa.DateTime(timezone=True), nullable=True), + ) + + +def downgrade() -> None: + op.drop_table("http_results") + op.drop_table("http_monitors") diff --git a/plugins/http/models.py b/plugins/http/models.py new file mode 100644 index 0000000..180effd --- /dev/null +++ b/plugins/http/models.py @@ -0,0 +1,63 @@ +# plugins/http/models.py +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from fabledscryer.models.base import Base + + +class HttpMonitor(Base): + """A configured HTTP endpoint to check periodically.""" + __tablename__ = "http_monitors" + + id: Mapped[str] = mapped_column( + String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + ) + name: Mapped[str] = mapped_column(String(128), nullable=False) + url: Mapped[str] = mapped_column(String(2048), nullable=False) + method: Mapped[str] = mapped_column(String(8), nullable=False, default="GET") + expected_status: Mapped[int] = mapped_column(Integer, nullable=False, default=200) + # Optional substring to find in the response body (empty = skip check) + content_match: Mapped[str] = mapped_column(String(512), nullable=False, default="") + # JSON dict of extra request headers, e.g. {"Authorization": "Bearer ..."} + headers_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}") + timeout_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=10) + # 0 = use global plugin check_interval_seconds + check_interval_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + follow_redirects: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + verify_ssl: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + # Updated after each check so the scheduler can compute next-run time efficiently + last_checked_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + +class HttpResult(Base): + """One check result for an HttpMonitor.""" + __tablename__ = "http_results" + + id: Mapped[str] = mapped_column( + String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + ) + monitor_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + checked_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + index=True, + ) + status_code: Mapped[int | None] = mapped_column(Integer, nullable=True) + response_ms: Mapped[float | None] = mapped_column(Float, nullable=True) + is_up: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + # None when no content_match is configured; True/False when it is + content_matched: Mapped[bool | None] = mapped_column(Boolean, nullable=True) + error_msg: Mapped[str | None] = mapped_column(String(512), nullable=True) + # TLS expiry if the endpoint is HTTPS; None for HTTP or on error + tls_expires_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) diff --git a/plugins/http/plugin.yaml b/plugins/http/plugin.yaml new file mode 100644 index 0000000..9aa1475 --- /dev/null +++ b/plugins/http/plugin.yaml @@ -0,0 +1,19 @@ +name: http +version: "1.0.0" +description: "Synthetic HTTP endpoint monitoring — status code, response time, content match, TLS expiry" +author: "FabledScryer" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/http" +tags: + - monitoring + - http + - synthetic + - uptime + +config: + check_interval_seconds: 60 + default_timeout_seconds: 10 + follow_redirects: true + verify_ssl: true diff --git a/plugins/http/routes.py b/plugins/http/routes.py new file mode 100644 index 0000000..eea5ce7 --- /dev/null +++ b/plugins/http/routes.py @@ -0,0 +1,229 @@ +# plugins/http/routes.py +from __future__ import annotations +import json +import uuid +from datetime import datetime, timezone +from quart import Blueprint, current_app, render_template, request, redirect, url_for +from sqlalchemy import Integer, cast, func, select + +from fabledscryer.auth.middleware import require_role +from fabledscryer.models.users import UserRole +from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds +from .models import HttpMonitor, HttpResult + +http_bp = Blueprint("http", __name__, template_folder="templates") + + +def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: + if len(values) < 2: + return f'' + mn, mx = min(values), max(values) + if mx == mn: + mx = mn + 1.0 + step = width / (len(values) - 1) + pts = [] + for i, v in enumerate(values): + x = i * step + y = height - (v - mn) / (mx - mn) * (height - 2) - 1 + pts.append(f"{x:.1f},{y:.1f}") + poly = " ".join(pts) + return ( + f'' + f'' + f'' + ) + + +async def _render_rows(range_key: str = DEFAULT_RANGE): + since, range_key = parse_range(range_key) + b_secs = bucket_seconds(since) + bucket_col = ( + cast(func.extract('epoch', HttpResult.checked_at), Integer) / b_secs + ).label("bucket") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(HttpMonitor).order_by(HttpMonitor.created_at) + ) + monitors = list(result.scalars()) + + # Latest result per monitor + latest_map: dict[str, HttpResult] = {} + histories: dict[str, list] = {} + for m in monitors: + r = await db.execute( + select(HttpResult) + .where(HttpResult.monitor_id == m.id) + .order_by(HttpResult.checked_at.desc()) + .limit(1) + ) + latest = r.scalar_one_or_none() + if latest: + latest_map[m.id] = latest + + r2 = await db.execute( + select( + func.avg(HttpResult.response_ms).label("response_ms"), + func.min(HttpResult.is_up.cast(Integer)).label("had_down"), + bucket_col, + ) + .where(HttpResult.monitor_id == m.id) + .where(HttpResult.checked_at >= since) + .group_by(bucket_col) + .order_by(bucket_col) + ) + histories[m.id] = r2.all() + + monitor_data = [] + for m in monitors: + hist = histories.get(m.id, []) + latest = latest_map.get(m.id) + now = datetime.now(timezone.utc) + tls_days = None + if latest and latest.tls_expires_at: + tls_days = (latest.tls_expires_at - now).total_seconds() / 86400.0 + monitor_data.append({ + "monitor": m, + "latest": latest, + "tls_days": tls_days, + "sparkline_ms": _sparkline([r.response_ms or 0 for r in hist]), + }) + + up = sum(1 for d in monitor_data if d["latest"] and d["latest"].is_up) + down = sum(1 for d in monitor_data if d["latest"] and not d["latest"].is_up) + return monitor_data, up, down, range_key + + +@http_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + current_range = request.args.get("range", DEFAULT_RANGE) + return await render_template( + "http/index.html", + poll_interval=poll_interval, + current_range=current_range, + ) + + +@http_bp.get("/rows") +@require_role(UserRole.viewer) +async def rows(): + """HTMX fragment: monitor list with status + sparklines.""" + monitor_data, up, down, range_key = await _render_rows( + request.args.get("range", DEFAULT_RANGE) + ) + return await render_template( + "http/rows.html", + monitor_data=monitor_data, + up=up, + down=down, + range_key=range_key, + ) + + +@http_bp.post("/add") +@require_role(UserRole.operator) +async def add_monitor(): + """Add a new HTTP monitor.""" + form = await request.form + url = form.get("url", "").strip() + if not url: + return redirect(url_for("http.index")) + if not url.startswith(("http://", "https://")): + url = "https://" + url + + monitor = HttpMonitor( + id=str(uuid.uuid4()), + name=form.get("name", "").strip() or url, + url=url, + method=form.get("method", "GET").upper(), + expected_status=int(form.get("expected_status", 200) or 200), + content_match=form.get("content_match", "").strip(), + headers_json="{}", + timeout_seconds=int(form.get("timeout_seconds", 10) or 10), + check_interval_seconds=int(form.get("check_interval_seconds", 0) or 0), + follow_redirects="follow_redirects" in form, + verify_ssl="verify_ssl" in form, + enabled=True, + created_at=datetime.now(timezone.utc), + ) + async with current_app.db_sessionmaker() as db: + async with db.begin(): + db.add(monitor) + + return redirect(url_for("http.index")) + + +@http_bp.post("//delete") +@require_role(UserRole.operator) +async def delete_monitor(monitor_id: str): + async with current_app.db_sessionmaker() as db: + async with db.begin(): + m = await db.get(HttpMonitor, monitor_id) + if m: + await db.delete(m) + return redirect(url_for("http.index")) + + +@http_bp.post("//toggle") +@require_role(UserRole.operator) +async def toggle_monitor(monitor_id: str): + async with current_app.db_sessionmaker() as db: + async with db.begin(): + m = await db.get(HttpMonitor, monitor_id) + if m: + m.enabled = not m.enabled + return redirect(url_for("http.index")) + + +@http_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + """HTMX dashboard widget: up/down counts + monitor list.""" + show_down_only = request.args.get("show_down_only", "no") == "yes" + limit = max(1, min(20, int(request.args.get("limit", 10) or 10))) + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(HttpMonitor) + .where(HttpMonitor.enabled == True) # noqa: E712 + .order_by(HttpMonitor.created_at) + ) + monitors = list(result.scalars()) + + latest_map: dict[str, HttpResult] = {} + for m in monitors: + r = await db.execute( + select(HttpResult) + .where(HttpResult.monitor_id == m.id) + .order_by(HttpResult.checked_at.desc()) + .limit(1) + ) + latest = r.scalar_one_or_none() + if latest: + latest_map[m.id] = latest + + monitor_data = [ + {"monitor": m, "latest": latest_map.get(m.id)} + for m in monitors + ] + + up = sum(1 for d in monitor_data if d["latest"] and d["latest"].is_up) + down = sum(1 for d in monitor_data if d["latest"] and not d["latest"].is_up) + pending = sum(1 for d in monitor_data if not d["latest"]) + + if show_down_only: + monitor_data = [d for d in monitor_data if not (d["latest"] and d["latest"].is_up)] + + return await render_template( + "http/widget.html", + monitor_data=monitor_data[:limit], + up=up, + down=down, + pending=pending, + show_down_only=show_down_only, + widget_id=widget_id, + ) diff --git a/plugins/http/scheduler.py b/plugins/http/scheduler.py new file mode 100644 index 0000000..f9edd67 --- /dev/null +++ b/plugins/http/scheduler.py @@ -0,0 +1,121 @@ +# plugins/http/scheduler.py +from __future__ import annotations +import asyncio +import json +import logging +from datetime import datetime, timezone, timedelta + +from fabledscryer.core.scheduler import ScheduledTask +from fabledscryer.core.alerts import record_metric + +logger = logging.getLogger(__name__) + + +def make_task(app) -> ScheduledTask: + interval = int( + app.config["PLUGINS"]["http"].get("check_interval_seconds", 60) + ) + + async def check(): + await _do_checks(app) + + return ScheduledTask( + name="http_check", + coro_factory=check, + interval_seconds=interval, + run_on_startup=True, + ) + + +async def _do_checks(app) -> None: + from sqlalchemy import select + from .models import HttpMonitor, HttpResult + from .checker import run_check + + cfg = app.config["PLUGINS"]["http"] + global_interval = int(cfg.get("check_interval_seconds", 60)) + global_timeout = int(cfg.get("default_timeout_seconds", 10)) + global_follow = bool(cfg.get("follow_redirects", True)) + global_verify = bool(cfg.get("verify_ssl", True)) + + now = datetime.now(timezone.utc) + + async with app.db_sessionmaker() as session: + result = await session.execute( + select(HttpMonitor).where(HttpMonitor.enabled == True) # noqa: E712 + ) + monitors = list(result.scalars()) + + # Filter to monitors whose next check time has passed + due: list[HttpMonitor] = [] + for m in monitors: + effective_interval = m.check_interval_seconds or global_interval + if m.last_checked_at is None: + due.append(m) + elif (now - m.last_checked_at).total_seconds() >= effective_interval: + due.append(m) + + if not due: + return + + async def _check_one(monitor: HttpMonitor) -> tuple[HttpMonitor, dict]: + try: + headers = json.loads(monitor.headers_json or "{}") + except (ValueError, TypeError): + headers = {} + result = await run_check( + url=monitor.url, + method=monitor.method, + expected_status=monitor.expected_status, + content_match=monitor.content_match, + headers=headers, + timeout_seconds=monitor.timeout_seconds or global_timeout, + follow_redirects=monitor.follow_redirects if monitor.follow_redirects is not None else global_follow, + verify_ssl=monitor.verify_ssl if monitor.verify_ssl is not None else global_verify, + ) + return monitor, result + + pairs = await asyncio.gather(*[_check_one(m) for m in due], return_exceptions=True) + + check_time = datetime.now(timezone.utc) + + async with app.db_sessionmaker() as session: + async with session.begin(): + for item in pairs: + if isinstance(item, Exception): + logger.error("HTTP check task error: %s", item) + continue + monitor, res = item + + session.add(HttpResult( + monitor_id=monitor.id, + checked_at=check_time, + status_code=res["status_code"], + response_ms=res["response_ms"], + is_up=res["is_up"], + content_matched=res["content_matched"], + error_msg=res["error_msg"], + tls_expires_at=res["tls_expires_at"], + )) + + # Update monitor's last_checked_at for next-run scheduling + db_monitor = await session.get(HttpMonitor, monitor.id) + if db_monitor: + db_monitor.last_checked_at = check_time + + # Alert pipeline + if res["response_ms"] is not None: + await record_metric( + session=session, + source_module="http", + resource_name=monitor.name, + metric_name="response_ms", + value=res["response_ms"], + ) + await record_metric( + session=session, + source_module="http", + resource_name=monitor.name, + metric_name="is_up", + value=1.0 if res["is_up"] else 0.0, + ) diff --git a/plugins/http/templates/http/index.html b/plugins/http/templates/http/index.html new file mode 100644 index 0000000..6d1ffd0 --- /dev/null +++ b/plugins/http/templates/http/index.html @@ -0,0 +1,77 @@ +{% extends "base.html" %} +{% block title %}HTTP Monitors — Fabled Scryer{% endblock %} +{% block content %} +
+

HTTP Monitors

+ {% include "_time_range.html" %} +
+ +{# ── Add monitor form ─────────────────────────────────────────────────────── #} +
+
Add Monitor
+
+
+
+ + +
+
+ + +
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ +
+
+
+ +{# ── Monitor list ─────────────────────────────────────────────────────────── #} +
+
Loading...
+
+{% endblock %} diff --git a/plugins/http/templates/http/rows.html b/plugins/http/templates/http/rows.html new file mode 100644 index 0000000..4666fa3 --- /dev/null +++ b/plugins/http/templates/http/rows.html @@ -0,0 +1,109 @@ +{# http/rows.html — HTMX fragment for HTTP monitor status list #} + +{# ── Summary ─────────────────────────────────────────────────────────────── #} +
+
+
Up
+ {{ up }} +
+
+
Down
+ {{ down }} +
+
+
Monitors
+ {{ monitor_data | length }} +
+
+ +{# ── Monitor list ─────────────────────────────────────────────────────────── #} +{% if monitor_data %} +
+ + + + + + + + + + + + + {% for item in monitor_data %} + {% set m = item.monitor %} + {% set latest = item.latest %} + + + + + + + + + {% endfor %} + +
MonitorStatusResponseHistoryTLS expiry
+
{{ m.name }}
+
+ {{ m.method }} {{ m.url }} +
+ {% if not m.enabled %} + paused + {% endif %} +
+ {% if not latest %} + pending + {% elif latest.is_up %} +
+ + {{ latest.status_code }} +
+ {% else %} +
+ + + {{ latest.status_code or latest.error_msg or "error" }} + +
+ {% if latest.content_matched == false %} +
content mismatch
+ {% endif %} + {% endif %} +
+ {% if latest and latest.response_ms is not none %} + + {{ "%.0f" | format(latest.response_ms) }}ms + + {% else %} + + {% endif %} + {{ item.sparkline_ms | safe }} + {% if item.tls_days is not none %} + + {{ item.tls_days | int }}d + + {% elif m.url.startswith('https://') %} + + {% endif %} + +
+ +
+
+ +
+
+
+{% else %} +
+
+ No monitors configured yet. Add one using the form above. +
+
+{% endif %} diff --git a/plugins/http/templates/http/widget.html b/plugins/http/templates/http/widget.html new file mode 100644 index 0000000..891f40b --- /dev/null +++ b/plugins/http/templates/http/widget.html @@ -0,0 +1,56 @@ +{# http/widget.html — dashboard widget: HTTP monitor summary #} +{% if not monitor_data and up == 0 and down == 0 and pending == 0 %} +
+ No monitors configured. Add monitors → +
+{% else %} + +
+ {% if up %} +
+ {{ up }} + up +
+ {% endif %} + {% if down %} +
+ {{ down }} + down +
+ {% endif %} + {% if pending %} +
+ {{ pending }} + pending +
+ {% endif %} +
+ +
+ {% for item in monitor_data %} + {% set latest = item.latest %} +
+ {% if not latest %} + + {% elif latest.is_up %} + + {% else %} + + {% endif %} + + {{ item.monitor.name }} + + {% if latest and latest.response_ms is not none %} + + {{ "%.0f" | format(latest.response_ms) }}ms + + {% elif latest and not latest.is_up %} + + {{ latest.status_code or "err" }} + + {% endif %} +
+ {% endfor %} +
+ +{% endif %} diff --git a/plugins/index.yaml b/plugins/index.yaml new file mode 100644 index 0000000..0192afe --- /dev/null +++ b/plugins/index.yaml @@ -0,0 +1,90 @@ +# fabledscryer-plugins / index.yaml +# +# Plugin catalog for Fabled Scryer. +# Fetched by the app at: Settings → Plugins → Plugin Catalog +# +# After updating an entry, commit and push — changes are live within 5 minutes +# (the app caches this file in-process for 300 seconds). +# +# See docs/plugins/index.yaml.example in the main app repo for the full schema. + +version: 1 +updated: "2026-03-22" + +plugins: + + - name: http + version: "1.0.0" + description: "Synthetic HTTP endpoint monitoring — status code, response time, content match, TLS expiry" + author: "FabledScryer" + license: "MIT" + min_app_version: "0.1.0" + repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/http" + download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/http-v1.0.0/http.zip" + checksum_sha256: "" + tags: + - monitoring + - http + - synthetic + - uptime + + - name: docker + version: "1.0.0" + description: "Docker container status, resource usage, and restart tracking via Docker socket" + author: "FabledScryer" + license: "MIT" + min_app_version: "0.1.0" + repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/docker" + download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/docker-v1.0.0/docker.zip" + checksum_sha256: "" + tags: + - containers + - docker + - infrastructure + + - name: traefik + version: "1.0.0" + description: "Traefik reverse proxy metrics and access log integration" + author: "FabledScryer" + license: "MIT" + min_app_version: "0.1.0" + repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/traefik" + download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/traefik-v1.0.0/traefik.zip" + checksum_sha256: "" + tags: + - proxy + - metrics + - access-log + + - name: unifi + version: "1.0.0" + description: "UniFi Network controller integration — WAN health, devices, clients, DPI" + author: "FabledScryer" + license: "MIT" + min_app_version: "0.1.0" + repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/unifi" + download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/unifi-v1.0.0/unifi.zip" + checksum_sha256: "" + tags: + - network + - unifi + - ubiquiti + + - name: ups + version: "1.0.0" + description: "UPS monitoring via NUT (Network UPS Tools) with Ansible shutdown automation" + author: "FabledScryer" + license: "MIT" + min_app_version: "0.1.0" + repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/ups" + download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/ups-v1.0.0/ups.zip" + checksum_sha256: "" + tags: + - ups + - power + - nut diff --git a/plugins/snmp/__init__.py b/plugins/snmp/__init__.py new file mode 100644 index 0000000..8655884 --- /dev/null +++ b/plugins/snmp/__init__.py @@ -0,0 +1,23 @@ +# plugins/snmp/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + + +def get_scheduled_tasks() -> list: + from .scheduler import make_poll_task + return [make_poll_task(_app)] + + +def get_blueprint(): + from .routes import snmp_bp + return snmp_bp diff --git a/plugins/snmp/plugin.yaml b/plugins/snmp/plugin.yaml new file mode 100644 index 0000000..5e98fce --- /dev/null +++ b/plugins/snmp/plugin.yaml @@ -0,0 +1,49 @@ +# plugins/snmp/plugin.yaml +name: snmp +version: "1.0.0" +description: "SNMP polling for network devices — switches, routers, printers, UPSes, etc." +author: "FabledScryer" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/snmp" +tags: + - snmp + - network + - monitoring + +config: + poll_interval_seconds: 60 + # Each device is polled independently. OIDs must resolve to numeric values. + devices: + - name: "core-switch" + host: "192.168.1.1" + port: 161 + community: "public" + version: "2c" # "1", "2c", or "3" + oids: + - oid: "1.3.6.1.2.1.1.3.0" + label: "uptime_centisecs" + - oid: "1.3.6.1.2.1.2.2.1.10.1" + label: "if1_in_octets" + - oid: "1.3.6.1.2.1.2.2.1.16.1" + label: "if1_out_octets" + # Example: network UPS with SNMP management card (RFC 1628 UPS-MIB). + # upsBatteryStatus values: 1=unknown, 2=batteryNormal, 3=batteryLow, 4=batteryDepleted. + # Uncomment and point at your UPS to enable. + # - name: "ups" + # host: "192.168.1.10" + # port: 161 + # community: "public" + # version: "2c" + # oids: + # - oid: "1.3.6.1.2.1.33.1.2.1.0" + # label: "battery_status" + # - oid: "1.3.6.1.2.1.33.1.2.2.0" + # label: "seconds_on_battery" + # - oid: "1.3.6.1.2.1.33.1.2.3.0" + # label: "runtime_minutes_remaining" + # - oid: "1.3.6.1.2.1.33.1.2.4.0" + # label: "battery_charge_pct" + # - oid: "1.3.6.1.2.1.33.1.3.3.1.3.1" + # label: "input_voltage_dV" diff --git a/plugins/snmp/poller.py b/plugins/snmp/poller.py new file mode 100644 index 0000000..2f7e86f --- /dev/null +++ b/plugins/snmp/poller.py @@ -0,0 +1,94 @@ +# plugins/snmp/poller.py +""" +Synchronous SNMP GET helper, run via executor. + +Requires pysnmp-lextudio (maintained pysnmp fork): + pip install 'fabledscryer[snmp]' + +If pysnmp is not installed, poll_device() returns an empty dict and logs a warning. +""" +from __future__ import annotations +import logging + +logger = logging.getLogger(__name__) + + +def _pysnmp_available() -> bool: + try: + import pysnmp # noqa: F401 + return True + except ImportError: + return False + + +def _mp_model(version: str) -> int: + """Map version string to pysnmp mpModel integer.""" + return 0 if version == "1" else 1 + + +def poll_device_sync( + host: str, + port: int, + community: str, + version: str, + oids: list[dict], +) -> dict[str, float]: + """ + Perform SNMP GET for each OID and return {label: float_value}. + Non-numeric OIDs (strings, etc.) are skipped. + Returns empty dict on any error. + """ + if not _pysnmp_available(): + logger.warning("pysnmp not installed — SNMP polling disabled. " + "Install with: pip install 'fabledscryer[snmp]'") + return {} + + from pysnmp.hlapi import ( + CommunityData, + ContextData, + ObjectIdentity, + ObjectType, + SnmpEngine, + UdpTransportTarget, + getCmd, + ) + + results: dict[str, float] = {} + engine = SnmpEngine() + + for oid_cfg in oids: + oid = oid_cfg["oid"] + label = oid_cfg.get("label") or oid + scale = float(oid_cfg.get("scale", 1.0)) + + try: + error_indication, error_status, error_index, var_binds = next( + getCmd( + engine, + CommunityData(community, mpModel=_mp_model(version)), + UdpTransportTarget((host, port), timeout=5, retries=1), + ContextData(), + ObjectType(ObjectIdentity(oid)), + ) + ) + except Exception as exc: + logger.debug("SNMP GET %s@%s OID %s failed: %s", host, port, oid, exc) + continue + + if error_indication: + logger.debug("SNMP error %s@%s OID %s: %s", host, port, oid, error_indication) + continue + if error_status: + logger.debug("SNMP status %s@%s OID %s: %s at %s", + host, port, oid, error_status.prettyPrint(), + error_index and var_binds[int(error_index) - 1][0] or "?") + continue + + for _, val in var_binds: + try: + results[label] = float(val) * scale + except (TypeError, ValueError): + # Non-numeric type (e.g. OctetString description) — skip + logger.debug("SNMP non-numeric value for %s label=%s: %r", oid, label, val) + + return results diff --git a/plugins/snmp/routes.py b/plugins/snmp/routes.py new file mode 100644 index 0000000..cca9697 --- /dev/null +++ b/plugins/snmp/routes.py @@ -0,0 +1,146 @@ +# plugins/snmp/routes.py +from __future__ import annotations +from datetime import datetime, timedelta, timezone + +from quart import Blueprint, current_app, render_template, request +from sqlalchemy import func, select + +from fabledscryer.auth.middleware import require_role +from fabledscryer.models.users import UserRole +from fabledscryer.models.metrics import PluginMetric + +snmp_bp = Blueprint("snmp", __name__, template_folder="templates") + + +async def _latest_readings(db, device_names: list[str]) -> dict[str, dict[str, float]]: + """Return {device_name: {label: latest_value}} for all configured devices.""" + if not device_names: + return {} + + # Latest value per (resource_name, metric_name) pair + subq = ( + select( + PluginMetric.resource_name, + PluginMetric.metric_name, + func.max(PluginMetric.recorded_at).label("latest_at"), + ) + .where(PluginMetric.source_module == "snmp") + .where(PluginMetric.resource_name.in_(device_names)) + .group_by(PluginMetric.resource_name, PluginMetric.metric_name) + .subquery() + ) + result = await db.execute( + select(PluginMetric).join( + subq, + (PluginMetric.resource_name == subq.c.resource_name) + & (PluginMetric.metric_name == subq.c.metric_name) + & (PluginMetric.recorded_at == subq.c.latest_at), + ).where(PluginMetric.source_module == "snmp") + ) + rows = result.scalars().all() + + out: dict[str, dict[str, float]] = {} + for row in rows: + out.setdefault(row.resource_name, {})[row.metric_name] = row.value + return out + + +async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]: + """Return {label: [(recorded_at, value), ...]} for a single device.""" + since = datetime.now(timezone.utc) - timedelta(hours=hours) + result = await db.execute( + select(PluginMetric) + .where( + PluginMetric.source_module == "snmp", + PluginMetric.resource_name == device_name, + PluginMetric.recorded_at >= since, + ) + .order_by(PluginMetric.recorded_at) + ) + rows = result.scalars().all() + + out: dict[str, list] = {} + for row in rows: + out.setdefault(row.metric_name, []).append( + (row.recorded_at.isoformat(), row.value) + ) + return out + + +@snmp_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", []) + device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg] + + async with current_app.db_sessionmaker() as db: + latest = await _latest_readings(db, device_names) + + # Merge config + latest readings for the template + devices = [] + for cfg in devices_cfg: + name = cfg.get("name") or cfg.get("host", "?") + devices.append({ + "name": name, + "host": cfg.get("host", ""), + "oids": cfg.get("oids", []), + "readings": latest.get(name, {}), + }) + + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + return await render_template("snmp/index.html", + devices=devices, + poll_interval=poll_interval) + + +@snmp_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", []) + device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg] + + async with current_app.db_sessionmaker() as db: + latest = await _latest_readings(db, device_names[:10]) + + devices = [] + for cfg in devices_cfg[:10]: + name = cfg.get("name") or cfg.get("host", "?") + devices.append({ + "name": name, + "oids": cfg.get("oids", []), + "readings": latest.get(name, {}), + }) + + return await render_template("snmp/widget.html", + devices=devices, + total=len(devices_cfg)) + + +@snmp_bp.get("/device/") +@require_role(UserRole.viewer) +async def device_detail(device_name: str): + devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", []) + device_cfg = next( + (d for d in devices_cfg if (d.get("name") or d.get("host")) == device_name), + None, + ) + if device_cfg is None: + from quart import abort + abort(404) + + hours = int(request.args.get("hours", 24)) + async with current_app.db_sessionmaker() as db: + hist = await _history(db, device_name, hours=hours) + + import json + history_json = json.dumps(hist) + oid_labels = [o.get("label") or o.get("oid") for o in device_cfg.get("oids", [])] + + return await render_template( + "snmp/device.html", + device=device_cfg, + device_name=device_name, + oid_labels=oid_labels, + history_json=history_json, + hours=hours, + ) diff --git a/plugins/snmp/scheduler.py b/plugins/snmp/scheduler.py new file mode 100644 index 0000000..b8809c3 --- /dev/null +++ b/plugins/snmp/scheduler.py @@ -0,0 +1,78 @@ +# plugins/snmp/scheduler.py +from __future__ import annotations +import asyncio +import logging +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +from fabledscryer.core.scheduler import ScheduledTask + +logger = logging.getLogger(__name__) + + +def make_poll_task(app: "Quart") -> ScheduledTask: + interval = app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60) + + async def poll() -> None: + await _do_poll(app) + + return ScheduledTask( + name="snmp_poll", + coro_factory=poll, + interval_seconds=int(interval), + run_on_startup=True, + ) + + +async def _do_poll(app: "Quart") -> None: + from datetime import datetime, timezone + from .poller import poll_device_sync + from fabledscryer.core.alerts import record_metric + + devices: list[dict] = app.config["PLUGINS"]["snmp"].get("devices", []) + if not devices: + return + + loop = asyncio.get_event_loop() + now = datetime.now(timezone.utc) + + async with app.db_sessionmaker() as session: + async with session.begin(): + for device in devices: + if not isinstance(device, dict): + continue + name = device.get("name") or device.get("host", "unknown") + host = device.get("host", "") + port = int(device.get("port", 161)) + community = device.get("community", "public") + version = str(device.get("version", "2c")) + oids = device.get("oids", []) + + if not host or not oids: + continue + + try: + readings = await loop.run_in_executor( + None, + poll_device_sync, + host, port, community, version, oids, + ) + except Exception: + logger.exception("SNMP poll failed for device %s (%s)", name, host) + continue + + for label, value in readings.items(): + await record_metric( + session=session, + source_module="snmp", + resource_name=name, + metric_name=label, + value=value, + ) + + if readings: + logger.debug("SNMP polled %s (%s): %d OID(s)", name, host, len(readings)) + else: + logger.debug("SNMP polled %s (%s): no readings", name, host) diff --git a/plugins/snmp/templates/snmp/device.html b/plugins/snmp/templates/snmp/device.html new file mode 100644 index 0000000..82022a9 --- /dev/null +++ b/plugins/snmp/templates/snmp/device.html @@ -0,0 +1,84 @@ +{# plugins/snmp/templates/snmp/device.html #} +{% extends "base.html" %} +{% block title %}SNMP — {{ device_name }} — Fabled Scryer{% endblock %} +{% block content %} +
+ ← SNMP +

{{ device_name }}

+ {{ device.host }} +
+ {% for h in [1, 6, 24, 168] %} + + {% if h < 24 %}{{ h }}h{% elif h == 24 %}24h{% else %}7d{% endif %} + + {% endfor %} +
+
+ +{% if oid_labels %} +
+ +
+ + +{% else %} +
+ No OIDs configured for this device. +
+{% endif %} +{% endblock %} diff --git a/plugins/snmp/templates/snmp/index.html b/plugins/snmp/templates/snmp/index.html new file mode 100644 index 0000000..86a1485 --- /dev/null +++ b/plugins/snmp/templates/snmp/index.html @@ -0,0 +1,57 @@ +{# plugins/snmp/templates/snmp/index.html #} +{% extends "base.html" %} +{% block title %}SNMP — Fabled Scryer{% endblock %} +{% block content %} +
+

SNMP Devices

+ {{ devices|length }} device(s) configured +
+ +{% if not devices %} +
+ No SNMP devices configured. Add devices to plugin.yaml. +
+{% else %} + +{% for device in devices %} +
+
+

{{ device.name }}

+ {{ device.host }} + {% if device.readings %} + ● reachable + {% else %} + ○ no data yet + {% endif %} + History +
+ + {% if device.readings %} +
+ {% for oid_cfg in device.oids %} + {% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %} + {% set val = device.readings.get(label) %} +
+
{{ label }}
+ {% if val is not none %} +
+ {% if val >= 1000000 %}{{ "%.2f"|format(val / 1000000) }}M + {% elif val >= 1000 %}{{ "%.1f"|format(val / 1000) }}K + {% else %}{{ "%.4g"|format(val) }}{% endif %} +
+ {% else %} +
+ {% endif %} +
+ {% endfor %} +
+ {% else %} +

+ No readings yet — waiting for next poll cycle. +

+ {% endif %} +
+{% endfor %} + +{% endif %} +{% endblock %} diff --git a/plugins/snmp/templates/snmp/widget.html b/plugins/snmp/templates/snmp/widget.html new file mode 100644 index 0000000..fc4d593 --- /dev/null +++ b/plugins/snmp/templates/snmp/widget.html @@ -0,0 +1,49 @@ +{# plugins/snmp/templates/snmp/widget.html #} +{% if devices %} +{% for device in devices %} +
+
+ + {{ device.name }} + + {% if device.readings %} + + {% else %} + + {% endif %} + detail → +
+ {% if device.readings %} +
+ {% for oid_cfg in device.oids[:4] %} + {% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %} + {% set val = device.readings.get(label) %} + {% if val is not none %} + + {{ label }} + + {% if val >= 1000000 %}{{ "%.1f"|format(val / 1000000) }}M + {% elif val >= 1000 %}{{ "%.0f"|format(val / 1000) }}K + {% else %}{{ "%.4g"|format(val) }}{% endif %} + + + {% endif %} + {% endfor %} + {% if device.oids|length > 4 %} + +{{ device.oids|length - 4 }} more + {% endif %} +
+ {% endif %} +
+{% if not loop.last %} +
+{% endif %} +{% endfor %} +{% if total > devices|length %} +
+ +{{ total - devices|length }} more — view all → +
+{% endif %} +{% else %} +

No SNMP devices configured.

+{% endif %} diff --git a/plugins/traefik/__init__.py b/plugins/traefik/__init__.py new file mode 100644 index 0000000..f2c7ac7 --- /dev/null +++ b/plugins/traefik/__init__.py @@ -0,0 +1,30 @@ +# plugins/traefik/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + from .models import TraefikMetric, TraefikCert, TraefikGlobalStat, TraefikAccessSummary # noqa: F401 + + +def get_scheduled_tasks() -> list: + from .scheduler import make_scrape_task, make_access_log_task + tasks = [make_scrape_task(_app)] + access_log_cfg = _app.config["PLUGINS"]["traefik"].get("access_log", {}) + if not isinstance(access_log_cfg, dict): + access_log_cfg = {} + if access_log_cfg.get("enabled", False): + tasks.append(make_access_log_task(_app)) + return tasks + + +def get_blueprint(): + from .routes import traefik_bp + return traefik_bp diff --git a/plugins/traefik/access_log.py b/plugins/traefik/access_log.py new file mode 100644 index 0000000..909b493 --- /dev/null +++ b/plugins/traefik/access_log.py @@ -0,0 +1,199 @@ +# plugins/traefik/access_log.py +"""Traefik JSON access log tailer and traffic-origin aggregator. + +Reads new lines from Traefik's JSON access log on each scheduler tick, +classifies source IPs as internal (RFC1918) or external, optionally +looks up country codes via MaxMind GeoLite2, and returns an aggregated +summary ready to persist in TraefikAccessSummary. +""" +from __future__ import annotations + +import ipaddress +import json +from collections import defaultdict +from pathlib import Path + +# ── Module-level file-position state ───────────────────────────────────────── +_log_position: int = 0 # byte offset of last read +_log_inode: int = 0 # detect rotation by inode change + +# ── Private IP ranges (RFC1918 + loopback + link-local) ────────────────────── +_PRIVATE_NETS = [ + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ipaddress.ip_network("127.0.0.0/8"), + ipaddress.ip_network("169.254.0.0/16"), + ipaddress.ip_network("::1/128"), + ipaddress.ip_network("fc00::/7"), + ipaddress.ip_network("fe80::/10"), +] + + +def _strip_port(raw: str) -> str: + """Strip trailing :port from an IPv4 address string. Leaves IPv6 untouched.""" + if not raw: + return raw + # Bracketed IPv6 with port: [::1]:12345 + if raw.startswith("["): + return raw[1:raw.index("]")] if "]" in raw else raw + # IPv4:port has exactly one colon + if raw.count(":") == 1: + return raw.rsplit(":", 1)[0] + return raw + + +def is_internal(raw_ip: str) -> bool: + """Return True if the address falls within a private/loopback range.""" + ip_str = _strip_port(raw_ip) + try: + addr = ipaddress.ip_address(ip_str) + return any(addr in net for net in _PRIVATE_NETS) + except ValueError: + return False + + +def _country(ip_str: str, geoip_db) -> str | None: + """Return ISO country code via an open maxminddb reader, or None.""" + if geoip_db is None: + return None + clean = _strip_port(ip_str) + try: + record = geoip_db.get(clean) + if record: + return record.get("country", {}).get("iso_code") + except Exception: + pass + return None + + +def read_new_lines(log_path: str) -> list[dict]: + """Return all new JSON-parsed access log entries since the last call. + + Handles log rotation: if the file shrinks or its inode changes, we + reset to the beginning of the new file. + """ + global _log_position, _log_inode + + path = Path(log_path) + if not path.exists(): + return [] + + try: + stat = path.stat() + except OSError: + return [] + + current_inode = stat.st_ino + current_size = stat.st_size + + # Rotation detected + if current_inode != _log_inode or current_size < _log_position: + _log_position = 0 + _log_inode = current_inode + + if current_size == _log_position: + return [] + + entries: list[dict] = [] + try: + with open(log_path, "rb") as f: + f.seek(_log_position) + raw = f.read(current_size - _log_position) + _log_position = current_size + except OSError: + return [] + + for raw_line in raw.split(b"\n"): + raw_line = raw_line.strip() + if not raw_line: + continue + try: + entries.append(json.loads(raw_line)) + except json.JSONDecodeError: + continue + + return entries + + +def aggregate(entries: list[dict], geoip_db=None, top_n: int = 10) -> dict | None: + """Aggregate a batch of access log entries into a summary dict. + + Returns None if there are no entries. + Result keys map directly to TraefikAccessSummary columns (excluding + id and period_start/period_end, which the caller provides). + """ + if not entries: + return None + + total = 0 + internal = 0 + external = 0 + total_bytes = 0.0 + + by_router: dict[str, int] = defaultdict(int) + ip_counts: dict[str, int] = defaultdict(int) + ip_internal: dict[str, bool] = {} + ip_country: dict[str, str | None] = {} + country_counts: dict[str, int] = defaultdict(int) + + for entry in entries: + total += 1 + raw_ip = entry.get("ClientHost", "") + router = entry.get("RouterName") or entry.get("ServiceName") or "unknown" + total_bytes += entry.get("DownstreamContentSize") or 0 + + by_router[router] += 1 + + internal_flag = is_internal(raw_ip) + if internal_flag: + internal += 1 + else: + external += 1 + if raw_ip not in ip_country: + ip_country[raw_ip] = _country(raw_ip, geoip_db) + c = ip_country[raw_ip] + if c: + country_counts[c] += 1 + + ip_counts[raw_ip] += 1 + ip_internal[raw_ip] = internal_flag + + top_ips = [ + { + "ip": ip, + "count": cnt, + "internal": ip_internal[ip], + "country": ip_country.get(ip), + } + for ip, cnt in sorted(ip_counts.items(), key=lambda x: x[1], reverse=True)[:top_n] + ] + top_countries = [ + {"country": c, "count": n} + for c, n in sorted(country_counts.items(), key=lambda x: x[1], reverse=True)[:top_n] + ] + top_routers = [ + {"router": r, "count": n} + for r, n in sorted(by_router.items(), key=lambda x: x[1], reverse=True)[:top_n] + ] + + return { + "total_requests": total, + "internal_requests": internal, + "external_requests": external, + "total_bytes": total_bytes, + "top_ips_json": json.dumps(top_ips), + "top_countries_json": json.dumps(top_countries), + "top_routers_json": json.dumps(top_routers), + } + + +def open_geoip(db_path: str | None): + """Open a MaxMind mmdb file if available. Returns None if not configured or missing.""" + if not db_path: + return None + try: + import maxminddb # type: ignore[import] + return maxminddb.open_database(db_path) + except Exception: + return None diff --git a/plugins/traefik/migrations/__init__.py b/plugins/traefik/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/traefik/migrations/env.py b/plugins/traefik/migrations/env.py new file mode 100644 index 0000000..1919fec --- /dev/null +++ b/plugins/traefik/migrations/env.py @@ -0,0 +1,75 @@ +# plugins/traefik/migrations/env.py +"""Alembic env.py for the Traefik plugin. + +For standalone use during plugin development. +At app startup, migration_runner.py uses the core env.py with version_locations. +""" +import asyncio +import os +import sys +from pathlib import Path +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config +from alembic import context + +# Make fabledscryer importable +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from fabledscryer.models.base import Base +import fabledscryer.models # noqa: F401 — core models +from plugins.traefik.models import TraefikMetric # noqa: F401 — plugin model + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _get_url() -> str: + import yaml + cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml") + try: + with open(cfg_path) as f: + cfg = yaml.safe_load(f) or {} + url = cfg.get("database", {}).get("url", "") + except FileNotFoundError: + url = "" + return os.environ.get("FABLEDSCRYER_DATABASE__URL", url) + + +def run_migrations_offline() -> None: + context.configure( + url=_get_url(), target_metadata=target_metadata, + literal_binds=True, dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + cfg = config.get_section(config.config_ini_section, {}) + cfg["sqlalchemy.url"] = _get_url() + connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/plugins/traefik/migrations/versions/__init__.py b/plugins/traefik/migrations/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/traefik/migrations/versions/traefik_001_initial.py b/plugins/traefik/migrations/versions/traefik_001_initial.py new file mode 100644 index 0000000..aa53f78 --- /dev/null +++ b/plugins/traefik/migrations/versions/traefik_001_initial.py @@ -0,0 +1,37 @@ +# plugins/traefik/migrations/versions/traefik_001_initial.py +"""Traefik metrics table + +Revision ID: traefik_001_initial +Revises: (none — this is a branch off the core head via depends_on) +Create Date: 2026-03-17 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "traefik_001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = "traefik" +depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",) + + +def upgrade() -> None: + op.create_table( + "traefik_metrics", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("router_name", sa.String(255), nullable=False), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("request_rate", sa.Float, nullable=False), + sa.Column("error_rate_4xx_pct", sa.Float, nullable=False), + sa.Column("error_rate_5xx_pct", sa.Float, nullable=False), + sa.Column("latency_p50_ms", sa.Float, nullable=False), + sa.Column("latency_p95_ms", sa.Float, nullable=False), + sa.Column("latency_p99_ms", sa.Float, nullable=False), + ) + op.create_index("ix_traefik_metrics_router_scraped", "traefik_metrics", + ["router_name", "scraped_at"]) + + +def downgrade() -> None: + op.drop_index("ix_traefik_metrics_router_scraped", "traefik_metrics") + op.drop_table("traefik_metrics") diff --git a/plugins/traefik/migrations/versions/traefik_002_stats.py b/plugins/traefik/migrations/versions/traefik_002_stats.py new file mode 100644 index 0000000..8320d53 --- /dev/null +++ b/plugins/traefik/migrations/versions/traefik_002_stats.py @@ -0,0 +1,48 @@ +"""Traefik extended stats: bandwidth, certs, global stats + +Revision ID: traefik_002_stats +Revises: traefik_001_initial +Create Date: 2026-03-19 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "traefik_002_stats" +down_revision: Union[str, None] = "traefik_001_initial" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "traefik_metrics", + sa.Column("response_bytes_rate", sa.Float, nullable=False, server_default="0"), + ) + + op.create_table( + "traefik_certs", + sa.Column("serial", sa.String(255), primary_key=True), + sa.Column("cn", sa.String(512), nullable=True), + sa.Column("sans", sa.Text, nullable=True), + sa.Column("not_after", sa.DateTime(timezone=True), nullable=False), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "traefik_global_stats", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("open_conns_total", sa.Float, nullable=True), + sa.Column("config_reloads_total", sa.Float, nullable=True), + sa.Column("config_last_reload_success", sa.Float, nullable=True), + sa.Column("process_memory_bytes", sa.Float, nullable=True), + ) + op.create_index("ix_traefik_global_stats_scraped", "traefik_global_stats", ["scraped_at"]) + + +def downgrade() -> None: + op.drop_index("ix_traefik_global_stats_scraped", "traefik_global_stats") + op.drop_table("traefik_global_stats") + op.drop_table("traefik_certs") + op.drop_column("traefik_metrics", "response_bytes_rate") diff --git a/plugins/traefik/migrations/versions/traefik_003_access_log.py b/plugins/traefik/migrations/versions/traefik_003_access_log.py new file mode 100644 index 0000000..fbc8009 --- /dev/null +++ b/plugins/traefik/migrations/versions/traefik_003_access_log.py @@ -0,0 +1,40 @@ +"""Traefik access log traffic summaries + +Revision ID: traefik_003_access_log +Revises: traefik_002_stats +Create Date: 2026-03-20 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "traefik_003_access_log" +down_revision: Union[str, None] = "traefik_002_stats" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "traefik_access_summaries", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("period_start", sa.DateTime(timezone=True), nullable=False), + sa.Column("period_end", sa.DateTime(timezone=True), nullable=False), + sa.Column("total_requests", sa.Integer, nullable=False, server_default="0"), + sa.Column("internal_requests", sa.Integer, nullable=False, server_default="0"), + sa.Column("external_requests", sa.Integer, nullable=False, server_default="0"), + sa.Column("total_bytes", sa.Float, nullable=False, server_default="0"), + sa.Column("top_ips_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("top_countries_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("top_routers_json", sa.Text, nullable=False, server_default="[]"), + ) + op.create_index( + "ix_traefik_access_summaries_period_end", + "traefik_access_summaries", + ["period_end"], + ) + + +def downgrade() -> None: + op.drop_index("ix_traefik_access_summaries_period_end", "traefik_access_summaries") + op.drop_table("traefik_access_summaries") diff --git a/plugins/traefik/models.py b/plugins/traefik/models.py new file mode 100644 index 0000000..1ba9e50 --- /dev/null +++ b/plugins/traefik/models.py @@ -0,0 +1,67 @@ +# plugins/traefik/models.py +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import DateTime, Float, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from fabledscryer.models.base import Base + + +class TraefikMetric(Base): + __tablename__ = "traefik_metrics" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + router_name: Mapped[str] = mapped_column(String(255), nullable=False) + scraped_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) + ) + request_rate: Mapped[float] = mapped_column(Float, nullable=False) + error_rate_4xx_pct: Mapped[float] = mapped_column(Float, nullable=False) + error_rate_5xx_pct: Mapped[float] = mapped_column(Float, nullable=False) + latency_p50_ms: Mapped[float] = mapped_column(Float, nullable=False) + latency_p95_ms: Mapped[float] = mapped_column(Float, nullable=False) + latency_p99_ms: Mapped[float] = mapped_column(Float, nullable=False) + response_bytes_rate: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + + +class TraefikCert(Base): + """TLS certificate expiry tracking — upserted by serial each scrape.""" + __tablename__ = "traefik_certs" + + serial: Mapped[str] = mapped_column(String(255), primary_key=True) + cn: Mapped[str | None] = mapped_column(String(512), nullable=True) + sans: Mapped[str | None] = mapped_column(Text, nullable=True) + not_after: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + scraped_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) + ) + + +class TraefikAccessSummary(Base): + """Aggregated traffic-origin summary — one row per scheduler tick.""" + __tablename__ = "traefik_access_summaries" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + period_start: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + period_end: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + total_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + internal_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + external_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + total_bytes: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + top_ips_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + top_countries_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + top_routers_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + + +class TraefikGlobalStat(Base): + """One row per scrape — process-level and config stats.""" + __tablename__ = "traefik_global_stats" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + scraped_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) + ) + open_conns_total: Mapped[float | None] = mapped_column(Float, nullable=True) + config_reloads_total: Mapped[float | None] = mapped_column(Float, nullable=True) + config_last_reload_success: Mapped[float | None] = mapped_column(Float, nullable=True) + process_memory_bytes: Mapped[float | None] = mapped_column(Float, nullable=True) diff --git a/plugins/traefik/plugin.yaml b/plugins/traefik/plugin.yaml new file mode 100644 index 0000000..392713b --- /dev/null +++ b/plugins/traefik/plugin.yaml @@ -0,0 +1,21 @@ +# plugins/traefik/plugin.yaml +name: traefik +version: "1.0.0" +description: "Traefik reverse proxy metrics and access log integration" +author: "FabledScryer" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/traefik" +tags: + - proxy + - metrics + - access-log + +config: + metrics_url: "http://localhost:8080/metrics" + scrape_interval_seconds: 60 + access_log: + enabled: false + path: "/var/log/traefik/access.log" + geoip_db: "" # optional: path to MaxMind GeoLite2-Country.mmdb diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py new file mode 100644 index 0000000..f7fe3ce --- /dev/null +++ b/plugins/traefik/routes.py @@ -0,0 +1,373 @@ +# plugins/traefik/routes.py +from __future__ import annotations +from collections import defaultdict +from datetime import datetime, timezone +from quart import Blueprint, current_app, render_template, request +from sqlalchemy import Integer, cast, func, select +import json + +from fabledscryer.auth.middleware import require_role +from fabledscryer.models.users import UserRole +from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample +from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric + +traefik_bp = Blueprint("traefik", __name__, template_folder="templates") + + +def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: + """Generate an inline SVG sparkline from a list of float values.""" + if len(values) < 2: + return f'' + mn, mx = min(values), max(values) + if mx == mn: + mx = mn + 1.0 + step = width / (len(values) - 1) + pts = [] + for i, v in enumerate(values): + x = i * step + y = height - (v - mn) / (mx - mn) * (height - 2) - 1 + pts.append(f"{x:.1f},{y:.1f}") + poly = " ".join(pts) + return ( + f'' + f'' + f'' + ) + + +@traefik_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + _al_cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {}) + if not isinstance(_al_cfg, dict): + _al_cfg = {} + access_log_enabled = _al_cfg.get("enabled", False) + current_range = request.args.get("range", DEFAULT_RANGE) + return await render_template( + "traefik/index.html", + poll_interval=poll_interval, + access_log_enabled=access_log_enabled, + current_range=current_range, + ) + + +@traefik_bp.get("/rows") +@require_role(UserRole.viewer) +async def rows(): + """HTMX fragment: service cards with sparklines scoped to selected time range.""" + since, range_key = parse_range(request.args.get("range")) + now = datetime.now(timezone.utc) + + async with current_app.db_sessionmaker() as db: + # All known routers + result = await db.execute( + select(TraefikMetric.router_name) + .distinct() + .order_by(TraefikMetric.router_name) + ) + routers = [row[0] for row in result.all()] + + b_secs = bucket_seconds(since) + bucket_col = ( + cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs + ).label("bucket") + + router_data = [] + for router in routers: + # Bucketed history for sparklines — always ~80 points regardless of range + result = await db.execute( + select( + func.avg(TraefikMetric.request_rate).label("request_rate"), + func.avg(TraefikMetric.latency_p95_ms).label("latency_p95_ms"), + func.avg(TraefikMetric.error_rate_5xx_pct).label("error_rate_5xx_pct"), + func.min(TraefikMetric.scraped_at).label("scraped_at"), + bucket_col, + ) + .where(TraefikMetric.router_name == router) + .where(TraefikMetric.scraped_at >= since) + .group_by(bucket_col) + .order_by(bucket_col) + ) + history = result.all() + + # Latest value — always the most recent raw row + result = await db.execute( + select(TraefikMetric) + .where(TraefikMetric.router_name == router) + .order_by(TraefikMetric.scraped_at.desc()) + .limit(1) + ) + latest = result.scalar_one_or_none() + if not latest: + continue + + router_data.append({ + "name": router, + "latest": latest, + "sparkline_req": _sparkline([r.request_rate for r in history]), + "sparkline_p95": _sparkline([r.latency_p95_ms for r in history]), + "sparkline_5xx": _sparkline([r.error_rate_5xx_pct for r in history]), + }) + + # Latest global stat (always most recent, not range-filtered) + result = await db.execute( + select(TraefikGlobalStat) + .order_by(TraefikGlobalStat.scraped_at.desc()) + .limit(1) + ) + global_stat = result.scalar_one_or_none() + + # All certs, sorted by soonest expiry + result = await db.execute( + select(TraefikCert).order_by(TraefikCert.not_after) + ) + raw_certs = result.scalars().all() + + certs = [] + for c in raw_certs: + days = (c.not_after - now).total_seconds() / 86400.0 + certs.append({ + "serial": c.serial, + "cn": c.cn or c.serial, + "sans": c.sans, + "not_after": c.not_after, + "days_remaining": days, + }) + + return await render_template( + "traefik/rows.html", + router_data=router_data, + global_stat=global_stat, + certs=certs, + range_key=range_key, + ) + + +@traefik_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + """HTMX dashboard widget fragment.""" + now = datetime.now(timezone.utc) + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(TraefikMetric.router_name).distinct().order_by(TraefikMetric.router_name) + ) + routers = [row[0] for row in result.all()] + router_data = [] + for router in routers: + result = await db.execute( + select(TraefikMetric) + .where(TraefikMetric.router_name == router) + .order_by(TraefikMetric.scraped_at.desc()) + .limit(1) + ) + latest = result.scalar_one_or_none() + if latest: + router_data.append({"name": router, "latest": latest}) + + router_data.sort(key=lambda r: ( + -(1 if r["latest"].error_rate_5xx_pct > 0.1 else 0), + -(1 if r["latest"].error_rate_4xx_pct > 5.0 else 0), + -r["latest"].request_rate, + )) + total_routers = len(router_data) + router_data = router_data[:5] + + result = await db.execute( + select(TraefikCert).order_by(TraefikCert.not_after) + ) + raw_certs = result.scalars().all() + + expiring_certs = [] + for c in raw_certs: + days = (c.not_after - now).total_seconds() / 86400.0 + if days < 30: + expiring_certs.append({"cn": c.cn or c.serial, "days_remaining": days}) + + return await render_template( + "traefik/widget.html", + router_data=router_data, + expiring_certs=expiring_certs, + total_routers=total_routers, + ) + + +@traefik_bp.get("/widget/access_log") +@require_role(UserRole.viewer) +async def widget_access_log(): + """HTMX dashboard widget: traffic origin summary with IP filter.""" + from collections import defaultdict + ip_filter = request.args.get("ip_filter", "all") + limit = max(1, min(50, int(request.args.get("limit", 10) or 10))) + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(TraefikAccessSummary) + .order_by(TraefikAccessSummary.period_end.desc()) + .limit(12) # last ~1 hour of 5-min windows + ) + rows = result.scalars().all() + + if not rows: + return await render_template("traefik/widget_access_log.html", + summary=None, widget_id=widget_id) + + total = sum(r.total_requests for r in rows) + internal = sum(r.internal_requests for r in rows) + external = sum(r.external_requests for r in rows) + + ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True}) + for row in rows: + for entry in json.loads(row.top_ips_json): + k = entry["ip"] + ip_counts[k]["count"] += entry["count"] + ip_counts[k]["internal"] = entry.get("internal", True) + + all_ips = sorted( + [{"ip": k, **v} for k, v in ip_counts.items()], + key=lambda x: x["count"], reverse=True, + ) + if ip_filter == "internal": + all_ips = [e for e in all_ips if e["internal"]] + elif ip_filter == "external": + all_ips = [e for e in all_ips if not e["internal"]] + + summary = { + "total": total, + "internal": internal, + "external": external, + "external_pct": (external / total * 100.0) if total else 0.0, + "top_ips": all_ips[:limit], + "ip_filter": ip_filter, + } + return await render_template("traefik/widget_access_log.html", + summary=summary, widget_id=widget_id) + + +@traefik_bp.get("/widget/request_chart") +@require_role(UserRole.viewer) +async def widget_request_chart(): + """HTMX dashboard widget: Chart.js request rate + error % over time.""" + from datetime import timedelta + hours = max(1, min(24, int(request.args.get("hours", 6) or 6))) + widget_id = request.args.get("wid", "0") + + since = datetime.now(timezone.utc) - timedelta(hours=hours) + b_secs = max(60, (hours * 3600) // 80) # ~80 buckets + + bucket_col = ( + cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs + ).label("bucket") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select( + func.avg(TraefikMetric.request_rate).label("req_rate"), + func.avg(TraefikMetric.error_rate_5xx_pct).label("err_pct"), + func.min(TraefikMetric.scraped_at).label("scraped_at"), + bucket_col, + ) + .where(TraefikMetric.scraped_at >= since) + .group_by(bucket_col) + .order_by(bucket_col) + ) + history = result.all() + + labels = list(range(len(history))) + req_rates = [round(r.req_rate or 0, 3) for r in history] + err_pcts = [round(r.err_pct or 0, 2) for r in history] + + return await render_template( + "traefik/widget_request_chart.html", + labels_json=json.dumps(labels), + req_rate_json=json.dumps(req_rates), + error_rate_json=json.dumps(err_pcts), + widget_id=widget_id, + hours=hours, + ) + + +@traefik_bp.get("/traffic") +@require_role(UserRole.viewer) +async def traffic(): + """HTMX fragment: traffic origin summary scoped to selected time range.""" + since, range_key = parse_range(request.args.get("range")) + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(TraefikAccessSummary) + .where(TraefikAccessSummary.period_end >= since) + .order_by(TraefikAccessSummary.period_end.asc()) + # No LIMIT — all summaries in range needed for accurate totals; + # sparkline subsamples to 80 points afterwards + ) + rows = result.scalars().all() + + if not rows: + return await render_template( + "traefik/traffic.html", summary=None, history=[], range_key=range_key + ) + + total = sum(r.total_requests for r in rows) + internal = sum(r.internal_requests for r in rows) + external = sum(r.external_requests for r in rows) + total_bytes = sum(r.total_bytes for r in rows) + + ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True, "country": None}) + country_counts: dict[str, int] = defaultdict(int) + router_counts: dict[str, int] = defaultdict(int) + + for row in rows: + for entry in json.loads(row.top_ips_json): + k = entry["ip"] + ip_counts[k]["count"] += entry["count"] + ip_counts[k]["internal"] = entry["internal"] + ip_counts[k]["country"] = entry.get("country") + for entry in json.loads(row.top_countries_json): + country_counts[entry["country"]] += entry["count"] + for entry in json.loads(row.top_routers_json): + router_counts[entry["router"]] += entry["count"] + + top_ips = sorted( + [{"ip": k, **v} for k, v in ip_counts.items()], + key=lambda x: x["count"], reverse=True + )[:10] + top_countries = sorted( + [{"country": k, "count": v} for k, v in country_counts.items()], + key=lambda x: x["count"], reverse=True + )[:10] + top_routers = sorted( + [{"router": k, "count": v} for k, v in router_counts.items()], + key=lambda x: x["count"], reverse=True + )[:10] + + history = [ + { + "period_end": r.period_end, + "external_pct": (r.external_requests / r.total_requests * 100.0) + if r.total_requests else 0.0, + } + for r in rows + ] + + summary = { + "total": total, + "internal": internal, + "external": external, + "total_bytes": total_bytes, + "external_pct": (external / total * 100.0) if total else 0.0, + "top_ips": top_ips, + "top_countries": top_countries, + "top_routers": top_routers, + "sparkline_external": _sparkline( + [h["external_pct"] for h in subsample(history, 80)] + ), + } + + return await render_template( + "traefik/traffic.html", summary=summary, history=history, range_key=range_key + ) diff --git a/plugins/traefik/scheduler.py b/plugins/traefik/scheduler.py new file mode 100644 index 0000000..b7eb050 --- /dev/null +++ b/plugins/traefik/scheduler.py @@ -0,0 +1,204 @@ +# plugins/traefik/scheduler.py +from __future__ import annotations +import logging +import time +from typing import TYPE_CHECKING + +from datetime import datetime +from fabledscryer.core.scheduler import ScheduledTask + +if TYPE_CHECKING: + from quart import Quart + +logger = logging.getLogger(__name__) + +# Module-level state: previous scrape data for delta computation +_prev_metrics: dict = {} +_prev_time: float = 0.0 + +# Access log state +_geoip_db = None +_access_log_last_tick: "datetime | None" = None + + +def make_scrape_task(app: "Quart") -> ScheduledTask: + """Return a ScheduledTask that scrapes Traefik metrics on the configured interval.""" + interval = app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"] + + async def scrape() -> None: + await _do_scrape(app) + + return ScheduledTask( + name="traefik_scrape", + coro_factory=scrape, + interval_seconds=int(interval), + run_on_startup=True, + ) + + +async def _do_scrape(app: "Quart") -> None: + """Fetch Traefik metrics, write to DB, and emit plugin_metrics.""" + global _prev_metrics, _prev_time + + from datetime import datetime, timezone + from sqlalchemy import select + from .models import TraefikCert, TraefikGlobalStat, TraefikMetric + from .scraper import ( + compute_global_stats, + compute_router_metrics, + extract_certs, + fetch_metrics, + ) + from fabledscryer.core.alerts import record_metric + + metrics_url: str = app.config["PLUGINS"]["traefik"]["metrics_url"] + + try: + current = await fetch_metrics(metrics_url) + except Exception: + logger.exception("Traefik scrape failed (url=%s)", metrics_url) + return + + now = time.monotonic() + elapsed = now - _prev_time if _prev_time else 60.0 + router_metrics = compute_router_metrics(current, _prev_metrics or None, elapsed) + global_stats = compute_global_stats(current, _prev_metrics or None) + cert_list = extract_certs(current) + + _prev_metrics = current + _prev_time = now + + scraped_at = datetime.now(timezone.utc) + + async with app.db_sessionmaker() as session: + async with session.begin(): + # ── Per-service metrics ─────────────────────────────────────────── + for router_name, metrics in router_metrics.items(): + row = TraefikMetric( + router_name=router_name, + scraped_at=scraped_at, + request_rate=metrics["request_rate"], + error_rate_4xx_pct=metrics["error_rate_4xx_pct"], + error_rate_5xx_pct=metrics["error_rate_5xx_pct"], + latency_p50_ms=metrics["latency_p50_ms"], + latency_p95_ms=metrics["latency_p95_ms"], + latency_p99_ms=metrics["latency_p99_ms"], + response_bytes_rate=metrics["response_bytes_rate"], + ) + session.add(row) + + for metric_name, value in [ + ("request_rate", metrics["request_rate"]), + ("error_rate_4xx_pct", metrics["error_rate_4xx_pct"]), + ("error_rate_5xx_pct", metrics["error_rate_5xx_pct"]), + ("latency_p50_ms", metrics["latency_p50_ms"]), + ("latency_p95_ms", metrics["latency_p95_ms"]), + ("latency_p99_ms", metrics["latency_p99_ms"]), + ("response_bytes_rate", metrics["response_bytes_rate"]), + ]: + await record_metric( + session=session, + source_module="traefik", + resource_name=router_name, + metric_name=metric_name, + value=value, + ) + + # ── Global stats ────────────────────────────────────────────────── + session.add(TraefikGlobalStat(scraped_at=scraped_at, **global_stats)) + + # ── TLS certs (upsert by serial) ────────────────────────────────── + for cert in cert_list: + existing = await session.get(TraefikCert, cert["serial"]) + if existing: + existing.cn = cert["cn"] + existing.sans = cert["sans"] + existing.not_after = cert["not_after"] + existing.scraped_at = scraped_at + else: + session.add(TraefikCert( + serial=cert["serial"], + cn=cert["cn"], + sans=cert["sans"], + not_after=cert["not_after"], + scraped_at=scraped_at, + )) + + # Emit cert_expiry_days for alert rules + days_remaining = (cert["not_after"] - scraped_at).total_seconds() / 86400.0 + await record_metric( + session=session, + source_module="traefik", + resource_name=cert["cn"] or cert["serial"], + metric_name="cert_expiry_days", + value=max(0.0, days_remaining), + ) + + logger.debug( + "Traefik scrape: %d router(s), %d cert(s)", + len(router_metrics), + len(cert_list), + ) + + +def make_access_log_task(app: "Quart") -> ScheduledTask: + """Return a ScheduledTask that processes new Traefik access log lines.""" + global _geoip_db + cfg = app.config["PLUGINS"]["traefik"].get("access_log", {}) + if not isinstance(cfg, dict): + cfg = {} + log_path: str = cfg.get("path", "/var/log/traefik/access.log") + geoip_path: str = cfg.get("geoip_db", "") + + from .access_log import open_geoip + _geoip_db = open_geoip(geoip_path or None) + if _geoip_db: + logger.info("Traefik access log: GeoIP database loaded from %s", geoip_path) + else: + logger.info("Traefik access log: GeoIP not configured, country lookup disabled") + + interval = app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"] + + async def process() -> None: + await _do_access_log(app, log_path) + + return ScheduledTask( + name="traefik_access_log", + coro_factory=process, + interval_seconds=int(interval), + run_on_startup=True, + ) + + +async def _do_access_log(app: "Quart", log_path: str) -> None: + """Read new access log lines, aggregate, and persist a summary.""" + global _access_log_last_tick + + from datetime import timezone + from .access_log import read_new_lines, aggregate + from .models import TraefikAccessSummary + + now = datetime.now(timezone.utc) + period_start = _access_log_last_tick or now + _access_log_last_tick = now + + entries = read_new_lines(log_path) + summary = aggregate(entries, geoip_db=_geoip_db) + + if summary is None: + return + + async with app.db_sessionmaker() as session: + async with session.begin(): + session.add(TraefikAccessSummary( + period_start=period_start, + period_end=now, + **summary, + )) + + logger.debug( + "Traefik access log: %d requests (%d internal, %d external)", + summary["total_requests"], + summary["internal_requests"], + summary["external_requests"], + ) diff --git a/plugins/traefik/scraper.py b/plugins/traefik/scraper.py new file mode 100644 index 0000000..6e06dd2 --- /dev/null +++ b/plugins/traefik/scraper.py @@ -0,0 +1,276 @@ +# plugins/traefik/scraper.py +"""Prometheus text-format scraper and per-service metrics calculator for Traefik. + +Traefik exposes Prometheus metrics at /metrics. This module: +1. Fetches the endpoint via httpx +2. Parses the Prometheus text format (counters + histograms) +3. Computes per-service request rate (delta/elapsed), error rates (% 4xx, % 5xx), + latency percentiles (p50, p95, p99) via linear histogram interpolation, + and bandwidth (bytes/sec from response bytes counter delta) +4. Extracts TLS certificate expiry info +5. Computes process-level and config health stats +""" +from __future__ import annotations +import math +import re +from collections import defaultdict +from datetime import datetime, timezone + +import httpx + + +# ────────────────────────────────────────────────────────────────────────────── +# Prometheus text format parser +# ────────────────────────────────────────────────────────────────────────────── + +# Parsed sample: {metric_name: {frozenset(label_items): float}} +ParsedMetrics = dict[str, dict[frozenset, float]] + +_SAMPLE_RE = re.compile( + r'^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+' + r'([+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?|[+-]?Inf|NaN)' +) +_LABEL_RE = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="([^"\\]*(?:\\.[^"\\]*)*)"') + + +def parse_prometheus(text: str) -> ParsedMetrics: + """Parse Prometheus text exposition format into nested dicts. + + Returns: {metric_name: {frozenset({(label, value), ...}): float}} + """ + metrics: ParsedMetrics = {} + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + m = _SAMPLE_RE.match(line) + if not m: + continue + name, labels_str, value_str = m.groups() + try: + value = float(value_str) + except ValueError: + continue + labels = frozenset( + (lm.group(1), lm.group(2)) + for lm in _LABEL_RE.finditer(labels_str or "") + ) + metrics.setdefault(name, {})[labels] = value + return metrics + + +# ────────────────────────────────────────────────────────────────────────────── +# Per-router metric computation +# ────────────────────────────────────────────────────────────────────────────── + +def compute_router_metrics( + current: ParsedMetrics, + previous: ParsedMetrics | None, + elapsed_seconds: float, +) -> dict[str, dict[str, float]]: + """Compute per-service derived metrics from two consecutive scrapes. + + Uses traefik_service_* metrics (available by default). + + Returns: + {service_name: { + "request_rate": float, # requests/sec + "error_rate_4xx_pct": float, # % 4xx of total + "error_rate_5xx_pct": float, # % 5xx of total + "latency_p50_ms": float, + "latency_p95_ms": float, + "latency_p99_ms": float, + "response_bytes_rate": float, # bytes/sec + }} + """ + prev = previous or {} + elapsed = max(elapsed_seconds, 1.0) + + # ── Request counters ────────────────────────────────────────────────────── + req_samples = current.get("traefik_service_requests_total", {}) + prev_req = prev.get("traefik_service_requests_total", {}) + + service_total: dict[str, float] = defaultdict(float) + service_4xx: dict[str, float] = defaultdict(float) + service_5xx: dict[str, float] = defaultdict(float) + + for labels, value in req_samples.items(): + label_dict = dict(labels) + service = label_dict.get("service", "") + if not service: + continue + code = label_dict.get("code", "") + prev_value = prev_req.get(labels, value) # no delta on first scrape + delta = max(0.0, value - prev_value) + service_total[service] += delta + if code.startswith("4"): + service_4xx[service] += delta + elif code.startswith("5"): + service_5xx[service] += delta + + # ── Response bytes counters ─────────────────────────────────────────────── + bytes_samples = current.get("traefik_service_responses_bytes_total", {}) + prev_bytes = prev.get("traefik_service_responses_bytes_total", {}) + + service_bytes: dict[str, float] = defaultdict(float) + for labels, value in bytes_samples.items(): + service = dict(labels).get("service", "") + if not service: + continue + prev_value = prev_bytes.get(labels, value) + service_bytes[service] += max(0.0, value - prev_value) + + # ── Histogram buckets ───────────────────────────────────────────────────── + hist_buckets = current.get("traefik_service_request_duration_seconds_bucket", {}) + + # Collect all known services + all_services: set[str] = set(service_total.keys()) | set(service_bytes.keys()) + for labels in hist_buckets: + service = dict(labels).get("service", "") + if service: + all_services.add(service) + + result: dict[str, dict[str, float]] = {} + + for service in all_services: + total = service_total.get(service, 0.0) + result[service] = { + "request_rate": total / elapsed, + "error_rate_4xx_pct": (service_4xx.get(service, 0.0) / total * 100.0) + if total > 0 else 0.0, + "error_rate_5xx_pct": (service_5xx.get(service, 0.0) / total * 100.0) + if total > 0 else 0.0, + "latency_p50_ms": _percentile_ms(hist_buckets, service, 0.50), + "latency_p95_ms": _percentile_ms(hist_buckets, service, 0.95), + "latency_p99_ms": _percentile_ms(hist_buckets, service, 0.99), + "response_bytes_rate": service_bytes.get(service, 0.0) / elapsed, + } + + return result + + +def compute_global_stats( + current: ParsedMetrics, + previous: ParsedMetrics | None, +) -> dict[str, float | None]: + """Extract process-level and config health metrics from a scrape. + + Returns: + { + "open_conns_total": float | None, + "config_reloads_total": float | None, + "config_last_reload_success": float | None, # 1.0 = success, 0.0 = failure + "process_memory_bytes": float | None, + } + """ + stats: dict[str, float | None] = { + "open_conns_total": None, + "config_reloads_total": None, + "config_last_reload_success": None, + "process_memory_bytes": None, + } + + # Open connections — sum across all entrypoints/protocols + open_conns = current.get("traefik_open_connections", {}) + if open_conns: + stats["open_conns_total"] = sum(open_conns.values()) + + # Config reloads — sum across result labels (success + failure) + reloads = current.get("traefik_config_reloads_total", {}) + if reloads: + stats["config_reloads_total"] = sum(reloads.values()) + + # Last reload success gauge + last_success = current.get("traefik_config_last_reload_success", {}) + if last_success: + # Usually a single sample with no labels + stats["config_last_reload_success"] = next(iter(last_success.values())) + + # Process RSS memory + mem = current.get("process_resident_memory_bytes", {}) + if mem: + stats["process_memory_bytes"] = next(iter(mem.values())) + + return stats + + +def extract_certs(current: ParsedMetrics) -> list[dict]: + """Extract TLS certificate expiry info from traefik_tls_certs_not_after. + + Returns a list of dicts: + [{"serial": str, "cn": str, "sans": str, "not_after": datetime}, ...] + """ + certs = [] + samples = current.get("traefik_tls_certs_not_after", {}) + for labels, ts_value in samples.items(): + label_dict = dict(labels) + serial = label_dict.get("serial", "") + if not serial: + continue + try: + not_after = datetime.fromtimestamp(ts_value, tz=timezone.utc) + except (ValueError, OSError, OverflowError): + continue + certs.append({ + "serial": serial, + "cn": label_dict.get("cn") or None, + "sans": label_dict.get("sans") or None, + "not_after": not_after, + }) + return certs + + +def _percentile_ms( + buckets: dict[frozenset, float], + service: str, + pct: float, +) -> float: + """Linear interpolation of a Prometheus histogram percentile, returned in ms.""" + router_buckets: list[tuple[float, float]] = [] + for labels, count in buckets.items(): + label_dict = dict(labels) + if label_dict.get("service") != service: + continue + le_str = label_dict.get("le", "") + try: + le = float("inf") if le_str == "+Inf" else float(le_str) + router_buckets.append((le, count)) + except ValueError: + continue + + if not router_buckets: + return 0.0 + + router_buckets.sort(key=lambda t: t[0]) + total = router_buckets[-1][1] # +Inf bucket count == total requests + if total == 0.0: + return 0.0 + + target = total * pct + for i, (le, count) in enumerate(router_buckets): + if count < target: + continue + if i == 0: + ratio = target / count if count > 0 else 0.0 + return le * ratio * 1000.0 + prev_le, prev_count = router_buckets[i - 1] + if count == prev_count: + return prev_le * 1000.0 + if math.isinf(le): + return prev_le * 1000.0 + fraction = (target - prev_count) / (count - prev_count) + return (prev_le + fraction * (le - prev_le)) * 1000.0 + + return 0.0 + + +# ────────────────────────────────────────────────────────────────────────────── +# HTTP fetch +# ────────────────────────────────────────────────────────────────────────────── + +async def fetch_metrics(metrics_url: str) -> ParsedMetrics: + """Fetch and parse Prometheus metrics from the given URL.""" + async with httpx.AsyncClient(timeout=10.0) as client: + response = await client.get(metrics_url) + response.raise_for_status() + return parse_prometheus(response.text) diff --git a/plugins/traefik/templates/traefik/index.html b/plugins/traefik/templates/traefik/index.html new file mode 100644 index 0000000..da66fc7 --- /dev/null +++ b/plugins/traefik/templates/traefik/index.html @@ -0,0 +1,31 @@ +{# plugins/traefik/templates/traefik/index.html #} +{% extends "base.html" %} +{% block title %}Traefik — Fabled Scryer{% endblock %} +{% block content %} +
+
+

Traefik Services

+ polling every {{ poll_interval }}s +
+ {% include "_time_range.html" %} +
+ +
+
+ +{% if access_log_enabled %} +
+

Traffic Origin

+
+
+
+{% endif %} +{% endblock %} diff --git a/plugins/traefik/templates/traefik/rows.html b/plugins/traefik/templates/traefik/rows.html new file mode 100644 index 0000000..c02b962 --- /dev/null +++ b/plugins/traefik/templates/traefik/rows.html @@ -0,0 +1,127 @@ +{# plugins/traefik/templates/traefik/rows.html #} + +{# ── Global stats bar ──────────────────────────────────────────────────────── #} +{% if global_stat %} +
+ + {% if global_stat.open_conns_total is not none %} + + Open conns + {{ global_stat.open_conns_total | int }} + + {% endif %} + + {% if global_stat.config_last_reload_success is not none %} + + Config + {% if global_stat.config_last_reload_success == 1.0 %} + ✓ OK + {% else %} + ✗ failed + {% endif %} + {% if global_stat.config_reloads_total is not none %} + ({{ global_stat.config_reloads_total | int }} reloads) + {% endif %} + + {% endif %} + + {% if global_stat.process_memory_bytes is not none %} + + Memory + {% set mb = (global_stat.process_memory_bytes / 1048576) %} + + {% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.0f"|format(mb) }} MB{% endif %} + + + {% endif %} + + + {{ global_stat.scraped_at.strftime("%H:%M:%S") }} UTC + +
+{% endif %} + +{# ── Per-service cards + cert card in shared columns layout ───────────────── #} +{% if router_data or certs %} +
+ +{% for r in router_data %} +
+

{{ r.name }}

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Req/s{{ "%.2f"|format(r.latest.request_rate) }}{{ r.sparkline_req | safe }}
Bandwidth + {% set bps = r.latest.response_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s + {% elif bps >= 1024 %}{{ "%.1f"|format(bps / 1024) }} KB/s + {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %} +
p95 ms{{ "%.1f"|format(r.latest.latency_p95_ms) }}{{ r.sparkline_p95 | safe }}
p99 ms{{ "%.1f"|format(r.latest.latency_p99_ms) }}
4xx % + {{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}% +
5xx % + {{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}% + {{ r.sparkline_5xx | safe }}
+
+ Last scraped {{ r.latest.scraped_at.strftime("%H:%M:%S") }} UTC +
+
+{% endfor %} + +{% if certs %} +
+
TLS Certificates
+ + {% for cert in certs %} + + + + + + + + {% endfor %} +
{{ cert.cn }} + {{ cert.days_remaining | int }}d +
+ {{ cert.not_after.strftime("%Y-%m-%d") }} +
+
+{% endif %} + +
+{% else %} +
+

No Traefik metrics yet. Configure metrics_url in config.yaml under plugins.traefik.

+
+{% endif %} diff --git a/plugins/traefik/templates/traefik/traffic.html b/plugins/traefik/templates/traefik/traffic.html new file mode 100644 index 0000000..dcdcb96 --- /dev/null +++ b/plugins/traefik/templates/traefik/traffic.html @@ -0,0 +1,101 @@ +{# plugins/traefik/templates/traefik/traffic.html #} +{% if summary %} + +{# ── Overview bar ─────────────────────────────────────────────────────────── #} +
+ + Total + {{ summary.total }} + + + Internal + {{ summary.internal }} + + + External + + {{ summary.external }} ({{ "%.1f"|format(summary.external_pct) }}%) + + + + Bytes + + {% set mb = summary.total_bytes / 1048576 %} + {% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.1f"|format(mb) }} MB{% endif %} + + + + {# External % sparkline #} + + ext % over time + {{ summary.sparkline_external | safe }} + +
+ +{# ── Internal/external fill bar ───────────────────────────────────────────── #} +{% set ext_pct = summary.external_pct %} +
+
+
+ +{# ── Detail cards in columns ──────────────────────────────────────────────── #} +
+ + {# Top source IPs #} +
+
Top Source IPs
+ + {% for entry in summary.top_ips %} + + + {% if entry.country %} + + {% endif %} + + + {% endfor %} +
{{ entry.ip }}{{ entry.country }} + + {{ entry.count }} + +
+
+ + {# Top routers #} +
+
Top Services Hit
+ + {% for entry in summary.top_routers %} + + + + + {% endfor %} +
{{ entry.router }}{{ entry.count }}
+
+ + {# Top countries (only shown if GeoIP is active) #} + {% if summary.top_countries %} +
+
External Traffic by Country
+ + {% for entry in summary.top_countries %} + + + + + {% endfor %} +
{{ entry.country }}{{ entry.count }}
+
+ {% endif %} + +
+ +{% else %} +
+ No access log data yet. + {% if not access_log_enabled %} + Enable access_log.enabled: true in your plugin config and bind-mount the Traefik log directory. + {% endif %} +
+{% endif %} diff --git a/plugins/traefik/templates/traefik/widget.html b/plugins/traefik/templates/traefik/widget.html new file mode 100644 index 0000000..aa43085 --- /dev/null +++ b/plugins/traefik/templates/traefik/widget.html @@ -0,0 +1,56 @@ +{# plugins/traefik/templates/traefik/widget.html #} +{% if expiring_certs %} +
+{% for cert in expiring_certs %} +
+ ⚠ {{ cert.cn }} + {{ cert.days_remaining | int }}d +
+{% endfor %} +
+{% endif %} + +{% if router_data %} +{% for r in router_data %} +
+
+ {{ r.name }} +
+
+ + req/s + {{ "%.2f"|format(r.latest.request_rate) }} + + + bw + + {% set bps = r.latest.response_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s + {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s + {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %} + + + + p95 + + {{ "%.0f"|format(r.latest.latency_p95_ms) }}ms + + + {% if r.latest.error_rate_5xx_pct > 0 %} + + {{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}% 5xx + + {% endif %} +
+
+{% endfor %} +{% if total_routers > 5 %} +
+ +{{ total_routers - 5 }} more — view all → +
+{% endif %} +{% else %} +

No Traefik data yet.

+{% endif %} diff --git a/plugins/traefik/templates/traefik/widget_access_log.html b/plugins/traefik/templates/traefik/widget_access_log.html new file mode 100644 index 0000000..899e09c --- /dev/null +++ b/plugins/traefik/templates/traefik/widget_access_log.html @@ -0,0 +1,40 @@ +{# traefik/widget_access_log.html — dashboard widget: traffic origin summary #} +{% if not summary %} +
No access log data yet.
+{% else %} + +
+
+ {{ summary.total | int }} + total req +
+
+ + {{ "%.1f" | format(summary.external_pct) }}% + + external +
+
+ +{% if summary.top_ips %} +
+ Top IPs + {% if summary.ip_filter != "all" %} + {{ summary.ip_filter }} + {% endif %} +
+
+ {% for ip in summary.top_ips %} +
+ + + {{ ip.ip }} + + {{ ip.count }} +
+ {% endfor %} +
+{% endif %} + +{% endif %} diff --git a/plugins/traefik/templates/traefik/widget_request_chart.html b/plugins/traefik/templates/traefik/widget_request_chart.html new file mode 100644 index 0000000..272ff39 --- /dev/null +++ b/plugins/traefik/templates/traefik/widget_request_chart.html @@ -0,0 +1,75 @@ +{# traefik/widget_request_chart.html — Chart.js request rate + error % over time #} +{% if not labels_json or labels_json == '[]' %} +
No data for last {{ hours }}h.
+{% else %} +
+ +
+ +
+ last {{ hours }}h +
+{% endif %} diff --git a/plugins/unifi/__init__.py b/plugins/unifi/__init__.py new file mode 100644 index 0000000..4bb589f --- /dev/null +++ b/plugins/unifi/__init__.py @@ -0,0 +1,28 @@ +# plugins/unifi/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + from .models import ( # noqa: F401 + UnifiWanStat, UnifiClientSnapshot, UnifiWlanStat, UnifiDpiSnapshot, + UnifiDevice, UnifiKnownClient, UnifiEvent, UnifiAlarm, + UnifiSpeedtestResult, UnifiNetwork, UnifiPortForward, UnifiFwRule, + ) + + +def get_scheduled_tasks() -> list: + from .scheduler import make_poll_task + return [make_poll_task(_app)] + + +def get_blueprint(): + from .routes import unifi_bp + return unifi_bp diff --git a/plugins/unifi/client.py b/plugins/unifi/client.py new file mode 100644 index 0000000..d18a7d8 --- /dev/null +++ b/plugins/unifi/client.py @@ -0,0 +1,170 @@ +# plugins/unifi/client.py +"""Async UniFi Network controller API client. + +Supports UDM/UDM-Pro (firmware 2+) using the /proxy/network/ path prefix +and Bearer-token auth via the /api/auth/login endpoint. + +On 401, re-authenticates automatically once before raising. +""" +from __future__ import annotations +import logging + +import httpx + +logger = logging.getLogger(__name__) + + +class UnifiClient: + def __init__( + self, + host: str, + username: str, + password: str, + site: str = "default", + verify_ssl: bool = False, + ) -> None: + self._host = host.rstrip("/") + self._username = username + self._password = password + self._site = site + self._verify_ssl = verify_ssl + self._http: httpx.AsyncClient | None = None + self._csrf_token: str = "" + + # ── Lifecycle ───────────────────────────────────────────────────────────── + + async def _ensure_http(self) -> None: + if self._http is None: + self._http = httpx.AsyncClient( + verify=self._verify_ssl, + timeout=15.0, + follow_redirects=True, + ) + + async def login(self) -> None: + await self._ensure_http() + resp = await self._http.post( + f"{self._host}/api/auth/login", + json={"username": self._username, "password": self._password, "rememberMe": False}, + ) + resp.raise_for_status() + self._csrf_token = resp.headers.get("X-CSRF-Token", "") + logger.debug("UniFi login OK (csrf=%s…)", self._csrf_token[:8] if self._csrf_token else "none") + + async def close(self) -> None: + if self._http: + await self._http.aclose() + self._http = None + + # ── API helpers ─────────────────────────────────────────────────────────── + + def _api_url(self, path: str) -> str: + return f"{self._host}/proxy/network/api/s/{self._site}{path}" + + def _headers(self) -> dict: + return {"X-CSRF-Token": self._csrf_token} if self._csrf_token else {} + + async def _get(self, path: str, params: dict | None = None) -> list | dict: + await self._ensure_http() + resp = await self._http.get(self._api_url(path), headers=self._headers(), params=params) + if resp.status_code == 401: + await self.login() + resp = await self._http.get(self._api_url(path), headers=self._headers(), params=params) + resp.raise_for_status() + data = resp.json() + return data.get("data", data) + + # ── Core monitoring ─────────────────────────────────────────────────────── + + async def get_health(self) -> list[dict]: + """Subsystem health (wan, wlan, lan, www).""" + result = await self._get("/stat/health") + return result if isinstance(result, list) else [] + + async def get_active_clients(self) -> list[dict]: + """Currently connected clients.""" + result = await self._get("/stat/sta") + return result if isinstance(result, list) else [] + + async def get_devices(self) -> list[dict]: + """Managed network devices (APs, switches, gateways) with full detail.""" + result = await self._get("/stat/device") + return result if isinstance(result, list) else [] + + # ── Events & alarms ─────────────────────────────────────────────────────── + + async def get_events(self, limit: int = 200) -> list[dict]: + """Recent site events (client connects, IDS alerts, device state changes, etc.).""" + result = await self._get("/stat/event", params={"_limit": limit, "_sort": "-time"}) + return result if isinstance(result, list) else [] + + async def get_alarms(self, archived: bool = False) -> list[dict]: + """Active (or archived) alarms.""" + result = await self._get("/stat/alarm") + if not isinstance(result, list): + return [] + return [a for a in result if a.get("archived", False) == archived] + + # ── Traffic & DPI ───────────────────────────────────────────────────────── + + async def get_dpi_stats(self) -> list[dict]: + """DPI per-client application/category traffic stats (cumulative counters).""" + result = await self._get("/stat/dpi") + return result if isinstance(result, list) else [] + + async def get_dpi_site_stats(self) -> list[dict]: + """DPI site-level category totals (not per-client).""" + result = await self._get("/dpi") + return result if isinstance(result, list) else [] + + async def get_speedtest_status(self) -> dict | None: + """Most recent WAN speed test result.""" + result = await self._get("/stat/speedtest-status") + if isinstance(result, list) and result: + return result[0] + if isinstance(result, dict): + return result + return None + + # ── Network inventory ───────────────────────────────────────────────────── + + async def get_known_clients(self) -> list[dict]: + """All historically seen clients (not just currently connected).""" + result = await self._get("/rest/user") + return result if isinstance(result, list) else [] + + async def get_networks(self) -> list[dict]: + """Network/VLAN configurations.""" + result = await self._get("/rest/networkconf") + return result if isinstance(result, list) else [] + + async def get_wlan_configs(self) -> list[dict]: + """WLAN (SSID) configurations.""" + result = await self._get("/rest/wlanconf") + return result if isinstance(result, list) else [] + + async def get_port_forwards(self) -> list[dict]: + """Port forwarding rules.""" + result = await self._get("/rest/portforwarding") + return result if isinstance(result, list) else [] + + async def get_firewall_rules(self) -> list[dict]: + """Firewall rules.""" + result = await self._get("/rest/firewallrule") + return result if isinstance(result, list) else [] + + async def get_firewall_groups(self) -> list[dict]: + """Firewall groups (IP sets / port sets referenced by rules).""" + result = await self._get("/rest/firewallgroup") + return result if isinstance(result, list) else [] + + # ── System ──────────────────────────────────────────────────────────────── + + async def get_sysinfo(self) -> dict | None: + """Controller system information (version, uptime, hostname).""" + result = await self._get("/stat/sysinfo") + if isinstance(result, list) and result: + return result[0] + if isinstance(result, dict): + return result + return None diff --git a/plugins/unifi/migrations/__init__.py b/plugins/unifi/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/unifi/migrations/env.py b/plugins/unifi/migrations/env.py new file mode 100644 index 0000000..56f15d5 --- /dev/null +++ b/plugins/unifi/migrations/env.py @@ -0,0 +1,72 @@ +# plugins/unifi/migrations/env.py +"""Alembic env.py for the UniFi plugin (standalone dev use). +At app startup, migration_runner.py uses the core env.py with version_locations. +""" +import asyncio +import os +import sys +from pathlib import Path +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config +from alembic import context + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from fabledscryer.models.base import Base +import fabledscryer.models # noqa: F401 +from plugins.unifi.models import UnifiWanStat, UnifiClientSnapshot, UnifiDevice # noqa: F401 + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _get_url() -> str: + import yaml + cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml") + try: + with open(cfg_path) as f: + cfg = yaml.safe_load(f) or {} + url = cfg.get("database", {}).get("url", "") + except FileNotFoundError: + url = "" + return os.environ.get("FABLEDSCRYER_DATABASE__URL", url) + + +def run_migrations_offline() -> None: + context.configure( + url=_get_url(), target_metadata=target_metadata, + literal_binds=True, dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + cfg = config.get_section(config.config_ini_section, {}) + cfg["sqlalchemy.url"] = _get_url() + connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/plugins/unifi/migrations/versions/__init__.py b/plugins/unifi/migrations/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/unifi/migrations/versions/unifi_001_initial.py b/plugins/unifi/migrations/versions/unifi_001_initial.py new file mode 100644 index 0000000..36158a3 --- /dev/null +++ b/plugins/unifi/migrations/versions/unifi_001_initial.py @@ -0,0 +1,62 @@ +"""UniFi plugin initial tables + +Revision ID: unifi_001_initial +Revises: (none — branch off core via depends_on) +Create Date: 2026-03-20 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "unifi_001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = "unifi" +depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",) + + +def upgrade() -> None: + op.create_table( + "unifi_wan_stats", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("is_up", sa.Boolean, nullable=False, server_default="false"), + sa.Column("wan_ip", sa.String(64), nullable=True), + sa.Column("latency_ms", sa.Float, nullable=True), + sa.Column("rx_bytes_rate", sa.Float, nullable=True), + sa.Column("tx_bytes_rate", sa.Float, nullable=True), + sa.Column("uptime_seconds", sa.Integer, nullable=True), + ) + op.create_index("ix_unifi_wan_stats_scraped", "unifi_wan_stats", ["scraped_at"]) + + op.create_table( + "unifi_client_snapshots", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("total_clients", sa.Integer, nullable=False, server_default="0"), + sa.Column("wireless_clients", sa.Integer, nullable=False, server_default="0"), + sa.Column("wired_clients", sa.Integer, nullable=False, server_default="0"), + sa.Column("guest_clients", sa.Integer, nullable=False, server_default="0"), + sa.Column("top_clients_json", sa.Text, nullable=False, server_default="[]"), + ) + op.create_index("ix_unifi_client_snapshots_scraped", "unifi_client_snapshots", ["scraped_at"]) + + op.create_table( + "unifi_devices", + sa.Column("mac", sa.String(32), primary_key=True), + sa.Column("name", sa.String(255), nullable=True), + sa.Column("model", sa.String(64), nullable=True), + sa.Column("device_type", sa.String(16), nullable=True), + sa.Column("ip", sa.String(64), nullable=True), + sa.Column("state", sa.Integer, nullable=False, server_default="0"), + sa.Column("uptime_seconds", sa.Integer, nullable=True), + sa.Column("last_seen", sa.DateTime(timezone=True), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("unifi_devices") + op.drop_index("ix_unifi_client_snapshots_scraped", "unifi_client_snapshots") + op.drop_table("unifi_client_snapshots") + op.drop_index("ix_unifi_wan_stats_scraped", "unifi_wan_stats") + op.drop_table("unifi_wan_stats") diff --git a/plugins/unifi/migrations/versions/unifi_002_expanded.py b/plugins/unifi/migrations/versions/unifi_002_expanded.py new file mode 100644 index 0000000..7336517 --- /dev/null +++ b/plugins/unifi/migrations/versions/unifi_002_expanded.py @@ -0,0 +1,156 @@ +"""UniFi expanded tables: wlan stats, dpi, events, alarms, known clients, +speedtest results, networks, port forwards, firewall rules. +Also adds missing 'version' column to unifi_devices. + +Revision ID: unifi_002_expanded +Revises: unifi_001_initial +Create Date: 2026-03-20 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "unifi_002_expanded" +down_revision: Union[str, None] = "unifi_001_initial" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ── Patch unifi_devices to add missing version column ───────────────────── + op.add_column("unifi_devices", sa.Column("version", sa.String(64), nullable=True)) + + # ── WLAN stats (time-series per SSID+radio) ─────────────────────────────── + op.create_table( + "unifi_wlan_stats", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("ssid", sa.String(255), nullable=False), + sa.Column("radio", sa.String(8), nullable=True), + sa.Column("num_sta", sa.Integer, nullable=False, server_default="0"), + sa.Column("tx_bytes", sa.Float, nullable=True), + sa.Column("rx_bytes", sa.Float, nullable=True), + sa.Column("channel", sa.Integer, nullable=True), + sa.Column("satisfaction", sa.Integer, nullable=True), + ) + op.create_index("ix_unifi_wlan_stats_scraped", "unifi_wlan_stats", ["scraped_at"]) + + # ── DPI snapshots (site-level category totals) ──────────────────────────── + op.create_table( + "unifi_dpi_snapshots", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("by_category_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("by_client_json", sa.Text, nullable=False, server_default="[]"), + ) + op.create_index("ix_unifi_dpi_scraped", "unifi_dpi_snapshots", ["scraped_at"]) + + # ── Known clients (upserted by MAC) ─────────────────────────────────────── + op.create_table( + "unifi_known_clients", + sa.Column("mac", sa.String(32), primary_key=True), + sa.Column("hostname", sa.String(255), nullable=True), + sa.Column("name", sa.String(255), nullable=True), + sa.Column("ip", sa.String(64), nullable=True), + sa.Column("mac_vendor", sa.String(128), nullable=True), + sa.Column("note", sa.Text, nullable=True), + sa.Column("is_blocked", sa.Boolean, nullable=False, server_default="false"), + sa.Column("first_seen", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_seen", sa.DateTime(timezone=True), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + # ── Events (upserted by UniFi event ID) ─────────────────────────────────── + op.create_table( + "unifi_events", + sa.Column("unifi_id", sa.String(64), primary_key=True), + sa.Column("event_time", sa.DateTime(timezone=True), nullable=False), + sa.Column("event_key", sa.String(64), nullable=False), + sa.Column("category", sa.String(16), nullable=False), + sa.Column("message", sa.Text, nullable=False), + sa.Column("source_mac", sa.String(32), nullable=True), + sa.Column("source_ip", sa.String(64), nullable=True), + sa.Column("details_json", sa.Text, nullable=False, server_default="{}"), + ) + op.create_index("ix_unifi_events_time", "unifi_events", ["event_time"]) + + # ── Alarms (upserted by UniFi alarm ID) ─────────────────────────────────── + op.create_table( + "unifi_alarms", + sa.Column("unifi_id", sa.String(64), primary_key=True), + sa.Column("alarm_time", sa.DateTime(timezone=True), nullable=False), + sa.Column("alarm_key", sa.String(64), nullable=False), + sa.Column("message", sa.Text, nullable=False), + sa.Column("archived", sa.Boolean, nullable=False, server_default="false"), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_index("ix_unifi_alarms_time", "unifi_alarms", ["alarm_time"]) + + # ── Speedtest results (upserted by run timestamp) ───────────────────────── + op.create_table( + "unifi_speedtest_results", + sa.Column("run_at", sa.DateTime(timezone=True), primary_key=True), + sa.Column("download_mbps", sa.Float, nullable=True), + sa.Column("upload_mbps", sa.Float, nullable=True), + sa.Column("latency_ms", sa.Float, nullable=True), + sa.Column("server_name", sa.String(255), nullable=True), + ) + + # ── Networks/VLANs (upserted by UniFi ID) ──────────────────────────────── + op.create_table( + "unifi_networks", + sa.Column("unifi_id", sa.String(64), primary_key=True), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("purpose", sa.String(32), nullable=True), + sa.Column("vlan", sa.Integer, nullable=True), + sa.Column("ip_subnet", sa.String(64), nullable=True), + sa.Column("dhcp_enabled", sa.Boolean, nullable=False, server_default="false"), + sa.Column("is_guest", sa.Boolean, nullable=False, server_default="false"), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + # ── Port forwards (upserted by UniFi ID) ───────────────────────────────── + op.create_table( + "unifi_port_forwards", + sa.Column("unifi_id", sa.String(64), primary_key=True), + sa.Column("name", sa.String(255), nullable=True), + sa.Column("proto", sa.String(8), nullable=False, server_default="tcp"), + sa.Column("dst_port", sa.String(32), nullable=False), + sa.Column("fwd_ip", sa.String(64), nullable=False), + sa.Column("fwd_port", sa.String(32), nullable=False), + sa.Column("src_filter", sa.String(255), nullable=True), + sa.Column("enabled", sa.Boolean, nullable=False, server_default="true"), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + # ── Firewall rules (upserted by UniFi ID) ──────────────────────────────── + op.create_table( + "unifi_fw_rules", + sa.Column("unifi_id", sa.String(64), primary_key=True), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("ruleset", sa.String(16), nullable=False), + sa.Column("rule_index", sa.Integer, nullable=True), + sa.Column("action", sa.String(16), nullable=False), + sa.Column("protocol", sa.String(16), nullable=True), + sa.Column("enabled", sa.Boolean, nullable=False, server_default="true"), + sa.Column("src_address", sa.String(255), nullable=True), + sa.Column("dst_address", sa.String(255), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("unifi_fw_rules") + op.drop_table("unifi_port_forwards") + op.drop_table("unifi_networks") + op.drop_table("unifi_speedtest_results") + op.drop_index("ix_unifi_alarms_time", "unifi_alarms") + op.drop_table("unifi_alarms") + op.drop_index("ix_unifi_events_time", "unifi_events") + op.drop_table("unifi_events") + op.drop_table("unifi_known_clients") + op.drop_index("ix_unifi_dpi_scraped", "unifi_dpi_snapshots") + op.drop_table("unifi_dpi_snapshots") + op.drop_index("ix_unifi_wlan_stats_scraped", "unifi_wlan_stats") + op.drop_table("unifi_wlan_stats") + op.drop_column("unifi_devices", "version") diff --git a/plugins/unifi/models.py b/plugins/unifi/models.py new file mode 100644 index 0000000..2ad90e9 --- /dev/null +++ b/plugins/unifi/models.py @@ -0,0 +1,182 @@ +# plugins/unifi/models.py +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from fabledscryer.models.base import Base + + +# ── Time-series (one row per scrape) ────────────────────────────────────────── + +class UnifiWanStat(Base): + """WAN interface health — one row per scrape.""" + __tablename__ = "unifi_wan_stats" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + is_up: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + wan_ip: Mapped[str | None] = mapped_column(String(64), nullable=True) + latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True) + rx_bytes_rate: Mapped[float | None] = mapped_column(Float, nullable=True) + tx_bytes_rate: Mapped[float | None] = mapped_column(Float, nullable=True) + uptime_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True) + + +class UnifiClientSnapshot(Base): + """Aggregated client counts + top talkers — one row per scrape.""" + __tablename__ = "unifi_client_snapshots" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + total_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + wireless_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + wired_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + guest_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + top_clients_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + # JSON: [{"mac","hostname","ip","type","tx_rate","rx_rate","signal","essid"}] + + +class UnifiWlanStat(Base): + """Per-SSID/radio stats — one row per scrape per SSID+radio combination.""" + __tablename__ = "unifi_wlan_stats" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + ssid: Mapped[str] = mapped_column(String(255), nullable=False) + radio: Mapped[str | None] = mapped_column(String(8), nullable=True) # ng=2.4GHz, na=5GHz, 6e=6GHz + num_sta: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + tx_bytes: Mapped[float | None] = mapped_column(Float, nullable=True) + rx_bytes: Mapped[float | None] = mapped_column(Float, nullable=True) + channel: Mapped[int | None] = mapped_column(Integer, nullable=True) + satisfaction: Mapped[int | None] = mapped_column(Integer, nullable=True) # 0-100 + + +class UnifiDpiSnapshot(Base): + """DPI application/category traffic snapshot — one row per scrape (JSON blob).""" + __tablename__ = "unifi_dpi_snapshots" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + # JSON: [{"cat_id", "cat_name", "rx_bytes", "tx_bytes"}] aggregated across all clients + by_category_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + # JSON: [{"mac", "hostname", "top_cats": [{"cat_name", "bytes"}]}] + by_client_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + + +# ── Upserted inventory (keyed by UniFi ID or MAC) ───────────────────────────── + +class UnifiDevice(Base): + """Network infrastructure devices — upserted by MAC each scrape.""" + __tablename__ = "unifi_devices" + + mac: Mapped[str] = mapped_column(String(32), primary_key=True) + name: Mapped[str | None] = mapped_column(String(255), nullable=True) + model: Mapped[str | None] = mapped_column(String(64), nullable=True) + device_type: Mapped[str | None] = mapped_column(String(16), nullable=True) + ip: Mapped[str | None] = mapped_column(String(64), nullable=True) + state: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + uptime_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True) + last_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + version: Mapped[str | None] = mapped_column(String(64), nullable=True) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class UnifiKnownClient(Base): + """All historically seen network clients — upserted by MAC.""" + __tablename__ = "unifi_known_clients" + + mac: Mapped[str] = mapped_column(String(32), primary_key=True) + hostname: Mapped[str | None] = mapped_column(String(255), nullable=True) + name: Mapped[str | None] = mapped_column(String(255), nullable=True) # user-set alias + ip: Mapped[str | None] = mapped_column(String(64), nullable=True) + mac_vendor: Mapped[str | None] = mapped_column(String(128), nullable=True) + note: Mapped[str | None] = mapped_column(Text, nullable=True) + is_blocked: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + first_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class UnifiEvent(Base): + """Site event log — upserted by UniFi event ID to avoid duplicates.""" + __tablename__ = "unifi_events" + + unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True) + event_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + event_key: Mapped[str] = mapped_column(String(64), nullable=False) # e.g. EVT_WU_Connected + category: Mapped[str] = mapped_column(String(16), nullable=False) # wlan, wired, ap, ids, wan, system + message: Mapped[str] = mapped_column(Text, nullable=False) + source_mac: Mapped[str | None] = mapped_column(String(32), nullable=True) + source_ip: Mapped[str | None] = mapped_column(String(64), nullable=True) + details_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}") + + +class UnifiAlarm(Base): + """Active alarms — upserted by UniFi alarm ID.""" + __tablename__ = "unifi_alarms" + + unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True) + alarm_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + alarm_key: Mapped[str] = mapped_column(String(64), nullable=False) + message: Mapped[str] = mapped_column(Text, nullable=False) + archived: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class UnifiSpeedtestResult(Base): + """WAN speed test results — upserted by run timestamp.""" + __tablename__ = "unifi_speedtest_results" + + run_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), primary_key=True) + download_mbps: Mapped[float | None] = mapped_column(Float, nullable=True) + upload_mbps: Mapped[float | None] = mapped_column(Float, nullable=True) + latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True) + server_name: Mapped[str | None] = mapped_column(String(255), nullable=True) + + +# ── Config inventory (upserted by UniFi ID) ─────────────────────────────────── + +class UnifiNetwork(Base): + """VLAN/network configurations — upserted by UniFi ID.""" + __tablename__ = "unifi_networks" + + unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + purpose: Mapped[str | None] = mapped_column(String(32), nullable=True) # corporate, guest, vlan-only + vlan: Mapped[int | None] = mapped_column(Integer, nullable=True) + ip_subnet: Mapped[str | None] = mapped_column(String(64), nullable=True) + dhcp_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_guest: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class UnifiPortForward(Base): + """Port forwarding rules — upserted by UniFi ID.""" + __tablename__ = "unifi_port_forwards" + + unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True) + name: Mapped[str | None] = mapped_column(String(255), nullable=True) + proto: Mapped[str] = mapped_column(String(8), nullable=False, default="tcp") + dst_port: Mapped[str] = mapped_column(String(32), nullable=False) + fwd_ip: Mapped[str] = mapped_column(String(64), nullable=False) + fwd_port: Mapped[str] = mapped_column(String(32), nullable=False) + src_filter: Mapped[str | None] = mapped_column(String(255), nullable=True) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class UnifiFwRule(Base): + """Firewall rules — upserted by UniFi ID.""" + __tablename__ = "unifi_fw_rules" + + unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + ruleset: Mapped[str] = mapped_column(String(16), nullable=False) # WAN_IN, LAN_IN, etc. + rule_index: Mapped[int | None] = mapped_column(Integer, nullable=True) + action: Mapped[str] = mapped_column(String(16), nullable=False) # accept, drop, reject + protocol: Mapped[str | None] = mapped_column(String(16), nullable=True) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + src_address: Mapped[str | None] = mapped_column(String(255), nullable=True) + dst_address: Mapped[str | None] = mapped_column(String(255), nullable=True) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) diff --git a/plugins/unifi/plugin.yaml b/plugins/unifi/plugin.yaml new file mode 100644 index 0000000..208ace5 --- /dev/null +++ b/plugins/unifi/plugin.yaml @@ -0,0 +1,21 @@ +name: unifi +version: "1.0.0" +description: "UniFi Network controller integration — WAN health, devices, clients, DPI" +author: "FabledScryer" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/unifi" +tags: + - network + - unifi + - ubiquiti + +config: + host: "https://192.168.1.1" + username: "admin" + password: "" + site: "default" + verify_ssl: false + poll_interval_seconds: 60 + top_clients: 10 # number of top-talker clients to store per snapshot diff --git a/plugins/unifi/routes.py b/plugins/unifi/routes.py new file mode 100644 index 0000000..2a57330 --- /dev/null +++ b/plugins/unifi/routes.py @@ -0,0 +1,357 @@ +# plugins/unifi/routes.py +from __future__ import annotations +import json +from datetime import datetime, timezone +from quart import Blueprint, current_app, render_template, request +from sqlalchemy import Integer, cast, func, select + +from fabledscryer.auth.middleware import require_role +from fabledscryer.models.users import UserRole +from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample +from .models import ( + UnifiAlarm, UnifiClientSnapshot, UnifiDevice, UnifiDpiSnapshot, + UnifiEvent, UnifiFwRule, UnifiKnownClient, UnifiNetwork, + UnifiPortForward, UnifiSpeedtestResult, UnifiWanStat, UnifiWlanStat, +) + +unifi_bp = Blueprint("unifi", __name__, template_folder="templates") + + +def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: + if len(values) < 2: + return f'' + mn, mx = min(values), max(values) + if mx == mn: + mx = mn + 1.0 + step = width / (len(values) - 1) + pts = [] + for i, v in enumerate(values): + x = i * step + y = height - (v - mn) / (mx - mn) * (height - 2) - 1 + pts.append(f"{x:.1f},{y:.1f}") + poly = " ".join(pts) + return ( + f'' + f'' + f'' + ) + + +@unifi_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + current_range = request.args.get("range", DEFAULT_RANGE) + return await render_template( + "unifi/index.html", + poll_interval=poll_interval, + current_range=current_range, + ) + + +@unifi_bp.get("/rows") +@require_role(UserRole.viewer) +async def rows(): + """HTMX fragment: WAN stats, devices, clients, WLAN — scoped to selected range.""" + since, range_key = parse_range(request.args.get("range")) + + b_secs = bucket_seconds(since) + + async with current_app.db_sessionmaker() as db: + # WAN history — bucketed for accurate full-range sparklines + wan_bucket = ( + cast(func.strftime('%s', UnifiWanStat.scraped_at), Integer) / b_secs + ).label("bucket") + result = await db.execute( + select( + func.avg(UnifiWanStat.latency_ms).label("latency_ms"), + func.avg(UnifiWanStat.rx_bytes_rate).label("rx_bytes_rate"), + func.avg(UnifiWanStat.tx_bytes_rate).label("tx_bytes_rate"), + func.min(UnifiWanStat.scraped_at).label("scraped_at"), + wan_bucket, + ) + .where(UnifiWanStat.scraped_at >= since) + .group_by(wan_bucket) + .order_by(wan_bucket) + ) + wan_history = result.all() + + # Latest WAN — always most recent raw row for current displayed values + result = await db.execute( + select(UnifiWanStat).order_by(UnifiWanStat.scraped_at.desc()).limit(1) + ) + latest_wan = result.scalar_one_or_none() + + # Latest client snapshot (always most recent) + result = await db.execute( + select(UnifiClientSnapshot) + .order_by(UnifiClientSnapshot.scraped_at.desc()) + .limit(1) + ) + latest_clients = result.scalar_one_or_none() + + # All devices + result = await db.execute( + select(UnifiDevice).order_by(UnifiDevice.state.desc(), UnifiDevice.name) + ) + devices = result.scalars().all() + + # Client count history — bucketed + cs_bucket = ( + cast(func.strftime('%s', UnifiClientSnapshot.scraped_at), Integer) / b_secs + ).label("bucket") + result = await db.execute( + select( + func.avg(UnifiClientSnapshot.total_clients).label("total_clients"), + func.min(UnifiClientSnapshot.scraped_at).label("scraped_at"), + cs_bucket, + ) + .where(UnifiClientSnapshot.scraped_at >= since) + .group_by(cs_bucket) + .order_by(cs_bucket) + ) + client_history = result.all() + + # Latest WLAN stats (always most recent per SSID) + result = await db.execute( + select(UnifiWlanStat.ssid).distinct().order_by(UnifiWlanStat.ssid) + ) + ssids = [row[0] for row in result.all()] + wlan_latest: list[UnifiWlanStat] = [] + for ssid in ssids: + result = await db.execute( + select(UnifiWlanStat) + .where(UnifiWlanStat.ssid == ssid) + .order_by(UnifiWlanStat.scraped_at.desc()) + .limit(1) + ) + w = result.scalar_one_or_none() + if w: + wlan_latest.append(w) + + # Latest speedtest + result = await db.execute( + select(UnifiSpeedtestResult) + .order_by(UnifiSpeedtestResult.run_at.desc()) + .limit(1) + ) + speedtest = result.scalar_one_or_none() + + top_clients = json.loads(latest_clients.top_clients_json) if latest_clients else [] + + sparkline_latency = _sparkline([r.latency_ms or 0 for r in wan_history]) + sparkline_rx = _sparkline([r.rx_bytes_rate or 0 for r in wan_history]) + sparkline_tx = _sparkline([r.tx_bytes_rate or 0 for r in wan_history]) + sparkline_clients = _sparkline([r.total_clients for r in client_history]) + + return await render_template( + "unifi/rows.html", + latest_wan=latest_wan, + latest_clients=latest_clients, + devices=devices, + top_clients=top_clients, + wlan_latest=wlan_latest, + speedtest=speedtest, + sparkline_latency=sparkline_latency, + sparkline_rx=sparkline_rx, + sparkline_tx=sparkline_tx, + sparkline_clients=sparkline_clients, + range_key=range_key, + ) + + +@unifi_bp.get("/events") +@require_role(UserRole.viewer) +async def events(): + """HTMX fragment: site events within selected range.""" + since, range_key = parse_range(request.args.get("range")) + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiEvent) + .where(UnifiEvent.event_time >= since) + .order_by(UnifiEvent.event_time.desc()) + .limit(500) + ) + event_rows = result.scalars().all() + + return await render_template( + "unifi/events.html", events=event_rows, range_key=range_key + ) + + +@unifi_bp.get("/dpi") +@require_role(UserRole.viewer) +async def dpi(): + """HTMX fragment: DPI traffic breakdown (latest snapshot).""" + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiDpiSnapshot) + .order_by(UnifiDpiSnapshot.scraped_at.desc()) + .limit(1) + ) + snapshot = result.scalar_one_or_none() + + categories = [] + top_clients = [] + if snapshot: + categories = json.loads(snapshot.by_category_json) + top_clients = json.loads(snapshot.by_client_json) + + return await render_template( + "unifi/dpi.html", categories=categories, top_clients=top_clients, snapshot=snapshot + ) + + +@unifi_bp.get("/security") +@require_role(UserRole.viewer) +async def security(): + """HTMX fragment: active and archived alarms.""" + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiAlarm) + .where(UnifiAlarm.archived == False) # noqa: E712 + .order_by(UnifiAlarm.alarm_time.desc()) + ) + active_alarms = result.scalars().all() + + result = await db.execute( + select(UnifiAlarm) + .where(UnifiAlarm.archived == True) # noqa: E712 + .order_by(UnifiAlarm.alarm_time.desc()) + .limit(20) + ) + archived_alarms = result.scalars().all() + + return await render_template( + "unifi/security.html", + active_alarms=active_alarms, + archived_alarms=archived_alarms, + ) + + +@unifi_bp.get("/inventory") +@require_role(UserRole.viewer) +async def inventory(): + """HTMX fragment: known clients, networks, port forwards, firewall rules.""" + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiKnownClient) + .order_by(UnifiKnownClient.last_seen.desc().nullslast()) + ) + known_clients = result.scalars().all() + + result = await db.execute( + select(UnifiNetwork).order_by(UnifiNetwork.name) + ) + networks = result.scalars().all() + + result = await db.execute( + select(UnifiPortForward).order_by(UnifiPortForward.name) + ) + port_forwards = result.scalars().all() + + result = await db.execute( + select(UnifiFwRule) + .order_by(UnifiFwRule.ruleset, UnifiFwRule.rule_index) + ) + fw_rules = result.scalars().all() + + return await render_template( + "unifi/inventory.html", + known_clients=known_clients, + networks=networks, + port_forwards=port_forwards, + fw_rules=fw_rules, + ) + + +@unifi_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + """HTMX dashboard widget fragment.""" + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiWanStat).order_by(UnifiWanStat.scraped_at.desc()).limit(1) + ) + latest_wan = result.scalar_one_or_none() + + result = await db.execute( + select(UnifiClientSnapshot).order_by(UnifiClientSnapshot.scraped_at.desc()).limit(1) + ) + latest_clients = result.scalar_one_or_none() + + result = await db.execute(select(UnifiDevice)) + devices = result.scalars().all() + + result = await db.execute( + select(UnifiAlarm).where(UnifiAlarm.archived == False) # noqa: E712 + ) + active_alarms = result.scalars().all() + + offline_devices = [d for d in devices if d.state != 1] + top_clients = json.loads(latest_clients.top_clients_json)[:5] if latest_clients else [] + + return await render_template( + "unifi/widget.html", + latest_wan=latest_wan, + latest_clients=latest_clients, + offline_devices=offline_devices, + active_alarms=active_alarms, + top_clients=top_clients, + ) + + +@unifi_bp.get("/widget/clients") +@require_role(UserRole.viewer) +async def widget_clients(): + """HTMX dashboard widget: top active clients.""" + limit = max(1, min(20, int(request.args.get("limit", 5) or 5))) + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiClientSnapshot) + .order_by(UnifiClientSnapshot.scraped_at.desc()) + .limit(1) + ) + snapshot = result.scalar_one_or_none() + + clients = json.loads(snapshot.top_clients_json)[:limit] if snapshot else [] + return await render_template( + "unifi/widget_clients.html", + clients=clients, + snapshot=snapshot, + widget_id=widget_id, + ) + + +@unifi_bp.get("/widget/devices") +@require_role(UserRole.viewer) +async def widget_devices(): + """HTMX dashboard widget: infrastructure devices with optional type filter.""" + type_filter = request.args.get("type_filter", "all") + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiDevice).order_by(UnifiDevice.state.desc(), UnifiDevice.name) + ) + all_devices = result.scalars().all() + + if type_filter == "wireless": + devices = [d for d in all_devices if d.device_type == "uap"] + elif type_filter == "wired": + devices = [d for d in all_devices if d.device_type != "uap"] + elif type_filter == "offline": + devices = [d for d in all_devices if d.state != 1] + else: + devices = list(all_devices) + + return await render_template( + "unifi/widget_devices.html", + devices=devices, + type_filter=type_filter, + widget_id=widget_id, + ) diff --git a/plugins/unifi/scheduler.py b/plugins/unifi/scheduler.py new file mode 100644 index 0000000..2a6345d --- /dev/null +++ b/plugins/unifi/scheduler.py @@ -0,0 +1,522 @@ +# plugins/unifi/scheduler.py +from __future__ import annotations +import json +import logging +from datetime import datetime, timezone +from typing import TYPE_CHECKING + +from fabledscryer.core.scheduler import ScheduledTask + +if TYPE_CHECKING: + from quart import Quart + +logger = logging.getLogger(__name__) + +_client = None # UnifiClient instance, initialised on first scrape + + +def make_poll_task(app: "Quart") -> ScheduledTask: + cfg = app.config["PLUGINS"]["unifi"] + interval = int(cfg.get("poll_interval_seconds", 60)) + + async def poll() -> None: + await _do_poll(app) + + return ScheduledTask( + name="unifi_poll", + coro_factory=poll, + interval_seconds=interval, + run_on_startup=True, + ) + + +async def _do_poll(app: "Quart") -> None: + global _client + + from .client import UnifiClient + from .models import UnifiClientSnapshot, UnifiDevice, UnifiWanStat + from fabledscryer.core.alerts import record_metric + + cfg = app.config["PLUGINS"]["unifi"] + + if _client is None: + _client = UnifiClient( + host=cfg["host"], + username=cfg["username"], + password=cfg["password"], + site=cfg.get("site", "default"), + verify_ssl=cfg.get("verify_ssl", False), + ) + try: + await _client.login() + except Exception as exc: + logger.warning("UniFi initial login failed: %s", exc) + _client = None + return + + try: + health = await _client.get_health() + clients = await _client.get_active_clients() + devices = await _client.get_devices() + except Exception as exc: + logger.warning("UniFi poll failed — will retry next tick: %s", exc) + _client = None # force re-auth on next tick + return + + scraped_at = datetime.now(timezone.utc) + top_n = int(cfg.get("top_clients", 10)) + + # ── WAN stat ────────────────────────────────────────────────────────────── + wan_data = next((h for h in health if h.get("subsystem") == "wan"), None) + wan_stat = UnifiWanStat(scraped_at=scraped_at) + if wan_data: + wan_stat.is_up = wan_data.get("status") == "ok" + wan_stat.wan_ip = wan_data.get("wan_ip") + wan_stat.latency_ms = wan_data.get("latency") + wan_stat.rx_bytes_rate = wan_data.get("rx_bytes-r") + wan_stat.tx_bytes_rate = wan_data.get("tx_bytes-r") + wan_stat.uptime_seconds = wan_data.get("uptime") + + # ── Client snapshot ─────────────────────────────────────────────────────── + wireless = [c for c in clients if not c.get("is_wired", True)] + wired = [c for c in clients if c.get("is_wired", True)] + guests = [c for c in clients if c.get("is_guest", False)] + + top_clients = sorted( + clients, + key=lambda c: (c.get("tx_bytes-r", 0) or 0) + (c.get("rx_bytes-r", 0) or 0), + reverse=True, + )[:top_n] + + top_clients_data = [ + { + "mac": c.get("mac", ""), + "hostname": c.get("hostname") or c.get("name") or c.get("mac", ""), + "ip": c.get("ip", ""), + "type": "wireless" if not c.get("is_wired", True) else "wired", + "tx_rate": c.get("tx_bytes-r", 0) or 0, + "rx_rate": c.get("rx_bytes-r", 0) or 0, + "signal": c.get("signal"), + "essid": c.get("essid"), + } + for c in top_clients + ] + + client_snapshot = UnifiClientSnapshot( + scraped_at=scraped_at, + total_clients=len(clients), + wireless_clients=len(wireless), + wired_clients=len(wired), + guest_clients=len(guests), + top_clients_json=json.dumps(top_clients_data), + ) + + async with app.db_sessionmaker() as session: + async with session.begin(): + session.add(wan_stat) + session.add(client_snapshot) + + # ── Devices (upsert by MAC) ─────────────────────────────────────── + for d in devices: + mac = d.get("mac", "") + if not mac: + continue + last_seen_ts = d.get("last_seen") + last_seen_dt = ( + datetime.fromtimestamp(last_seen_ts, tz=timezone.utc) + if last_seen_ts else None + ) + existing = await session.get(UnifiDevice, mac) + if existing: + existing.name = d.get("name") + existing.model = d.get("model") + existing.device_type = d.get("type") + existing.ip = d.get("ip") + existing.state = d.get("state", 0) + existing.uptime_seconds = d.get("uptime") + existing.last_seen = last_seen_dt + existing.version = d.get("version") + existing.scraped_at = scraped_at + else: + session.add(UnifiDevice( + mac=mac, + name=d.get("name"), + model=d.get("model"), + device_type=d.get("type"), + ip=d.get("ip"), + state=d.get("state", 0), + uptime_seconds=d.get("uptime"), + last_seen=last_seen_dt, + version=d.get("version"), + scraped_at=scraped_at, + )) + + # ── Alert metrics ───────────────────────────────────────────────── + if wan_stat.latency_ms is not None: + await record_metric( + session=session, + source_module="unifi", + resource_name="wan", + metric_name="latency_ms", + value=wan_stat.latency_ms, + ) + await record_metric( + session=session, + source_module="unifi", + resource_name="wan", + metric_name="is_up", + value=1.0 if wan_stat.is_up else 0.0, + ) + await record_metric( + session=session, + source_module="unifi", + resource_name="clients", + metric_name="total_clients", + value=float(len(clients)), + ) + + logger.debug( + "UniFi poll: wan=%s, clients=%d, devices=%d", + "up" if wan_stat.is_up else "down", + len(clients), + len(devices), + ) + + # ── Expanded data (best-effort, failures don't abort core poll) ─────────── + try: + await _do_expanded(app, scraped_at) + except Exception as exc: + logger.warning("UniFi expanded poll failed: %s", exc) + + +async def _do_expanded(app: "Quart", scraped_at: datetime) -> None: + """Fetch supplementary UniFi data: WLAN stats, events, alarms, DPI, + known clients, speedtest, networks, port forwards, firewall rules.""" + from .models import ( + UnifiAlarm, UnifiDpiSnapshot, UnifiEvent, UnifiFwRule, + UnifiKnownClient, UnifiNetwork, UnifiPortForward, UnifiSpeedtestResult, + UnifiWlanStat, + ) + + # ── Fetch from controller ───────────────────────────────────────────────── + try: + wlan_configs = await _client.get_wlan_configs() + except Exception: + wlan_configs = [] + + try: + events = await _client.get_events(limit=200) + except Exception: + events = [] + + try: + alarms = await _client.get_alarms(archived=False) + except Exception: + alarms = [] + + try: + dpi_cats = await _client.get_dpi_site_stats() + except Exception: + dpi_cats = [] + + try: + dpi_clients = await _client.get_dpi_stats() + except Exception: + dpi_clients = [] + + try: + known_clients = await _client.get_known_clients() + except Exception: + known_clients = [] + + try: + speedtest = await _client.get_speedtest_status() + except Exception: + speedtest = None + + try: + networks = await _client.get_networks() + except Exception: + networks = [] + + try: + port_forwards = await _client.get_port_forwards() + except Exception: + port_forwards = [] + + try: + fw_rules = await _client.get_firewall_rules() + except Exception: + fw_rules = [] + + # ── Derive WLAN stats from device VAP table ─────────────────────────────── + # WLAN usage stats live on the device objects' vap_table, not wlan configs. + # We get per-radio SSID data from the active clients grouped by essid. + # Store one row per SSID using wlan_configs for channel/satisfaction. + wlan_rows: list[UnifiWlanStat] = [] + for wc in wlan_configs: + ssid = wc.get("name") or wc.get("x_passphrase", "") + if not ssid: + continue + wlan_rows.append(UnifiWlanStat( + scraped_at=scraped_at, + ssid=ssid, + radio=None, + num_sta=wc.get("num_sta", 0) or 0, + tx_bytes=wc.get("tx_bytes"), + rx_bytes=wc.get("rx_bytes"), + channel=None, + satisfaction=wc.get("satisfaction"), + )) + + # ── Build DPI category summary ──────────────────────────────────────────── + cat_summary = [] + for entry in dpi_cats: + cat_id = entry.get("cat") or entry.get("cat_id") + cat_name = entry.get("cat_name") or str(cat_id) + rx = entry.get("rx_bytes", 0) or 0 + tx = entry.get("tx_bytes", 0) or 0 + if rx or tx: + cat_summary.append({"cat_id": cat_id, "cat_name": cat_name, "rx_bytes": rx, "tx_bytes": tx}) + cat_summary.sort(key=lambda x: x["rx_bytes"] + x["tx_bytes"], reverse=True) + + # Top 10 clients by DPI total + client_dpi: dict[str, dict] = {} + for entry in dpi_clients: + mac = entry.get("mac", "") + if not mac: + continue + if mac not in client_dpi: + client_dpi[mac] = {"mac": mac, "hostname": entry.get("hostname", mac), "bytes": 0, "top_cats": []} + client_dpi[mac]["bytes"] += (entry.get("rx_bytes", 0) or 0) + (entry.get("tx_bytes", 0) or 0) + top_dpi_clients = sorted(client_dpi.values(), key=lambda x: x["bytes"], reverse=True)[:10] + + # ── Write to DB ─────────────────────────────────────────────────────────── + async with app.db_sessionmaker() as session: + async with session.begin(): + # WLAN stats + for row in wlan_rows: + session.add(row) + + # DPI snapshot (only if we got data) + if cat_summary or top_dpi_clients: + session.add(UnifiDpiSnapshot( + scraped_at=scraped_at, + by_category_json=json.dumps(cat_summary), + by_client_json=json.dumps(top_dpi_clients), + )) + + # Events (upsert by _id) + _category_map = { + "wlan": "wlan", "wireless": "wlan", + "wired": "wired", + "ugw": "wan", "wan": "wan", + "ids": "ids", + "system": "system", + "ap": "ap", + } + for ev in events: + uid = ev.get("_id", "") + if not uid: + continue + ts_raw = ev.get("datetime") or ev.get("time") + try: + if isinstance(ts_raw, (int, float)): + event_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc) + else: + event_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00")) + except Exception: + event_time = scraped_at + subsystem = ev.get("subsystem", "system") + category = _category_map.get(subsystem, subsystem[:16]) + details = {k: v for k, v in ev.items() + if k not in ("_id", "datetime", "time", "key", "subsystem", "msg", "user", "src_ip")} + existing = await session.get(UnifiEvent, uid) + if not existing: + session.add(UnifiEvent( + unifi_id=uid, + event_time=event_time, + event_key=ev.get("key", ""), + category=category, + message=ev.get("msg", ""), + source_mac=ev.get("user") or ev.get("src_mac"), + source_ip=ev.get("src_ip"), + details_json=json.dumps(details), + )) + + # Alarms (upsert) + for al in alarms: + uid = al.get("_id", "") + if not uid: + continue + ts_raw = al.get("datetime") or al.get("time") + try: + if isinstance(ts_raw, (int, float)): + alarm_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc) + else: + alarm_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00")) + except Exception: + alarm_time = scraped_at + existing = await session.get(UnifiAlarm, uid) + if existing: + existing.archived = al.get("archived", False) + existing.scraped_at = scraped_at + else: + session.add(UnifiAlarm( + unifi_id=uid, + alarm_time=alarm_time, + alarm_key=al.get("key", ""), + message=al.get("msg", ""), + archived=al.get("archived", False), + scraped_at=scraped_at, + )) + + # Known clients (upsert by MAC) + for kc in known_clients: + mac = kc.get("mac", "") + if not mac: + continue + def _ts(val): + if not val: + return None + try: + if isinstance(val, (int, float)): + return datetime.fromtimestamp(val, tz=timezone.utc) + return datetime.fromisoformat(str(val).replace("Z", "+00:00")) + except Exception: + return None + existing = await session.get(UnifiKnownClient, mac) + if existing: + existing.hostname = kc.get("hostname") + existing.name = kc.get("name") + existing.ip = kc.get("ip") + existing.mac_vendor = kc.get("oui") + existing.note = kc.get("note") + existing.is_blocked = kc.get("blocked", False) + existing.first_seen = _ts(kc.get("first_seen")) + existing.last_seen = _ts(kc.get("last_seen")) + existing.scraped_at = scraped_at + else: + session.add(UnifiKnownClient( + mac=mac, + hostname=kc.get("hostname"), + name=kc.get("name"), + ip=kc.get("ip"), + mac_vendor=kc.get("oui"), + note=kc.get("note"), + is_blocked=kc.get("blocked", False), + first_seen=_ts(kc.get("first_seen")), + last_seen=_ts(kc.get("last_seen")), + scraped_at=scraped_at, + )) + + # Speedtest (upsert by run_at) + if speedtest: + ts_raw = speedtest.get("rundate") or speedtest.get("time") + try: + if isinstance(ts_raw, (int, float)): + run_at = datetime.fromtimestamp(ts_raw, tz=timezone.utc) + else: + run_at = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00")) + except Exception: + run_at = scraped_at + existing = await session.get(UnifiSpeedtestResult, run_at) + if not existing: + session.add(UnifiSpeedtestResult( + run_at=run_at, + download_mbps=speedtest.get("xput_download"), + upload_mbps=speedtest.get("xput_upload"), + latency_ms=speedtest.get("latency"), + server_name=speedtest.get("server_name"), + )) + + # Networks (upsert by UniFi ID) + for net in networks: + uid = net.get("_id", "") + if not uid: + continue + existing = await session.get(UnifiNetwork, uid) + if existing: + existing.name = net.get("name", "") + existing.purpose = net.get("purpose") + existing.vlan = net.get("vlan") + existing.ip_subnet = net.get("ip_subnet") + existing.dhcp_enabled = net.get("dhcpd_enabled", False) + existing.is_guest = net.get("is_guest", False) + existing.scraped_at = scraped_at + else: + session.add(UnifiNetwork( + unifi_id=uid, + name=net.get("name", ""), + purpose=net.get("purpose"), + vlan=net.get("vlan"), + ip_subnet=net.get("ip_subnet"), + dhcp_enabled=net.get("dhcpd_enabled", False), + is_guest=net.get("is_guest", False), + scraped_at=scraped_at, + )) + + # Port forwards (upsert by UniFi ID) + for pf in port_forwards: + uid = pf.get("_id", "") + if not uid: + continue + existing = await session.get(UnifiPortForward, uid) + if existing: + existing.name = pf.get("name") + existing.proto = pf.get("proto", "tcp") + existing.dst_port = pf.get("dst_port", "") + existing.fwd_ip = pf.get("fwd", "") + existing.fwd_port = pf.get("fwd_port", "") + existing.src_filter = pf.get("src") + existing.enabled = pf.get("enabled", True) + existing.scraped_at = scraped_at + else: + session.add(UnifiPortForward( + unifi_id=uid, + name=pf.get("name"), + proto=pf.get("proto", "tcp"), + dst_port=pf.get("dst_port", ""), + fwd_ip=pf.get("fwd", ""), + fwd_port=pf.get("fwd_port", ""), + src_filter=pf.get("src"), + enabled=pf.get("enabled", True), + scraped_at=scraped_at, + )) + + # Firewall rules (upsert by UniFi ID) + for rule in fw_rules: + uid = rule.get("_id", "") + if not uid: + continue + existing = await session.get(UnifiFwRule, uid) + if existing: + existing.name = rule.get("name", "") + existing.ruleset = rule.get("ruleset", "") + existing.rule_index = rule.get("rule_index") + existing.action = rule.get("action", "") + existing.protocol = rule.get("protocol") + existing.enabled = rule.get("enabled", True) + existing.src_address = rule.get("src_address") + existing.dst_address = rule.get("dst_address") + existing.scraped_at = scraped_at + else: + session.add(UnifiFwRule( + unifi_id=uid, + name=rule.get("name", ""), + ruleset=rule.get("ruleset", ""), + rule_index=rule.get("rule_index"), + action=rule.get("action", ""), + protocol=rule.get("protocol"), + enabled=rule.get("enabled", True), + src_address=rule.get("src_address"), + dst_address=rule.get("dst_address"), + scraped_at=scraped_at, + )) + + logger.debug( + "UniFi expanded: wlans=%d, events=%d, alarms=%d, known_clients=%d, " + "networks=%d, port_fwds=%d, fw_rules=%d", + len(wlan_rows), len(events), len(alarms), len(known_clients), + len(networks), len(port_forwards), len(fw_rules), + ) diff --git a/plugins/unifi/templates/unifi/dpi.html b/plugins/unifi/templates/unifi/dpi.html new file mode 100644 index 0000000..6e085b7 --- /dev/null +++ b/plugins/unifi/templates/unifi/dpi.html @@ -0,0 +1,65 @@ +{# plugins/unifi/templates/unifi/dpi.html #} +{% if not snapshot %} +
+

No DPI data yet. DPI must be enabled on the UniFi controller.

+
+{% else %} +
+ + {# Category breakdown #} + {% if categories %} +
+
+ Traffic by Category + {{ snapshot.scraped_at.strftime("%H:%M") }} +
+ {% set total_bytes = categories | sum(attribute='rx_bytes') + categories | sum(attribute='tx_bytes') %} + + {% for cat in categories[:20] %} + {% set bytes = cat.rx_bytes + cat.tx_bytes %} + {% set pct = (bytes / total_bytes * 100) if total_bytes else 0 %} + + + + + + {% endfor %} +
{{ cat.cat_name or cat.cat_id }} + {% if bytes >= 1073741824 %}{{ "%.1f"|format(bytes / 1073741824) }} GB + {% elif bytes >= 1048576 %}{{ "%.1f"|format(bytes / 1048576) }} MB + {% elif bytes >= 1024 %}{{ "%.0f"|format(bytes / 1024) }} KB + {% else %}{{ bytes }} B{% endif %} + +
+
+
+
+
+ {% endif %} + + {# Top clients by DPI #} + {% if top_clients %} +
+
Top Clients by Traffic
+ + {% for c in top_clients %} + + + + + {% endfor %} +
+ {{ c.hostname or c.mac }} + {{ c.mac }} + + {% set bytes = c.bytes %} + {% if bytes >= 1073741824 %}{{ "%.1f"|format(bytes / 1073741824) }} GB + {% elif bytes >= 1048576 %}{{ "%.1f"|format(bytes / 1048576) }} MB + {% elif bytes >= 1024 %}{{ "%.0f"|format(bytes / 1024) }} KB + {% else %}{{ bytes }} B{% endif %} +
+
+ {% endif %} + +
+{% endif %} diff --git a/plugins/unifi/templates/unifi/events.html b/plugins/unifi/templates/unifi/events.html new file mode 100644 index 0000000..79c4a21 --- /dev/null +++ b/plugins/unifi/templates/unifi/events.html @@ -0,0 +1,52 @@ +{# plugins/unifi/templates/unifi/events.html #} +{% if not events %} +
+

No events in the last {{ range_key }}.

+
+{% else %} +
+
+ Site Events + {{ events|length }} in {{ range_key }} +
+ + + + + + + + + + + + + + {% for ev in events %} + + + + + + + {% endfor %} +
TimeCategoryKeyMessage
+ {{ ev.event_time.strftime("%Y-%m-%d %H:%M:%S") }} + + + {{ ev.category }} + + + {{ ev.event_key.replace("EVT_", "") if ev.event_key else "" }} + + {{ ev.message }} + {% if ev.source_mac %} + {{ ev.source_mac }} + {% endif %} +
+
+{% endif %} diff --git a/plugins/unifi/templates/unifi/index.html b/plugins/unifi/templates/unifi/index.html new file mode 100644 index 0000000..b2113b6 --- /dev/null +++ b/plugins/unifi/templates/unifi/index.html @@ -0,0 +1,63 @@ +{# plugins/unifi/templates/unifi/index.html #} +{% extends "base.html" %} +{% block title %}UniFi — Fabled Scryer{% endblock %} +{% block content %} +
+
+

UniFi Network

+ polling every {{ poll_interval }}s +
+ {% include "_time_range.html" %} +
+ +{# Tab nav #} +
+ {% set tabs = [ + ("overview", "Overview", "/plugins/unifi/rows"), + ("events", "Events", "/plugins/unifi/events"), + ("dpi", "Traffic", "/plugins/unifi/dpi"), + ("security", "Security", "/plugins/unifi/security"), + ("inventory", "Inventory", "/plugins/unifi/inventory"), + ] %} + {% for tab_id, label, url in tabs %} + + {% endfor %} +
+ +
+
+ + +{% endblock %} diff --git a/plugins/unifi/templates/unifi/inventory.html b/plugins/unifi/templates/unifi/inventory.html new file mode 100644 index 0000000..62e99f7 --- /dev/null +++ b/plugins/unifi/templates/unifi/inventory.html @@ -0,0 +1,146 @@ +{# plugins/unifi/templates/unifi/inventory.html #} +
+ + {# Networks / VLANs #} + {% if networks %} +
+
Networks / VLANs
+ + + + + + + {% for net in networks %} + + + + + + {% endfor %} +
NameVLANSubnet
+ {{ net.name }} + {% if net.is_guest %} guest{% endif %} + + {{ net.vlan or "—" }} + + {{ net.ip_subnet or "—" }} +
+
+ {% endif %} + + {# Port forwards #} + {% if port_forwards %} +
+
Port Forwards
+ + {% for pf in port_forwards %} + + + + + {% endfor %} +
+ + {{ pf.name or "—" }} + + {% if not pf.enabled %} disabled{% endif %} + + {{ pf.proto }}:{{ pf.dst_port }} → {{ pf.fwd_ip }}:{{ pf.fwd_port }} +
+
+ {% endif %} + + {# Firewall rules, grouped by ruleset #} + {% if fw_rules %} + {% set ns = namespace(current_rs=None) %} +
+
Firewall Rules
+ + {% for rule in fw_rules %} + {% if rule.ruleset != ns.current_rs %} + {% set ns.current_rs = rule.ruleset %} + + + + {% endif %} + + + + + + {% endfor %} +
+ {{ rule.ruleset }} +
+ + {{ rule.name }} + + + + {{ rule.action }} + + + {% if rule.protocol %}{{ rule.protocol }}{% endif %} + {% if rule.src_address %} src:{{ rule.src_address }}{% endif %} + {% if rule.dst_address %} dst:{{ rule.dst_address }}{% endif %} +
+
+ {% endif %} + + {# Known clients #} + {% if known_clients %} +
+
+ Known Clients + {{ known_clients|length }} total +
+ + + + + + + {% for kc in known_clients[:50] %} + + + + + + {% endfor %} + {% if known_clients|length > 50 %} + + + + {% endif %} +
Name / MACIPLast Seen
+
+ {{ kc.name or kc.hostname or kc.mac }} + {% if kc.is_blocked %} blocked{% endif %} +
+ {% if kc.name or kc.hostname %} +
{{ kc.mac }}
+ {% endif %} + {% if kc.mac_vendor %} +
{{ kc.mac_vendor }}
+ {% endif %} +
+ {{ kc.ip or "—" }} + + {% if kc.last_seen %}{{ kc.last_seen.strftime("%m-%d %H:%M") }}{% else %}—{% endif %} +
+ +{{ known_clients|length - 50 }} more +
+
+ {% endif %} + + {% if not networks and not port_forwards and not fw_rules and not known_clients %} +
+

No inventory data yet. This populates on the next poll cycle.

+
+ {% endif %} + +
diff --git a/plugins/unifi/templates/unifi/rows.html b/plugins/unifi/templates/unifi/rows.html new file mode 100644 index 0000000..9de46ce --- /dev/null +++ b/plugins/unifi/templates/unifi/rows.html @@ -0,0 +1,203 @@ +{# plugins/unifi/templates/unifi/rows.html #} + +{# ── WAN health bar ───────────────────────────────────────────────────────── #} +{% if latest_wan %} +
+ + WAN + + {% if latest_wan.is_up %}✓ Up{% else %}✗ Down{% endif %} + + {% if latest_wan.wan_ip %} + {{ latest_wan.wan_ip }} + {% endif %} + + + {% if latest_wan.latency_ms is not none %} + + Latency + + {{ "%.1f"|format(latest_wan.latency_ms) }}ms + + {{ sparkline_latency | safe }} + + {% endif %} + + {% if latest_wan.rx_bytes_rate is not none %} + + ↓ RX + + {% set bps = latest_wan.rx_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s + {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }} KB/s + {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %} + + {{ sparkline_rx | safe }} + + + ↑ TX + + {% set bps = latest_wan.tx_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s + {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }} KB/s + {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %} + + {{ sparkline_tx | safe }} + + {% endif %} + + {% if latest_wan.uptime_seconds is not none %} + + up {{ (latest_wan.uptime_seconds // 3600) }}h {{ ((latest_wan.uptime_seconds % 3600) // 60) }}m + + {% endif %} +
+{% endif %} + +{# ── Content columns ──────────────────────────────────────────────────────── #} +
+ + {# Client summary card #} + {% if latest_clients %} +
+
+ Clients + {{ sparkline_clients | safe }} +
+
+ + {{ latest_clients.total_clients }} + total + + + {{ latest_clients.wireless_clients }}↗ wireless   + {{ latest_clients.wired_clients }}↔ wired + {% if latest_clients.guest_clients %}  {{ latest_clients.guest_clients }} guest{% endif %} + +
+
+ {% endif %} + + {# Infrastructure devices card #} + {% if devices %} +
+
Infrastructure Devices
+ + {% for d in devices %} + + + + + + {% if d.state == 1 and d.uptime_seconds %} + + + + {% endif %} + {% endfor %} +
+ + {{ d.name or d.mac }} + {{ d.model or d.device_type or "" }}{{ d.ip or "" }}
+ up {{ (d.uptime_seconds // 86400) }}d {{ ((d.uptime_seconds % 86400) // 3600) }}h +
+
+ {% endif %} + + {# Top talkers card #} + {% if top_clients %} +
+
Top Talkers
+ + {% for c in top_clients %} + + + + + {% endfor %} +
+ {{ c.hostname }} + {% if c.type == "wireless" and c.signal %} + {{ c.signal }}dBm + {% endif %} + + {% set rx = c.rx_rate %}{% set tx = c.tx_rate %} + + {% if rx >= 1048576 %}{{ "%.1f"|format(rx/1048576) }}M + {% elif rx >= 1024 %}{{ "%.0f"|format(rx/1024) }}K + {% else %}{{ "%.0f"|format(rx) }}B{% endif %} + + {% if tx >= 1048576 %}{{ "%.1f"|format(tx/1048576) }}M + {% elif tx >= 1024 %}{{ "%.0f"|format(tx/1024) }}K + {% else %}{{ "%.0f"|format(tx) }}B{% endif %} +
+
+ {% endif %} + + {# WLAN SSIDs card #} + {% if wlan_latest %} +
+
Wireless Networks
+ + + + + + + {% for w in wlan_latest %} + + + + + + {% endfor %} +
SSIDClientsScore
{{ w.ssid }}{{ w.num_sta }} + {% if w.satisfaction is not none %}{{ w.satisfaction }}%{% else %}—{% endif %} +
+
+ {% endif %} + + {# Speedtest card #} + {% if speedtest %} +
+
Last Speed Test
+
+ {% if speedtest.download_mbps is not none %} + + ↓ DL + {{ "%.1f"|format(speedtest.download_mbps) }} + Mbps + + {% endif %} + {% if speedtest.upload_mbps is not none %} + + ↑ UL + {{ "%.1f"|format(speedtest.upload_mbps) }} + Mbps + + {% endif %} + {% if speedtest.latency_ms is not none %} + + Latency + {{ "%.0f"|format(speedtest.latency_ms) }}ms + + {% endif %} +
+ {% if speedtest.server_name %} +
via {{ speedtest.server_name }}
+ {% endif %} +
{{ speedtest.run_at.strftime("%Y-%m-%d %H:%M") }} UTC
+
+ {% endif %} + +
+ +{% if not latest_wan and not devices %} +
+

No UniFi data yet. Check your host, username, and password in the plugin config.

+
+{% endif %} diff --git a/plugins/unifi/templates/unifi/security.html b/plugins/unifi/templates/unifi/security.html new file mode 100644 index 0000000..f28d4c0 --- /dev/null +++ b/plugins/unifi/templates/unifi/security.html @@ -0,0 +1,56 @@ +{# plugins/unifi/templates/unifi/security.html #} +
+ + {# Active alarms #} +
+
+ Active Alarms + {% if active_alarms %} + {{ active_alarms|length }} + {% endif %} +
+ {% if not active_alarms %} +

✓ No active alarms

+ {% else %} + + {% for al in active_alarms %} + + + + {% endfor %} +
+
+ ⚠ {{ al.alarm_key }} +
+
{{ al.message }}
+
+ {{ al.alarm_time.strftime("%Y-%m-%d %H:%M") }} UTC +
+
+ {% endif %} +
+ + {# Recent archived alarms #} + {% if archived_alarms %} +
+
+ Recent Archived Alarms + last {{ archived_alarms|length }} +
+ + {% for al in archived_alarms %} + + + + + {% endfor %} +
+
{{ al.alarm_key }}
+
{{ al.message }}
+
+ {{ al.alarm_time.strftime("%m-%d %H:%M") }} +
+
+ {% endif %} + +
diff --git a/plugins/unifi/templates/unifi/widget.html b/plugins/unifi/templates/unifi/widget.html new file mode 100644 index 0000000..2993ea6 --- /dev/null +++ b/plugins/unifi/templates/unifi/widget.html @@ -0,0 +1,95 @@ +{# plugins/unifi/templates/unifi/widget.html #} + +{# Active alarms #} +{% for al in active_alarms %} +
+ ⚠ {{ al.alarm_key }} + alarm +
+{% endfor %} +{# Offline device alerts #} +{% for d in offline_devices %} +
+ ⚠ {{ d.name or d.mac }} + offline +
+{% endfor %} +{% if active_alarms or offline_devices %}
{% endif %} + +{% if latest_wan %} +
+
+ WAN +
+
+ + {% if latest_wan.is_up %}Up{% else %}Down{% endif %} + + {% if latest_wan.latency_ms is not none %} + + lat + + {{ "%.0f"|format(latest_wan.latency_ms) }}ms + + + {% endif %} + {% if latest_wan.rx_bytes_rate is not none %} + + + {% set bps = latest_wan.rx_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s + {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s + {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %} + + + + {% set bps = latest_wan.tx_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s + {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s + {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %} + + {% endif %} +
+
+{% endif %} + +{% if latest_clients %} +
+
+ Clients +
+
+ {{ latest_clients.total_clients }} + {{ latest_clients.wireless_clients }}w / {{ latest_clients.wired_clients }}e +
+
+{% endif %} + +{% if top_clients %} +
+ {% for c in top_clients %} +
+
+ {{ c.hostname }} +
+
+ {% set total_bps = c.tx_rate + c.rx_rate %} + + {% set rx = c.rx_rate %} + {% if rx >= 1048576 %}{{ "%.1f"|format(rx / 1048576) }}M + {% elif rx >= 1024 %}{{ "%.0f"|format(rx / 1024) }}K + {% else %}{{ "%.0f"|format(rx) }}B{% endif %} + + {% set tx = c.tx_rate %} + {% if tx >= 1048576 %}{{ "%.1f"|format(tx / 1048576) }}M + {% elif tx >= 1024 %}{{ "%.0f"|format(tx / 1024) }}K + {% else %}{{ "%.0f"|format(tx) }}B{% endif %} +
+
+ {% endfor %} +
+{% endif %} + +{% if not latest_wan and not latest_clients and not active_alarms and not offline_devices %} +

No UniFi data yet.

+{% endif %} diff --git a/plugins/unifi/templates/unifi/widget_clients.html b/plugins/unifi/templates/unifi/widget_clients.html new file mode 100644 index 0000000..99a5892 --- /dev/null +++ b/plugins/unifi/templates/unifi/widget_clients.html @@ -0,0 +1,42 @@ +{# unifi/widget_clients.html — dashboard widget: top active clients #} +{% if not snapshot %} +
No client data yet.
+{% else %} + +
+
+ {{ snapshot.total_clients }} + total +
+
+ {{ snapshot.wireless_clients }} + wireless +
+
+ {{ snapshot.wired_clients }} + wired +
+
+ +{% if clients %} +
+ {% for c in clients %} +
+ + + {{ c.hostname or c.ip or c.mac }} + + {% if c.tx_rate or c.rx_rate %} + + {% if c.tx_rate %}↑{{ "%.0f" | format(c.tx_rate / 1000) }}k{% endif %} + {% if c.rx_rate %} ↓{{ "%.0f" | format(c.rx_rate / 1000) }}k{% endif %} + + {% endif %} +
+ {% endfor %} +
+{% else %} +
No clients connected.
+{% endif %} + +{% endif %} diff --git a/plugins/unifi/templates/unifi/widget_devices.html b/plugins/unifi/templates/unifi/widget_devices.html new file mode 100644 index 0000000..90994ed --- /dev/null +++ b/plugins/unifi/templates/unifi/widget_devices.html @@ -0,0 +1,36 @@ +{# unifi/widget_devices.html — dashboard widget: infrastructure devices #} +{% if not devices %} +
+ {% if type_filter != "all" %}No {{ type_filter }} devices found.{% else %}No devices found.{% endif %} +
+{% else %} + +{% set online = devices | selectattr("state", "equalto", 1) | list %} +{% set offline = devices | rejectattr("state", "equalto", 1) | list %} + +
+
+ {{ online | length }} + online +
+ {% if offline %} +
+ {{ offline | length }} + offline +
+ {% endif %} +
+ +
+ {% for d in devices %} +
+ + + {{ d.name or d.mac }} + + {{ d.model or d.device_type or "" }} +
+ {% endfor %} +
+ +{% endif %} diff --git a/steward/app.py b/steward/app.py index 676593a..b1ad710 100644 --- a/steward/app.py +++ b/steward/app.py @@ -20,13 +20,15 @@ def create_app( bootstrap = { "database_url": "postgresql+asyncpg://test/test", "secret_key": "test-secret-key", - "plugin_dir": "plugins", + "plugin_dirs": ["plugins"], + "plugin_install_dir": "plugins", } app.config.update( SECRET_KEY=bootstrap["secret_key"], DATABASE_URL=bootstrap["database_url"], - PLUGIN_DIR=bootstrap["plugin_dir"], + PLUGIN_DIRS=bootstrap["plugin_dirs"], + PLUGIN_INSTALL_DIR=bootstrap["plugin_install_dir"], TESTING=testing, ) @@ -42,7 +44,7 @@ def create_app( from .core.migration_runner import run_core_migrations run_core_migrations( app.config["DATABASE_URL"], - plugin_dir=Path(app.config["PLUGIN_DIR"]).resolve(), + plugin_dirs=[Path(d).resolve() for d in app.config["PLUGIN_DIRS"]], ) # ── 4. Load all settings from DB → populate app.config ──────────────────── @@ -81,9 +83,9 @@ def create_app( # so this is a no-op on normal startup. We still run it with all discovered dirs so # Alembic can resolve the full revision graph regardless of which plugins are enabled. if not testing: - from .core.migration_runner import run_plugin_migrations, discover_all_plugin_migration_dirs - _plugin_dir = Path(app.config["PLUGIN_DIR"]).resolve() - _all_plugin_dirs = discover_all_plugin_migration_dirs(_plugin_dir) + from .core.migration_runner import run_plugin_migrations, discover_all_in + _plugin_dirs = [Path(d).resolve() for d in app.config["PLUGIN_DIRS"]] + _all_plugin_dirs = discover_all_in(_plugin_dirs) run_plugin_migrations(app.config["DATABASE_URL"], _all_plugin_dirs) # ── 6. Alert pipeline ────────────────────────────────────────────────────── diff --git a/steward/config.py b/steward/config.py index b119a25..9c8f1fd 100644 --- a/steward/config.py +++ b/steward/config.py @@ -15,7 +15,7 @@ def _env(suffix: str) -> str | None: return os.environ.get(f"STEWARD_{suffix}") -def load_bootstrap(config_path: Path | str | None = None) -> dict[str, str]: +def load_bootstrap(config_path: Path | str | None = None) -> dict[str, Any]: """Return the minimum bootstrap config: database_url and secret_key. This is the only config read from files/env vars at startup. @@ -48,15 +48,29 @@ def load_bootstrap(config_path: Path | str | None = None) -> dict[str, str]: ) secret_key = _resolve_secret_key(raw) - plugin_dir = ( + + # Plugin discovery spans two roots (see load_plugins / migration_runner): + # • bundled — first-party plugins shipped inside the image at repo-root + # `plugins/`; they version atomically with core and are read-only at runtime. + # • external — operator-mounted dir for third-party plugins, persisted in the + # /data volume. Downloads/installs land here, never in the bundled dir. + # Bundled is scanned first, so on a name collision the first-party plugin wins. + bundled_plugin_dir = raw.get("plugin_dir", "plugins") + external_plugin_dir = ( _env("PLUGIN_DIR") - or raw.get("plugin_dir", "plugins") + or raw.get("external_plugin_dir") + or "/data/plugins" ) + plugin_dirs = [bundled_plugin_dir] + if external_plugin_dir and external_plugin_dir != bundled_plugin_dir: + plugin_dirs.append(external_plugin_dir) return { "database_url": database_url, "secret_key": secret_key, - "plugin_dir": plugin_dir, + "plugin_dirs": plugin_dirs, + # Installs/downloads target the external (writable, persistent) dir. + "plugin_install_dir": external_plugin_dir or bundled_plugin_dir, } diff --git a/steward/core/migration_runner.py b/steward/core/migration_runner.py index d8e513c..c382ff5 100644 --- a/steward/core/migration_runner.py +++ b/steward/core/migration_runner.py @@ -24,14 +24,26 @@ def discover_all_plugin_migration_dirs(plugin_dir: Path) -> list[Path]: return dirs -def run_core_migrations(db_url: str, plugin_dir: Path | None = None) -> None: +def discover_all_in(plugin_dirs: list[Path]) -> list[Path]: + """discover_all_plugin_migration_dirs across multiple plugin roots. + + Plugins now live in more than one root (bundled + external), so the full + revision graph is the union of every root's migration dirs. + """ + dirs: list[Path] = [] + for pd in plugin_dirs: + dirs.extend(discover_all_plugin_migration_dirs(pd)) + return dirs + + +def run_core_migrations(db_url: str, plugin_dirs: list[Path] | None = None) -> None: """Run core Alembic migrations. - Includes all discovered plugin migration dirs (if plugin_dir given) so + Includes all discovered plugin migration dirs (across every plugin root) so Alembic can resolve any previously-applied plugin revisions in the graph. Called first so the app_settings table exists before loading settings. """ - dirs = discover_all_plugin_migration_dirs(plugin_dir) if plugin_dir else [] + dirs = discover_all_in(plugin_dirs) if plugin_dirs else [] _run(db_url, plugin_migration_dirs=dirs) diff --git a/steward/core/plugin_manager.py b/steward/core/plugin_manager.py index 2bc0e2c..b0c6e6d 100644 --- a/steward/core/plugin_manager.py +++ b/steward/core/plugin_manager.py @@ -31,6 +31,19 @@ def get_plugin_failures() -> dict[str, str]: return dict(_FAILED_PLUGINS) +def resolve_plugin_path(plugin_dirs: list[Path], name: str) -> Path | None: + """Return the first plugin root that contains `name`, else None. + + Roots are searched in order, so a bundled (first-party) plugin shadows an + external plugin of the same name. Pure function — no app/IO beyond exists(). + """ + for pd in plugin_dirs: + candidate = pd / name + if candidate.exists(): + return candidate + return None + + def _import_plugin(name: str, plugin_path: Path): """Load a plugin module by file path, avoiding sys.modules stdlib collisions. @@ -80,22 +93,24 @@ def load_plugins(app: "Quart") -> None: """ import steward - plugin_dir = Path(app.config["PLUGIN_DIR"]) + plugin_dirs = [Path(d) for d in app.config["PLUGIN_DIRS"]] plugins_cfg: dict = app.config["PLUGINS"] - # Ensure plugin_dir is on sys.path so plugins are importable by name - plugin_dir_str = str(plugin_dir.resolve()) - if plugin_dir_str not in sys.path: - sys.path.insert(0, plugin_dir_str) + # Ensure every plugin root is on sys.path so plugins are importable by name + for pd in plugin_dirs: + pd_str = str(pd.resolve()) + if pd_str not in sys.path: + sys.path.insert(0, pd_str) for name, cfg in list(plugins_cfg.items()): if not cfg.get("enabled", False): continue - plugin_path = plugin_dir / name - if not plugin_path.exists(): - _FAILED_PLUGINS[name] = f"Plugin directory not found: {plugin_path}" - logger.error("Plugin %r: directory %s not found, skipping", name, plugin_path) + plugin_path = resolve_plugin_path(plugin_dirs, name) + if plugin_path is None: + roots = ", ".join(str(d) for d in plugin_dirs) + _FAILED_PLUGINS[name] = f"Plugin directory not found in: {roots}" + logger.error("Plugin %r: not found in any plugin root (%s), skipping", name, roots) continue # Load and validate plugin.yaml @@ -224,10 +239,12 @@ async def download_and_install_plugin( """ import httpx - plugin_dir = Path(app.config["PLUGIN_DIR"]).resolve() + # Downloads always land in the external (writable, persistent) install dir, + # never the read-only bundled dir. + plugin_dir = Path(app.config["PLUGIN_INSTALL_DIR"]).resolve() - # Ensure plugin_dir is on sys.path (may not be set yet if no plugins were - # enabled at startup) + # Ensure the install dir is on sys.path (may not be set yet if no plugins + # were enabled at startup) plugin_dir_str = str(plugin_dir) if plugin_dir_str not in sys.path: sys.path.insert(0, plugin_dir_str) @@ -304,9 +321,11 @@ async def download_and_install_plugin( if mdir.exists(): from steward.core.migration_runner import ( run_plugin_migrations, - discover_all_plugin_migration_dirs, + discover_all_in, ) - all_dirs = discover_all_plugin_migration_dirs(plugin_dir) + # Span every plugin root so Alembic can resolve bundled-plugin + # revisions already stamped in alembic_version. + all_dirs = discover_all_in([Path(d).resolve() for d in app.config["PLUGIN_DIRS"]]) run_plugin_migrations(app.config["DATABASE_URL"], all_dirs) except Exception: logger.exception("Plugin %r: migration failed after install", name) @@ -330,11 +349,11 @@ def hot_reload_plugin(app: "Quart", name: str) -> tuple[bool, str]: if name in _LOADED_PLUGINS: return False, "Plugin already loaded — restart required to apply updates" - plugin_dir = Path(app.config["PLUGIN_DIR"]).resolve() - plugin_path = plugin_dir / name + plugin_dirs = [Path(d).resolve() for d in app.config["PLUGIN_DIRS"]] + plugin_path = resolve_plugin_path(plugin_dirs, name) - if not plugin_path.exists(): - return False, f"Plugin directory {plugin_path} not found" + if plugin_path is None: + return False, f"Plugin {name!r} not found in any plugin root" yaml_path = plugin_path / "plugin.yaml" if not yaml_path.exists(): @@ -383,9 +402,9 @@ def hot_reload_plugin(app: "Quart", name: str) -> tuple[bool, str]: try: from steward.core.migration_runner import ( run_plugin_migrations, - discover_all_plugin_migration_dirs, + discover_all_in, ) - all_dirs = discover_all_plugin_migration_dirs(plugin_dir) + all_dirs = discover_all_in(plugin_dirs) run_plugin_migrations(app.config["DATABASE_URL"], all_dirs) except Exception: logger.exception("Plugin %r: migration failed during hot-reload", name) diff --git a/steward/settings/routes.py b/steward/settings/routes.py index c3e5b58..8434b70 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -19,22 +19,32 @@ logger = logging.getLogger(__name__) def _discover_plugins() -> list[dict]: - """Scan PLUGIN_DIR for plugin.yaml files.""" + """Scan every plugin root for plugin.yaml files. + + Roots are scanned in order (bundled first), and a plugin name found in an + earlier root shadows the same name in a later one — matching load_plugins. + """ import yaml - plugin_dir = Path(current_app.config.get("PLUGIN_DIR", "plugins")).resolve() - plugins = [] - if not plugin_dir.exists(): - return plugins - for entry in sorted(plugin_dir.iterdir()): - yaml_path = entry / "plugin.yaml" - if entry.is_dir() and yaml_path.exists(): - try: - with yaml_path.open() as f: - meta = yaml.safe_load(f) or {} - meta["_dir"] = entry.name - plugins.append(meta) - except Exception: - logger.warning("Could not read %s", yaml_path) + plugin_dirs = [ + Path(d).resolve() + for d in current_app.config.get("PLUGIN_DIRS", ["plugins"]) + ] + plugins: list[dict] = [] + seen: set[str] = set() + for plugin_dir in plugin_dirs: + if not plugin_dir.exists(): + continue + for entry in sorted(plugin_dir.iterdir()): + yaml_path = entry / "plugin.yaml" + if entry.is_dir() and yaml_path.exists() and entry.name not in seen: + try: + with yaml_path.open() as f: + meta = yaml.safe_load(f) or {} + meta["_dir"] = entry.name + plugins.append(meta) + seen.add(entry.name) + except Exception: + logger.warning("Could not read %s", yaml_path) return plugins diff --git a/tests/test_plugin_dirs.py b/tests/test_plugin_dirs.py new file mode 100644 index 0000000..65d7cee --- /dev/null +++ b/tests/test_plugin_dirs.py @@ -0,0 +1,84 @@ +"""Unit tests for the bundled + external multi-root plugin model. + +Pure-function coverage (no DB, no app): path resolution, multi-root migration +discovery, and the config split into bundled/external roots + install dir. +""" +from __future__ import annotations + +from pathlib import Path + +from steward.core.migration_runner import discover_all_in +from steward.core.plugin_manager import resolve_plugin_path + + +def _make_plugin(root: Path, name: str, with_migrations: bool = False) -> None: + pdir = root / name + pdir.mkdir(parents=True) + (pdir / "plugin.yaml").write_text(f"name: {name}\n") + if with_migrations: + (pdir / "migrations" / "versions").mkdir(parents=True) + + +def test_resolve_plugin_path_prefers_earlier_root(tmp_path): + bundled = tmp_path / "bundled" + external = tmp_path / "external" + _make_plugin(bundled, "dupe") + _make_plugin(external, "dupe") + _make_plugin(external, "only_external") + + # Bundled is listed first → it shadows the external copy of the same name. + assert resolve_plugin_path([bundled, external], "dupe") == bundled / "dupe" + # A name only present externally still resolves. + assert resolve_plugin_path([bundled, external], "only_external") == external / "only_external" + + +def test_resolve_plugin_path_missing_returns_none(tmp_path): + assert resolve_plugin_path([tmp_path], "nope") is None + + +def test_discover_all_in_unions_roots(tmp_path): + bundled = tmp_path / "bundled" + external = tmp_path / "external" + _make_plugin(bundled, "a", with_migrations=True) + _make_plugin(bundled, "no_mig") # no migrations dir → not discovered + _make_plugin(external, "b", with_migrations=True) + + found = discover_all_in([bundled, external]) + assert (bundled / "a" / "migrations") in found + assert (external / "b" / "migrations") in found + assert (bundled / "no_mig" / "migrations") not in found + + +def test_discover_all_in_skips_missing_root(tmp_path): + bundled = tmp_path / "bundled" + _make_plugin(bundled, "a", with_migrations=True) + # A non-existent external root must not raise. + found = discover_all_in([bundled, tmp_path / "does_not_exist"]) + assert found == [bundled / "a" / "migrations"] + + +def test_bootstrap_splits_bundled_and_external(tmp_path, monkeypatch): + from steward.config import load_bootstrap + + monkeypatch.setenv("STEWARD_DATABASE_URL", "postgresql+asyncpg://x/y") + monkeypatch.setenv("STEWARD_SECRET_KEY", "k" * 32) + monkeypatch.setenv("STEWARD_PLUGIN_DIR", "/srv/external-plugins") + + boot = load_bootstrap(config_path=tmp_path / "absent.yaml") + + assert boot["plugin_dirs"] == ["plugins", "/srv/external-plugins"] + # Installs target the external (writable) root, never the bundled one. + assert boot["plugin_install_dir"] == "/srv/external-plugins" + + +def test_bootstrap_external_defaults_to_data_plugins(tmp_path, monkeypatch): + from steward.config import load_bootstrap + + monkeypatch.setenv("STEWARD_DATABASE_URL", "postgresql+asyncpg://x/y") + monkeypatch.setenv("STEWARD_SECRET_KEY", "k" * 32) + monkeypatch.delenv("STEWARD_PLUGIN_DIR", raising=False) + + boot = load_bootstrap(config_path=tmp_path / "absent.yaml") + + assert boot["plugin_dirs"] == ["plugins", "/data/plugins"] + assert boot["plugin_install_dir"] == "/data/plugins"