diff --git a/.gitignore b/.gitignore
index fb2b9af..b4b5ccd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,9 +1,10 @@
# Planning files
docs/superpowers/
-# Plugin directory — managed as a separate repo (bvandeusen/steward-plugins)
-# PLUGIN_DIR in config.yaml points here at runtime
-/plugins/
+# First-party plugins are tracked in-tree under plugins/ and ship in the image.
+# Third-party plugins are mounted at runtime into the external dir
+# (STEWARD_PLUGIN_DIR, default /data/plugins) and are not tracked here.
+/plugins/**/*.zip
# Python
__pycache__/
@@ -50,3 +51,6 @@ Thumbs.db
# Playbook cache (runtime data)
playbook_cache/
+
+# Superpowers brainstorm scratch (visual companion mockups, state)
+.superpowers/
diff --git a/Dockerfile b/Dockerfile
index 938501c..4172e1d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -17,6 +17,10 @@ COPY steward/ steward/
RUN pip install --no-cache-dir .
COPY alembic.ini .
+# First-party plugins ship inside the image (bundled root at /app/plugins).
+# Third-party plugins are mounted at runtime into the external root
+# (STEWARD_PLUGIN_DIR, default /data/plugins).
+COPY plugins/ plugins/
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
diff --git a/docker-compose.yml b/docker-compose.yml
index faedd9f..3e1ce35 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -6,8 +6,10 @@ services:
ports:
- "5000:5000"
volumes:
+ # First-party plugins are baked into the image at /app/plugins.
+ # Third-party plugins persist in the external root /data/plugins
+ # (inside app_data); STEWARD_PLUGIN_DIR defaults there, no extra mount needed.
- app_data:/data
- - ./plugins:/app/plugins:ro
- /mnt/Data/traefik/log:/var/log/traefik:ro
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
diff --git a/plugins/__init__.py b/plugins/__init__.py
new file mode 100644
index 0000000..1648f8e
--- /dev/null
+++ b/plugins/__init__.py
@@ -0,0 +1 @@
+# plugins/__init__.py
diff --git a/plugins/docker/__init__.py b/plugins/docker/__init__.py
new file mode 100644
index 0000000..d569556
--- /dev/null
+++ b/plugins/docker/__init__.py
@@ -0,0 +1,24 @@
+# plugins/docker/__init__.py
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from quart import Quart
+
+_app: "Quart | None" = None
+
+
+def setup(app: "Quart") -> None:
+ global _app
+ _app = app
+ from .models import DockerContainer, DockerMetric # noqa: F401 — register with Base
+
+
+def get_scheduled_tasks() -> list:
+ from .scheduler import make_task
+ return [make_task(_app)]
+
+
+def get_blueprint():
+ from .routes import docker_bp
+ return docker_bp
diff --git a/plugins/docker/migrations/__init__.py b/plugins/docker/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/plugins/docker/migrations/env.py b/plugins/docker/migrations/env.py
new file mode 100644
index 0000000..5c20755
--- /dev/null
+++ b/plugins/docker/migrations/env.py
@@ -0,0 +1,70 @@
+# plugins/docker/migrations/env.py
+"""Alembic env.py for the Docker plugin."""
+import asyncio
+import os
+import sys
+from pathlib import Path
+from logging.config import fileConfig
+
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+from alembic import context
+
+sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
+
+from fabledscryer.models.base import Base
+import fabledscryer.models # noqa: F401
+from plugins.docker.models import DockerContainer, DockerMetric # noqa: F401
+
+config = context.config
+if config.config_file_name is not None:
+ fileConfig(config.config_file_name)
+
+target_metadata = Base.metadata
+
+
+def _get_url() -> str:
+ import yaml
+ cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml")
+ try:
+ with open(cfg_path) as f:
+ cfg = yaml.safe_load(f) or {}
+ url = cfg.get("database", {}).get("url", "")
+ except FileNotFoundError:
+ url = ""
+ return os.environ.get("FABLEDSCRYER_DATABASE__URL", url)
+
+
+def run_migrations_offline() -> None:
+ context.configure(
+ url=_get_url(), target_metadata=target_metadata,
+ literal_binds=True, dialect_opts={"paramstyle": "named"},
+ )
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+ context.configure(connection=connection, target_metadata=target_metadata)
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+ cfg = config.get_section(config.config_ini_section, {})
+ cfg["sqlalchemy.url"] = _get_url()
+ connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool)
+ async with connectable.connect() as connection:
+ await connection.run_sync(do_run_migrations)
+ await connectable.dispose()
+
+
+def run_migrations_online() -> None:
+ asyncio.run(run_async_migrations())
+
+
+if context.is_offline_mode():
+ run_migrations_offline()
+else:
+ run_migrations_online()
diff --git a/plugins/docker/migrations/versions/__init__.py b/plugins/docker/migrations/versions/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/plugins/docker/migrations/versions/docker_001_initial.py b/plugins/docker/migrations/versions/docker_001_initial.py
new file mode 100644
index 0000000..90e5298
--- /dev/null
+++ b/plugins/docker/migrations/versions/docker_001_initial.py
@@ -0,0 +1,47 @@
+"""Docker plugin initial tables
+
+Revision ID: docker_001_initial
+Revises: (none — branch off core via depends_on)
+Create Date: 2026-03-22
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_001_initial"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = "docker"
+depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
+
+
+def upgrade() -> None:
+ op.create_table(
+ "docker_containers",
+ sa.Column("name", sa.String(255), primary_key=True),
+ sa.Column("container_id", sa.String(64), nullable=False, server_default=""),
+ sa.Column("image", sa.String(512), nullable=False, server_default=""),
+ sa.Column("status", sa.String(32), nullable=False, server_default="unknown"),
+ sa.Column("cpu_pct", sa.Float, nullable=True),
+ sa.Column("mem_usage_bytes", sa.Integer, nullable=True),
+ sa.Column("mem_limit_bytes", sa.Integer, nullable=True),
+ sa.Column("mem_pct", sa.Float, nullable=True),
+ sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"),
+ sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"),
+ sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+ )
+
+ op.create_table(
+ "docker_metrics",
+ sa.Column("id", sa.String(36), primary_key=True),
+ sa.Column("container_name", sa.String(255), nullable=False, index=True),
+ sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False, index=True),
+ sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"),
+ sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"),
+ sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"),
+ )
+
+
+def downgrade() -> None:
+ op.drop_table("docker_metrics")
+ op.drop_table("docker_containers")
diff --git a/plugins/docker/models.py b/plugins/docker/models.py
new file mode 100644
index 0000000..09c1cf3
--- /dev/null
+++ b/plugins/docker/models.py
@@ -0,0 +1,49 @@
+# plugins/docker/models.py
+from __future__ import annotations
+import uuid
+from datetime import datetime, timezone
+from sqlalchemy import DateTime, Float, Integer, String, Text
+from sqlalchemy.orm import Mapped, mapped_column
+from fabledscryer.models.base import Base
+
+
+class DockerContainer(Base):
+ """Latest known state per container — upserted by name on each scrape."""
+ __tablename__ = "docker_containers"
+
+ # Container name is the stable natural key; container_id changes on recreation
+ name: Mapped[str] = mapped_column(String(255), primary_key=True)
+ container_id: Mapped[str] = mapped_column(String(64), nullable=False, default="")
+ image: Mapped[str] = mapped_column(String(512), nullable=False, default="")
+ status: Mapped[str] = mapped_column(String(32), nullable=False, default="unknown")
+ # running | stopped | paused | exited | dead
+ cpu_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
+ mem_usage_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
+ mem_limit_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
+ mem_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
+ restart_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ ports_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
+ # JSON: [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}]
+ started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+ scraped_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), nullable=False,
+ default=lambda: datetime.now(timezone.utc),
+ )
+
+
+class DockerMetric(Base):
+ """Time-series CPU/memory per container — one row per scrape per running container."""
+ __tablename__ = "docker_metrics"
+
+ id: Mapped[str] = mapped_column(
+ String(36), primary_key=True, default=lambda: str(uuid.uuid4())
+ )
+ container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
+ scraped_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), nullable=False,
+ default=lambda: datetime.now(timezone.utc),
+ index=True,
+ )
+ cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
+ mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
+ mem_usage_bytes: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
diff --git a/plugins/docker/plugin.yaml b/plugins/docker/plugin.yaml
new file mode 100644
index 0000000..8745302
--- /dev/null
+++ b/plugins/docker/plugin.yaml
@@ -0,0 +1,17 @@
+name: docker
+version: "1.0.0"
+description: "Docker container status, resource usage, restart tracking via Docker socket"
+author: "FabledScryer"
+license: "MIT"
+min_app_version: "0.1.0"
+repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins"
+homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/docker"
+tags:
+ - containers
+ - docker
+ - infrastructure
+
+config:
+ socket_path: /var/run/docker.sock
+ scrape_interval_seconds: 60
+ include_stopped: false
diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py
new file mode 100644
index 0000000..12d0383
--- /dev/null
+++ b/plugins/docker/routes.py
@@ -0,0 +1,161 @@
+# plugins/docker/routes.py
+from __future__ import annotations
+import json
+from datetime import datetime, timezone
+from quart import Blueprint, current_app, render_template, request
+from sqlalchemy import Integer, cast, func, select
+
+from fabledscryer.auth.middleware import require_role
+from fabledscryer.models.users import UserRole
+from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds
+from .models import DockerContainer, DockerMetric
+
+docker_bp = Blueprint("docker", __name__, template_folder="templates")
+
+
+def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
+ if len(values) < 2:
+ return f' '
+ mn, mx = min(values), max(values)
+ if mx == mn:
+ mx = mn + 1.0
+ step = width / (len(values) - 1)
+ pts = []
+ for i, v in enumerate(values):
+ x = i * step
+ y = height - (v - mn) / (mx - mn) * (height - 2) - 1
+ pts.append(f"{x:.1f},{y:.1f}")
+ poly = " ".join(pts)
+ return (
+ f''
+ f' '
+ f' '
+ )
+
+
+@docker_bp.get("/")
+@require_role(UserRole.viewer)
+async def index():
+ poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
+ current_range = request.args.get("range", DEFAULT_RANGE)
+ return await render_template(
+ "docker/index.html",
+ poll_interval=poll_interval,
+ current_range=current_range,
+ )
+
+
+@docker_bp.get("/rows")
+@require_role(UserRole.viewer)
+async def rows():
+ """HTMX fragment: container list with status and resource sparklines."""
+ since, range_key = parse_range(request.args.get("range"))
+ b_secs = bucket_seconds(since)
+
+ bucket_col = (
+ cast(func.strftime('%s', DockerMetric.scraped_at), Integer) / b_secs
+ ).label("bucket")
+
+ async with current_app.db_sessionmaker() as db:
+ # All known containers ordered by running first, then name
+ result = await db.execute(
+ select(DockerContainer)
+ .order_by(
+ # running first
+ (DockerContainer.status == "running").desc(),
+ DockerContainer.name,
+ )
+ )
+ containers = list(result.scalars())
+
+ # Build per-container sparkline histories
+ histories: dict[str, list] = {}
+ for c in containers:
+ result = await db.execute(
+ select(
+ func.avg(DockerMetric.cpu_pct).label("cpu_pct"),
+ func.avg(DockerMetric.mem_pct).label("mem_pct"),
+ bucket_col,
+ )
+ .where(DockerMetric.container_name == c.name)
+ .where(DockerMetric.scraped_at >= since)
+ .group_by(bucket_col)
+ .order_by(bucket_col)
+ )
+ histories[c.name] = result.all()
+
+ running = sum(1 for c in containers if c.status == "running")
+ stopped = len(containers) - running
+
+ container_data = []
+ for c in containers:
+ hist = histories.get(c.name, [])
+ ports = json.loads(c.ports_json) if c.ports_json else []
+ container_data.append({
+ "container": c,
+ "ports": ports,
+ "sparkline_cpu": _sparkline([r.cpu_pct or 0 for r in hist]),
+ "sparkline_mem": _sparkline([r.mem_pct or 0 for r in hist]),
+ })
+
+ return await render_template(
+ "docker/rows.html",
+ container_data=container_data,
+ running=running,
+ stopped=stopped,
+ range_key=range_key,
+ )
+
+
+@docker_bp.get("/widget")
+@require_role(UserRole.viewer)
+async def widget():
+ """HTMX dashboard widget: container status overview."""
+ show_stopped = request.args.get("show_stopped", "no") == "yes"
+ widget_id = request.args.get("wid", "0")
+
+ async with current_app.db_sessionmaker() as db:
+ result = await db.execute(
+ select(DockerContainer)
+ .order_by(
+ (DockerContainer.status == "running").desc(),
+ DockerContainer.name,
+ )
+ )
+ all_containers = list(result.scalars())
+
+ running = [c for c in all_containers if c.status == "running"]
+ stopped = [c for c in all_containers if c.status != "running"]
+ display = all_containers if show_stopped else running
+
+ return await render_template(
+ "docker/widget.html",
+ containers=display,
+ running_count=len(running),
+ stopped_count=len(stopped),
+ show_stopped=show_stopped,
+ widget_id=widget_id,
+ )
+
+
+@docker_bp.get("/widget/resources")
+@require_role(UserRole.viewer)
+async def widget_resources():
+ """HTMX dashboard widget: CPU + memory usage for running containers."""
+ limit = max(1, min(20, int(request.args.get("limit", 10) or 10)))
+ widget_id = request.args.get("wid", "0")
+
+ async with current_app.db_sessionmaker() as db:
+ result = await db.execute(
+ select(DockerContainer)
+ .where(DockerContainer.status == "running")
+ .order_by(DockerContainer.cpu_pct.desc().nullslast())
+ )
+ containers = list(result.scalars())[:limit]
+
+ return await render_template(
+ "docker/widget_resources.html",
+ containers=containers,
+ widget_id=widget_id,
+ )
diff --git a/plugins/docker/scheduler.py b/plugins/docker/scheduler.py
new file mode 100644
index 0000000..eef8c61
--- /dev/null
+++ b/plugins/docker/scheduler.py
@@ -0,0 +1,93 @@
+# plugins/docker/scheduler.py
+from __future__ import annotations
+import json
+import logging
+from datetime import datetime, timezone
+
+from fabledscryer.core.scheduler import ScheduledTask
+from fabledscryer.core.alerts import record_metric
+
+logger = logging.getLogger(__name__)
+
+
+def make_task(app) -> ScheduledTask:
+ interval = int(
+ app.config["PLUGINS"]["docker"].get("scrape_interval_seconds", 60)
+ )
+
+ async def scrape():
+ await _do_scrape(app)
+
+ return ScheduledTask(
+ name="docker_scrape",
+ coro_factory=scrape,
+ interval_seconds=interval,
+ run_on_startup=True,
+ )
+
+
+async def _do_scrape(app) -> None:
+ from .scraper import scrape_docker
+ from .models import DockerContainer, DockerMetric
+
+ cfg = app.config["PLUGINS"]["docker"]
+ socket_path = cfg.get("socket_path", "/var/run/docker.sock")
+ include_stopped = bool(cfg.get("include_stopped", False))
+
+ try:
+ containers = await scrape_docker(socket_path, include_stopped)
+ except ConnectionError as exc:
+ logger.error("Docker scrape failed: %s", exc)
+ return
+ except Exception:
+ logger.exception("Docker scrape error")
+ return
+
+ now = datetime.now(timezone.utc)
+
+ async with app.db_sessionmaker() as session:
+ async with session.begin():
+ for c in containers:
+ # Upsert container state
+ existing = await session.get(DockerContainer, c["name"])
+ if existing is None:
+ existing = DockerContainer(name=c["name"])
+ session.add(existing)
+ existing.container_id = c["container_id"]
+ existing.image = c["image"]
+ existing.status = c["status"]
+ existing.cpu_pct = c["cpu_pct"]
+ existing.mem_usage_bytes = c["mem_usage_bytes"]
+ existing.mem_limit_bytes = c["mem_limit_bytes"]
+ existing.mem_pct = c["mem_pct"]
+ existing.restart_count = c["restart_count"]
+ existing.ports_json = json.dumps(c["ports"])
+ existing.started_at = c["started_at"]
+ existing.scraped_at = now
+
+ # Time-series metric (running containers only)
+ if c["status"] == "running" and c["cpu_pct"] is not None:
+ session.add(DockerMetric(
+ container_name=c["name"],
+ scraped_at=now,
+ cpu_pct=c["cpu_pct"],
+ mem_pct=c["mem_pct"] or 0.0,
+ mem_usage_bytes=c["mem_usage_bytes"] or 0,
+ ))
+
+ # Feed alert pipeline
+ await record_metric(
+ session=session,
+ source_module="docker",
+ resource_name=c["name"],
+ metric_name="cpu_pct",
+ value=c["cpu_pct"],
+ )
+ if c["mem_pct"] is not None:
+ await record_metric(
+ session=session,
+ source_module="docker",
+ resource_name=c["name"],
+ metric_name="mem_pct",
+ value=c["mem_pct"],
+ )
diff --git a/plugins/docker/scraper.py b/plugins/docker/scraper.py
new file mode 100644
index 0000000..96f601b
--- /dev/null
+++ b/plugins/docker/scraper.py
@@ -0,0 +1,140 @@
+# plugins/docker/scraper.py
+"""Docker API client via Unix socket."""
+from __future__ import annotations
+import asyncio
+import logging
+from datetime import datetime, timezone
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+def _short_id(container_id: str) -> str:
+ return container_id[:12] if container_id else ""
+
+
+def _calc_cpu_pct(stats: dict) -> float:
+ """Calculate CPU % from a Docker stats snapshot (one-shot)."""
+ try:
+ cpu = stats.get("cpu_stats", {})
+ precpu = stats.get("precpu_stats", {})
+ cpu_delta = (
+ cpu["cpu_usage"]["total_usage"] - precpu["cpu_usage"]["total_usage"]
+ )
+ sys_delta = cpu.get("system_cpu_usage", 0) - precpu.get("system_cpu_usage", 0)
+ num_cpus = cpu.get("online_cpus") or len(
+ cpu["cpu_usage"].get("percpu_usage") or [None]
+ )
+ if sys_delta <= 0 or cpu_delta < 0:
+ return 0.0
+ return round((cpu_delta / sys_delta) * num_cpus * 100.0, 2)
+ except (KeyError, TypeError, ZeroDivisionError):
+ return 0.0
+
+
+def _calc_mem(stats: dict) -> tuple[int, int, float]:
+ """Return (usage_bytes, limit_bytes, mem_pct) from Docker stats."""
+ try:
+ mem = stats.get("memory_stats", {})
+ usage = mem.get("usage", 0)
+ limit = mem.get("limit", 0)
+ # Subtract page cache for working set (cgroup v1: "cache", v2: "inactive_file")
+ cache = mem.get("stats", {}).get("cache", 0) or \
+ mem.get("stats", {}).get("inactive_file", 0)
+ actual = max(0, usage - cache)
+ pct = round(actual / limit * 100.0, 2) if limit > 0 else 0.0
+ return actual, limit, pct
+ except (KeyError, TypeError):
+ return 0, 0, 0.0
+
+
+def _parse_ports(ports: list) -> list[dict]:
+ """Normalise Docker port bindings to a compact list."""
+ result = []
+ for p in (ports or []):
+ if p.get("PublicPort"):
+ result.append({
+ "host_port": p["PublicPort"],
+ "container_port": p["PrivatePort"],
+ "protocol": p.get("Type", "tcp"),
+ })
+ return result
+
+
+async def scrape_docker(socket_path: str, include_stopped: bool) -> list[dict]:
+ """
+ Fetch all containers + resource stats from the Docker daemon.
+ Returns a list of normalised dicts ready for the scheduler to persist.
+ Raises ConnectionError if the socket is unreachable.
+ """
+ transport = httpx.AsyncHTTPTransport(uds=socket_path)
+ try:
+ async with httpx.AsyncClient(
+ transport=transport,
+ base_url="http://docker",
+ timeout=10.0,
+ ) as client:
+ resp = await client.get(
+ "/containers/json",
+ params={"all": "true" if include_stopped else "false"},
+ )
+ resp.raise_for_status()
+ containers = resp.json()
+
+ async def _get_stats(c: dict) -> tuple[dict, dict | None]:
+ if c.get("State") != "running":
+ return c, None
+ try:
+ r = await client.get(
+ f"/containers/{c['Id']}/stats",
+ params={"stream": "false", "one-shot": "true"},
+ timeout=5.0,
+ )
+ return c, r.json()
+ except Exception as exc:
+ logger.debug(
+ "Stats unavailable for %s: %s", _short_id(c.get("Id", "")), exc
+ )
+ return c, None
+
+ pairs = await asyncio.gather(*[_get_stats(c) for c in containers])
+
+ except httpx.ConnectError as exc:
+ raise ConnectionError(
+ f"Cannot connect to Docker socket at {socket_path}: {exc}"
+ ) from exc
+
+ results = []
+ now = datetime.now(timezone.utc)
+ for container, stats in pairs:
+ names = container.get("Names") or []
+ name = names[0].lstrip("/") if names else _short_id(container.get("Id", ""))
+
+ cpu_pct = mem_usage = mem_limit = mem_pct = None
+ if stats is not None:
+ cpu_pct = _calc_cpu_pct(stats)
+ mem_usage, mem_limit, mem_pct = _calc_mem(stats)
+
+ # Created is a Unix epoch int from /containers/json
+ created_ts = container.get("Created")
+ started_at = (
+ datetime.fromtimestamp(created_ts, tz=timezone.utc)
+ if isinstance(created_ts, (int, float)) else None
+ )
+
+ results.append({
+ "name": name,
+ "container_id": _short_id(container.get("Id", "")),
+ "image": container.get("Image", ""),
+ "status": container.get("State", "unknown"),
+ "cpu_pct": cpu_pct,
+ "mem_usage_bytes": mem_usage,
+ "mem_limit_bytes": mem_limit,
+ "mem_pct": mem_pct,
+ "restart_count": 0, # would require /inspect; left as 0 for now
+ "ports": _parse_ports(container.get("Ports", [])),
+ "started_at": started_at,
+ })
+
+ return results
diff --git a/plugins/docker/templates/docker/index.html b/plugins/docker/templates/docker/index.html
new file mode 100644
index 0000000..42c9bb5
--- /dev/null
+++ b/plugins/docker/templates/docker/index.html
@@ -0,0 +1,16 @@
+{% extends "base.html" %}
+{% block title %}Docker — Fabled Scryer{% endblock %}
+{% block content %}
+
+
Docker
+ {% include "_time_range.html" %}
+
+
+
+{% endblock %}
diff --git a/plugins/docker/templates/docker/rows.html b/plugins/docker/templates/docker/rows.html
new file mode 100644
index 0000000..3b6c84e
--- /dev/null
+++ b/plugins/docker/templates/docker/rows.html
@@ -0,0 +1,86 @@
+{# docker/rows.html — HTMX fragment for the Docker main page #}
+
+{# ── Summary strip ─────────────────────────────────────────────────────────── #}
+
+
+
Running
+
{{ running }}
+
+
+
Stopped
+
{{ stopped }}
+
+
+
Total
+
{{ container_data | length }}
+
+
+
+{# ── Container table ─────────────────────────────────────────────────────── #}
+{% if container_data %}
+
+
+
+
+ Container
+ Image
+ Ports
+ CPU %
+ CPU history
+ Mem %
+ Mem history
+
+
+
+ {% for item in container_data %}
+ {% set c = item.container %}
+
+
+
+
+
+
{{ c.name }}
+
{{ c.status }}
+
+
+
+
+ {{ c.image }}
+
+
+ {% for p in item.ports %}
+ {{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}
+ {% endfor %}
+
+
+ {% if c.cpu_pct is not none %}
+
+ {{ "%.1f" | format(c.cpu_pct) }}%
+
+ {% else %}
+ —
+ {% endif %}
+
+ {{ item.sparkline_cpu | safe }}
+
+ {% if c.mem_pct is not none %}
+
+ {{ "%.1f" | format(c.mem_pct) }}%
+
+ {% else %}
+ —
+ {% endif %}
+
+ {{ item.sparkline_mem | safe }}
+
+ {% endfor %}
+
+
+
+{% else %}
+
+
+ No containers found. Make sure the Docker socket is accessible and the plugin is configured correctly.
+
+
+{% endif %}
diff --git a/plugins/docker/templates/docker/widget.html b/plugins/docker/templates/docker/widget.html
new file mode 100644
index 0000000..427018f
--- /dev/null
+++ b/plugins/docker/templates/docker/widget.html
@@ -0,0 +1,35 @@
+{# docker/widget.html — dashboard widget: container status overview #}
+{% if not containers and running_count == 0 and stopped_count == 0 %}
+No containers found.
+{% else %}
+
+
+
+ {{ running_count }}
+ running
+
+ {% if stopped_count %}
+
+ {{ stopped_count }}
+ stopped
+
+ {% endif %}
+
+
+
+ {% for c in containers %}
+
+
+
+ {{ c.name }}
+
+ {% if c.cpu_pct is not none %}
+
+ {{ "%.1f" | format(c.cpu_pct) }}%
+
+ {% endif %}
+
+ {% endfor %}
+
+
+{% endif %}
diff --git a/plugins/docker/templates/docker/widget_resources.html b/plugins/docker/templates/docker/widget_resources.html
new file mode 100644
index 0000000..39c90dc
--- /dev/null
+++ b/plugins/docker/templates/docker/widget_resources.html
@@ -0,0 +1,36 @@
+{# docker/widget_resources.html — dashboard widget: CPU + memory bars for running containers #}
+{% if not containers %}
+No running containers.
+{% else %}
+
+ {% for c in containers %}
+
+
+
+ {{ c.name }}
+
+
+ {% if c.cpu_pct is not none %}CPU {{ "%.1f" | format(c.cpu_pct) }}%{% endif %}
+ {% if c.mem_pct is not none %} · Mem {{ "%.1f" | format(c.mem_pct) }}%{% endif %}
+
+
+ {% if c.cpu_pct is not none %}
+
+ {% endif %}
+ {% if c.mem_pct is not none %}
+
+ {% endif %}
+
+ {% endfor %}
+
+{% endif %}
diff --git a/plugins/host_agent/__init__.py b/plugins/host_agent/__init__.py
new file mode 100644
index 0000000..6a7b229
--- /dev/null
+++ b/plugins/host_agent/__init__.py
@@ -0,0 +1,23 @@
+# plugins/host_agent/__init__.py
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from quart import Quart
+
+_app: "Quart | None" = None
+
+
+def setup(app: "Quart") -> None:
+ global _app
+ _app = app
+
+
+def get_scheduled_tasks() -> list:
+ from .scheduler import make_task
+ return [make_task(_app)]
+
+
+def get_blueprint():
+ from .routes import host_agent_bp
+ return host_agent_bp
diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py
new file mode 100644
index 0000000..34f52d6
--- /dev/null
+++ b/plugins/host_agent/agent.py
@@ -0,0 +1,370 @@
+# plugins/host_agent/agent.py
+"""Steward host agent — pushes resource metrics to a Steward instance.
+
+Python 3.8+ stdlib only. Target ~300 lines. Served to targets at
+GET /plugins/host_agent/agent.py.
+"""
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import signal
+import socket
+import sys
+import time
+import urllib.error
+import urllib.request
+from collections import deque
+from datetime import datetime, timezone
+
+AGENT_VERSION = "1.0.0"
+
+
+class ConfigError(Exception):
+ pass
+
+
+REQUIRED_KEYS = ("url", "token")
+INT_KEYS = ("interval_seconds",)
+LIST_KEYS = ("mounts",)
+
+
+def read_config(path: str) -> dict:
+ """Parse a flat `key = value` config file."""
+ cfg: dict = {}
+ try:
+ with open(path, "r", encoding="utf-8") as f:
+ for lineno, raw in enumerate(f, 1):
+ line = raw.strip()
+ if not line or line.startswith("#"):
+ continue
+ if "=" not in line:
+ raise ConfigError(f"{path}:{lineno}: expected 'key = value'")
+ key, _, value = line.partition("=")
+ key = key.strip()
+ value = value.strip()
+ if key in INT_KEYS:
+ try:
+ cfg[key] = int(value)
+ except ValueError:
+ raise ConfigError(f"{path}:{lineno}: {key} must be int")
+ elif key in LIST_KEYS:
+ cfg[key] = [v.strip() for v in value.split(",") if v.strip()]
+ else:
+ cfg[key] = value
+ except FileNotFoundError:
+ raise ConfigError(f"{path}: not found")
+
+ missing = [k for k in REQUIRED_KEYS if k not in cfg]
+ if missing:
+ raise ConfigError(f"{path}: missing required key(s): {', '.join(missing)}")
+
+ cfg.setdefault("interval_seconds", 30)
+ return cfg
+
+
+# ─── collectors ──────────────────────────────────────────────────────────────
+
+STAT_PATH = "/proc/stat"
+MEMINFO_PATH = "/proc/meminfo"
+LOADAVG_PATH = "/proc/loadavg"
+UPTIME_PATH = "/proc/uptime"
+OS_RELEASE_PATH = "/etc/os-release"
+
+
+def _read_file(path: str) -> str:
+ with open(path, "r", encoding="utf-8") as f:
+ return f.read()
+
+
+def _parse_cpu_line(stat_text: str) -> tuple[int, int]:
+ """Return (total_jiffies, idle_jiffies) for the aggregate cpu line."""
+ for line in stat_text.splitlines():
+ if line.startswith("cpu "):
+ parts = line.split()
+ fields = [int(x) for x in parts[1:]]
+ idle = fields[3] + (fields[4] if len(fields) > 4 else 0) # idle + iowait
+ total = sum(fields)
+ return total, idle
+ raise RuntimeError("no aggregate cpu line in /proc/stat")
+
+
+def collect_cpu(sample_window: float = 0.2) -> float:
+ """Return CPU utilization % over sample_window seconds."""
+ t1, i1 = _parse_cpu_line(_read_file(STAT_PATH))
+ time.sleep(sample_window)
+ t2, i2 = _parse_cpu_line(_read_file(STAT_PATH))
+ dt = t2 - t1
+ di = i2 - i1
+ if dt <= 0:
+ return 0.0
+ return round(100.0 * (dt - di) / dt, 2)
+
+
+def collect_memory() -> dict:
+ info: dict[str, int] = {}
+ for line in _read_file(MEMINFO_PATH).splitlines():
+ if ":" not in line:
+ continue
+ key, _, rest = line.partition(":")
+ parts = rest.strip().split()
+ if not parts:
+ continue
+ try:
+ value_kb = int(parts[0])
+ except ValueError:
+ continue
+ info[key] = value_kb * 1024 # bytes
+ total = info.get("MemTotal", 0)
+ available = info.get("MemAvailable", 0)
+ swap_total = info.get("SwapTotal", 0)
+ swap_free = info.get("SwapFree", 0)
+ return {
+ "total_bytes": total,
+ "used_bytes": max(total - available, 0),
+ "available_bytes": available,
+ "swap_used_bytes": max(swap_total - swap_free, 0),
+ }
+
+
+def collect_storage(mounts: list[str]) -> list[dict]:
+ out: list[dict] = []
+ for m in mounts:
+ try:
+ u = shutil.disk_usage(m)
+ except (FileNotFoundError, PermissionError):
+ continue
+ out.append({"mount": m, "total_bytes": u.total, "used_bytes": u.used})
+ return out
+
+
+def collect_load() -> dict:
+ parts = _read_file(LOADAVG_PATH).split()
+ return {"1m": float(parts[0]), "5m": float(parts[1]), "15m": float(parts[2])}
+
+
+def collect_uptime() -> int:
+ return int(float(_read_file(UPTIME_PATH).split()[0]))
+
+
+def collect_metadata() -> dict:
+ u = os.uname()
+ distro = "unknown"
+ try:
+ for line in _read_file(OS_RELEASE_PATH).splitlines():
+ if line.startswith("PRETTY_NAME="):
+ distro = line.split("=", 1)[1].strip().strip('"')
+ break
+ except (OSError, FileNotFoundError):
+ pass
+ return {"kernel": u.release, "distro": distro, "arch": u.machine}
+
+
+def default_mounts() -> list[str]:
+ """Return real mount points from /proc/mounts, skipping pseudo filesystems."""
+ skip_types = {"tmpfs", "devtmpfs", "proc", "sysfs", "cgroup", "cgroup2",
+ "overlay", "squashfs", "ramfs", "devpts", "mqueue", "pstore",
+ "securityfs", "debugfs", "tracefs", "hugetlbfs", "fusectl",
+ "configfs", "bpf", "autofs", "efivarfs", "binfmt_misc"}
+ mounts: list[str] = []
+ try:
+ with open("/proc/mounts", "r") as f:
+ for line in f:
+ parts = line.split()
+ if len(parts) < 3:
+ continue
+ if parts[2] in skip_types:
+ continue
+ mounts.append(parts[1])
+ except OSError:
+ return ["/"]
+ return mounts or ["/"]
+
+
+# ─── ring buffer ─────────────────────────────────────────────────────────────
+
+
+class RingBuffer:
+ def __init__(self, maxlen: int = 20) -> None:
+ self._dq: deque = deque(maxlen=maxlen)
+
+ def push(self, item) -> None:
+ self._dq.append(item)
+
+ def drain(self) -> list:
+ out = list(self._dq)
+ self._dq.clear()
+ return out
+
+ def __len__(self) -> int:
+ return len(self._dq)
+
+
+# ─── payload ─────────────────────────────────────────────────────────────────
+
+
+def build_sample(mounts: list[str]) -> dict:
+ """Collect one full sample. Partial samples allowed if a collector fails."""
+ sample: dict = {"ts": datetime.now(timezone.utc).isoformat()}
+ try:
+ sample["cpu_pct"] = collect_cpu()
+ except Exception:
+ sample["cpu_pct"] = None
+ try:
+ sample["mem"] = collect_memory()
+ except Exception:
+ sample["mem"] = None
+ try:
+ sample["load"] = collect_load()
+ except Exception:
+ sample["load"] = None
+ try:
+ sample["uptime_secs"] = collect_uptime()
+ except Exception:
+ sample["uptime_secs"] = None
+ try:
+ sample["storage"] = collect_storage(mounts)
+ except Exception:
+ sample["storage"] = []
+ return sample
+
+
+def build_payload(samples: list[dict], hostname: str, metadata: dict) -> dict:
+ return {
+ "agent_version": AGENT_VERSION,
+ "hostname": hostname,
+ "metadata": metadata,
+ "samples": samples,
+ }
+
+
+# ─── POST + backoff ──────────────────────────────────────────────────────────
+
+
+BACKOFF_CAP = 300
+
+
+def next_backoff(current: int) -> int:
+ if current <= 0:
+ return 30
+ return min(current * 2, BACKOFF_CAP)
+
+
+def post_payload(url: str, token: str, payload: dict) -> tuple[bool, int | None]:
+ body = json.dumps(payload).encode("utf-8")
+ req = urllib.request.Request(
+ url.rstrip("/") + "/plugins/host_agent/ingest",
+ data=body,
+ headers={
+ "Content-Type": "application/json",
+ "Authorization": f"Bearer {token}",
+ "User-Agent": f"steward-host-agent/{AGENT_VERSION}",
+ },
+ method="POST",
+ )
+ try:
+ with urllib.request.urlopen(req, timeout=10) as resp:
+ return (200 <= resp.status < 300, resp.status)
+ except urllib.error.HTTPError as e:
+ return (False, e.code)
+ except (urllib.error.URLError, TimeoutError, socket.timeout, OSError):
+ return (False, None)
+
+
+# ─── main loop ───────────────────────────────────────────────────────────────
+
+
+_reload_requested = False
+_shutdown_requested = False
+
+
+def _handle_hup(_signum, _frame):
+ global _reload_requested
+ _reload_requested = True
+
+
+def _handle_term(_signum, _frame):
+ global _shutdown_requested
+ _shutdown_requested = True
+
+
+def _log(level: str, msg: str) -> None:
+ sys.stderr.write(f"[{level}] {msg}\n")
+ sys.stderr.flush()
+
+
+def main_loop(conf_path: str) -> int:
+ global _reload_requested, _shutdown_requested
+ signal.signal(signal.SIGHUP, _handle_hup)
+ signal.signal(signal.SIGTERM, _handle_term)
+
+ try:
+ cfg = read_config(conf_path)
+ except ConfigError as e:
+ _log("ERROR", str(e))
+ return 2
+
+ metadata = collect_metadata()
+ hostname = cfg.get("hostname") or socket.gethostname()
+ mounts = cfg.get("mounts") or default_mounts()
+ buffer = RingBuffer(maxlen=20)
+ backoff = 0
+
+ _log("INFO", f"steward-host-agent {AGENT_VERSION} starting "
+ f"(url={cfg['url']}, interval={cfg['interval_seconds']}s)")
+
+ while not _shutdown_requested:
+ if _reload_requested:
+ try:
+ cfg = read_config(conf_path)
+ mounts = cfg.get("mounts") or default_mounts()
+ _log("INFO", "config reloaded")
+ except ConfigError as e:
+ _log("ERROR", f"reload failed: {e}")
+ _reload_requested = False
+
+ sample = build_sample(mounts)
+ buffered = buffer.drain()
+ payload = build_payload(
+ samples=buffered + [sample],
+ hostname=hostname,
+ metadata=metadata,
+ )
+ ok, status = post_payload(cfg["url"], cfg["token"], payload)
+
+ if ok:
+ if buffered:
+ _log("INFO", f"flushed {len(buffered)} buffered samples")
+ backoff = 0
+ sleep_for = cfg["interval_seconds"]
+ else:
+ if status == 400:
+ _log("ERROR", "server rejected payload (400) — dropping sample")
+ elif status == 401:
+ _log("ERROR", "token rejected (401) — check config + UI")
+ buffer.push(sample)
+ else:
+ _log("WARN", f"POST failed (status={status}); buffering")
+ for s in buffered:
+ buffer.push(s)
+ buffer.push(sample)
+ backoff = next_backoff(backoff)
+ sleep_for = backoff
+
+ slept = 0.0
+ while slept < sleep_for and not _shutdown_requested and not _reload_requested:
+ time.sleep(min(1.0, sleep_for - slept))
+ slept += 1.0
+
+ _log("INFO", "SIGTERM — flushing and exiting")
+ final = buffer.drain()
+ if final:
+ post_payload(cfg["url"], cfg["token"],
+ build_payload(final, hostname, metadata))
+ return 0
+
+
+if __name__ == "__main__":
+ conf = os.environ.get("STEWARD_AGENT_CONFIG", "/etc/steward-agent.conf")
+ sys.exit(main_loop(conf))
diff --git a/plugins/host_agent/migrations/__init__.py b/plugins/host_agent/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/plugins/host_agent/migrations/env.py b/plugins/host_agent/migrations/env.py
new file mode 100644
index 0000000..6545c0e
--- /dev/null
+++ b/plugins/host_agent/migrations/env.py
@@ -0,0 +1,70 @@
+# plugins/host_agent/migrations/env.py
+"""Alembic env.py for the host_agent plugin."""
+import asyncio
+import os
+import sys
+from pathlib import Path
+from logging.config import fileConfig
+
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+from alembic import context
+
+sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
+
+from steward.models.base import Base
+import steward.models # noqa: F401
+from plugins.host_agent.models import HostAgentRegistration # noqa: F401
+
+config = context.config
+if config.config_file_name is not None:
+ fileConfig(config.config_file_name)
+
+target_metadata = Base.metadata
+
+
+def _get_url() -> str:
+ import yaml
+ cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml")
+ try:
+ with open(cfg_path) as f:
+ cfg = yaml.safe_load(f) or {}
+ url = cfg.get("database", {}).get("url", "")
+ except FileNotFoundError:
+ url = ""
+ return os.environ.get("STEWARD_DATABASE__URL", url)
+
+
+def run_migrations_offline() -> None:
+ context.configure(
+ url=_get_url(), target_metadata=target_metadata,
+ literal_binds=True, dialect_opts={"paramstyle": "named"},
+ )
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+ context.configure(connection=connection, target_metadata=target_metadata)
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+ cfg = config.get_section(config.config_ini_section, {})
+ cfg["sqlalchemy.url"] = _get_url()
+ connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool)
+ async with connectable.connect() as connection:
+ await connection.run_sync(do_run_migrations)
+ await connectable.dispose()
+
+
+def run_migrations_online() -> None:
+ asyncio.run(run_async_migrations())
+
+
+if context.is_offline_mode():
+ run_migrations_offline()
+else:
+ run_migrations_online()
diff --git a/plugins/host_agent/migrations/versions/__init__.py b/plugins/host_agent/migrations/versions/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/plugins/host_agent/migrations/versions/host_agent_001_initial.py b/plugins/host_agent/migrations/versions/host_agent_001_initial.py
new file mode 100644
index 0000000..d241257
--- /dev/null
+++ b/plugins/host_agent/migrations/versions/host_agent_001_initial.py
@@ -0,0 +1,36 @@
+# plugins/host_agent/migrations/versions/host_agent_001_initial.py
+"""host_agent plugin initial tables
+
+Revision ID: host_agent_001_initial
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_001_initial"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = "host_agent"
+depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
+
+
+def upgrade() -> None:
+ op.create_table(
+ "host_agent_registrations",
+ sa.Column("id", sa.String(36), primary_key=True),
+ sa.Column("host_id", sa.String(36),
+ sa.ForeignKey("hosts.id", ondelete="CASCADE"),
+ nullable=False, unique=True),
+ sa.Column("token_hash", sa.String(64), nullable=False, unique=True, index=True),
+ sa.Column("token_created_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("agent_version", sa.String(32), nullable=True),
+ sa.Column("kernel", sa.String(128), nullable=True),
+ sa.Column("distro", sa.String(128), nullable=True),
+ sa.Column("arch", sa.String(32), nullable=True),
+ sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+ )
+
+
+def downgrade() -> None:
+ op.drop_table("host_agent_registrations")
diff --git a/plugins/host_agent/models.py b/plugins/host_agent/models.py
new file mode 100644
index 0000000..b93d725
--- /dev/null
+++ b/plugins/host_agent/models.py
@@ -0,0 +1,32 @@
+# plugins/host_agent/models.py
+from __future__ import annotations
+import uuid
+from datetime import datetime, timezone
+from sqlalchemy import Column, String, DateTime, ForeignKey
+from steward.models.base import Base
+
+
+def _uuid() -> str:
+ return str(uuid.uuid4())
+
+
+def _utcnow() -> datetime:
+ return datetime.now(timezone.utc)
+
+
+class HostAgentRegistration(Base):
+ __tablename__ = "host_agent_registrations"
+
+ id = Column(String(36), primary_key=True, default=_uuid)
+ host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"),
+ unique=True, nullable=False)
+ token_hash = Column(String(64), nullable=False, unique=True, index=True)
+ token_created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
+ agent_version = Column(String(32), nullable=True)
+ kernel = Column(String(128), nullable=True)
+ distro = Column(String(128), nullable=True)
+ arch = Column(String(32), nullable=True)
+ last_seen_at = Column(DateTime(timezone=True), nullable=True)
+ created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
+ updated_at = Column(DateTime(timezone=True), nullable=False,
+ default=_utcnow, onupdate=_utcnow)
diff --git a/plugins/host_agent/plugin.yaml b/plugins/host_agent/plugin.yaml
new file mode 100644
index 0000000..09b9022
--- /dev/null
+++ b/plugins/host_agent/plugin.yaml
@@ -0,0 +1,19 @@
+# plugins/host_agent/plugin.yaml
+name: host_agent
+version: "1.0.0"
+description: "Remote Linux host resource monitoring via a lightweight Python push agent (CPU, memory, storage, load, uptime)"
+author: "Steward"
+license: "MIT"
+min_app_version: "0.1.0"
+repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
+homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/host_agent"
+tags:
+ - host
+ - monitoring
+ - cpu
+ - memory
+ - storage
+
+config:
+ stale_after_seconds: 180
+ default_interval_seconds: 30
diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py
new file mode 100644
index 0000000..f023811
--- /dev/null
+++ b/plugins/host_agent/routes.py
@@ -0,0 +1,398 @@
+# plugins/host_agent/routes.py
+from __future__ import annotations
+
+import hashlib
+from datetime import datetime, timezone
+from typing import Any
+
+from pathlib import Path
+
+import secrets
+from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for
+from steward.auth.middleware import require_role
+from steward.models.users import UserRole
+from sqlalchemy import select, func
+from datetime import timedelta
+
+from steward.core.settings import public_base_url
+from steward.models.hosts import Host
+from steward.models.metrics import PluginMetric
+from .models import HostAgentRegistration
+
+host_agent_bp = Blueprint("host_agent", __name__, template_folder="templates")
+
+SOURCE_MODULE = "host_agent"
+
+
+def _hash_token(raw: str) -> str:
+ return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+
+def _parse_ts(ts: str) -> datetime:
+ if ts.endswith("Z"):
+ ts = ts[:-1] + "+00:00"
+ return datetime.fromisoformat(ts)
+
+
+def _expand_sample_to_metrics(
+ sample: dict, host_name: str, recorded_at: datetime
+) -> list[PluginMetric]:
+ rows: list[PluginMetric] = []
+
+ def row(metric: str, resource: str, value: float) -> None:
+ rows.append(PluginMetric(
+ source_module=SOURCE_MODULE,
+ resource_name=resource,
+ metric_name=metric,
+ value=float(value),
+ recorded_at=recorded_at,
+ ))
+
+ if sample.get("cpu_pct") is not None:
+ row("cpu_pct", host_name, sample["cpu_pct"])
+
+ mem = sample.get("mem") or {}
+ if mem.get("total_bytes"):
+ total = mem["total_bytes"]
+ available = mem.get("available_bytes", 0)
+ used_pct = 100.0 * (total - available) / total if total else 0.0
+ row("mem_used_pct", host_name, used_pct)
+ row("mem_available_bytes", host_name, available)
+ row("swap_used_bytes", host_name, mem.get("swap_used_bytes", 0))
+
+ load = sample.get("load") or {}
+ for key in ("1m", "5m", "15m"):
+ if key in load:
+ row(f"load_{key}", host_name, load[key])
+
+ if sample.get("uptime_secs") is not None:
+ row("uptime_secs", host_name, sample["uptime_secs"])
+
+ worst_pct = 0.0
+ for disk in sample.get("storage") or []:
+ total = disk.get("total_bytes", 0)
+ used = disk.get("used_bytes", 0)
+ pct = 100.0 * used / total if total else 0.0
+ mount_res = f"{host_name}:{disk['mount']}"
+ row("disk_used_pct", mount_res, pct)
+ row("disk_used_bytes", mount_res, used)
+ row("disk_total_bytes", mount_res, total)
+ if pct > worst_pct:
+ worst_pct = pct
+ if sample.get("storage"):
+ row("disk_used_pct_worst", host_name, worst_pct)
+
+ return rows
+
+
+def _error(status: int, code: str, detail: str | None = None) -> tuple[Any, int]:
+ body: dict = {"ok": False, "error": code}
+ if detail:
+ body["detail"] = detail
+ return jsonify(body), status
+
+
+@host_agent_bp.post("/ingest")
+async def ingest():
+ auth = request.headers.get("Authorization", "")
+ if not auth.startswith("Bearer "):
+ return _error(401, "invalid_token")
+ raw = auth[len("Bearer "):].strip()
+ if not raw:
+ return _error(401, "invalid_token")
+
+ try:
+ payload = await request.get_json(force=True)
+ except Exception:
+ return _error(400, "malformed_payload", "invalid JSON")
+ if not isinstance(payload, dict) or "samples" not in payload:
+ return _error(400, "malformed_payload", "missing 'samples'")
+
+ samples = payload.get("samples") or []
+ if not isinstance(samples, list) or not samples:
+ return _error(400, "malformed_payload", "'samples' must be a non-empty list")
+
+ token_hash = _hash_token(raw)
+ async with current_app.db_sessionmaker() as session:
+ reg = (await session.execute(
+ select(HostAgentRegistration).where(
+ HostAgentRegistration.token_hash == token_hash)
+ )).scalar_one_or_none()
+ if reg is None:
+ return _error(401, "invalid_token")
+ host = (await session.execute(
+ select(Host).where(Host.id == reg.host_id)
+ )).scalar_one_or_none()
+ if host is None:
+ return _error(401, "invalid_token")
+
+ accepted = 0
+ latest_ts: datetime | None = None
+ for sample in samples:
+ try:
+ recorded_at = _parse_ts(sample["ts"])
+ except (KeyError, ValueError):
+ continue
+ metrics = _expand_sample_to_metrics(sample, host.name, recorded_at)
+ for m in metrics:
+ session.add(m)
+ accepted += 1
+ if latest_ts is None or recorded_at > latest_ts:
+ latest_ts = recorded_at
+
+ if accepted == 0:
+ await session.rollback()
+ return _error(400, "malformed_payload", "no valid samples")
+
+ md = payload.get("metadata") or {}
+ changed = False
+ if latest_ts and (reg.last_seen_at is None or latest_ts > reg.last_seen_at):
+ reg.last_seen_at = latest_ts
+ changed = True
+ version = payload.get("agent_version")
+ if version and reg.agent_version != version:
+ reg.agent_version = version
+ changed = True
+ for field in ("kernel", "distro", "arch"):
+ if field in md and getattr(reg, field) != md[field]:
+ setattr(reg, field, md[field])
+ changed = True
+ if changed:
+ reg.updated_at = datetime.now(timezone.utc)
+
+ if latest_ts:
+ skew = abs((datetime.now(timezone.utc) - latest_ts).total_seconds())
+ if skew > 300:
+ current_app.logger.warning(
+ "host_agent ingest: clock skew %.0fs for host=%s", skew, host.name)
+
+ await session.commit()
+
+ return jsonify({"ok": True, "samples_accepted": accepted}), 200
+
+
+AGENT_SOURCE_PATH = Path(__file__).parent / "agent.py"
+
+
+def _agent_version() -> str:
+ try:
+ for line in AGENT_SOURCE_PATH.read_text().splitlines():
+ if line.startswith("AGENT_VERSION"):
+ return line.split("=", 1)[1].strip().strip('"').strip("'")
+ except OSError:
+ pass
+ return "unknown"
+
+
+@host_agent_bp.get("/install.sh")
+async def install_script():
+ token = request.args.get("token", "")
+ if not token:
+ return _error(401, "invalid_token")
+ async with current_app.db_sessionmaker() as session:
+ reg = (await session.execute(
+ select(HostAgentRegistration).where(
+ HostAgentRegistration.token_hash == _hash_token(token))
+ )).scalar_one_or_none()
+ if reg is None:
+ return _error(401, "invalid_token")
+ host = (await session.execute(
+ select(Host).where(Host.id == reg.host_id)
+ )).scalar_one_or_none()
+ if host is None:
+ return _error(401, "invalid_token")
+
+ url = public_base_url(request)
+
+ rendered = await render_template(
+ "install.sh.j2",
+ url=url,
+ token=token,
+ agent_version=_agent_version(),
+ host_name=host.name,
+ host_address=host.address,
+ )
+ return Response(rendered, mimetype="text/plain")
+
+
+@host_agent_bp.get("/agent.py")
+async def agent_source():
+ try:
+ body = AGENT_SOURCE_PATH.read_text(encoding="utf-8")
+ except OSError:
+ return _error(500, "agent_missing")
+ return Response(body, mimetype="text/x-python")
+
+
+@host_agent_bp.get("/widget")
+async def widget_table():
+ """Fleet-glance table: one row per monitored host, latest metrics."""
+ async with current_app.db_sessionmaker() as session:
+ regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
+ host_ids = [r.host_id for r in regs]
+ hosts = {
+ h.id: h for h in (await session.execute(
+ select(Host).where(Host.id.in_(host_ids))
+ )).scalars().all()
+ } if host_ids else {}
+
+ subq = (
+ select(
+ PluginMetric.resource_name,
+ PluginMetric.metric_name,
+ func.max(PluginMetric.recorded_at).label("max_ts"),
+ )
+ .where(PluginMetric.source_module == SOURCE_MODULE)
+ .group_by(PluginMetric.resource_name, PluginMetric.metric_name)
+ ).subquery()
+ latest_rows = (await session.execute(
+ select(PluginMetric).join(
+ subq,
+ (PluginMetric.resource_name == subq.c.resource_name) &
+ (PluginMetric.metric_name == subq.c.metric_name) &
+ (PluginMetric.recorded_at == subq.c.max_ts),
+ ).where(PluginMetric.source_module == SOURCE_MODULE)
+ )).scalars().all()
+
+ latest: dict[str, dict[str, float]] = {}
+ for row in latest_rows:
+ if ":" in row.resource_name:
+ continue
+ latest.setdefault(row.resource_name, {})[row.metric_name] = row.value
+
+ rows = []
+ for reg in regs:
+ host = hosts.get(reg.host_id)
+ if host is None:
+ continue
+ m = latest.get(host.name, {})
+ rows.append({
+ "host": host,
+ "reg": reg,
+ "cpu_pct": m.get("cpu_pct"),
+ "mem_used_pct": m.get("mem_used_pct"),
+ "disk_worst": m.get("disk_used_pct_worst"),
+ "load_1m": m.get("load_1m"),
+ })
+
+ return await render_template("widget_table.html", rows=rows)
+
+
+@host_agent_bp.get("/widget/history")
+async def widget_history():
+ host_id = request.args.get("host_id", "")
+ hours = int(request.args.get("hours", "6"))
+ cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
+
+ async with current_app.db_sessionmaker() as session:
+ host = (await session.execute(
+ select(Host).where(Host.id == host_id))).scalar_one_or_none()
+ if host is None:
+ return _error(404, "not_found")
+ points = (await session.execute(
+ select(PluginMetric).where(
+ PluginMetric.source_module == SOURCE_MODULE,
+ PluginMetric.resource_name == host.name,
+ PluginMetric.metric_name.in_(
+ ("cpu_pct", "mem_used_pct", "disk_used_pct_worst")),
+ PluginMetric.recorded_at >= cutoff,
+ ).order_by(PluginMetric.recorded_at)
+ )).scalars().all()
+
+ series: dict[str, list[dict]] = {"cpu_pct": [], "mem_used_pct": [], "disk_used_pct_worst": []}
+ for p in points:
+ series[p.metric_name].append({"t": p.recorded_at.isoformat(), "v": p.value})
+
+ return await render_template("widget_history.html", host=host, series=series, hours=hours)
+
+
+def _new_token_pair() -> tuple[str, str]:
+ raw = secrets.token_urlsafe(32)
+ return raw, _hash_token(raw)
+
+
+@host_agent_bp.get("/settings/")
+@require_role(UserRole.admin)
+async def settings_list():
+ async with current_app.db_sessionmaker() as session:
+ regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
+ hosts_by_id = {
+ h.id: h for h in (await session.execute(
+ select(Host).where(Host.id.in_([r.host_id for r in regs])))).scalars().all()
+ } if regs else {}
+
+ new_token = request.args.get("new_token")
+ new_host_id = request.args.get("host_id")
+ install_url = None
+ if new_token and new_host_id:
+ install_url = f"{public_base_url(request)}/plugins/host_agent/install.sh?token={new_token}"
+
+ return await render_template(
+ "settings_list.html",
+ registrations=[
+ {"reg": r, "host": hosts_by_id.get(r.host_id)} for r in regs
+ ],
+ new_token=new_token,
+ install_url=install_url,
+ )
+
+
+@host_agent_bp.post("/settings/add-host")
+@require_role(UserRole.admin)
+async def add_host():
+ form = await request.form
+ name = (form.get("name") or "").strip()
+ address = (form.get("address") or "").strip()
+ if not name:
+ return _error(400, "missing_name")
+
+ async with current_app.db_sessionmaker() as session:
+ host = (await session.execute(
+ select(Host).where(Host.name == name))).scalar_one_or_none()
+ if host is None:
+ host = Host(name=name, address=address)
+ session.add(host)
+ await session.flush()
+ existing = (await session.execute(
+ select(HostAgentRegistration).where(
+ HostAgentRegistration.host_id == host.id))).scalar_one_or_none()
+ if existing is not None:
+ await session.rollback()
+ return _error(400, "already_registered")
+
+ raw, hashed = _new_token_pair()
+ reg = HostAgentRegistration(host_id=host.id, token_hash=hashed)
+ session.add(reg)
+ await session.commit()
+ host_id = host.id
+
+ return redirect(url_for("host_agent.settings_list") + f"?new_token={raw}&host_id={host_id}")
+
+
+@host_agent_bp.post("/settings//rotate-token")
+@require_role(UserRole.admin)
+async def rotate_token(host_id: str):
+ async with current_app.db_sessionmaker() as session:
+ reg = (await session.execute(
+ select(HostAgentRegistration).where(
+ HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
+ if reg is None:
+ return _error(404, "not_found")
+ raw, hashed = _new_token_pair()
+ reg.token_hash = hashed
+ reg.token_created_at = datetime.now(timezone.utc)
+ await session.commit()
+ return redirect(url_for("host_agent.settings_list") + f"?new_token={raw}&host_id={host_id}")
+
+
+@host_agent_bp.post("/settings//delete")
+@require_role(UserRole.admin)
+async def delete_registration(host_id: str):
+ async with current_app.db_sessionmaker() as session:
+ reg = (await session.execute(
+ select(HostAgentRegistration).where(
+ HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
+ if reg is not None:
+ await session.delete(reg)
+ await session.commit()
+ return redirect(url_for("host_agent.settings_list"))
diff --git a/plugins/host_agent/scheduler.py b/plugins/host_agent/scheduler.py
new file mode 100644
index 0000000..85056c1
--- /dev/null
+++ b/plugins/host_agent/scheduler.py
@@ -0,0 +1,78 @@
+# plugins/host_agent/scheduler.py
+from __future__ import annotations
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Iterable
+
+from sqlalchemy import select
+
+from steward.core.scheduler import ScheduledTask
+from steward.models.hosts import Host
+from .models import HostAgentRegistration
+
+logger = logging.getLogger(__name__)
+
+
+def _filter_stale(
+ regs: Iterable,
+ *,
+ now: datetime,
+ stale_after_seconds: int,
+) -> list:
+ """Pure staleness filter: returns the subset with last_seen_at strictly
+ older than (now - stale_after_seconds). Rows with last_seen_at=None are
+ never stale (they are unregistered-in-practice)."""
+ cutoff = now - timedelta(seconds=stale_after_seconds)
+ return [
+ r for r in regs
+ if r.last_seen_at is not None and r.last_seen_at < cutoff
+ ]
+
+
+async def find_stale_registrations(app, stale_after_seconds: int = 180) -> list[dict]:
+ async with app.db_sessionmaker() as session:
+ all_regs = (await session.execute(
+ select(HostAgentRegistration)
+ )).scalars().all()
+ stale_rows = _filter_stale(
+ all_regs,
+ now=datetime.now(timezone.utc),
+ stale_after_seconds=stale_after_seconds,
+ )
+ if not stale_rows:
+ return []
+ hosts = {
+ h.id: h for h in (await session.execute(
+ select(Host).where(Host.id.in_([r.host_id for r in stale_rows]))
+ )).scalars().all()
+ }
+ return [
+ {
+ "host_id": r.host_id,
+ "host_name": hosts[r.host_id].name if r.host_id in hosts else "?",
+ "last_seen_at": r.last_seen_at,
+ }
+ for r in stale_rows
+ ]
+
+
+def make_task(app) -> ScheduledTask:
+ async def _check_stale():
+ try:
+ stale = await find_stale_registrations(app)
+ except Exception:
+ logger.exception("host_agent stale check failed")
+ return
+ if stale:
+ logger.info(
+ "host_agent: %d stale agent(s): %s",
+ len(stale),
+ [s["host_name"] for s in stale],
+ )
+
+ return ScheduledTask(
+ name="host_agent_stale_check",
+ coro_factory=_check_stale,
+ interval_seconds=60,
+ run_on_startup=False,
+ )
diff --git a/plugins/host_agent/templates/install.sh.j2 b/plugins/host_agent/templates/install.sh.j2
new file mode 100644
index 0000000..02d55be
--- /dev/null
+++ b/plugins/host_agent/templates/install.sh.j2
@@ -0,0 +1,81 @@
+#!/bin/sh
+# Steward host agent installer
+# Generated for: {{ host_name }} ({{ host_address }})
+# Steward URL: {{ url }}
+set -e
+
+STEWARD_URL="{{ url }}"
+AGENT_TOKEN="{{ token }}"
+AGENT_VERSION="{{ agent_version }}"
+
+AGENT_USER="steward-agent"
+AGENT_DIR="/usr/local/lib/steward-agent"
+CONF_FILE="/etc/steward-agent.conf"
+UNIT_FILE="/etc/systemd/system/steward-agent.service"
+
+# ── preflight ────────────────────────────────────────────────────────────────
+[ "$(id -u)" = "0" ] || { echo "must run as root (use sudo)"; exit 1; }
+command -v systemctl >/dev/null 2>&1 || { echo "systemd not found — unsupported init system"; exit 1; }
+command -v python3 >/dev/null 2>&1 || { echo "python3 not found — install python3 first"; exit 1; }
+
+# Handle --uninstall
+if [ "${1:-}" = "--uninstall" ]; then
+ systemctl disable --now steward-agent.service 2>/dev/null || true
+ rm -f "$UNIT_FILE" "$CONF_FILE"
+ rm -rf "$AGENT_DIR"
+ systemctl daemon-reload
+ # keep the system user — removing it risks orphaned files elsewhere
+ echo "uninstalled"
+ exit 0
+fi
+
+# ── create system user ───────────────────────────────────────────────────────
+if ! id "$AGENT_USER" >/dev/null 2>&1; then
+ useradd --system --no-create-home --shell /usr/sbin/nologin "$AGENT_USER"
+fi
+
+# ── drop the agent file ──────────────────────────────────────────────────────
+mkdir -p "$AGENT_DIR"
+curl -sSL "${STEWARD_URL}/plugins/host_agent/agent.py" -o "$AGENT_DIR/agent.py"
+chmod 0755 "$AGENT_DIR/agent.py"
+chown root:root "$AGENT_DIR/agent.py"
+
+# ── write config ─────────────────────────────────────────────────────────────
+cat > "$CONF_FILE" < "$UNIT_FILE" <Host Agent — Registered Hosts
+
+{% if new_token %}
+
+
New token — copy this install command now
+
+ This is the only time the raw token is shown. Rotate if you lose it.
+
+
curl -sSL '{{ install_url }}' | sudo sh
+
+{% endif %}
+
+
+
+
+
+
+
+ Host Agent version Distro
+ Last seen Actions
+
+
+
+ {% for item in registrations %}
+
+ {{ item.host.name if item.host else item.reg.host_id }}
+ {{ item.reg.agent_version or "—" }}
+ {{ item.reg.distro or "—" }}
+ {{ item.reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") if item.reg.last_seen_at else "never" }}
+
+
+
+
+
+ {% else %}
+ No hosts registered. Add one above.
+ {% endfor %}
+
+
+
+{% endblock %}
diff --git a/plugins/host_agent/templates/widget_history.html b/plugins/host_agent/templates/widget_history.html
new file mode 100644
index 0000000..35e363f
--- /dev/null
+++ b/plugins/host_agent/templates/widget_history.html
@@ -0,0 +1,24 @@
+
+
{{ host.name }} — last {{ hours }}h
+
+
+
diff --git a/plugins/host_agent/templates/widget_table.html b/plugins/host_agent/templates/widget_table.html
new file mode 100644
index 0000000..a646fad
--- /dev/null
+++ b/plugins/host_agent/templates/widget_table.html
@@ -0,0 +1,34 @@
+{# host_agent widget — fleet glance, flex rows (no header wrap) #}
+{% if rows %}
+
+ {% for r in rows %}
+ {% set stale = not r.reg.last_seen_at %}
+
+
+
+ {{ r.host.name }}
+
+
+ {% if r.cpu_pct is not none %}
+ cpu {{ "%.0f"|format(r.cpu_pct) }}%
+ {% endif %}
+ {% if r.mem_used_pct is not none %}
+ mem {{ "%.0f"|format(r.mem_used_pct) }}%
+ {% endif %}
+ {% if r.disk_worst is not none %}
+ disk {{ "%.0f"|format(r.disk_worst) }}%
+ {% endif %}
+ {% if r.load_1m is not none %}
+ load {{ "%.2f"|format(r.load_1m) }}
+ {% endif %}
+
+
+ {{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }}
+
+
+ {% endfor %}
+
+{% else %}
+No hosts with agent data yet.
+{% endif %}
diff --git a/plugins/http/__init__.py b/plugins/http/__init__.py
new file mode 100644
index 0000000..cb79685
--- /dev/null
+++ b/plugins/http/__init__.py
@@ -0,0 +1,24 @@
+# plugins/http/__init__.py
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from quart import Quart
+
+_app: "Quart | None" = None
+
+
+def setup(app: "Quart") -> None:
+ global _app
+ _app = app
+ from .models import HttpMonitor, HttpResult # noqa: F401
+
+
+def get_scheduled_tasks() -> list:
+ from .scheduler import make_task
+ return [make_task(_app)]
+
+
+def get_blueprint():
+ from .routes import http_bp
+ return http_bp
diff --git a/plugins/http/checker.py b/plugins/http/checker.py
new file mode 100644
index 0000000..7eec743
--- /dev/null
+++ b/plugins/http/checker.py
@@ -0,0 +1,105 @@
+# plugins/http/checker.py
+"""Core HTTP check logic — runs a single monitor check and returns a result dict."""
+from __future__ import annotations
+import asyncio
+import json
+import logging
+import ssl
+import time
+from datetime import datetime, timezone
+from urllib.parse import urlparse
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+async def _get_tls_expiry(hostname: str, port: int) -> datetime | None:
+ """Attempt a bare TLS handshake to extract the certificate expiry date."""
+ ctx = ssl.create_default_context()
+ try:
+ reader, writer = await asyncio.wait_for(
+ asyncio.open_connection(hostname, port, ssl=ctx),
+ timeout=5.0,
+ )
+ cert = writer.get_extra_info("ssl_object").getpeercert()
+ writer.close()
+ try:
+ await writer.wait_closed()
+ except Exception:
+ pass
+ exp_str = cert.get("notAfter", "")
+ if not exp_str:
+ return None
+ # Format: "Mar 22 12:00:00 2027 GMT"
+ return datetime.strptime(exp_str, "%b %d %H:%M:%S %Y %Z").replace(
+ tzinfo=timezone.utc
+ )
+ except Exception as exc:
+ logger.debug("TLS check failed for %s:%d — %s", hostname, port, exc)
+ return None
+
+
+async def run_check(
+ url: str,
+ method: str = "GET",
+ expected_status: int = 200,
+ content_match: str = "",
+ headers: dict | None = None,
+ timeout_seconds: int = 10,
+ follow_redirects: bool = True,
+ verify_ssl: bool = True,
+) -> dict:
+ """
+ Perform one HTTP check. Returns a dict with:
+ is_up, status_code, response_ms, content_matched, error_msg, tls_expires_at
+ """
+ result: dict = {
+ "is_up": False,
+ "status_code": None,
+ "response_ms": None,
+ "content_matched": None,
+ "error_msg": None,
+ "tls_expires_at": None,
+ }
+
+ parsed = urlparse(url)
+ is_https = parsed.scheme.lower() == "https"
+
+ t0 = time.monotonic()
+ try:
+ async with httpx.AsyncClient(
+ follow_redirects=follow_redirects,
+ verify=verify_ssl,
+ timeout=timeout_seconds,
+ ) as client:
+ response = await client.request(
+ method,
+ url,
+ headers=headers or {},
+ )
+ result["response_ms"] = round((time.monotonic() - t0) * 1000, 1)
+ result["status_code"] = response.status_code
+
+ status_ok = response.status_code == expected_status
+ if content_match:
+ matched = content_match in response.text
+ result["content_matched"] = matched
+ result["is_up"] = status_ok and matched
+ else:
+ result["is_up"] = status_ok
+
+ except httpx.TimeoutException:
+ result["response_ms"] = round((time.monotonic() - t0) * 1000, 1)
+ result["error_msg"] = "Timeout"
+ except httpx.ConnectError as exc:
+ result["error_msg"] = f"Connection error: {exc}"
+ except Exception as exc:
+ result["error_msg"] = str(exc)[:512]
+
+ # TLS expiry — only for HTTPS, and only when the check succeeded or we got a response
+ if is_https and result["status_code"] is not None:
+ port = parsed.port or 443
+ result["tls_expires_at"] = await _get_tls_expiry(parsed.hostname, port)
+
+ return result
diff --git a/plugins/http/migrations/__init__.py b/plugins/http/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/plugins/http/migrations/env.py b/plugins/http/migrations/env.py
new file mode 100644
index 0000000..fd308a2
--- /dev/null
+++ b/plugins/http/migrations/env.py
@@ -0,0 +1,70 @@
+# plugins/http/migrations/env.py
+"""Alembic env.py for the HTTP monitoring plugin."""
+import asyncio
+import os
+import sys
+from pathlib import Path
+from logging.config import fileConfig
+
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+from alembic import context
+
+sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
+
+from fabledscryer.models.base import Base
+import fabledscryer.models # noqa: F401
+from plugins.http.models import HttpMonitor, HttpResult # noqa: F401
+
+config = context.config
+if config.config_file_name is not None:
+ fileConfig(config.config_file_name)
+
+target_metadata = Base.metadata
+
+
+def _get_url() -> str:
+ import yaml
+ cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml")
+ try:
+ with open(cfg_path) as f:
+ cfg = yaml.safe_load(f) or {}
+ url = cfg.get("database", {}).get("url", "")
+ except FileNotFoundError:
+ url = ""
+ return os.environ.get("FABLEDSCRYER_DATABASE__URL", url)
+
+
+def run_migrations_offline() -> None:
+ context.configure(
+ url=_get_url(), target_metadata=target_metadata,
+ literal_binds=True, dialect_opts={"paramstyle": "named"},
+ )
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+ context.configure(connection=connection, target_metadata=target_metadata)
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+ cfg = config.get_section(config.config_ini_section, {})
+ cfg["sqlalchemy.url"] = _get_url()
+ connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool)
+ async with connectable.connect() as connection:
+ await connection.run_sync(do_run_migrations)
+ await connectable.dispose()
+
+
+def run_migrations_online() -> None:
+ asyncio.run(run_async_migrations())
+
+
+if context.is_offline_mode():
+ run_migrations_offline()
+else:
+ run_migrations_online()
diff --git a/plugins/http/migrations/versions/__init__.py b/plugins/http/migrations/versions/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/plugins/http/migrations/versions/http_001_initial.py b/plugins/http/migrations/versions/http_001_initial.py
new file mode 100644
index 0000000..c546023
--- /dev/null
+++ b/plugins/http/migrations/versions/http_001_initial.py
@@ -0,0 +1,52 @@
+"""HTTP monitoring plugin initial tables
+
+Revision ID: http_001_initial
+Revises: (none — branch off core via depends_on)
+Create Date: 2026-03-22
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "http_001_initial"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = "http"
+depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
+
+
+def upgrade() -> None:
+ op.create_table(
+ "http_monitors",
+ sa.Column("id", sa.String(36), primary_key=True),
+ sa.Column("name", sa.String(128), nullable=False),
+ sa.Column("url", sa.String(2048), nullable=False),
+ sa.Column("method", sa.String(8), nullable=False, server_default="GET"),
+ sa.Column("expected_status", sa.Integer, nullable=False, server_default="200"),
+ sa.Column("content_match", sa.String(512), nullable=False, server_default=""),
+ sa.Column("headers_json", sa.Text, nullable=False, server_default="{}"),
+ sa.Column("timeout_seconds", sa.Integer, nullable=False, server_default="10"),
+ sa.Column("check_interval_seconds", sa.Integer, nullable=False, server_default="0"),
+ sa.Column("follow_redirects", sa.Boolean, nullable=False, server_default="1"),
+ sa.Column("verify_ssl", sa.Boolean, nullable=False, server_default="1"),
+ sa.Column("enabled", sa.Boolean, nullable=False, server_default="1"),
+ sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+ )
+
+ op.create_table(
+ "http_results",
+ sa.Column("id", sa.String(36), primary_key=True),
+ sa.Column("monitor_id", sa.String(36), nullable=False, index=True),
+ sa.Column("checked_at", sa.DateTime(timezone=True), nullable=False, index=True),
+ sa.Column("status_code", sa.Integer, nullable=True),
+ sa.Column("response_ms", sa.Float, nullable=True),
+ sa.Column("is_up", sa.Boolean, nullable=False, server_default="0"),
+ sa.Column("content_matched", sa.Boolean, nullable=True),
+ sa.Column("error_msg", sa.String(512), nullable=True),
+ sa.Column("tls_expires_at", sa.DateTime(timezone=True), nullable=True),
+ )
+
+
+def downgrade() -> None:
+ op.drop_table("http_results")
+ op.drop_table("http_monitors")
diff --git a/plugins/http/models.py b/plugins/http/models.py
new file mode 100644
index 0000000..180effd
--- /dev/null
+++ b/plugins/http/models.py
@@ -0,0 +1,63 @@
+# plugins/http/models.py
+from __future__ import annotations
+import uuid
+from datetime import datetime, timezone
+from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text
+from sqlalchemy.orm import Mapped, mapped_column
+from fabledscryer.models.base import Base
+
+
+class HttpMonitor(Base):
+ """A configured HTTP endpoint to check periodically."""
+ __tablename__ = "http_monitors"
+
+ id: Mapped[str] = mapped_column(
+ String(36), primary_key=True, default=lambda: str(uuid.uuid4())
+ )
+ name: Mapped[str] = mapped_column(String(128), nullable=False)
+ url: Mapped[str] = mapped_column(String(2048), nullable=False)
+ method: Mapped[str] = mapped_column(String(8), nullable=False, default="GET")
+ expected_status: Mapped[int] = mapped_column(Integer, nullable=False, default=200)
+ # Optional substring to find in the response body (empty = skip check)
+ content_match: Mapped[str] = mapped_column(String(512), nullable=False, default="")
+ # JSON dict of extra request headers, e.g. {"Authorization": "Bearer ..."}
+ headers_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
+ timeout_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=10)
+ # 0 = use global plugin check_interval_seconds
+ check_interval_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ follow_redirects: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+ verify_ssl: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+ enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+ # Updated after each check so the scheduler can compute next-run time efficiently
+ last_checked_at: Mapped[datetime | None] = mapped_column(
+ DateTime(timezone=True), nullable=True
+ )
+ created_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), nullable=False,
+ default=lambda: datetime.now(timezone.utc),
+ )
+
+
+class HttpResult(Base):
+ """One check result for an HttpMonitor."""
+ __tablename__ = "http_results"
+
+ id: Mapped[str] = mapped_column(
+ String(36), primary_key=True, default=lambda: str(uuid.uuid4())
+ )
+ monitor_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
+ checked_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), nullable=False,
+ default=lambda: datetime.now(timezone.utc),
+ index=True,
+ )
+ status_code: Mapped[int | None] = mapped_column(Integer, nullable=True)
+ response_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
+ is_up: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+ # None when no content_match is configured; True/False when it is
+ content_matched: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
+ error_msg: Mapped[str | None] = mapped_column(String(512), nullable=True)
+ # TLS expiry if the endpoint is HTTPS; None for HTTP or on error
+ tls_expires_at: Mapped[datetime | None] = mapped_column(
+ DateTime(timezone=True), nullable=True
+ )
diff --git a/plugins/http/plugin.yaml b/plugins/http/plugin.yaml
new file mode 100644
index 0000000..9aa1475
--- /dev/null
+++ b/plugins/http/plugin.yaml
@@ -0,0 +1,19 @@
+name: http
+version: "1.0.0"
+description: "Synthetic HTTP endpoint monitoring — status code, response time, content match, TLS expiry"
+author: "FabledScryer"
+license: "MIT"
+min_app_version: "0.1.0"
+repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins"
+homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/http"
+tags:
+ - monitoring
+ - http
+ - synthetic
+ - uptime
+
+config:
+ check_interval_seconds: 60
+ default_timeout_seconds: 10
+ follow_redirects: true
+ verify_ssl: true
diff --git a/plugins/http/routes.py b/plugins/http/routes.py
new file mode 100644
index 0000000..eea5ce7
--- /dev/null
+++ b/plugins/http/routes.py
@@ -0,0 +1,229 @@
+# plugins/http/routes.py
+from __future__ import annotations
+import json
+import uuid
+from datetime import datetime, timezone
+from quart import Blueprint, current_app, render_template, request, redirect, url_for
+from sqlalchemy import Integer, cast, func, select
+
+from fabledscryer.auth.middleware import require_role
+from fabledscryer.models.users import UserRole
+from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds
+from .models import HttpMonitor, HttpResult
+
+http_bp = Blueprint("http", __name__, template_folder="templates")
+
+
+def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
+ if len(values) < 2:
+ return f' '
+ mn, mx = min(values), max(values)
+ if mx == mn:
+ mx = mn + 1.0
+ step = width / (len(values) - 1)
+ pts = []
+ for i, v in enumerate(values):
+ x = i * step
+ y = height - (v - mn) / (mx - mn) * (height - 2) - 1
+ pts.append(f"{x:.1f},{y:.1f}")
+ poly = " ".join(pts)
+ return (
+ f''
+ f' '
+ f' '
+ )
+
+
+async def _render_rows(range_key: str = DEFAULT_RANGE):
+ since, range_key = parse_range(range_key)
+ b_secs = bucket_seconds(since)
+ bucket_col = (
+ cast(func.extract('epoch', HttpResult.checked_at), Integer) / b_secs
+ ).label("bucket")
+
+ async with current_app.db_sessionmaker() as db:
+ result = await db.execute(
+ select(HttpMonitor).order_by(HttpMonitor.created_at)
+ )
+ monitors = list(result.scalars())
+
+ # Latest result per monitor
+ latest_map: dict[str, HttpResult] = {}
+ histories: dict[str, list] = {}
+ for m in monitors:
+ r = await db.execute(
+ select(HttpResult)
+ .where(HttpResult.monitor_id == m.id)
+ .order_by(HttpResult.checked_at.desc())
+ .limit(1)
+ )
+ latest = r.scalar_one_or_none()
+ if latest:
+ latest_map[m.id] = latest
+
+ r2 = await db.execute(
+ select(
+ func.avg(HttpResult.response_ms).label("response_ms"),
+ func.min(HttpResult.is_up.cast(Integer)).label("had_down"),
+ bucket_col,
+ )
+ .where(HttpResult.monitor_id == m.id)
+ .where(HttpResult.checked_at >= since)
+ .group_by(bucket_col)
+ .order_by(bucket_col)
+ )
+ histories[m.id] = r2.all()
+
+ monitor_data = []
+ for m in monitors:
+ hist = histories.get(m.id, [])
+ latest = latest_map.get(m.id)
+ now = datetime.now(timezone.utc)
+ tls_days = None
+ if latest and latest.tls_expires_at:
+ tls_days = (latest.tls_expires_at - now).total_seconds() / 86400.0
+ monitor_data.append({
+ "monitor": m,
+ "latest": latest,
+ "tls_days": tls_days,
+ "sparkline_ms": _sparkline([r.response_ms or 0 for r in hist]),
+ })
+
+ up = sum(1 for d in monitor_data if d["latest"] and d["latest"].is_up)
+ down = sum(1 for d in monitor_data if d["latest"] and not d["latest"].is_up)
+ return monitor_data, up, down, range_key
+
+
+@http_bp.get("/")
+@require_role(UserRole.viewer)
+async def index():
+ poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
+ current_range = request.args.get("range", DEFAULT_RANGE)
+ return await render_template(
+ "http/index.html",
+ poll_interval=poll_interval,
+ current_range=current_range,
+ )
+
+
+@http_bp.get("/rows")
+@require_role(UserRole.viewer)
+async def rows():
+ """HTMX fragment: monitor list with status + sparklines."""
+ monitor_data, up, down, range_key = await _render_rows(
+ request.args.get("range", DEFAULT_RANGE)
+ )
+ return await render_template(
+ "http/rows.html",
+ monitor_data=monitor_data,
+ up=up,
+ down=down,
+ range_key=range_key,
+ )
+
+
+@http_bp.post("/add")
+@require_role(UserRole.operator)
+async def add_monitor():
+ """Add a new HTTP monitor."""
+ form = await request.form
+ url = form.get("url", "").strip()
+ if not url:
+ return redirect(url_for("http.index"))
+ if not url.startswith(("http://", "https://")):
+ url = "https://" + url
+
+ monitor = HttpMonitor(
+ id=str(uuid.uuid4()),
+ name=form.get("name", "").strip() or url,
+ url=url,
+ method=form.get("method", "GET").upper(),
+ expected_status=int(form.get("expected_status", 200) or 200),
+ content_match=form.get("content_match", "").strip(),
+ headers_json="{}",
+ timeout_seconds=int(form.get("timeout_seconds", 10) or 10),
+ check_interval_seconds=int(form.get("check_interval_seconds", 0) or 0),
+ follow_redirects="follow_redirects" in form,
+ verify_ssl="verify_ssl" in form,
+ enabled=True,
+ created_at=datetime.now(timezone.utc),
+ )
+ async with current_app.db_sessionmaker() as db:
+ async with db.begin():
+ db.add(monitor)
+
+ return redirect(url_for("http.index"))
+
+
+@http_bp.post("//delete")
+@require_role(UserRole.operator)
+async def delete_monitor(monitor_id: str):
+ async with current_app.db_sessionmaker() as db:
+ async with db.begin():
+ m = await db.get(HttpMonitor, monitor_id)
+ if m:
+ await db.delete(m)
+ return redirect(url_for("http.index"))
+
+
+@http_bp.post("//toggle")
+@require_role(UserRole.operator)
+async def toggle_monitor(monitor_id: str):
+ async with current_app.db_sessionmaker() as db:
+ async with db.begin():
+ m = await db.get(HttpMonitor, monitor_id)
+ if m:
+ m.enabled = not m.enabled
+ return redirect(url_for("http.index"))
+
+
+@http_bp.get("/widget")
+@require_role(UserRole.viewer)
+async def widget():
+ """HTMX dashboard widget: up/down counts + monitor list."""
+ show_down_only = request.args.get("show_down_only", "no") == "yes"
+ limit = max(1, min(20, int(request.args.get("limit", 10) or 10)))
+ widget_id = request.args.get("wid", "0")
+
+ async with current_app.db_sessionmaker() as db:
+ result = await db.execute(
+ select(HttpMonitor)
+ .where(HttpMonitor.enabled == True) # noqa: E712
+ .order_by(HttpMonitor.created_at)
+ )
+ monitors = list(result.scalars())
+
+ latest_map: dict[str, HttpResult] = {}
+ for m in monitors:
+ r = await db.execute(
+ select(HttpResult)
+ .where(HttpResult.monitor_id == m.id)
+ .order_by(HttpResult.checked_at.desc())
+ .limit(1)
+ )
+ latest = r.scalar_one_or_none()
+ if latest:
+ latest_map[m.id] = latest
+
+ monitor_data = [
+ {"monitor": m, "latest": latest_map.get(m.id)}
+ for m in monitors
+ ]
+
+ up = sum(1 for d in monitor_data if d["latest"] and d["latest"].is_up)
+ down = sum(1 for d in monitor_data if d["latest"] and not d["latest"].is_up)
+ pending = sum(1 for d in monitor_data if not d["latest"])
+
+ if show_down_only:
+ monitor_data = [d for d in monitor_data if not (d["latest"] and d["latest"].is_up)]
+
+ return await render_template(
+ "http/widget.html",
+ monitor_data=monitor_data[:limit],
+ up=up,
+ down=down,
+ pending=pending,
+ show_down_only=show_down_only,
+ widget_id=widget_id,
+ )
diff --git a/plugins/http/scheduler.py b/plugins/http/scheduler.py
new file mode 100644
index 0000000..f9edd67
--- /dev/null
+++ b/plugins/http/scheduler.py
@@ -0,0 +1,121 @@
+# plugins/http/scheduler.py
+from __future__ import annotations
+import asyncio
+import json
+import logging
+from datetime import datetime, timezone, timedelta
+
+from fabledscryer.core.scheduler import ScheduledTask
+from fabledscryer.core.alerts import record_metric
+
+logger = logging.getLogger(__name__)
+
+
+def make_task(app) -> ScheduledTask:
+ interval = int(
+ app.config["PLUGINS"]["http"].get("check_interval_seconds", 60)
+ )
+
+ async def check():
+ await _do_checks(app)
+
+ return ScheduledTask(
+ name="http_check",
+ coro_factory=check,
+ interval_seconds=interval,
+ run_on_startup=True,
+ )
+
+
+async def _do_checks(app) -> None:
+ from sqlalchemy import select
+ from .models import HttpMonitor, HttpResult
+ from .checker import run_check
+
+ cfg = app.config["PLUGINS"]["http"]
+ global_interval = int(cfg.get("check_interval_seconds", 60))
+ global_timeout = int(cfg.get("default_timeout_seconds", 10))
+ global_follow = bool(cfg.get("follow_redirects", True))
+ global_verify = bool(cfg.get("verify_ssl", True))
+
+ now = datetime.now(timezone.utc)
+
+ async with app.db_sessionmaker() as session:
+ result = await session.execute(
+ select(HttpMonitor).where(HttpMonitor.enabled == True) # noqa: E712
+ )
+ monitors = list(result.scalars())
+
+ # Filter to monitors whose next check time has passed
+ due: list[HttpMonitor] = []
+ for m in monitors:
+ effective_interval = m.check_interval_seconds or global_interval
+ if m.last_checked_at is None:
+ due.append(m)
+ elif (now - m.last_checked_at).total_seconds() >= effective_interval:
+ due.append(m)
+
+ if not due:
+ return
+
+ async def _check_one(monitor: HttpMonitor) -> tuple[HttpMonitor, dict]:
+ try:
+ headers = json.loads(monitor.headers_json or "{}")
+ except (ValueError, TypeError):
+ headers = {}
+ result = await run_check(
+ url=monitor.url,
+ method=monitor.method,
+ expected_status=monitor.expected_status,
+ content_match=monitor.content_match,
+ headers=headers,
+ timeout_seconds=monitor.timeout_seconds or global_timeout,
+ follow_redirects=monitor.follow_redirects if monitor.follow_redirects is not None else global_follow,
+ verify_ssl=monitor.verify_ssl if monitor.verify_ssl is not None else global_verify,
+ )
+ return monitor, result
+
+ pairs = await asyncio.gather(*[_check_one(m) for m in due], return_exceptions=True)
+
+ check_time = datetime.now(timezone.utc)
+
+ async with app.db_sessionmaker() as session:
+ async with session.begin():
+ for item in pairs:
+ if isinstance(item, Exception):
+ logger.error("HTTP check task error: %s", item)
+ continue
+ monitor, res = item
+
+ session.add(HttpResult(
+ monitor_id=monitor.id,
+ checked_at=check_time,
+ status_code=res["status_code"],
+ response_ms=res["response_ms"],
+ is_up=res["is_up"],
+ content_matched=res["content_matched"],
+ error_msg=res["error_msg"],
+ tls_expires_at=res["tls_expires_at"],
+ ))
+
+ # Update monitor's last_checked_at for next-run scheduling
+ db_monitor = await session.get(HttpMonitor, monitor.id)
+ if db_monitor:
+ db_monitor.last_checked_at = check_time
+
+ # Alert pipeline
+ if res["response_ms"] is not None:
+ await record_metric(
+ session=session,
+ source_module="http",
+ resource_name=monitor.name,
+ metric_name="response_ms",
+ value=res["response_ms"],
+ )
+ await record_metric(
+ session=session,
+ source_module="http",
+ resource_name=monitor.name,
+ metric_name="is_up",
+ value=1.0 if res["is_up"] else 0.0,
+ )
diff --git a/plugins/http/templates/http/index.html b/plugins/http/templates/http/index.html
new file mode 100644
index 0000000..6d1ffd0
--- /dev/null
+++ b/plugins/http/templates/http/index.html
@@ -0,0 +1,77 @@
+{% extends "base.html" %}
+{% block title %}HTTP Monitors — Fabled Scryer{% endblock %}
+{% block content %}
+
+
HTTP Monitors
+ {% include "_time_range.html" %}
+
+
+{# ── Add monitor form ─────────────────────────────────────────────────────── #}
+
+
+{# ── Monitor list ─────────────────────────────────────────────────────────── #}
+
+{% endblock %}
diff --git a/plugins/http/templates/http/rows.html b/plugins/http/templates/http/rows.html
new file mode 100644
index 0000000..4666fa3
--- /dev/null
+++ b/plugins/http/templates/http/rows.html
@@ -0,0 +1,109 @@
+{# http/rows.html — HTMX fragment for HTTP monitor status list #}
+
+{# ── Summary ─────────────────────────────────────────────────────────────── #}
+
+
+
+
+
Monitors
+
{{ monitor_data | length }}
+
+
+
+{# ── Monitor list ─────────────────────────────────────────────────────────── #}
+{% if monitor_data %}
+
+
+
+
+ Monitor
+ Status
+ Response
+ History
+ TLS expiry
+
+
+
+
+ {% for item in monitor_data %}
+ {% set m = item.monitor %}
+ {% set latest = item.latest %}
+
+
+ {{ m.name }}
+
+ {{ m.method }} {{ m.url }}
+
+ {% if not m.enabled %}
+ paused
+ {% endif %}
+
+
+ {% if not latest %}
+ pending
+ {% elif latest.is_up %}
+
+
+ {{ latest.status_code }}
+
+ {% else %}
+
+
+
+ {{ latest.status_code or latest.error_msg or "error" }}
+
+
+ {% if latest.content_matched == false %}
+ content mismatch
+ {% endif %}
+ {% endif %}
+
+
+ {% if latest and latest.response_ms is not none %}
+
+ {{ "%.0f" | format(latest.response_ms) }}ms
+
+ {% else %}
+ —
+ {% endif %}
+
+ {{ item.sparkline_ms | safe }}
+
+ {% if item.tls_days is not none %}
+
+ {{ item.tls_days | int }}d
+
+ {% elif m.url.startswith('https://') %}
+ —
+ {% endif %}
+
+
+
+
+
+
+ {% endfor %}
+
+
+
+{% else %}
+
+
+ No monitors configured yet. Add one using the form above.
+
+
+{% endif %}
diff --git a/plugins/http/templates/http/widget.html b/plugins/http/templates/http/widget.html
new file mode 100644
index 0000000..891f40b
--- /dev/null
+++ b/plugins/http/templates/http/widget.html
@@ -0,0 +1,56 @@
+{# http/widget.html — dashboard widget: HTTP monitor summary #}
+{% if not monitor_data and up == 0 and down == 0 and pending == 0 %}
+
+{% else %}
+
+
+ {% if up %}
+
+ {{ up }}
+ up
+
+ {% endif %}
+ {% if down %}
+
+ {{ down }}
+ down
+
+ {% endif %}
+ {% if pending %}
+
+ {{ pending }}
+ pending
+
+ {% endif %}
+
+
+
+ {% for item in monitor_data %}
+ {% set latest = item.latest %}
+
+ {% if not latest %}
+
+ {% elif latest.is_up %}
+
+ {% else %}
+
+ {% endif %}
+
+ {{ item.monitor.name }}
+
+ {% if latest and latest.response_ms is not none %}
+
+ {{ "%.0f" | format(latest.response_ms) }}ms
+
+ {% elif latest and not latest.is_up %}
+
+ {{ latest.status_code or "err" }}
+
+ {% endif %}
+
+ {% endfor %}
+
+
+{% endif %}
diff --git a/plugins/index.yaml b/plugins/index.yaml
new file mode 100644
index 0000000..0192afe
--- /dev/null
+++ b/plugins/index.yaml
@@ -0,0 +1,90 @@
+# fabledscryer-plugins / index.yaml
+#
+# Plugin catalog for Fabled Scryer.
+# Fetched by the app at: Settings → Plugins → Plugin Catalog
+#
+# After updating an entry, commit and push — changes are live within 5 minutes
+# (the app caches this file in-process for 300 seconds).
+#
+# See docs/plugins/index.yaml.example in the main app repo for the full schema.
+
+version: 1
+updated: "2026-03-22"
+
+plugins:
+
+ - name: http
+ version: "1.0.0"
+ description: "Synthetic HTTP endpoint monitoring — status code, response time, content match, TLS expiry"
+ author: "FabledScryer"
+ license: "MIT"
+ min_app_version: "0.1.0"
+ repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins"
+ homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/http"
+ download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/http-v1.0.0/http.zip"
+ checksum_sha256: ""
+ tags:
+ - monitoring
+ - http
+ - synthetic
+ - uptime
+
+ - name: docker
+ version: "1.0.0"
+ description: "Docker container status, resource usage, and restart tracking via Docker socket"
+ author: "FabledScryer"
+ license: "MIT"
+ min_app_version: "0.1.0"
+ repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins"
+ homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/docker"
+ download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/docker-v1.0.0/docker.zip"
+ checksum_sha256: ""
+ tags:
+ - containers
+ - docker
+ - infrastructure
+
+ - name: traefik
+ version: "1.0.0"
+ description: "Traefik reverse proxy metrics and access log integration"
+ author: "FabledScryer"
+ license: "MIT"
+ min_app_version: "0.1.0"
+ repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins"
+ homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/traefik"
+ download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/traefik-v1.0.0/traefik.zip"
+ checksum_sha256: ""
+ tags:
+ - proxy
+ - metrics
+ - access-log
+
+ - name: unifi
+ version: "1.0.0"
+ description: "UniFi Network controller integration — WAN health, devices, clients, DPI"
+ author: "FabledScryer"
+ license: "MIT"
+ min_app_version: "0.1.0"
+ repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins"
+ homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/unifi"
+ download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/unifi-v1.0.0/unifi.zip"
+ checksum_sha256: ""
+ tags:
+ - network
+ - unifi
+ - ubiquiti
+
+ - name: ups
+ version: "1.0.0"
+ description: "UPS monitoring via NUT (Network UPS Tools) with Ansible shutdown automation"
+ author: "FabledScryer"
+ license: "MIT"
+ min_app_version: "0.1.0"
+ repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins"
+ homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/ups"
+ download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/ups-v1.0.0/ups.zip"
+ checksum_sha256: ""
+ tags:
+ - ups
+ - power
+ - nut
diff --git a/plugins/snmp/__init__.py b/plugins/snmp/__init__.py
new file mode 100644
index 0000000..8655884
--- /dev/null
+++ b/plugins/snmp/__init__.py
@@ -0,0 +1,23 @@
+# plugins/snmp/__init__.py
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from quart import Quart
+
+_app: "Quart | None" = None
+
+
+def setup(app: "Quart") -> None:
+ global _app
+ _app = app
+
+
+def get_scheduled_tasks() -> list:
+ from .scheduler import make_poll_task
+ return [make_poll_task(_app)]
+
+
+def get_blueprint():
+ from .routes import snmp_bp
+ return snmp_bp
diff --git a/plugins/snmp/plugin.yaml b/plugins/snmp/plugin.yaml
new file mode 100644
index 0000000..5e98fce
--- /dev/null
+++ b/plugins/snmp/plugin.yaml
@@ -0,0 +1,49 @@
+# plugins/snmp/plugin.yaml
+name: snmp
+version: "1.0.0"
+description: "SNMP polling for network devices — switches, routers, printers, UPSes, etc."
+author: "FabledScryer"
+license: "MIT"
+min_app_version: "0.1.0"
+repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins"
+homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/snmp"
+tags:
+ - snmp
+ - network
+ - monitoring
+
+config:
+ poll_interval_seconds: 60
+ # Each device is polled independently. OIDs must resolve to numeric values.
+ devices:
+ - name: "core-switch"
+ host: "192.168.1.1"
+ port: 161
+ community: "public"
+ version: "2c" # "1", "2c", or "3"
+ oids:
+ - oid: "1.3.6.1.2.1.1.3.0"
+ label: "uptime_centisecs"
+ - oid: "1.3.6.1.2.1.2.2.1.10.1"
+ label: "if1_in_octets"
+ - oid: "1.3.6.1.2.1.2.2.1.16.1"
+ label: "if1_out_octets"
+ # Example: network UPS with SNMP management card (RFC 1628 UPS-MIB).
+ # upsBatteryStatus values: 1=unknown, 2=batteryNormal, 3=batteryLow, 4=batteryDepleted.
+ # Uncomment and point at your UPS to enable.
+ # - name: "ups"
+ # host: "192.168.1.10"
+ # port: 161
+ # community: "public"
+ # version: "2c"
+ # oids:
+ # - oid: "1.3.6.1.2.1.33.1.2.1.0"
+ # label: "battery_status"
+ # - oid: "1.3.6.1.2.1.33.1.2.2.0"
+ # label: "seconds_on_battery"
+ # - oid: "1.3.6.1.2.1.33.1.2.3.0"
+ # label: "runtime_minutes_remaining"
+ # - oid: "1.3.6.1.2.1.33.1.2.4.0"
+ # label: "battery_charge_pct"
+ # - oid: "1.3.6.1.2.1.33.1.3.3.1.3.1"
+ # label: "input_voltage_dV"
diff --git a/plugins/snmp/poller.py b/plugins/snmp/poller.py
new file mode 100644
index 0000000..2f7e86f
--- /dev/null
+++ b/plugins/snmp/poller.py
@@ -0,0 +1,94 @@
+# plugins/snmp/poller.py
+"""
+Synchronous SNMP GET helper, run via executor.
+
+Requires pysnmp-lextudio (maintained pysnmp fork):
+ pip install 'fabledscryer[snmp]'
+
+If pysnmp is not installed, poll_device() returns an empty dict and logs a warning.
+"""
+from __future__ import annotations
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def _pysnmp_available() -> bool:
+ try:
+ import pysnmp # noqa: F401
+ return True
+ except ImportError:
+ return False
+
+
+def _mp_model(version: str) -> int:
+ """Map version string to pysnmp mpModel integer."""
+ return 0 if version == "1" else 1
+
+
+def poll_device_sync(
+ host: str,
+ port: int,
+ community: str,
+ version: str,
+ oids: list[dict],
+) -> dict[str, float]:
+ """
+ Perform SNMP GET for each OID and return {label: float_value}.
+ Non-numeric OIDs (strings, etc.) are skipped.
+ Returns empty dict on any error.
+ """
+ if not _pysnmp_available():
+ logger.warning("pysnmp not installed — SNMP polling disabled. "
+ "Install with: pip install 'fabledscryer[snmp]'")
+ return {}
+
+ from pysnmp.hlapi import (
+ CommunityData,
+ ContextData,
+ ObjectIdentity,
+ ObjectType,
+ SnmpEngine,
+ UdpTransportTarget,
+ getCmd,
+ )
+
+ results: dict[str, float] = {}
+ engine = SnmpEngine()
+
+ for oid_cfg in oids:
+ oid = oid_cfg["oid"]
+ label = oid_cfg.get("label") or oid
+ scale = float(oid_cfg.get("scale", 1.0))
+
+ try:
+ error_indication, error_status, error_index, var_binds = next(
+ getCmd(
+ engine,
+ CommunityData(community, mpModel=_mp_model(version)),
+ UdpTransportTarget((host, port), timeout=5, retries=1),
+ ContextData(),
+ ObjectType(ObjectIdentity(oid)),
+ )
+ )
+ except Exception as exc:
+ logger.debug("SNMP GET %s@%s OID %s failed: %s", host, port, oid, exc)
+ continue
+
+ if error_indication:
+ logger.debug("SNMP error %s@%s OID %s: %s", host, port, oid, error_indication)
+ continue
+ if error_status:
+ logger.debug("SNMP status %s@%s OID %s: %s at %s",
+ host, port, oid, error_status.prettyPrint(),
+ error_index and var_binds[int(error_index) - 1][0] or "?")
+ continue
+
+ for _, val in var_binds:
+ try:
+ results[label] = float(val) * scale
+ except (TypeError, ValueError):
+ # Non-numeric type (e.g. OctetString description) — skip
+ logger.debug("SNMP non-numeric value for %s label=%s: %r", oid, label, val)
+
+ return results
diff --git a/plugins/snmp/routes.py b/plugins/snmp/routes.py
new file mode 100644
index 0000000..cca9697
--- /dev/null
+++ b/plugins/snmp/routes.py
@@ -0,0 +1,146 @@
+# plugins/snmp/routes.py
+from __future__ import annotations
+from datetime import datetime, timedelta, timezone
+
+from quart import Blueprint, current_app, render_template, request
+from sqlalchemy import func, select
+
+from fabledscryer.auth.middleware import require_role
+from fabledscryer.models.users import UserRole
+from fabledscryer.models.metrics import PluginMetric
+
+snmp_bp = Blueprint("snmp", __name__, template_folder="templates")
+
+
+async def _latest_readings(db, device_names: list[str]) -> dict[str, dict[str, float]]:
+ """Return {device_name: {label: latest_value}} for all configured devices."""
+ if not device_names:
+ return {}
+
+ # Latest value per (resource_name, metric_name) pair
+ subq = (
+ select(
+ PluginMetric.resource_name,
+ PluginMetric.metric_name,
+ func.max(PluginMetric.recorded_at).label("latest_at"),
+ )
+ .where(PluginMetric.source_module == "snmp")
+ .where(PluginMetric.resource_name.in_(device_names))
+ .group_by(PluginMetric.resource_name, PluginMetric.metric_name)
+ .subquery()
+ )
+ result = await db.execute(
+ select(PluginMetric).join(
+ subq,
+ (PluginMetric.resource_name == subq.c.resource_name)
+ & (PluginMetric.metric_name == subq.c.metric_name)
+ & (PluginMetric.recorded_at == subq.c.latest_at),
+ ).where(PluginMetric.source_module == "snmp")
+ )
+ rows = result.scalars().all()
+
+ out: dict[str, dict[str, float]] = {}
+ for row in rows:
+ out.setdefault(row.resource_name, {})[row.metric_name] = row.value
+ return out
+
+
+async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]:
+ """Return {label: [(recorded_at, value), ...]} for a single device."""
+ since = datetime.now(timezone.utc) - timedelta(hours=hours)
+ result = await db.execute(
+ select(PluginMetric)
+ .where(
+ PluginMetric.source_module == "snmp",
+ PluginMetric.resource_name == device_name,
+ PluginMetric.recorded_at >= since,
+ )
+ .order_by(PluginMetric.recorded_at)
+ )
+ rows = result.scalars().all()
+
+ out: dict[str, list] = {}
+ for row in rows:
+ out.setdefault(row.metric_name, []).append(
+ (row.recorded_at.isoformat(), row.value)
+ )
+ return out
+
+
+@snmp_bp.get("/")
+@require_role(UserRole.viewer)
+async def index():
+ devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
+ device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg]
+
+ async with current_app.db_sessionmaker() as db:
+ latest = await _latest_readings(db, device_names)
+
+ # Merge config + latest readings for the template
+ devices = []
+ for cfg in devices_cfg:
+ name = cfg.get("name") or cfg.get("host", "?")
+ devices.append({
+ "name": name,
+ "host": cfg.get("host", ""),
+ "oids": cfg.get("oids", []),
+ "readings": latest.get(name, {}),
+ })
+
+ poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
+ return await render_template("snmp/index.html",
+ devices=devices,
+ poll_interval=poll_interval)
+
+
+@snmp_bp.get("/widget")
+@require_role(UserRole.viewer)
+async def widget():
+ devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
+ device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg]
+
+ async with current_app.db_sessionmaker() as db:
+ latest = await _latest_readings(db, device_names[:10])
+
+ devices = []
+ for cfg in devices_cfg[:10]:
+ name = cfg.get("name") or cfg.get("host", "?")
+ devices.append({
+ "name": name,
+ "oids": cfg.get("oids", []),
+ "readings": latest.get(name, {}),
+ })
+
+ return await render_template("snmp/widget.html",
+ devices=devices,
+ total=len(devices_cfg))
+
+
+@snmp_bp.get("/device/")
+@require_role(UserRole.viewer)
+async def device_detail(device_name: str):
+ devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
+ device_cfg = next(
+ (d for d in devices_cfg if (d.get("name") or d.get("host")) == device_name),
+ None,
+ )
+ if device_cfg is None:
+ from quart import abort
+ abort(404)
+
+ hours = int(request.args.get("hours", 24))
+ async with current_app.db_sessionmaker() as db:
+ hist = await _history(db, device_name, hours=hours)
+
+ import json
+ history_json = json.dumps(hist)
+ oid_labels = [o.get("label") or o.get("oid") for o in device_cfg.get("oids", [])]
+
+ return await render_template(
+ "snmp/device.html",
+ device=device_cfg,
+ device_name=device_name,
+ oid_labels=oid_labels,
+ history_json=history_json,
+ hours=hours,
+ )
diff --git a/plugins/snmp/scheduler.py b/plugins/snmp/scheduler.py
new file mode 100644
index 0000000..b8809c3
--- /dev/null
+++ b/plugins/snmp/scheduler.py
@@ -0,0 +1,78 @@
+# plugins/snmp/scheduler.py
+from __future__ import annotations
+import asyncio
+import logging
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from quart import Quart
+
+from fabledscryer.core.scheduler import ScheduledTask
+
+logger = logging.getLogger(__name__)
+
+
+def make_poll_task(app: "Quart") -> ScheduledTask:
+ interval = app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60)
+
+ async def poll() -> None:
+ await _do_poll(app)
+
+ return ScheduledTask(
+ name="snmp_poll",
+ coro_factory=poll,
+ interval_seconds=int(interval),
+ run_on_startup=True,
+ )
+
+
+async def _do_poll(app: "Quart") -> None:
+ from datetime import datetime, timezone
+ from .poller import poll_device_sync
+ from fabledscryer.core.alerts import record_metric
+
+ devices: list[dict] = app.config["PLUGINS"]["snmp"].get("devices", [])
+ if not devices:
+ return
+
+ loop = asyncio.get_event_loop()
+ now = datetime.now(timezone.utc)
+
+ async with app.db_sessionmaker() as session:
+ async with session.begin():
+ for device in devices:
+ if not isinstance(device, dict):
+ continue
+ name = device.get("name") or device.get("host", "unknown")
+ host = device.get("host", "")
+ port = int(device.get("port", 161))
+ community = device.get("community", "public")
+ version = str(device.get("version", "2c"))
+ oids = device.get("oids", [])
+
+ if not host or not oids:
+ continue
+
+ try:
+ readings = await loop.run_in_executor(
+ None,
+ poll_device_sync,
+ host, port, community, version, oids,
+ )
+ except Exception:
+ logger.exception("SNMP poll failed for device %s (%s)", name, host)
+ continue
+
+ for label, value in readings.items():
+ await record_metric(
+ session=session,
+ source_module="snmp",
+ resource_name=name,
+ metric_name=label,
+ value=value,
+ )
+
+ if readings:
+ logger.debug("SNMP polled %s (%s): %d OID(s)", name, host, len(readings))
+ else:
+ logger.debug("SNMP polled %s (%s): no readings", name, host)
diff --git a/plugins/snmp/templates/snmp/device.html b/plugins/snmp/templates/snmp/device.html
new file mode 100644
index 0000000..82022a9
--- /dev/null
+++ b/plugins/snmp/templates/snmp/device.html
@@ -0,0 +1,84 @@
+{# plugins/snmp/templates/snmp/device.html #}
+{% extends "base.html" %}
+{% block title %}SNMP — {{ device_name }} — Fabled Scryer{% endblock %}
+{% block content %}
+
+
← SNMP
+
{{ device_name }}
+
{{ device.host }}
+
+
+
+{% if oid_labels %}
+
+
+
+
+
+{% else %}
+
+ No OIDs configured for this device.
+
+{% endif %}
+{% endblock %}
diff --git a/plugins/snmp/templates/snmp/index.html b/plugins/snmp/templates/snmp/index.html
new file mode 100644
index 0000000..86a1485
--- /dev/null
+++ b/plugins/snmp/templates/snmp/index.html
@@ -0,0 +1,57 @@
+{# plugins/snmp/templates/snmp/index.html #}
+{% extends "base.html" %}
+{% block title %}SNMP — Fabled Scryer{% endblock %}
+{% block content %}
+
+
SNMP Devices
+ {{ devices|length }} device(s) configured
+
+
+{% if not devices %}
+
+ No SNMP devices configured. Add devices to plugin.yaml.
+
+{% else %}
+
+{% for device in devices %}
+
+
+
{{ device.name }}
+
{{ device.host }}
+ {% if device.readings %}
+
● reachable
+ {% else %}
+
○ no data yet
+ {% endif %}
+
History
+
+
+ {% if device.readings %}
+
+ {% for oid_cfg in device.oids %}
+ {% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %}
+ {% set val = device.readings.get(label) %}
+
+
{{ label }}
+ {% if val is not none %}
+
+ {% if val >= 1000000 %}{{ "%.2f"|format(val / 1000000) }}M
+ {% elif val >= 1000 %}{{ "%.1f"|format(val / 1000) }}K
+ {% else %}{{ "%.4g"|format(val) }}{% endif %}
+
+ {% else %}
+
—
+ {% endif %}
+
+ {% endfor %}
+
+ {% else %}
+
+ No readings yet — waiting for next poll cycle.
+
+ {% endif %}
+
+{% endfor %}
+
+{% endif %}
+{% endblock %}
diff --git a/plugins/snmp/templates/snmp/widget.html b/plugins/snmp/templates/snmp/widget.html
new file mode 100644
index 0000000..fc4d593
--- /dev/null
+++ b/plugins/snmp/templates/snmp/widget.html
@@ -0,0 +1,49 @@
+{# plugins/snmp/templates/snmp/widget.html #}
+{% if devices %}
+{% for device in devices %}
+
+
+
+ {{ device.name }}
+
+ {% if device.readings %}
+
●
+ {% else %}
+
○
+ {% endif %}
+
detail →
+
+ {% if device.readings %}
+
+ {% for oid_cfg in device.oids[:4] %}
+ {% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %}
+ {% set val = device.readings.get(label) %}
+ {% if val is not none %}
+
+ {{ label }}
+
+ {% if val >= 1000000 %}{{ "%.1f"|format(val / 1000000) }}M
+ {% elif val >= 1000 %}{{ "%.0f"|format(val / 1000) }}K
+ {% else %}{{ "%.4g"|format(val) }}{% endif %}
+
+
+ {% endif %}
+ {% endfor %}
+ {% if device.oids|length > 4 %}
+ +{{ device.oids|length - 4 }} more
+ {% endif %}
+
+ {% endif %}
+
+{% if not loop.last %}
+
+{% endif %}
+{% endfor %}
+{% if total > devices|length %}
+
+ +{{ total - devices|length }} more —
view all →
+
+{% endif %}
+{% else %}
+No SNMP devices configured.
+{% endif %}
diff --git a/plugins/traefik/__init__.py b/plugins/traefik/__init__.py
new file mode 100644
index 0000000..f2c7ac7
--- /dev/null
+++ b/plugins/traefik/__init__.py
@@ -0,0 +1,30 @@
+# plugins/traefik/__init__.py
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from quart import Quart
+
+_app: "Quart | None" = None
+
+
+def setup(app: "Quart") -> None:
+ global _app
+ _app = app
+ from .models import TraefikMetric, TraefikCert, TraefikGlobalStat, TraefikAccessSummary # noqa: F401
+
+
+def get_scheduled_tasks() -> list:
+ from .scheduler import make_scrape_task, make_access_log_task
+ tasks = [make_scrape_task(_app)]
+ access_log_cfg = _app.config["PLUGINS"]["traefik"].get("access_log", {})
+ if not isinstance(access_log_cfg, dict):
+ access_log_cfg = {}
+ if access_log_cfg.get("enabled", False):
+ tasks.append(make_access_log_task(_app))
+ return tasks
+
+
+def get_blueprint():
+ from .routes import traefik_bp
+ return traefik_bp
diff --git a/plugins/traefik/access_log.py b/plugins/traefik/access_log.py
new file mode 100644
index 0000000..909b493
--- /dev/null
+++ b/plugins/traefik/access_log.py
@@ -0,0 +1,199 @@
+# plugins/traefik/access_log.py
+"""Traefik JSON access log tailer and traffic-origin aggregator.
+
+Reads new lines from Traefik's JSON access log on each scheduler tick,
+classifies source IPs as internal (RFC1918) or external, optionally
+looks up country codes via MaxMind GeoLite2, and returns an aggregated
+summary ready to persist in TraefikAccessSummary.
+"""
+from __future__ import annotations
+
+import ipaddress
+import json
+from collections import defaultdict
+from pathlib import Path
+
+# ── Module-level file-position state ─────────────────────────────────────────
+_log_position: int = 0 # byte offset of last read
+_log_inode: int = 0 # detect rotation by inode change
+
+# ── Private IP ranges (RFC1918 + loopback + link-local) ──────────────────────
+_PRIVATE_NETS = [
+ ipaddress.ip_network("10.0.0.0/8"),
+ ipaddress.ip_network("172.16.0.0/12"),
+ ipaddress.ip_network("192.168.0.0/16"),
+ ipaddress.ip_network("127.0.0.0/8"),
+ ipaddress.ip_network("169.254.0.0/16"),
+ ipaddress.ip_network("::1/128"),
+ ipaddress.ip_network("fc00::/7"),
+ ipaddress.ip_network("fe80::/10"),
+]
+
+
+def _strip_port(raw: str) -> str:
+ """Strip trailing :port from an IPv4 address string. Leaves IPv6 untouched."""
+ if not raw:
+ return raw
+ # Bracketed IPv6 with port: [::1]:12345
+ if raw.startswith("["):
+ return raw[1:raw.index("]")] if "]" in raw else raw
+ # IPv4:port has exactly one colon
+ if raw.count(":") == 1:
+ return raw.rsplit(":", 1)[0]
+ return raw
+
+
+def is_internal(raw_ip: str) -> bool:
+ """Return True if the address falls within a private/loopback range."""
+ ip_str = _strip_port(raw_ip)
+ try:
+ addr = ipaddress.ip_address(ip_str)
+ return any(addr in net for net in _PRIVATE_NETS)
+ except ValueError:
+ return False
+
+
+def _country(ip_str: str, geoip_db) -> str | None:
+ """Return ISO country code via an open maxminddb reader, or None."""
+ if geoip_db is None:
+ return None
+ clean = _strip_port(ip_str)
+ try:
+ record = geoip_db.get(clean)
+ if record:
+ return record.get("country", {}).get("iso_code")
+ except Exception:
+ pass
+ return None
+
+
+def read_new_lines(log_path: str) -> list[dict]:
+ """Return all new JSON-parsed access log entries since the last call.
+
+ Handles log rotation: if the file shrinks or its inode changes, we
+ reset to the beginning of the new file.
+ """
+ global _log_position, _log_inode
+
+ path = Path(log_path)
+ if not path.exists():
+ return []
+
+ try:
+ stat = path.stat()
+ except OSError:
+ return []
+
+ current_inode = stat.st_ino
+ current_size = stat.st_size
+
+ # Rotation detected
+ if current_inode != _log_inode or current_size < _log_position:
+ _log_position = 0
+ _log_inode = current_inode
+
+ if current_size == _log_position:
+ return []
+
+ entries: list[dict] = []
+ try:
+ with open(log_path, "rb") as f:
+ f.seek(_log_position)
+ raw = f.read(current_size - _log_position)
+ _log_position = current_size
+ except OSError:
+ return []
+
+ for raw_line in raw.split(b"\n"):
+ raw_line = raw_line.strip()
+ if not raw_line:
+ continue
+ try:
+ entries.append(json.loads(raw_line))
+ except json.JSONDecodeError:
+ continue
+
+ return entries
+
+
+def aggregate(entries: list[dict], geoip_db=None, top_n: int = 10) -> dict | None:
+ """Aggregate a batch of access log entries into a summary dict.
+
+ Returns None if there are no entries.
+ Result keys map directly to TraefikAccessSummary columns (excluding
+ id and period_start/period_end, which the caller provides).
+ """
+ if not entries:
+ return None
+
+ total = 0
+ internal = 0
+ external = 0
+ total_bytes = 0.0
+
+ by_router: dict[str, int] = defaultdict(int)
+ ip_counts: dict[str, int] = defaultdict(int)
+ ip_internal: dict[str, bool] = {}
+ ip_country: dict[str, str | None] = {}
+ country_counts: dict[str, int] = defaultdict(int)
+
+ for entry in entries:
+ total += 1
+ raw_ip = entry.get("ClientHost", "")
+ router = entry.get("RouterName") or entry.get("ServiceName") or "unknown"
+ total_bytes += entry.get("DownstreamContentSize") or 0
+
+ by_router[router] += 1
+
+ internal_flag = is_internal(raw_ip)
+ if internal_flag:
+ internal += 1
+ else:
+ external += 1
+ if raw_ip not in ip_country:
+ ip_country[raw_ip] = _country(raw_ip, geoip_db)
+ c = ip_country[raw_ip]
+ if c:
+ country_counts[c] += 1
+
+ ip_counts[raw_ip] += 1
+ ip_internal[raw_ip] = internal_flag
+
+ top_ips = [
+ {
+ "ip": ip,
+ "count": cnt,
+ "internal": ip_internal[ip],
+ "country": ip_country.get(ip),
+ }
+ for ip, cnt in sorted(ip_counts.items(), key=lambda x: x[1], reverse=True)[:top_n]
+ ]
+ top_countries = [
+ {"country": c, "count": n}
+ for c, n in sorted(country_counts.items(), key=lambda x: x[1], reverse=True)[:top_n]
+ ]
+ top_routers = [
+ {"router": r, "count": n}
+ for r, n in sorted(by_router.items(), key=lambda x: x[1], reverse=True)[:top_n]
+ ]
+
+ return {
+ "total_requests": total,
+ "internal_requests": internal,
+ "external_requests": external,
+ "total_bytes": total_bytes,
+ "top_ips_json": json.dumps(top_ips),
+ "top_countries_json": json.dumps(top_countries),
+ "top_routers_json": json.dumps(top_routers),
+ }
+
+
+def open_geoip(db_path: str | None):
+ """Open a MaxMind mmdb file if available. Returns None if not configured or missing."""
+ if not db_path:
+ return None
+ try:
+ import maxminddb # type: ignore[import]
+ return maxminddb.open_database(db_path)
+ except Exception:
+ return None
diff --git a/plugins/traefik/migrations/__init__.py b/plugins/traefik/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/plugins/traefik/migrations/env.py b/plugins/traefik/migrations/env.py
new file mode 100644
index 0000000..1919fec
--- /dev/null
+++ b/plugins/traefik/migrations/env.py
@@ -0,0 +1,75 @@
+# plugins/traefik/migrations/env.py
+"""Alembic env.py for the Traefik plugin.
+
+For standalone use during plugin development.
+At app startup, migration_runner.py uses the core env.py with version_locations.
+"""
+import asyncio
+import os
+import sys
+from pathlib import Path
+from logging.config import fileConfig
+
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+from alembic import context
+
+# Make fabledscryer importable
+sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
+
+from fabledscryer.models.base import Base
+import fabledscryer.models # noqa: F401 — core models
+from plugins.traefik.models import TraefikMetric # noqa: F401 — plugin model
+
+config = context.config
+if config.config_file_name is not None:
+ fileConfig(config.config_file_name)
+
+target_metadata = Base.metadata
+
+
+def _get_url() -> str:
+ import yaml
+ cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml")
+ try:
+ with open(cfg_path) as f:
+ cfg = yaml.safe_load(f) or {}
+ url = cfg.get("database", {}).get("url", "")
+ except FileNotFoundError:
+ url = ""
+ return os.environ.get("FABLEDSCRYER_DATABASE__URL", url)
+
+
+def run_migrations_offline() -> None:
+ context.configure(
+ url=_get_url(), target_metadata=target_metadata,
+ literal_binds=True, dialect_opts={"paramstyle": "named"},
+ )
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+ context.configure(connection=connection, target_metadata=target_metadata)
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+ cfg = config.get_section(config.config_ini_section, {})
+ cfg["sqlalchemy.url"] = _get_url()
+ connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool)
+ async with connectable.connect() as connection:
+ await connection.run_sync(do_run_migrations)
+ await connectable.dispose()
+
+
+def run_migrations_online() -> None:
+ asyncio.run(run_async_migrations())
+
+
+if context.is_offline_mode():
+ run_migrations_offline()
+else:
+ run_migrations_online()
diff --git a/plugins/traefik/migrations/versions/__init__.py b/plugins/traefik/migrations/versions/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/plugins/traefik/migrations/versions/traefik_001_initial.py b/plugins/traefik/migrations/versions/traefik_001_initial.py
new file mode 100644
index 0000000..aa53f78
--- /dev/null
+++ b/plugins/traefik/migrations/versions/traefik_001_initial.py
@@ -0,0 +1,37 @@
+# plugins/traefik/migrations/versions/traefik_001_initial.py
+"""Traefik metrics table
+
+Revision ID: traefik_001_initial
+Revises: (none — this is a branch off the core head via depends_on)
+Create Date: 2026-03-17
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "traefik_001_initial"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = "traefik"
+depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
+
+
+def upgrade() -> None:
+ op.create_table(
+ "traefik_metrics",
+ sa.Column("id", sa.String(36), primary_key=True),
+ sa.Column("router_name", sa.String(255), nullable=False),
+ sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("request_rate", sa.Float, nullable=False),
+ sa.Column("error_rate_4xx_pct", sa.Float, nullable=False),
+ sa.Column("error_rate_5xx_pct", sa.Float, nullable=False),
+ sa.Column("latency_p50_ms", sa.Float, nullable=False),
+ sa.Column("latency_p95_ms", sa.Float, nullable=False),
+ sa.Column("latency_p99_ms", sa.Float, nullable=False),
+ )
+ op.create_index("ix_traefik_metrics_router_scraped", "traefik_metrics",
+ ["router_name", "scraped_at"])
+
+
+def downgrade() -> None:
+ op.drop_index("ix_traefik_metrics_router_scraped", "traefik_metrics")
+ op.drop_table("traefik_metrics")
diff --git a/plugins/traefik/migrations/versions/traefik_002_stats.py b/plugins/traefik/migrations/versions/traefik_002_stats.py
new file mode 100644
index 0000000..8320d53
--- /dev/null
+++ b/plugins/traefik/migrations/versions/traefik_002_stats.py
@@ -0,0 +1,48 @@
+"""Traefik extended stats: bandwidth, certs, global stats
+
+Revision ID: traefik_002_stats
+Revises: traefik_001_initial
+Create Date: 2026-03-19
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "traefik_002_stats"
+down_revision: Union[str, None] = "traefik_001_initial"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ op.add_column(
+ "traefik_metrics",
+ sa.Column("response_bytes_rate", sa.Float, nullable=False, server_default="0"),
+ )
+
+ op.create_table(
+ "traefik_certs",
+ sa.Column("serial", sa.String(255), primary_key=True),
+ sa.Column("cn", sa.String(512), nullable=True),
+ sa.Column("sans", sa.Text, nullable=True),
+ sa.Column("not_after", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+ )
+
+ op.create_table(
+ "traefik_global_stats",
+ sa.Column("id", sa.String(36), primary_key=True),
+ sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("open_conns_total", sa.Float, nullable=True),
+ sa.Column("config_reloads_total", sa.Float, nullable=True),
+ sa.Column("config_last_reload_success", sa.Float, nullable=True),
+ sa.Column("process_memory_bytes", sa.Float, nullable=True),
+ )
+ op.create_index("ix_traefik_global_stats_scraped", "traefik_global_stats", ["scraped_at"])
+
+
+def downgrade() -> None:
+ op.drop_index("ix_traefik_global_stats_scraped", "traefik_global_stats")
+ op.drop_table("traefik_global_stats")
+ op.drop_table("traefik_certs")
+ op.drop_column("traefik_metrics", "response_bytes_rate")
diff --git a/plugins/traefik/migrations/versions/traefik_003_access_log.py b/plugins/traefik/migrations/versions/traefik_003_access_log.py
new file mode 100644
index 0000000..fbc8009
--- /dev/null
+++ b/plugins/traefik/migrations/versions/traefik_003_access_log.py
@@ -0,0 +1,40 @@
+"""Traefik access log traffic summaries
+
+Revision ID: traefik_003_access_log
+Revises: traefik_002_stats
+Create Date: 2026-03-20
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "traefik_003_access_log"
+down_revision: Union[str, None] = "traefik_002_stats"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "traefik_access_summaries",
+ sa.Column("id", sa.String(36), primary_key=True),
+ sa.Column("period_start", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("period_end", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("total_requests", sa.Integer, nullable=False, server_default="0"),
+ sa.Column("internal_requests", sa.Integer, nullable=False, server_default="0"),
+ sa.Column("external_requests", sa.Integer, nullable=False, server_default="0"),
+ sa.Column("total_bytes", sa.Float, nullable=False, server_default="0"),
+ sa.Column("top_ips_json", sa.Text, nullable=False, server_default="[]"),
+ sa.Column("top_countries_json", sa.Text, nullable=False, server_default="[]"),
+ sa.Column("top_routers_json", sa.Text, nullable=False, server_default="[]"),
+ )
+ op.create_index(
+ "ix_traefik_access_summaries_period_end",
+ "traefik_access_summaries",
+ ["period_end"],
+ )
+
+
+def downgrade() -> None:
+ op.drop_index("ix_traefik_access_summaries_period_end", "traefik_access_summaries")
+ op.drop_table("traefik_access_summaries")
diff --git a/plugins/traefik/models.py b/plugins/traefik/models.py
new file mode 100644
index 0000000..1ba9e50
--- /dev/null
+++ b/plugins/traefik/models.py
@@ -0,0 +1,67 @@
+# plugins/traefik/models.py
+from __future__ import annotations
+import uuid
+from datetime import datetime, timezone
+from sqlalchemy import DateTime, Float, Integer, String, Text
+from sqlalchemy.orm import Mapped, mapped_column
+from fabledscryer.models.base import Base
+
+
+class TraefikMetric(Base):
+ __tablename__ = "traefik_metrics"
+
+ id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+ router_name: Mapped[str] = mapped_column(String(255), nullable=False)
+ scraped_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
+ )
+ request_rate: Mapped[float] = mapped_column(Float, nullable=False)
+ error_rate_4xx_pct: Mapped[float] = mapped_column(Float, nullable=False)
+ error_rate_5xx_pct: Mapped[float] = mapped_column(Float, nullable=False)
+ latency_p50_ms: Mapped[float] = mapped_column(Float, nullable=False)
+ latency_p95_ms: Mapped[float] = mapped_column(Float, nullable=False)
+ latency_p99_ms: Mapped[float] = mapped_column(Float, nullable=False)
+ response_bytes_rate: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
+
+
+class TraefikCert(Base):
+ """TLS certificate expiry tracking — upserted by serial each scrape."""
+ __tablename__ = "traefik_certs"
+
+ serial: Mapped[str] = mapped_column(String(255), primary_key=True)
+ cn: Mapped[str | None] = mapped_column(String(512), nullable=True)
+ sans: Mapped[str | None] = mapped_column(Text, nullable=True)
+ not_after: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+ scraped_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
+ )
+
+
+class TraefikAccessSummary(Base):
+ """Aggregated traffic-origin summary — one row per scheduler tick."""
+ __tablename__ = "traefik_access_summaries"
+
+ id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+ period_start: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+ period_end: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+ total_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ internal_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ external_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ total_bytes: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
+ top_ips_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
+ top_countries_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
+ top_routers_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
+
+
+class TraefikGlobalStat(Base):
+ """One row per scrape — process-level and config stats."""
+ __tablename__ = "traefik_global_stats"
+
+ id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+ scraped_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
+ )
+ open_conns_total: Mapped[float | None] = mapped_column(Float, nullable=True)
+ config_reloads_total: Mapped[float | None] = mapped_column(Float, nullable=True)
+ config_last_reload_success: Mapped[float | None] = mapped_column(Float, nullable=True)
+ process_memory_bytes: Mapped[float | None] = mapped_column(Float, nullable=True)
diff --git a/plugins/traefik/plugin.yaml b/plugins/traefik/plugin.yaml
new file mode 100644
index 0000000..392713b
--- /dev/null
+++ b/plugins/traefik/plugin.yaml
@@ -0,0 +1,21 @@
+# plugins/traefik/plugin.yaml
+name: traefik
+version: "1.0.0"
+description: "Traefik reverse proxy metrics and access log integration"
+author: "FabledScryer"
+license: "MIT"
+min_app_version: "0.1.0"
+repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins"
+homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/traefik"
+tags:
+ - proxy
+ - metrics
+ - access-log
+
+config:
+ metrics_url: "http://localhost:8080/metrics"
+ scrape_interval_seconds: 60
+ access_log:
+ enabled: false
+ path: "/var/log/traefik/access.log"
+ geoip_db: "" # optional: path to MaxMind GeoLite2-Country.mmdb
diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py
new file mode 100644
index 0000000..f7fe3ce
--- /dev/null
+++ b/plugins/traefik/routes.py
@@ -0,0 +1,373 @@
+# plugins/traefik/routes.py
+from __future__ import annotations
+from collections import defaultdict
+from datetime import datetime, timezone
+from quart import Blueprint, current_app, render_template, request
+from sqlalchemy import Integer, cast, func, select
+import json
+
+from fabledscryer.auth.middleware import require_role
+from fabledscryer.models.users import UserRole
+from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample
+from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric
+
+traefik_bp = Blueprint("traefik", __name__, template_folder="templates")
+
+
+def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
+ """Generate an inline SVG sparkline from a list of float values."""
+ if len(values) < 2:
+ return f' '
+ mn, mx = min(values), max(values)
+ if mx == mn:
+ mx = mn + 1.0
+ step = width / (len(values) - 1)
+ pts = []
+ for i, v in enumerate(values):
+ x = i * step
+ y = height - (v - mn) / (mx - mn) * (height - 2) - 1
+ pts.append(f"{x:.1f},{y:.1f}")
+ poly = " ".join(pts)
+ return (
+ f''
+ f' '
+ f' '
+ )
+
+
+@traefik_bp.get("/")
+@require_role(UserRole.viewer)
+async def index():
+ poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
+ _al_cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {})
+ if not isinstance(_al_cfg, dict):
+ _al_cfg = {}
+ access_log_enabled = _al_cfg.get("enabled", False)
+ current_range = request.args.get("range", DEFAULT_RANGE)
+ return await render_template(
+ "traefik/index.html",
+ poll_interval=poll_interval,
+ access_log_enabled=access_log_enabled,
+ current_range=current_range,
+ )
+
+
+@traefik_bp.get("/rows")
+@require_role(UserRole.viewer)
+async def rows():
+ """HTMX fragment: service cards with sparklines scoped to selected time range."""
+ since, range_key = parse_range(request.args.get("range"))
+ now = datetime.now(timezone.utc)
+
+ async with current_app.db_sessionmaker() as db:
+ # All known routers
+ result = await db.execute(
+ select(TraefikMetric.router_name)
+ .distinct()
+ .order_by(TraefikMetric.router_name)
+ )
+ routers = [row[0] for row in result.all()]
+
+ b_secs = bucket_seconds(since)
+ bucket_col = (
+ cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs
+ ).label("bucket")
+
+ router_data = []
+ for router in routers:
+ # Bucketed history for sparklines — always ~80 points regardless of range
+ result = await db.execute(
+ select(
+ func.avg(TraefikMetric.request_rate).label("request_rate"),
+ func.avg(TraefikMetric.latency_p95_ms).label("latency_p95_ms"),
+ func.avg(TraefikMetric.error_rate_5xx_pct).label("error_rate_5xx_pct"),
+ func.min(TraefikMetric.scraped_at).label("scraped_at"),
+ bucket_col,
+ )
+ .where(TraefikMetric.router_name == router)
+ .where(TraefikMetric.scraped_at >= since)
+ .group_by(bucket_col)
+ .order_by(bucket_col)
+ )
+ history = result.all()
+
+ # Latest value — always the most recent raw row
+ result = await db.execute(
+ select(TraefikMetric)
+ .where(TraefikMetric.router_name == router)
+ .order_by(TraefikMetric.scraped_at.desc())
+ .limit(1)
+ )
+ latest = result.scalar_one_or_none()
+ if not latest:
+ continue
+
+ router_data.append({
+ "name": router,
+ "latest": latest,
+ "sparkline_req": _sparkline([r.request_rate for r in history]),
+ "sparkline_p95": _sparkline([r.latency_p95_ms for r in history]),
+ "sparkline_5xx": _sparkline([r.error_rate_5xx_pct for r in history]),
+ })
+
+ # Latest global stat (always most recent, not range-filtered)
+ result = await db.execute(
+ select(TraefikGlobalStat)
+ .order_by(TraefikGlobalStat.scraped_at.desc())
+ .limit(1)
+ )
+ global_stat = result.scalar_one_or_none()
+
+ # All certs, sorted by soonest expiry
+ result = await db.execute(
+ select(TraefikCert).order_by(TraefikCert.not_after)
+ )
+ raw_certs = result.scalars().all()
+
+ certs = []
+ for c in raw_certs:
+ days = (c.not_after - now).total_seconds() / 86400.0
+ certs.append({
+ "serial": c.serial,
+ "cn": c.cn or c.serial,
+ "sans": c.sans,
+ "not_after": c.not_after,
+ "days_remaining": days,
+ })
+
+ return await render_template(
+ "traefik/rows.html",
+ router_data=router_data,
+ global_stat=global_stat,
+ certs=certs,
+ range_key=range_key,
+ )
+
+
+@traefik_bp.get("/widget")
+@require_role(UserRole.viewer)
+async def widget():
+ """HTMX dashboard widget fragment."""
+ now = datetime.now(timezone.utc)
+
+ async with current_app.db_sessionmaker() as db:
+ result = await db.execute(
+ select(TraefikMetric.router_name).distinct().order_by(TraefikMetric.router_name)
+ )
+ routers = [row[0] for row in result.all()]
+ router_data = []
+ for router in routers:
+ result = await db.execute(
+ select(TraefikMetric)
+ .where(TraefikMetric.router_name == router)
+ .order_by(TraefikMetric.scraped_at.desc())
+ .limit(1)
+ )
+ latest = result.scalar_one_or_none()
+ if latest:
+ router_data.append({"name": router, "latest": latest})
+
+ router_data.sort(key=lambda r: (
+ -(1 if r["latest"].error_rate_5xx_pct > 0.1 else 0),
+ -(1 if r["latest"].error_rate_4xx_pct > 5.0 else 0),
+ -r["latest"].request_rate,
+ ))
+ total_routers = len(router_data)
+ router_data = router_data[:5]
+
+ result = await db.execute(
+ select(TraefikCert).order_by(TraefikCert.not_after)
+ )
+ raw_certs = result.scalars().all()
+
+ expiring_certs = []
+ for c in raw_certs:
+ days = (c.not_after - now).total_seconds() / 86400.0
+ if days < 30:
+ expiring_certs.append({"cn": c.cn or c.serial, "days_remaining": days})
+
+ return await render_template(
+ "traefik/widget.html",
+ router_data=router_data,
+ expiring_certs=expiring_certs,
+ total_routers=total_routers,
+ )
+
+
+@traefik_bp.get("/widget/access_log")
+@require_role(UserRole.viewer)
+async def widget_access_log():
+ """HTMX dashboard widget: traffic origin summary with IP filter."""
+ from collections import defaultdict
+ ip_filter = request.args.get("ip_filter", "all")
+ limit = max(1, min(50, int(request.args.get("limit", 10) or 10)))
+ widget_id = request.args.get("wid", "0")
+
+ async with current_app.db_sessionmaker() as db:
+ result = await db.execute(
+ select(TraefikAccessSummary)
+ .order_by(TraefikAccessSummary.period_end.desc())
+ .limit(12) # last ~1 hour of 5-min windows
+ )
+ rows = result.scalars().all()
+
+ if not rows:
+ return await render_template("traefik/widget_access_log.html",
+ summary=None, widget_id=widget_id)
+
+ total = sum(r.total_requests for r in rows)
+ internal = sum(r.internal_requests for r in rows)
+ external = sum(r.external_requests for r in rows)
+
+ ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True})
+ for row in rows:
+ for entry in json.loads(row.top_ips_json):
+ k = entry["ip"]
+ ip_counts[k]["count"] += entry["count"]
+ ip_counts[k]["internal"] = entry.get("internal", True)
+
+ all_ips = sorted(
+ [{"ip": k, **v} for k, v in ip_counts.items()],
+ key=lambda x: x["count"], reverse=True,
+ )
+ if ip_filter == "internal":
+ all_ips = [e for e in all_ips if e["internal"]]
+ elif ip_filter == "external":
+ all_ips = [e for e in all_ips if not e["internal"]]
+
+ summary = {
+ "total": total,
+ "internal": internal,
+ "external": external,
+ "external_pct": (external / total * 100.0) if total else 0.0,
+ "top_ips": all_ips[:limit],
+ "ip_filter": ip_filter,
+ }
+ return await render_template("traefik/widget_access_log.html",
+ summary=summary, widget_id=widget_id)
+
+
+@traefik_bp.get("/widget/request_chart")
+@require_role(UserRole.viewer)
+async def widget_request_chart():
+ """HTMX dashboard widget: Chart.js request rate + error % over time."""
+ from datetime import timedelta
+ hours = max(1, min(24, int(request.args.get("hours", 6) or 6)))
+ widget_id = request.args.get("wid", "0")
+
+ since = datetime.now(timezone.utc) - timedelta(hours=hours)
+ b_secs = max(60, (hours * 3600) // 80) # ~80 buckets
+
+ bucket_col = (
+ cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs
+ ).label("bucket")
+
+ async with current_app.db_sessionmaker() as db:
+ result = await db.execute(
+ select(
+ func.avg(TraefikMetric.request_rate).label("req_rate"),
+ func.avg(TraefikMetric.error_rate_5xx_pct).label("err_pct"),
+ func.min(TraefikMetric.scraped_at).label("scraped_at"),
+ bucket_col,
+ )
+ .where(TraefikMetric.scraped_at >= since)
+ .group_by(bucket_col)
+ .order_by(bucket_col)
+ )
+ history = result.all()
+
+ labels = list(range(len(history)))
+ req_rates = [round(r.req_rate or 0, 3) for r in history]
+ err_pcts = [round(r.err_pct or 0, 2) for r in history]
+
+ return await render_template(
+ "traefik/widget_request_chart.html",
+ labels_json=json.dumps(labels),
+ req_rate_json=json.dumps(req_rates),
+ error_rate_json=json.dumps(err_pcts),
+ widget_id=widget_id,
+ hours=hours,
+ )
+
+
+@traefik_bp.get("/traffic")
+@require_role(UserRole.viewer)
+async def traffic():
+ """HTMX fragment: traffic origin summary scoped to selected time range."""
+ since, range_key = parse_range(request.args.get("range"))
+
+ async with current_app.db_sessionmaker() as db:
+ result = await db.execute(
+ select(TraefikAccessSummary)
+ .where(TraefikAccessSummary.period_end >= since)
+ .order_by(TraefikAccessSummary.period_end.asc())
+ # No LIMIT — all summaries in range needed for accurate totals;
+ # sparkline subsamples to 80 points afterwards
+ )
+ rows = result.scalars().all()
+
+ if not rows:
+ return await render_template(
+ "traefik/traffic.html", summary=None, history=[], range_key=range_key
+ )
+
+ total = sum(r.total_requests for r in rows)
+ internal = sum(r.internal_requests for r in rows)
+ external = sum(r.external_requests for r in rows)
+ total_bytes = sum(r.total_bytes for r in rows)
+
+ ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True, "country": None})
+ country_counts: dict[str, int] = defaultdict(int)
+ router_counts: dict[str, int] = defaultdict(int)
+
+ for row in rows:
+ for entry in json.loads(row.top_ips_json):
+ k = entry["ip"]
+ ip_counts[k]["count"] += entry["count"]
+ ip_counts[k]["internal"] = entry["internal"]
+ ip_counts[k]["country"] = entry.get("country")
+ for entry in json.loads(row.top_countries_json):
+ country_counts[entry["country"]] += entry["count"]
+ for entry in json.loads(row.top_routers_json):
+ router_counts[entry["router"]] += entry["count"]
+
+ top_ips = sorted(
+ [{"ip": k, **v} for k, v in ip_counts.items()],
+ key=lambda x: x["count"], reverse=True
+ )[:10]
+ top_countries = sorted(
+ [{"country": k, "count": v} for k, v in country_counts.items()],
+ key=lambda x: x["count"], reverse=True
+ )[:10]
+ top_routers = sorted(
+ [{"router": k, "count": v} for k, v in router_counts.items()],
+ key=lambda x: x["count"], reverse=True
+ )[:10]
+
+ history = [
+ {
+ "period_end": r.period_end,
+ "external_pct": (r.external_requests / r.total_requests * 100.0)
+ if r.total_requests else 0.0,
+ }
+ for r in rows
+ ]
+
+ summary = {
+ "total": total,
+ "internal": internal,
+ "external": external,
+ "total_bytes": total_bytes,
+ "external_pct": (external / total * 100.0) if total else 0.0,
+ "top_ips": top_ips,
+ "top_countries": top_countries,
+ "top_routers": top_routers,
+ "sparkline_external": _sparkline(
+ [h["external_pct"] for h in subsample(history, 80)]
+ ),
+ }
+
+ return await render_template(
+ "traefik/traffic.html", summary=summary, history=history, range_key=range_key
+ )
diff --git a/plugins/traefik/scheduler.py b/plugins/traefik/scheduler.py
new file mode 100644
index 0000000..b7eb050
--- /dev/null
+++ b/plugins/traefik/scheduler.py
@@ -0,0 +1,204 @@
+# plugins/traefik/scheduler.py
+from __future__ import annotations
+import logging
+import time
+from typing import TYPE_CHECKING
+
+from datetime import datetime
+from fabledscryer.core.scheduler import ScheduledTask
+
+if TYPE_CHECKING:
+ from quart import Quart
+
+logger = logging.getLogger(__name__)
+
+# Module-level state: previous scrape data for delta computation
+_prev_metrics: dict = {}
+_prev_time: float = 0.0
+
+# Access log state
+_geoip_db = None
+_access_log_last_tick: "datetime | None" = None
+
+
+def make_scrape_task(app: "Quart") -> ScheduledTask:
+ """Return a ScheduledTask that scrapes Traefik metrics on the configured interval."""
+ interval = app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
+
+ async def scrape() -> None:
+ await _do_scrape(app)
+
+ return ScheduledTask(
+ name="traefik_scrape",
+ coro_factory=scrape,
+ interval_seconds=int(interval),
+ run_on_startup=True,
+ )
+
+
+async def _do_scrape(app: "Quart") -> None:
+ """Fetch Traefik metrics, write to DB, and emit plugin_metrics."""
+ global _prev_metrics, _prev_time
+
+ from datetime import datetime, timezone
+ from sqlalchemy import select
+ from .models import TraefikCert, TraefikGlobalStat, TraefikMetric
+ from .scraper import (
+ compute_global_stats,
+ compute_router_metrics,
+ extract_certs,
+ fetch_metrics,
+ )
+ from fabledscryer.core.alerts import record_metric
+
+ metrics_url: str = app.config["PLUGINS"]["traefik"]["metrics_url"]
+
+ try:
+ current = await fetch_metrics(metrics_url)
+ except Exception:
+ logger.exception("Traefik scrape failed (url=%s)", metrics_url)
+ return
+
+ now = time.monotonic()
+ elapsed = now - _prev_time if _prev_time else 60.0
+ router_metrics = compute_router_metrics(current, _prev_metrics or None, elapsed)
+ global_stats = compute_global_stats(current, _prev_metrics or None)
+ cert_list = extract_certs(current)
+
+ _prev_metrics = current
+ _prev_time = now
+
+ scraped_at = datetime.now(timezone.utc)
+
+ async with app.db_sessionmaker() as session:
+ async with session.begin():
+ # ── Per-service metrics ───────────────────────────────────────────
+ for router_name, metrics in router_metrics.items():
+ row = TraefikMetric(
+ router_name=router_name,
+ scraped_at=scraped_at,
+ request_rate=metrics["request_rate"],
+ error_rate_4xx_pct=metrics["error_rate_4xx_pct"],
+ error_rate_5xx_pct=metrics["error_rate_5xx_pct"],
+ latency_p50_ms=metrics["latency_p50_ms"],
+ latency_p95_ms=metrics["latency_p95_ms"],
+ latency_p99_ms=metrics["latency_p99_ms"],
+ response_bytes_rate=metrics["response_bytes_rate"],
+ )
+ session.add(row)
+
+ for metric_name, value in [
+ ("request_rate", metrics["request_rate"]),
+ ("error_rate_4xx_pct", metrics["error_rate_4xx_pct"]),
+ ("error_rate_5xx_pct", metrics["error_rate_5xx_pct"]),
+ ("latency_p50_ms", metrics["latency_p50_ms"]),
+ ("latency_p95_ms", metrics["latency_p95_ms"]),
+ ("latency_p99_ms", metrics["latency_p99_ms"]),
+ ("response_bytes_rate", metrics["response_bytes_rate"]),
+ ]:
+ await record_metric(
+ session=session,
+ source_module="traefik",
+ resource_name=router_name,
+ metric_name=metric_name,
+ value=value,
+ )
+
+ # ── Global stats ──────────────────────────────────────────────────
+ session.add(TraefikGlobalStat(scraped_at=scraped_at, **global_stats))
+
+ # ── TLS certs (upsert by serial) ──────────────────────────────────
+ for cert in cert_list:
+ existing = await session.get(TraefikCert, cert["serial"])
+ if existing:
+ existing.cn = cert["cn"]
+ existing.sans = cert["sans"]
+ existing.not_after = cert["not_after"]
+ existing.scraped_at = scraped_at
+ else:
+ session.add(TraefikCert(
+ serial=cert["serial"],
+ cn=cert["cn"],
+ sans=cert["sans"],
+ not_after=cert["not_after"],
+ scraped_at=scraped_at,
+ ))
+
+ # Emit cert_expiry_days for alert rules
+ days_remaining = (cert["not_after"] - scraped_at).total_seconds() / 86400.0
+ await record_metric(
+ session=session,
+ source_module="traefik",
+ resource_name=cert["cn"] or cert["serial"],
+ metric_name="cert_expiry_days",
+ value=max(0.0, days_remaining),
+ )
+
+ logger.debug(
+ "Traefik scrape: %d router(s), %d cert(s)",
+ len(router_metrics),
+ len(cert_list),
+ )
+
+
+def make_access_log_task(app: "Quart") -> ScheduledTask:
+ """Return a ScheduledTask that processes new Traefik access log lines."""
+ global _geoip_db
+ cfg = app.config["PLUGINS"]["traefik"].get("access_log", {})
+ if not isinstance(cfg, dict):
+ cfg = {}
+ log_path: str = cfg.get("path", "/var/log/traefik/access.log")
+ geoip_path: str = cfg.get("geoip_db", "")
+
+ from .access_log import open_geoip
+ _geoip_db = open_geoip(geoip_path or None)
+ if _geoip_db:
+ logger.info("Traefik access log: GeoIP database loaded from %s", geoip_path)
+ else:
+ logger.info("Traefik access log: GeoIP not configured, country lookup disabled")
+
+ interval = app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
+
+ async def process() -> None:
+ await _do_access_log(app, log_path)
+
+ return ScheduledTask(
+ name="traefik_access_log",
+ coro_factory=process,
+ interval_seconds=int(interval),
+ run_on_startup=True,
+ )
+
+
+async def _do_access_log(app: "Quart", log_path: str) -> None:
+ """Read new access log lines, aggregate, and persist a summary."""
+ global _access_log_last_tick
+
+ from datetime import timezone
+ from .access_log import read_new_lines, aggregate
+ from .models import TraefikAccessSummary
+
+ now = datetime.now(timezone.utc)
+ period_start = _access_log_last_tick or now
+ _access_log_last_tick = now
+
+ entries = read_new_lines(log_path)
+ summary = aggregate(entries, geoip_db=_geoip_db)
+
+ if summary is None:
+ return
+
+ async with app.db_sessionmaker() as session:
+ async with session.begin():
+ session.add(TraefikAccessSummary(
+ period_start=period_start,
+ period_end=now,
+ **summary,
+ ))
+
+ logger.debug(
+ "Traefik access log: %d requests (%d internal, %d external)",
+ summary["total_requests"],
+ summary["internal_requests"],
+ summary["external_requests"],
+ )
diff --git a/plugins/traefik/scraper.py b/plugins/traefik/scraper.py
new file mode 100644
index 0000000..6e06dd2
--- /dev/null
+++ b/plugins/traefik/scraper.py
@@ -0,0 +1,276 @@
+# plugins/traefik/scraper.py
+"""Prometheus text-format scraper and per-service metrics calculator for Traefik.
+
+Traefik exposes Prometheus metrics at /metrics. This module:
+1. Fetches the endpoint via httpx
+2. Parses the Prometheus text format (counters + histograms)
+3. Computes per-service request rate (delta/elapsed), error rates (% 4xx, % 5xx),
+ latency percentiles (p50, p95, p99) via linear histogram interpolation,
+ and bandwidth (bytes/sec from response bytes counter delta)
+4. Extracts TLS certificate expiry info
+5. Computes process-level and config health stats
+"""
+from __future__ import annotations
+import math
+import re
+from collections import defaultdict
+from datetime import datetime, timezone
+
+import httpx
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Prometheus text format parser
+# ──────────────────────────────────────────────────────────────────────────────
+
+# Parsed sample: {metric_name: {frozenset(label_items): float}}
+ParsedMetrics = dict[str, dict[frozenset, float]]
+
+_SAMPLE_RE = re.compile(
+ r'^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+'
+ r'([+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?|[+-]?Inf|NaN)'
+)
+_LABEL_RE = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="([^"\\]*(?:\\.[^"\\]*)*)"')
+
+
+def parse_prometheus(text: str) -> ParsedMetrics:
+ """Parse Prometheus text exposition format into nested dicts.
+
+ Returns: {metric_name: {frozenset({(label, value), ...}): float}}
+ """
+ metrics: ParsedMetrics = {}
+ for line in text.splitlines():
+ line = line.strip()
+ if not line or line.startswith("#"):
+ continue
+ m = _SAMPLE_RE.match(line)
+ if not m:
+ continue
+ name, labels_str, value_str = m.groups()
+ try:
+ value = float(value_str)
+ except ValueError:
+ continue
+ labels = frozenset(
+ (lm.group(1), lm.group(2))
+ for lm in _LABEL_RE.finditer(labels_str or "")
+ )
+ metrics.setdefault(name, {})[labels] = value
+ return metrics
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Per-router metric computation
+# ──────────────────────────────────────────────────────────────────────────────
+
+def compute_router_metrics(
+ current: ParsedMetrics,
+ previous: ParsedMetrics | None,
+ elapsed_seconds: float,
+) -> dict[str, dict[str, float]]:
+ """Compute per-service derived metrics from two consecutive scrapes.
+
+ Uses traefik_service_* metrics (available by default).
+
+ Returns:
+ {service_name: {
+ "request_rate": float, # requests/sec
+ "error_rate_4xx_pct": float, # % 4xx of total
+ "error_rate_5xx_pct": float, # % 5xx of total
+ "latency_p50_ms": float,
+ "latency_p95_ms": float,
+ "latency_p99_ms": float,
+ "response_bytes_rate": float, # bytes/sec
+ }}
+ """
+ prev = previous or {}
+ elapsed = max(elapsed_seconds, 1.0)
+
+ # ── Request counters ──────────────────────────────────────────────────────
+ req_samples = current.get("traefik_service_requests_total", {})
+ prev_req = prev.get("traefik_service_requests_total", {})
+
+ service_total: dict[str, float] = defaultdict(float)
+ service_4xx: dict[str, float] = defaultdict(float)
+ service_5xx: dict[str, float] = defaultdict(float)
+
+ for labels, value in req_samples.items():
+ label_dict = dict(labels)
+ service = label_dict.get("service", "")
+ if not service:
+ continue
+ code = label_dict.get("code", "")
+ prev_value = prev_req.get(labels, value) # no delta on first scrape
+ delta = max(0.0, value - prev_value)
+ service_total[service] += delta
+ if code.startswith("4"):
+ service_4xx[service] += delta
+ elif code.startswith("5"):
+ service_5xx[service] += delta
+
+ # ── Response bytes counters ───────────────────────────────────────────────
+ bytes_samples = current.get("traefik_service_responses_bytes_total", {})
+ prev_bytes = prev.get("traefik_service_responses_bytes_total", {})
+
+ service_bytes: dict[str, float] = defaultdict(float)
+ for labels, value in bytes_samples.items():
+ service = dict(labels).get("service", "")
+ if not service:
+ continue
+ prev_value = prev_bytes.get(labels, value)
+ service_bytes[service] += max(0.0, value - prev_value)
+
+ # ── Histogram buckets ─────────────────────────────────────────────────────
+ hist_buckets = current.get("traefik_service_request_duration_seconds_bucket", {})
+
+ # Collect all known services
+ all_services: set[str] = set(service_total.keys()) | set(service_bytes.keys())
+ for labels in hist_buckets:
+ service = dict(labels).get("service", "")
+ if service:
+ all_services.add(service)
+
+ result: dict[str, dict[str, float]] = {}
+
+ for service in all_services:
+ total = service_total.get(service, 0.0)
+ result[service] = {
+ "request_rate": total / elapsed,
+ "error_rate_4xx_pct": (service_4xx.get(service, 0.0) / total * 100.0)
+ if total > 0 else 0.0,
+ "error_rate_5xx_pct": (service_5xx.get(service, 0.0) / total * 100.0)
+ if total > 0 else 0.0,
+ "latency_p50_ms": _percentile_ms(hist_buckets, service, 0.50),
+ "latency_p95_ms": _percentile_ms(hist_buckets, service, 0.95),
+ "latency_p99_ms": _percentile_ms(hist_buckets, service, 0.99),
+ "response_bytes_rate": service_bytes.get(service, 0.0) / elapsed,
+ }
+
+ return result
+
+
+def compute_global_stats(
+ current: ParsedMetrics,
+ previous: ParsedMetrics | None,
+) -> dict[str, float | None]:
+ """Extract process-level and config health metrics from a scrape.
+
+ Returns:
+ {
+ "open_conns_total": float | None,
+ "config_reloads_total": float | None,
+ "config_last_reload_success": float | None, # 1.0 = success, 0.0 = failure
+ "process_memory_bytes": float | None,
+ }
+ """
+ stats: dict[str, float | None] = {
+ "open_conns_total": None,
+ "config_reloads_total": None,
+ "config_last_reload_success": None,
+ "process_memory_bytes": None,
+ }
+
+ # Open connections — sum across all entrypoints/protocols
+ open_conns = current.get("traefik_open_connections", {})
+ if open_conns:
+ stats["open_conns_total"] = sum(open_conns.values())
+
+ # Config reloads — sum across result labels (success + failure)
+ reloads = current.get("traefik_config_reloads_total", {})
+ if reloads:
+ stats["config_reloads_total"] = sum(reloads.values())
+
+ # Last reload success gauge
+ last_success = current.get("traefik_config_last_reload_success", {})
+ if last_success:
+ # Usually a single sample with no labels
+ stats["config_last_reload_success"] = next(iter(last_success.values()))
+
+ # Process RSS memory
+ mem = current.get("process_resident_memory_bytes", {})
+ if mem:
+ stats["process_memory_bytes"] = next(iter(mem.values()))
+
+ return stats
+
+
+def extract_certs(current: ParsedMetrics) -> list[dict]:
+ """Extract TLS certificate expiry info from traefik_tls_certs_not_after.
+
+ Returns a list of dicts:
+ [{"serial": str, "cn": str, "sans": str, "not_after": datetime}, ...]
+ """
+ certs = []
+ samples = current.get("traefik_tls_certs_not_after", {})
+ for labels, ts_value in samples.items():
+ label_dict = dict(labels)
+ serial = label_dict.get("serial", "")
+ if not serial:
+ continue
+ try:
+ not_after = datetime.fromtimestamp(ts_value, tz=timezone.utc)
+ except (ValueError, OSError, OverflowError):
+ continue
+ certs.append({
+ "serial": serial,
+ "cn": label_dict.get("cn") or None,
+ "sans": label_dict.get("sans") or None,
+ "not_after": not_after,
+ })
+ return certs
+
+
+def _percentile_ms(
+ buckets: dict[frozenset, float],
+ service: str,
+ pct: float,
+) -> float:
+ """Linear interpolation of a Prometheus histogram percentile, returned in ms."""
+ router_buckets: list[tuple[float, float]] = []
+ for labels, count in buckets.items():
+ label_dict = dict(labels)
+ if label_dict.get("service") != service:
+ continue
+ le_str = label_dict.get("le", "")
+ try:
+ le = float("inf") if le_str == "+Inf" else float(le_str)
+ router_buckets.append((le, count))
+ except ValueError:
+ continue
+
+ if not router_buckets:
+ return 0.0
+
+ router_buckets.sort(key=lambda t: t[0])
+ total = router_buckets[-1][1] # +Inf bucket count == total requests
+ if total == 0.0:
+ return 0.0
+
+ target = total * pct
+ for i, (le, count) in enumerate(router_buckets):
+ if count < target:
+ continue
+ if i == 0:
+ ratio = target / count if count > 0 else 0.0
+ return le * ratio * 1000.0
+ prev_le, prev_count = router_buckets[i - 1]
+ if count == prev_count:
+ return prev_le * 1000.0
+ if math.isinf(le):
+ return prev_le * 1000.0
+ fraction = (target - prev_count) / (count - prev_count)
+ return (prev_le + fraction * (le - prev_le)) * 1000.0
+
+ return 0.0
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# HTTP fetch
+# ──────────────────────────────────────────────────────────────────────────────
+
+async def fetch_metrics(metrics_url: str) -> ParsedMetrics:
+ """Fetch and parse Prometheus metrics from the given URL."""
+ async with httpx.AsyncClient(timeout=10.0) as client:
+ response = await client.get(metrics_url)
+ response.raise_for_status()
+ return parse_prometheus(response.text)
diff --git a/plugins/traefik/templates/traefik/index.html b/plugins/traefik/templates/traefik/index.html
new file mode 100644
index 0000000..da66fc7
--- /dev/null
+++ b/plugins/traefik/templates/traefik/index.html
@@ -0,0 +1,31 @@
+{# plugins/traefik/templates/traefik/index.html #}
+{% extends "base.html" %}
+{% block title %}Traefik — Fabled Scryer{% endblock %}
+{% block content %}
+
+
+
Traefik Services
+ polling every {{ poll_interval }}s
+
+ {% include "_time_range.html" %}
+
+
+
+
+
+{% if access_log_enabled %}
+
+{% endif %}
+{% endblock %}
diff --git a/plugins/traefik/templates/traefik/rows.html b/plugins/traefik/templates/traefik/rows.html
new file mode 100644
index 0000000..c02b962
--- /dev/null
+++ b/plugins/traefik/templates/traefik/rows.html
@@ -0,0 +1,127 @@
+{# plugins/traefik/templates/traefik/rows.html #}
+
+{# ── Global stats bar ──────────────────────────────────────────────────────── #}
+{% if global_stat %}
+
+
+ {% if global_stat.open_conns_total is not none %}
+
+ Open conns
+ {{ global_stat.open_conns_total | int }}
+
+ {% endif %}
+
+ {% if global_stat.config_last_reload_success is not none %}
+
+ Config
+ {% if global_stat.config_last_reload_success == 1.0 %}
+ ✓ OK
+ {% else %}
+ ✗ failed
+ {% endif %}
+ {% if global_stat.config_reloads_total is not none %}
+ ({{ global_stat.config_reloads_total | int }} reloads)
+ {% endif %}
+
+ {% endif %}
+
+ {% if global_stat.process_memory_bytes is not none %}
+
+ Memory
+ {% set mb = (global_stat.process_memory_bytes / 1048576) %}
+
+ {% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.0f"|format(mb) }} MB{% endif %}
+
+
+ {% endif %}
+
+
+ {{ global_stat.scraped_at.strftime("%H:%M:%S") }} UTC
+
+
+{% endif %}
+
+{# ── Per-service cards + cert card in shared columns layout ───────────────── #}
+{% if router_data or certs %}
+
+
+{% for r in router_data %}
+
+
{{ r.name }}
+
+
+ Req/s
+ {{ "%.2f"|format(r.latest.request_rate) }}
+ {{ r.sparkline_req | safe }}
+
+
+ Bandwidth
+
+ {% set bps = r.latest.response_bytes_rate %}
+ {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s
+ {% elif bps >= 1024 %}{{ "%.1f"|format(bps / 1024) }} KB/s
+ {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %}
+
+
+
+
+ p95 ms
+ {{ "%.1f"|format(r.latest.latency_p95_ms) }}
+ {{ r.sparkline_p95 | safe }}
+
+
+ p99 ms
+ {{ "%.1f"|format(r.latest.latency_p99_ms) }}
+
+
+
+ 4xx %
+
+ {{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%
+
+
+
+
+ 5xx %
+
+ {{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%
+
+ {{ r.sparkline_5xx | safe }}
+
+
+
+ Last scraped {{ r.latest.scraped_at.strftime("%H:%M:%S") }} UTC
+
+
+{% endfor %}
+
+{% if certs %}
+
+
TLS Certificates
+
+ {% for cert in certs %}
+
+ {{ cert.cn }}
+
+ {{ cert.days_remaining | int }}d
+
+
+
+
+ {{ cert.not_after.strftime("%Y-%m-%d") }}
+
+
+ {% endfor %}
+
+
+{% endif %}
+
+
+{% else %}
+
+
No Traefik metrics yet. Configure metrics_url in config.yaml under plugins.traefik.
+
+{% endif %}
diff --git a/plugins/traefik/templates/traefik/traffic.html b/plugins/traefik/templates/traefik/traffic.html
new file mode 100644
index 0000000..dcdcb96
--- /dev/null
+++ b/plugins/traefik/templates/traefik/traffic.html
@@ -0,0 +1,101 @@
+{# plugins/traefik/templates/traefik/traffic.html #}
+{% if summary %}
+
+{# ── Overview bar ─────────────────────────────────────────────────────────── #}
+
+
+ Total
+ {{ summary.total }}
+
+
+ Internal
+ {{ summary.internal }}
+
+
+ External
+
+ {{ summary.external }} ({{ "%.1f"|format(summary.external_pct) }}%)
+
+
+
+ Bytes
+
+ {% set mb = summary.total_bytes / 1048576 %}
+ {% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.1f"|format(mb) }} MB{% endif %}
+
+
+
+ {# External % sparkline #}
+
+ ext % over time
+ {{ summary.sparkline_external | safe }}
+
+
+
+{# ── Internal/external fill bar ───────────────────────────────────────────── #}
+{% set ext_pct = summary.external_pct %}
+
+
+{# ── Detail cards in columns ──────────────────────────────────────────────── #}
+
+
+ {# Top source IPs #}
+
+
Top Source IPs
+
+ {% for entry in summary.top_ips %}
+
+ {{ entry.ip }}
+ {% if entry.country %}
+ {{ entry.country }}
+ {% endif %}
+
+
+ {{ entry.count }}
+
+
+
+ {% endfor %}
+
+
+
+ {# Top routers #}
+
+
Top Services Hit
+
+ {% for entry in summary.top_routers %}
+
+ {{ entry.router }}
+ {{ entry.count }}
+
+ {% endfor %}
+
+
+
+ {# Top countries (only shown if GeoIP is active) #}
+ {% if summary.top_countries %}
+
+
External Traffic by Country
+
+ {% for entry in summary.top_countries %}
+
+ {{ entry.country }}
+ {{ entry.count }}
+
+ {% endfor %}
+
+
+ {% endif %}
+
+
+
+{% else %}
+
+ No access log data yet.
+ {% if not access_log_enabled %}
+ Enable access_log.enabled: true in your plugin config and bind-mount the Traefik log directory.
+ {% endif %}
+
+{% endif %}
diff --git a/plugins/traefik/templates/traefik/widget.html b/plugins/traefik/templates/traefik/widget.html
new file mode 100644
index 0000000..aa43085
--- /dev/null
+++ b/plugins/traefik/templates/traefik/widget.html
@@ -0,0 +1,56 @@
+{# plugins/traefik/templates/traefik/widget.html #}
+{% if expiring_certs %}
+
+{% for cert in expiring_certs %}
+
+ ⚠ {{ cert.cn }}
+ {{ cert.days_remaining | int }}d
+
+{% endfor %}
+
+{% endif %}
+
+{% if router_data %}
+{% for r in router_data %}
+
+
+ {{ r.name }}
+
+
+
+ req/s
+ {{ "%.2f"|format(r.latest.request_rate) }}
+
+
+ bw
+
+ {% set bps = r.latest.response_bytes_rate %}
+ {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s
+ {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s
+ {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %}
+
+
+
+ p95
+
+ {{ "%.0f"|format(r.latest.latency_p95_ms) }}ms
+
+
+ {% if r.latest.error_rate_5xx_pct > 0 %}
+
+ {{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}% 5xx
+
+ {% endif %}
+
+
+{% endfor %}
+{% if total_routers > 5 %}
+
+{% endif %}
+{% else %}
+No Traefik data yet.
+{% endif %}
diff --git a/plugins/traefik/templates/traefik/widget_access_log.html b/plugins/traefik/templates/traefik/widget_access_log.html
new file mode 100644
index 0000000..899e09c
--- /dev/null
+++ b/plugins/traefik/templates/traefik/widget_access_log.html
@@ -0,0 +1,40 @@
+{# traefik/widget_access_log.html — dashboard widget: traffic origin summary #}
+{% if not summary %}
+No access log data yet.
+{% else %}
+
+
+
+ {{ summary.total | int }}
+ total req
+
+
+
+ {{ "%.1f" | format(summary.external_pct) }}%
+
+ external
+
+
+
+{% if summary.top_ips %}
+
+ Top IPs
+ {% if summary.ip_filter != "all" %}
+ {{ summary.ip_filter }}
+ {% endif %}
+
+
+ {% for ip in summary.top_ips %}
+
+
+
+ {{ ip.ip }}
+
+ {{ ip.count }}
+
+ {% endfor %}
+
+{% endif %}
+
+{% endif %}
diff --git a/plugins/traefik/templates/traefik/widget_request_chart.html b/plugins/traefik/templates/traefik/widget_request_chart.html
new file mode 100644
index 0000000..272ff39
--- /dev/null
+++ b/plugins/traefik/templates/traefik/widget_request_chart.html
@@ -0,0 +1,75 @@
+{# traefik/widget_request_chart.html — Chart.js request rate + error % over time #}
+{% if not labels_json or labels_json == '[]' %}
+No data for last {{ hours }}h.
+{% else %}
+
+
+
+
+
+ last {{ hours }}h
+
+{% endif %}
diff --git a/plugins/unifi/__init__.py b/plugins/unifi/__init__.py
new file mode 100644
index 0000000..4bb589f
--- /dev/null
+++ b/plugins/unifi/__init__.py
@@ -0,0 +1,28 @@
+# plugins/unifi/__init__.py
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from quart import Quart
+
+_app: "Quart | None" = None
+
+
+def setup(app: "Quart") -> None:
+ global _app
+ _app = app
+ from .models import ( # noqa: F401
+ UnifiWanStat, UnifiClientSnapshot, UnifiWlanStat, UnifiDpiSnapshot,
+ UnifiDevice, UnifiKnownClient, UnifiEvent, UnifiAlarm,
+ UnifiSpeedtestResult, UnifiNetwork, UnifiPortForward, UnifiFwRule,
+ )
+
+
+def get_scheduled_tasks() -> list:
+ from .scheduler import make_poll_task
+ return [make_poll_task(_app)]
+
+
+def get_blueprint():
+ from .routes import unifi_bp
+ return unifi_bp
diff --git a/plugins/unifi/client.py b/plugins/unifi/client.py
new file mode 100644
index 0000000..d18a7d8
--- /dev/null
+++ b/plugins/unifi/client.py
@@ -0,0 +1,170 @@
+# plugins/unifi/client.py
+"""Async UniFi Network controller API client.
+
+Supports UDM/UDM-Pro (firmware 2+) using the /proxy/network/ path prefix
+and Bearer-token auth via the /api/auth/login endpoint.
+
+On 401, re-authenticates automatically once before raising.
+"""
+from __future__ import annotations
+import logging
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+class UnifiClient:
+ def __init__(
+ self,
+ host: str,
+ username: str,
+ password: str,
+ site: str = "default",
+ verify_ssl: bool = False,
+ ) -> None:
+ self._host = host.rstrip("/")
+ self._username = username
+ self._password = password
+ self._site = site
+ self._verify_ssl = verify_ssl
+ self._http: httpx.AsyncClient | None = None
+ self._csrf_token: str = ""
+
+ # ── Lifecycle ─────────────────────────────────────────────────────────────
+
+ async def _ensure_http(self) -> None:
+ if self._http is None:
+ self._http = httpx.AsyncClient(
+ verify=self._verify_ssl,
+ timeout=15.0,
+ follow_redirects=True,
+ )
+
+ async def login(self) -> None:
+ await self._ensure_http()
+ resp = await self._http.post(
+ f"{self._host}/api/auth/login",
+ json={"username": self._username, "password": self._password, "rememberMe": False},
+ )
+ resp.raise_for_status()
+ self._csrf_token = resp.headers.get("X-CSRF-Token", "")
+ logger.debug("UniFi login OK (csrf=%s…)", self._csrf_token[:8] if self._csrf_token else "none")
+
+ async def close(self) -> None:
+ if self._http:
+ await self._http.aclose()
+ self._http = None
+
+ # ── API helpers ───────────────────────────────────────────────────────────
+
+ def _api_url(self, path: str) -> str:
+ return f"{self._host}/proxy/network/api/s/{self._site}{path}"
+
+ def _headers(self) -> dict:
+ return {"X-CSRF-Token": self._csrf_token} if self._csrf_token else {}
+
+ async def _get(self, path: str, params: dict | None = None) -> list | dict:
+ await self._ensure_http()
+ resp = await self._http.get(self._api_url(path), headers=self._headers(), params=params)
+ if resp.status_code == 401:
+ await self.login()
+ resp = await self._http.get(self._api_url(path), headers=self._headers(), params=params)
+ resp.raise_for_status()
+ data = resp.json()
+ return data.get("data", data)
+
+ # ── Core monitoring ───────────────────────────────────────────────────────
+
+ async def get_health(self) -> list[dict]:
+ """Subsystem health (wan, wlan, lan, www)."""
+ result = await self._get("/stat/health")
+ return result if isinstance(result, list) else []
+
+ async def get_active_clients(self) -> list[dict]:
+ """Currently connected clients."""
+ result = await self._get("/stat/sta")
+ return result if isinstance(result, list) else []
+
+ async def get_devices(self) -> list[dict]:
+ """Managed network devices (APs, switches, gateways) with full detail."""
+ result = await self._get("/stat/device")
+ return result if isinstance(result, list) else []
+
+ # ── Events & alarms ───────────────────────────────────────────────────────
+
+ async def get_events(self, limit: int = 200) -> list[dict]:
+ """Recent site events (client connects, IDS alerts, device state changes, etc.)."""
+ result = await self._get("/stat/event", params={"_limit": limit, "_sort": "-time"})
+ return result if isinstance(result, list) else []
+
+ async def get_alarms(self, archived: bool = False) -> list[dict]:
+ """Active (or archived) alarms."""
+ result = await self._get("/stat/alarm")
+ if not isinstance(result, list):
+ return []
+ return [a for a in result if a.get("archived", False) == archived]
+
+ # ── Traffic & DPI ─────────────────────────────────────────────────────────
+
+ async def get_dpi_stats(self) -> list[dict]:
+ """DPI per-client application/category traffic stats (cumulative counters)."""
+ result = await self._get("/stat/dpi")
+ return result if isinstance(result, list) else []
+
+ async def get_dpi_site_stats(self) -> list[dict]:
+ """DPI site-level category totals (not per-client)."""
+ result = await self._get("/dpi")
+ return result if isinstance(result, list) else []
+
+ async def get_speedtest_status(self) -> dict | None:
+ """Most recent WAN speed test result."""
+ result = await self._get("/stat/speedtest-status")
+ if isinstance(result, list) and result:
+ return result[0]
+ if isinstance(result, dict):
+ return result
+ return None
+
+ # ── Network inventory ─────────────────────────────────────────────────────
+
+ async def get_known_clients(self) -> list[dict]:
+ """All historically seen clients (not just currently connected)."""
+ result = await self._get("/rest/user")
+ return result if isinstance(result, list) else []
+
+ async def get_networks(self) -> list[dict]:
+ """Network/VLAN configurations."""
+ result = await self._get("/rest/networkconf")
+ return result if isinstance(result, list) else []
+
+ async def get_wlan_configs(self) -> list[dict]:
+ """WLAN (SSID) configurations."""
+ result = await self._get("/rest/wlanconf")
+ return result if isinstance(result, list) else []
+
+ async def get_port_forwards(self) -> list[dict]:
+ """Port forwarding rules."""
+ result = await self._get("/rest/portforwarding")
+ return result if isinstance(result, list) else []
+
+ async def get_firewall_rules(self) -> list[dict]:
+ """Firewall rules."""
+ result = await self._get("/rest/firewallrule")
+ return result if isinstance(result, list) else []
+
+ async def get_firewall_groups(self) -> list[dict]:
+ """Firewall groups (IP sets / port sets referenced by rules)."""
+ result = await self._get("/rest/firewallgroup")
+ return result if isinstance(result, list) else []
+
+ # ── System ────────────────────────────────────────────────────────────────
+
+ async def get_sysinfo(self) -> dict | None:
+ """Controller system information (version, uptime, hostname)."""
+ result = await self._get("/stat/sysinfo")
+ if isinstance(result, list) and result:
+ return result[0]
+ if isinstance(result, dict):
+ return result
+ return None
diff --git a/plugins/unifi/migrations/__init__.py b/plugins/unifi/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/plugins/unifi/migrations/env.py b/plugins/unifi/migrations/env.py
new file mode 100644
index 0000000..56f15d5
--- /dev/null
+++ b/plugins/unifi/migrations/env.py
@@ -0,0 +1,72 @@
+# plugins/unifi/migrations/env.py
+"""Alembic env.py for the UniFi plugin (standalone dev use).
+At app startup, migration_runner.py uses the core env.py with version_locations.
+"""
+import asyncio
+import os
+import sys
+from pathlib import Path
+from logging.config import fileConfig
+
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+from alembic import context
+
+sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
+
+from fabledscryer.models.base import Base
+import fabledscryer.models # noqa: F401
+from plugins.unifi.models import UnifiWanStat, UnifiClientSnapshot, UnifiDevice # noqa: F401
+
+config = context.config
+if config.config_file_name is not None:
+ fileConfig(config.config_file_name)
+
+target_metadata = Base.metadata
+
+
+def _get_url() -> str:
+ import yaml
+ cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml")
+ try:
+ with open(cfg_path) as f:
+ cfg = yaml.safe_load(f) or {}
+ url = cfg.get("database", {}).get("url", "")
+ except FileNotFoundError:
+ url = ""
+ return os.environ.get("FABLEDSCRYER_DATABASE__URL", url)
+
+
+def run_migrations_offline() -> None:
+ context.configure(
+ url=_get_url(), target_metadata=target_metadata,
+ literal_binds=True, dialect_opts={"paramstyle": "named"},
+ )
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+ context.configure(connection=connection, target_metadata=target_metadata)
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+ cfg = config.get_section(config.config_ini_section, {})
+ cfg["sqlalchemy.url"] = _get_url()
+ connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool)
+ async with connectable.connect() as connection:
+ await connection.run_sync(do_run_migrations)
+ await connectable.dispose()
+
+
+def run_migrations_online() -> None:
+ asyncio.run(run_async_migrations())
+
+
+if context.is_offline_mode():
+ run_migrations_offline()
+else:
+ run_migrations_online()
diff --git a/plugins/unifi/migrations/versions/__init__.py b/plugins/unifi/migrations/versions/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/plugins/unifi/migrations/versions/unifi_001_initial.py b/plugins/unifi/migrations/versions/unifi_001_initial.py
new file mode 100644
index 0000000..36158a3
--- /dev/null
+++ b/plugins/unifi/migrations/versions/unifi_001_initial.py
@@ -0,0 +1,62 @@
+"""UniFi plugin initial tables
+
+Revision ID: unifi_001_initial
+Revises: (none — branch off core via depends_on)
+Create Date: 2026-03-20
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "unifi_001_initial"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = "unifi"
+depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
+
+
+def upgrade() -> None:
+ op.create_table(
+ "unifi_wan_stats",
+ sa.Column("id", sa.String(36), primary_key=True),
+ sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("is_up", sa.Boolean, nullable=False, server_default="false"),
+ sa.Column("wan_ip", sa.String(64), nullable=True),
+ sa.Column("latency_ms", sa.Float, nullable=True),
+ sa.Column("rx_bytes_rate", sa.Float, nullable=True),
+ sa.Column("tx_bytes_rate", sa.Float, nullable=True),
+ sa.Column("uptime_seconds", sa.Integer, nullable=True),
+ )
+ op.create_index("ix_unifi_wan_stats_scraped", "unifi_wan_stats", ["scraped_at"])
+
+ op.create_table(
+ "unifi_client_snapshots",
+ sa.Column("id", sa.String(36), primary_key=True),
+ sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("total_clients", sa.Integer, nullable=False, server_default="0"),
+ sa.Column("wireless_clients", sa.Integer, nullable=False, server_default="0"),
+ sa.Column("wired_clients", sa.Integer, nullable=False, server_default="0"),
+ sa.Column("guest_clients", sa.Integer, nullable=False, server_default="0"),
+ sa.Column("top_clients_json", sa.Text, nullable=False, server_default="[]"),
+ )
+ op.create_index("ix_unifi_client_snapshots_scraped", "unifi_client_snapshots", ["scraped_at"])
+
+ op.create_table(
+ "unifi_devices",
+ sa.Column("mac", sa.String(32), primary_key=True),
+ sa.Column("name", sa.String(255), nullable=True),
+ sa.Column("model", sa.String(64), nullable=True),
+ sa.Column("device_type", sa.String(16), nullable=True),
+ sa.Column("ip", sa.String(64), nullable=True),
+ sa.Column("state", sa.Integer, nullable=False, server_default="0"),
+ sa.Column("uptime_seconds", sa.Integer, nullable=True),
+ sa.Column("last_seen", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+ )
+
+
+def downgrade() -> None:
+ op.drop_table("unifi_devices")
+ op.drop_index("ix_unifi_client_snapshots_scraped", "unifi_client_snapshots")
+ op.drop_table("unifi_client_snapshots")
+ op.drop_index("ix_unifi_wan_stats_scraped", "unifi_wan_stats")
+ op.drop_table("unifi_wan_stats")
diff --git a/plugins/unifi/migrations/versions/unifi_002_expanded.py b/plugins/unifi/migrations/versions/unifi_002_expanded.py
new file mode 100644
index 0000000..7336517
--- /dev/null
+++ b/plugins/unifi/migrations/versions/unifi_002_expanded.py
@@ -0,0 +1,156 @@
+"""UniFi expanded tables: wlan stats, dpi, events, alarms, known clients,
+speedtest results, networks, port forwards, firewall rules.
+Also adds missing 'version' column to unifi_devices.
+
+Revision ID: unifi_002_expanded
+Revises: unifi_001_initial
+Create Date: 2026-03-20
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "unifi_002_expanded"
+down_revision: Union[str, None] = "unifi_001_initial"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ── Patch unifi_devices to add missing version column ─────────────────────
+ op.add_column("unifi_devices", sa.Column("version", sa.String(64), nullable=True))
+
+ # ── WLAN stats (time-series per SSID+radio) ───────────────────────────────
+ op.create_table(
+ "unifi_wlan_stats",
+ sa.Column("id", sa.String(36), primary_key=True),
+ sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("ssid", sa.String(255), nullable=False),
+ sa.Column("radio", sa.String(8), nullable=True),
+ sa.Column("num_sta", sa.Integer, nullable=False, server_default="0"),
+ sa.Column("tx_bytes", sa.Float, nullable=True),
+ sa.Column("rx_bytes", sa.Float, nullable=True),
+ sa.Column("channel", sa.Integer, nullable=True),
+ sa.Column("satisfaction", sa.Integer, nullable=True),
+ )
+ op.create_index("ix_unifi_wlan_stats_scraped", "unifi_wlan_stats", ["scraped_at"])
+
+ # ── DPI snapshots (site-level category totals) ────────────────────────────
+ op.create_table(
+ "unifi_dpi_snapshots",
+ sa.Column("id", sa.String(36), primary_key=True),
+ sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("by_category_json", sa.Text, nullable=False, server_default="[]"),
+ sa.Column("by_client_json", sa.Text, nullable=False, server_default="[]"),
+ )
+ op.create_index("ix_unifi_dpi_scraped", "unifi_dpi_snapshots", ["scraped_at"])
+
+ # ── Known clients (upserted by MAC) ───────────────────────────────────────
+ op.create_table(
+ "unifi_known_clients",
+ sa.Column("mac", sa.String(32), primary_key=True),
+ sa.Column("hostname", sa.String(255), nullable=True),
+ sa.Column("name", sa.String(255), nullable=True),
+ sa.Column("ip", sa.String(64), nullable=True),
+ sa.Column("mac_vendor", sa.String(128), nullable=True),
+ sa.Column("note", sa.Text, nullable=True),
+ sa.Column("is_blocked", sa.Boolean, nullable=False, server_default="false"),
+ sa.Column("first_seen", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("last_seen", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+ )
+
+ # ── Events (upserted by UniFi event ID) ───────────────────────────────────
+ op.create_table(
+ "unifi_events",
+ sa.Column("unifi_id", sa.String(64), primary_key=True),
+ sa.Column("event_time", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("event_key", sa.String(64), nullable=False),
+ sa.Column("category", sa.String(16), nullable=False),
+ sa.Column("message", sa.Text, nullable=False),
+ sa.Column("source_mac", sa.String(32), nullable=True),
+ sa.Column("source_ip", sa.String(64), nullable=True),
+ sa.Column("details_json", sa.Text, nullable=False, server_default="{}"),
+ )
+ op.create_index("ix_unifi_events_time", "unifi_events", ["event_time"])
+
+ # ── Alarms (upserted by UniFi alarm ID) ───────────────────────────────────
+ op.create_table(
+ "unifi_alarms",
+ sa.Column("unifi_id", sa.String(64), primary_key=True),
+ sa.Column("alarm_time", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("alarm_key", sa.String(64), nullable=False),
+ sa.Column("message", sa.Text, nullable=False),
+ sa.Column("archived", sa.Boolean, nullable=False, server_default="false"),
+ sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+ )
+ op.create_index("ix_unifi_alarms_time", "unifi_alarms", ["alarm_time"])
+
+ # ── Speedtest results (upserted by run timestamp) ─────────────────────────
+ op.create_table(
+ "unifi_speedtest_results",
+ sa.Column("run_at", sa.DateTime(timezone=True), primary_key=True),
+ sa.Column("download_mbps", sa.Float, nullable=True),
+ sa.Column("upload_mbps", sa.Float, nullable=True),
+ sa.Column("latency_ms", sa.Float, nullable=True),
+ sa.Column("server_name", sa.String(255), nullable=True),
+ )
+
+ # ── Networks/VLANs (upserted by UniFi ID) ────────────────────────────────
+ op.create_table(
+ "unifi_networks",
+ sa.Column("unifi_id", sa.String(64), primary_key=True),
+ sa.Column("name", sa.String(255), nullable=False),
+ sa.Column("purpose", sa.String(32), nullable=True),
+ sa.Column("vlan", sa.Integer, nullable=True),
+ sa.Column("ip_subnet", sa.String(64), nullable=True),
+ sa.Column("dhcp_enabled", sa.Boolean, nullable=False, server_default="false"),
+ sa.Column("is_guest", sa.Boolean, nullable=False, server_default="false"),
+ sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+ )
+
+ # ── Port forwards (upserted by UniFi ID) ─────────────────────────────────
+ op.create_table(
+ "unifi_port_forwards",
+ sa.Column("unifi_id", sa.String(64), primary_key=True),
+ sa.Column("name", sa.String(255), nullable=True),
+ sa.Column("proto", sa.String(8), nullable=False, server_default="tcp"),
+ sa.Column("dst_port", sa.String(32), nullable=False),
+ sa.Column("fwd_ip", sa.String(64), nullable=False),
+ sa.Column("fwd_port", sa.String(32), nullable=False),
+ sa.Column("src_filter", sa.String(255), nullable=True),
+ sa.Column("enabled", sa.Boolean, nullable=False, server_default="true"),
+ sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+ )
+
+ # ── Firewall rules (upserted by UniFi ID) ────────────────────────────────
+ op.create_table(
+ "unifi_fw_rules",
+ sa.Column("unifi_id", sa.String(64), primary_key=True),
+ sa.Column("name", sa.String(255), nullable=False),
+ sa.Column("ruleset", sa.String(16), nullable=False),
+ sa.Column("rule_index", sa.Integer, nullable=True),
+ sa.Column("action", sa.String(16), nullable=False),
+ sa.Column("protocol", sa.String(16), nullable=True),
+ sa.Column("enabled", sa.Boolean, nullable=False, server_default="true"),
+ sa.Column("src_address", sa.String(255), nullable=True),
+ sa.Column("dst_address", sa.String(255), nullable=True),
+ sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+ )
+
+
+def downgrade() -> None:
+ op.drop_table("unifi_fw_rules")
+ op.drop_table("unifi_port_forwards")
+ op.drop_table("unifi_networks")
+ op.drop_table("unifi_speedtest_results")
+ op.drop_index("ix_unifi_alarms_time", "unifi_alarms")
+ op.drop_table("unifi_alarms")
+ op.drop_index("ix_unifi_events_time", "unifi_events")
+ op.drop_table("unifi_events")
+ op.drop_table("unifi_known_clients")
+ op.drop_index("ix_unifi_dpi_scraped", "unifi_dpi_snapshots")
+ op.drop_table("unifi_dpi_snapshots")
+ op.drop_index("ix_unifi_wlan_stats_scraped", "unifi_wlan_stats")
+ op.drop_table("unifi_wlan_stats")
+ op.drop_column("unifi_devices", "version")
diff --git a/plugins/unifi/models.py b/plugins/unifi/models.py
new file mode 100644
index 0000000..2ad90e9
--- /dev/null
+++ b/plugins/unifi/models.py
@@ -0,0 +1,182 @@
+# plugins/unifi/models.py
+from __future__ import annotations
+import uuid
+from datetime import datetime, timezone
+from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text
+from sqlalchemy.orm import Mapped, mapped_column
+from fabledscryer.models.base import Base
+
+
+# ── Time-series (one row per scrape) ──────────────────────────────────────────
+
+class UnifiWanStat(Base):
+ """WAN interface health — one row per scrape."""
+ __tablename__ = "unifi_wan_stats"
+
+ id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+ scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
+ is_up: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+ wan_ip: Mapped[str | None] = mapped_column(String(64), nullable=True)
+ latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
+ rx_bytes_rate: Mapped[float | None] = mapped_column(Float, nullable=True)
+ tx_bytes_rate: Mapped[float | None] = mapped_column(Float, nullable=True)
+ uptime_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True)
+
+
+class UnifiClientSnapshot(Base):
+ """Aggregated client counts + top talkers — one row per scrape."""
+ __tablename__ = "unifi_client_snapshots"
+
+ id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+ scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
+ total_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ wireless_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ wired_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ guest_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ top_clients_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
+ # JSON: [{"mac","hostname","ip","type","tx_rate","rx_rate","signal","essid"}]
+
+
+class UnifiWlanStat(Base):
+ """Per-SSID/radio stats — one row per scrape per SSID+radio combination."""
+ __tablename__ = "unifi_wlan_stats"
+
+ id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+ scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
+ ssid: Mapped[str] = mapped_column(String(255), nullable=False)
+ radio: Mapped[str | None] = mapped_column(String(8), nullable=True) # ng=2.4GHz, na=5GHz, 6e=6GHz
+ num_sta: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ tx_bytes: Mapped[float | None] = mapped_column(Float, nullable=True)
+ rx_bytes: Mapped[float | None] = mapped_column(Float, nullable=True)
+ channel: Mapped[int | None] = mapped_column(Integer, nullable=True)
+ satisfaction: Mapped[int | None] = mapped_column(Integer, nullable=True) # 0-100
+
+
+class UnifiDpiSnapshot(Base):
+ """DPI application/category traffic snapshot — one row per scrape (JSON blob)."""
+ __tablename__ = "unifi_dpi_snapshots"
+
+ id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+ scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
+ # JSON: [{"cat_id", "cat_name", "rx_bytes", "tx_bytes"}] aggregated across all clients
+ by_category_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
+ # JSON: [{"mac", "hostname", "top_cats": [{"cat_name", "bytes"}]}]
+ by_client_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
+
+
+# ── Upserted inventory (keyed by UniFi ID or MAC) ─────────────────────────────
+
+class UnifiDevice(Base):
+ """Network infrastructure devices — upserted by MAC each scrape."""
+ __tablename__ = "unifi_devices"
+
+ mac: Mapped[str] = mapped_column(String(32), primary_key=True)
+ name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+ model: Mapped[str | None] = mapped_column(String(64), nullable=True)
+ device_type: Mapped[str | None] = mapped_column(String(16), nullable=True)
+ ip: Mapped[str | None] = mapped_column(String(64), nullable=True)
+ state: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ uptime_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True)
+ last_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+ version: Mapped[str | None] = mapped_column(String(64), nullable=True)
+ scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
+
+
+class UnifiKnownClient(Base):
+ """All historically seen network clients — upserted by MAC."""
+ __tablename__ = "unifi_known_clients"
+
+ mac: Mapped[str] = mapped_column(String(32), primary_key=True)
+ hostname: Mapped[str | None] = mapped_column(String(255), nullable=True)
+ name: Mapped[str | None] = mapped_column(String(255), nullable=True) # user-set alias
+ ip: Mapped[str | None] = mapped_column(String(64), nullable=True)
+ mac_vendor: Mapped[str | None] = mapped_column(String(128), nullable=True)
+ note: Mapped[str | None] = mapped_column(Text, nullable=True)
+ is_blocked: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+ first_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+ last_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+ scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
+
+
+class UnifiEvent(Base):
+ """Site event log — upserted by UniFi event ID to avoid duplicates."""
+ __tablename__ = "unifi_events"
+
+ unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True)
+ event_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+ event_key: Mapped[str] = mapped_column(String(64), nullable=False) # e.g. EVT_WU_Connected
+ category: Mapped[str] = mapped_column(String(16), nullable=False) # wlan, wired, ap, ids, wan, system
+ message: Mapped[str] = mapped_column(Text, nullable=False)
+ source_mac: Mapped[str | None] = mapped_column(String(32), nullable=True)
+ source_ip: Mapped[str | None] = mapped_column(String(64), nullable=True)
+ details_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
+
+
+class UnifiAlarm(Base):
+ """Active alarms — upserted by UniFi alarm ID."""
+ __tablename__ = "unifi_alarms"
+
+ unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True)
+ alarm_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+ alarm_key: Mapped[str] = mapped_column(String(64), nullable=False)
+ message: Mapped[str] = mapped_column(Text, nullable=False)
+ archived: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+ scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
+
+
+class UnifiSpeedtestResult(Base):
+ """WAN speed test results — upserted by run timestamp."""
+ __tablename__ = "unifi_speedtest_results"
+
+ run_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), primary_key=True)
+ download_mbps: Mapped[float | None] = mapped_column(Float, nullable=True)
+ upload_mbps: Mapped[float | None] = mapped_column(Float, nullable=True)
+ latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
+ server_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+
+
+# ── Config inventory (upserted by UniFi ID) ───────────────────────────────────
+
+class UnifiNetwork(Base):
+ """VLAN/network configurations — upserted by UniFi ID."""
+ __tablename__ = "unifi_networks"
+
+ unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True)
+ name: Mapped[str] = mapped_column(String(255), nullable=False)
+ purpose: Mapped[str | None] = mapped_column(String(32), nullable=True) # corporate, guest, vlan-only
+ vlan: Mapped[int | None] = mapped_column(Integer, nullable=True)
+ ip_subnet: Mapped[str | None] = mapped_column(String(64), nullable=True)
+ dhcp_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+ is_guest: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+ scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
+
+
+class UnifiPortForward(Base):
+ """Port forwarding rules — upserted by UniFi ID."""
+ __tablename__ = "unifi_port_forwards"
+
+ unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True)
+ name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+ proto: Mapped[str] = mapped_column(String(8), nullable=False, default="tcp")
+ dst_port: Mapped[str] = mapped_column(String(32), nullable=False)
+ fwd_ip: Mapped[str] = mapped_column(String(64), nullable=False)
+ fwd_port: Mapped[str] = mapped_column(String(32), nullable=False)
+ src_filter: Mapped[str | None] = mapped_column(String(255), nullable=True)
+ enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+ scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
+
+
+class UnifiFwRule(Base):
+ """Firewall rules — upserted by UniFi ID."""
+ __tablename__ = "unifi_fw_rules"
+
+ unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True)
+ name: Mapped[str] = mapped_column(String(255), nullable=False)
+ ruleset: Mapped[str] = mapped_column(String(16), nullable=False) # WAN_IN, LAN_IN, etc.
+ rule_index: Mapped[int | None] = mapped_column(Integer, nullable=True)
+ action: Mapped[str] = mapped_column(String(16), nullable=False) # accept, drop, reject
+ protocol: Mapped[str | None] = mapped_column(String(16), nullable=True)
+ enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+ src_address: Mapped[str | None] = mapped_column(String(255), nullable=True)
+ dst_address: Mapped[str | None] = mapped_column(String(255), nullable=True)
+ scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
diff --git a/plugins/unifi/plugin.yaml b/plugins/unifi/plugin.yaml
new file mode 100644
index 0000000..208ace5
--- /dev/null
+++ b/plugins/unifi/plugin.yaml
@@ -0,0 +1,21 @@
+name: unifi
+version: "1.0.0"
+description: "UniFi Network controller integration — WAN health, devices, clients, DPI"
+author: "FabledScryer"
+license: "MIT"
+min_app_version: "0.1.0"
+repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins"
+homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/unifi"
+tags:
+ - network
+ - unifi
+ - ubiquiti
+
+config:
+ host: "https://192.168.1.1"
+ username: "admin"
+ password: ""
+ site: "default"
+ verify_ssl: false
+ poll_interval_seconds: 60
+ top_clients: 10 # number of top-talker clients to store per snapshot
diff --git a/plugins/unifi/routes.py b/plugins/unifi/routes.py
new file mode 100644
index 0000000..2a57330
--- /dev/null
+++ b/plugins/unifi/routes.py
@@ -0,0 +1,357 @@
+# plugins/unifi/routes.py
+from __future__ import annotations
+import json
+from datetime import datetime, timezone
+from quart import Blueprint, current_app, render_template, request
+from sqlalchemy import Integer, cast, func, select
+
+from fabledscryer.auth.middleware import require_role
+from fabledscryer.models.users import UserRole
+from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample
+from .models import (
+ UnifiAlarm, UnifiClientSnapshot, UnifiDevice, UnifiDpiSnapshot,
+ UnifiEvent, UnifiFwRule, UnifiKnownClient, UnifiNetwork,
+ UnifiPortForward, UnifiSpeedtestResult, UnifiWanStat, UnifiWlanStat,
+)
+
+unifi_bp = Blueprint("unifi", __name__, template_folder="templates")
+
+
+def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
+ if len(values) < 2:
+ return f' '
+ mn, mx = min(values), max(values)
+ if mx == mn:
+ mx = mn + 1.0
+ step = width / (len(values) - 1)
+ pts = []
+ for i, v in enumerate(values):
+ x = i * step
+ y = height - (v - mn) / (mx - mn) * (height - 2) - 1
+ pts.append(f"{x:.1f},{y:.1f}")
+ poly = " ".join(pts)
+ return (
+ f''
+ f' '
+ f' '
+ )
+
+
+@unifi_bp.get("/")
+@require_role(UserRole.viewer)
+async def index():
+ poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
+ current_range = request.args.get("range", DEFAULT_RANGE)
+ return await render_template(
+ "unifi/index.html",
+ poll_interval=poll_interval,
+ current_range=current_range,
+ )
+
+
+@unifi_bp.get("/rows")
+@require_role(UserRole.viewer)
+async def rows():
+ """HTMX fragment: WAN stats, devices, clients, WLAN — scoped to selected range."""
+ since, range_key = parse_range(request.args.get("range"))
+
+ b_secs = bucket_seconds(since)
+
+ async with current_app.db_sessionmaker() as db:
+ # WAN history — bucketed for accurate full-range sparklines
+ wan_bucket = (
+ cast(func.strftime('%s', UnifiWanStat.scraped_at), Integer) / b_secs
+ ).label("bucket")
+ result = await db.execute(
+ select(
+ func.avg(UnifiWanStat.latency_ms).label("latency_ms"),
+ func.avg(UnifiWanStat.rx_bytes_rate).label("rx_bytes_rate"),
+ func.avg(UnifiWanStat.tx_bytes_rate).label("tx_bytes_rate"),
+ func.min(UnifiWanStat.scraped_at).label("scraped_at"),
+ wan_bucket,
+ )
+ .where(UnifiWanStat.scraped_at >= since)
+ .group_by(wan_bucket)
+ .order_by(wan_bucket)
+ )
+ wan_history = result.all()
+
+ # Latest WAN — always most recent raw row for current displayed values
+ result = await db.execute(
+ select(UnifiWanStat).order_by(UnifiWanStat.scraped_at.desc()).limit(1)
+ )
+ latest_wan = result.scalar_one_or_none()
+
+ # Latest client snapshot (always most recent)
+ result = await db.execute(
+ select(UnifiClientSnapshot)
+ .order_by(UnifiClientSnapshot.scraped_at.desc())
+ .limit(1)
+ )
+ latest_clients = result.scalar_one_or_none()
+
+ # All devices
+ result = await db.execute(
+ select(UnifiDevice).order_by(UnifiDevice.state.desc(), UnifiDevice.name)
+ )
+ devices = result.scalars().all()
+
+ # Client count history — bucketed
+ cs_bucket = (
+ cast(func.strftime('%s', UnifiClientSnapshot.scraped_at), Integer) / b_secs
+ ).label("bucket")
+ result = await db.execute(
+ select(
+ func.avg(UnifiClientSnapshot.total_clients).label("total_clients"),
+ func.min(UnifiClientSnapshot.scraped_at).label("scraped_at"),
+ cs_bucket,
+ )
+ .where(UnifiClientSnapshot.scraped_at >= since)
+ .group_by(cs_bucket)
+ .order_by(cs_bucket)
+ )
+ client_history = result.all()
+
+ # Latest WLAN stats (always most recent per SSID)
+ result = await db.execute(
+ select(UnifiWlanStat.ssid).distinct().order_by(UnifiWlanStat.ssid)
+ )
+ ssids = [row[0] for row in result.all()]
+ wlan_latest: list[UnifiWlanStat] = []
+ for ssid in ssids:
+ result = await db.execute(
+ select(UnifiWlanStat)
+ .where(UnifiWlanStat.ssid == ssid)
+ .order_by(UnifiWlanStat.scraped_at.desc())
+ .limit(1)
+ )
+ w = result.scalar_one_or_none()
+ if w:
+ wlan_latest.append(w)
+
+ # Latest speedtest
+ result = await db.execute(
+ select(UnifiSpeedtestResult)
+ .order_by(UnifiSpeedtestResult.run_at.desc())
+ .limit(1)
+ )
+ speedtest = result.scalar_one_or_none()
+
+ top_clients = json.loads(latest_clients.top_clients_json) if latest_clients else []
+
+ sparkline_latency = _sparkline([r.latency_ms or 0 for r in wan_history])
+ sparkline_rx = _sparkline([r.rx_bytes_rate or 0 for r in wan_history])
+ sparkline_tx = _sparkline([r.tx_bytes_rate or 0 for r in wan_history])
+ sparkline_clients = _sparkline([r.total_clients for r in client_history])
+
+ return await render_template(
+ "unifi/rows.html",
+ latest_wan=latest_wan,
+ latest_clients=latest_clients,
+ devices=devices,
+ top_clients=top_clients,
+ wlan_latest=wlan_latest,
+ speedtest=speedtest,
+ sparkline_latency=sparkline_latency,
+ sparkline_rx=sparkline_rx,
+ sparkline_tx=sparkline_tx,
+ sparkline_clients=sparkline_clients,
+ range_key=range_key,
+ )
+
+
+@unifi_bp.get("/events")
+@require_role(UserRole.viewer)
+async def events():
+ """HTMX fragment: site events within selected range."""
+ since, range_key = parse_range(request.args.get("range"))
+
+ async with current_app.db_sessionmaker() as db:
+ result = await db.execute(
+ select(UnifiEvent)
+ .where(UnifiEvent.event_time >= since)
+ .order_by(UnifiEvent.event_time.desc())
+ .limit(500)
+ )
+ event_rows = result.scalars().all()
+
+ return await render_template(
+ "unifi/events.html", events=event_rows, range_key=range_key
+ )
+
+
+@unifi_bp.get("/dpi")
+@require_role(UserRole.viewer)
+async def dpi():
+ """HTMX fragment: DPI traffic breakdown (latest snapshot)."""
+ async with current_app.db_sessionmaker() as db:
+ result = await db.execute(
+ select(UnifiDpiSnapshot)
+ .order_by(UnifiDpiSnapshot.scraped_at.desc())
+ .limit(1)
+ )
+ snapshot = result.scalar_one_or_none()
+
+ categories = []
+ top_clients = []
+ if snapshot:
+ categories = json.loads(snapshot.by_category_json)
+ top_clients = json.loads(snapshot.by_client_json)
+
+ return await render_template(
+ "unifi/dpi.html", categories=categories, top_clients=top_clients, snapshot=snapshot
+ )
+
+
+@unifi_bp.get("/security")
+@require_role(UserRole.viewer)
+async def security():
+ """HTMX fragment: active and archived alarms."""
+ async with current_app.db_sessionmaker() as db:
+ result = await db.execute(
+ select(UnifiAlarm)
+ .where(UnifiAlarm.archived == False) # noqa: E712
+ .order_by(UnifiAlarm.alarm_time.desc())
+ )
+ active_alarms = result.scalars().all()
+
+ result = await db.execute(
+ select(UnifiAlarm)
+ .where(UnifiAlarm.archived == True) # noqa: E712
+ .order_by(UnifiAlarm.alarm_time.desc())
+ .limit(20)
+ )
+ archived_alarms = result.scalars().all()
+
+ return await render_template(
+ "unifi/security.html",
+ active_alarms=active_alarms,
+ archived_alarms=archived_alarms,
+ )
+
+
+@unifi_bp.get("/inventory")
+@require_role(UserRole.viewer)
+async def inventory():
+ """HTMX fragment: known clients, networks, port forwards, firewall rules."""
+ async with current_app.db_sessionmaker() as db:
+ result = await db.execute(
+ select(UnifiKnownClient)
+ .order_by(UnifiKnownClient.last_seen.desc().nullslast())
+ )
+ known_clients = result.scalars().all()
+
+ result = await db.execute(
+ select(UnifiNetwork).order_by(UnifiNetwork.name)
+ )
+ networks = result.scalars().all()
+
+ result = await db.execute(
+ select(UnifiPortForward).order_by(UnifiPortForward.name)
+ )
+ port_forwards = result.scalars().all()
+
+ result = await db.execute(
+ select(UnifiFwRule)
+ .order_by(UnifiFwRule.ruleset, UnifiFwRule.rule_index)
+ )
+ fw_rules = result.scalars().all()
+
+ return await render_template(
+ "unifi/inventory.html",
+ known_clients=known_clients,
+ networks=networks,
+ port_forwards=port_forwards,
+ fw_rules=fw_rules,
+ )
+
+
+@unifi_bp.get("/widget")
+@require_role(UserRole.viewer)
+async def widget():
+ """HTMX dashboard widget fragment."""
+ async with current_app.db_sessionmaker() as db:
+ result = await db.execute(
+ select(UnifiWanStat).order_by(UnifiWanStat.scraped_at.desc()).limit(1)
+ )
+ latest_wan = result.scalar_one_or_none()
+
+ result = await db.execute(
+ select(UnifiClientSnapshot).order_by(UnifiClientSnapshot.scraped_at.desc()).limit(1)
+ )
+ latest_clients = result.scalar_one_or_none()
+
+ result = await db.execute(select(UnifiDevice))
+ devices = result.scalars().all()
+
+ result = await db.execute(
+ select(UnifiAlarm).where(UnifiAlarm.archived == False) # noqa: E712
+ )
+ active_alarms = result.scalars().all()
+
+ offline_devices = [d for d in devices if d.state != 1]
+ top_clients = json.loads(latest_clients.top_clients_json)[:5] if latest_clients else []
+
+ return await render_template(
+ "unifi/widget.html",
+ latest_wan=latest_wan,
+ latest_clients=latest_clients,
+ offline_devices=offline_devices,
+ active_alarms=active_alarms,
+ top_clients=top_clients,
+ )
+
+
+@unifi_bp.get("/widget/clients")
+@require_role(UserRole.viewer)
+async def widget_clients():
+ """HTMX dashboard widget: top active clients."""
+ limit = max(1, min(20, int(request.args.get("limit", 5) or 5)))
+ widget_id = request.args.get("wid", "0")
+
+ async with current_app.db_sessionmaker() as db:
+ result = await db.execute(
+ select(UnifiClientSnapshot)
+ .order_by(UnifiClientSnapshot.scraped_at.desc())
+ .limit(1)
+ )
+ snapshot = result.scalar_one_or_none()
+
+ clients = json.loads(snapshot.top_clients_json)[:limit] if snapshot else []
+ return await render_template(
+ "unifi/widget_clients.html",
+ clients=clients,
+ snapshot=snapshot,
+ widget_id=widget_id,
+ )
+
+
+@unifi_bp.get("/widget/devices")
+@require_role(UserRole.viewer)
+async def widget_devices():
+ """HTMX dashboard widget: infrastructure devices with optional type filter."""
+ type_filter = request.args.get("type_filter", "all")
+ widget_id = request.args.get("wid", "0")
+
+ async with current_app.db_sessionmaker() as db:
+ result = await db.execute(
+ select(UnifiDevice).order_by(UnifiDevice.state.desc(), UnifiDevice.name)
+ )
+ all_devices = result.scalars().all()
+
+ if type_filter == "wireless":
+ devices = [d for d in all_devices if d.device_type == "uap"]
+ elif type_filter == "wired":
+ devices = [d for d in all_devices if d.device_type != "uap"]
+ elif type_filter == "offline":
+ devices = [d for d in all_devices if d.state != 1]
+ else:
+ devices = list(all_devices)
+
+ return await render_template(
+ "unifi/widget_devices.html",
+ devices=devices,
+ type_filter=type_filter,
+ widget_id=widget_id,
+ )
diff --git a/plugins/unifi/scheduler.py b/plugins/unifi/scheduler.py
new file mode 100644
index 0000000..2a6345d
--- /dev/null
+++ b/plugins/unifi/scheduler.py
@@ -0,0 +1,522 @@
+# plugins/unifi/scheduler.py
+from __future__ import annotations
+import json
+import logging
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING
+
+from fabledscryer.core.scheduler import ScheduledTask
+
+if TYPE_CHECKING:
+ from quart import Quart
+
+logger = logging.getLogger(__name__)
+
+_client = None # UnifiClient instance, initialised on first scrape
+
+
+def make_poll_task(app: "Quart") -> ScheduledTask:
+ cfg = app.config["PLUGINS"]["unifi"]
+ interval = int(cfg.get("poll_interval_seconds", 60))
+
+ async def poll() -> None:
+ await _do_poll(app)
+
+ return ScheduledTask(
+ name="unifi_poll",
+ coro_factory=poll,
+ interval_seconds=interval,
+ run_on_startup=True,
+ )
+
+
+async def _do_poll(app: "Quart") -> None:
+ global _client
+
+ from .client import UnifiClient
+ from .models import UnifiClientSnapshot, UnifiDevice, UnifiWanStat
+ from fabledscryer.core.alerts import record_metric
+
+ cfg = app.config["PLUGINS"]["unifi"]
+
+ if _client is None:
+ _client = UnifiClient(
+ host=cfg["host"],
+ username=cfg["username"],
+ password=cfg["password"],
+ site=cfg.get("site", "default"),
+ verify_ssl=cfg.get("verify_ssl", False),
+ )
+ try:
+ await _client.login()
+ except Exception as exc:
+ logger.warning("UniFi initial login failed: %s", exc)
+ _client = None
+ return
+
+ try:
+ health = await _client.get_health()
+ clients = await _client.get_active_clients()
+ devices = await _client.get_devices()
+ except Exception as exc:
+ logger.warning("UniFi poll failed — will retry next tick: %s", exc)
+ _client = None # force re-auth on next tick
+ return
+
+ scraped_at = datetime.now(timezone.utc)
+ top_n = int(cfg.get("top_clients", 10))
+
+ # ── WAN stat ──────────────────────────────────────────────────────────────
+ wan_data = next((h for h in health if h.get("subsystem") == "wan"), None)
+ wan_stat = UnifiWanStat(scraped_at=scraped_at)
+ if wan_data:
+ wan_stat.is_up = wan_data.get("status") == "ok"
+ wan_stat.wan_ip = wan_data.get("wan_ip")
+ wan_stat.latency_ms = wan_data.get("latency")
+ wan_stat.rx_bytes_rate = wan_data.get("rx_bytes-r")
+ wan_stat.tx_bytes_rate = wan_data.get("tx_bytes-r")
+ wan_stat.uptime_seconds = wan_data.get("uptime")
+
+ # ── Client snapshot ───────────────────────────────────────────────────────
+ wireless = [c for c in clients if not c.get("is_wired", True)]
+ wired = [c for c in clients if c.get("is_wired", True)]
+ guests = [c for c in clients if c.get("is_guest", False)]
+
+ top_clients = sorted(
+ clients,
+ key=lambda c: (c.get("tx_bytes-r", 0) or 0) + (c.get("rx_bytes-r", 0) or 0),
+ reverse=True,
+ )[:top_n]
+
+ top_clients_data = [
+ {
+ "mac": c.get("mac", ""),
+ "hostname": c.get("hostname") or c.get("name") or c.get("mac", ""),
+ "ip": c.get("ip", ""),
+ "type": "wireless" if not c.get("is_wired", True) else "wired",
+ "tx_rate": c.get("tx_bytes-r", 0) or 0,
+ "rx_rate": c.get("rx_bytes-r", 0) or 0,
+ "signal": c.get("signal"),
+ "essid": c.get("essid"),
+ }
+ for c in top_clients
+ ]
+
+ client_snapshot = UnifiClientSnapshot(
+ scraped_at=scraped_at,
+ total_clients=len(clients),
+ wireless_clients=len(wireless),
+ wired_clients=len(wired),
+ guest_clients=len(guests),
+ top_clients_json=json.dumps(top_clients_data),
+ )
+
+ async with app.db_sessionmaker() as session:
+ async with session.begin():
+ session.add(wan_stat)
+ session.add(client_snapshot)
+
+ # ── Devices (upsert by MAC) ───────────────────────────────────────
+ for d in devices:
+ mac = d.get("mac", "")
+ if not mac:
+ continue
+ last_seen_ts = d.get("last_seen")
+ last_seen_dt = (
+ datetime.fromtimestamp(last_seen_ts, tz=timezone.utc)
+ if last_seen_ts else None
+ )
+ existing = await session.get(UnifiDevice, mac)
+ if existing:
+ existing.name = d.get("name")
+ existing.model = d.get("model")
+ existing.device_type = d.get("type")
+ existing.ip = d.get("ip")
+ existing.state = d.get("state", 0)
+ existing.uptime_seconds = d.get("uptime")
+ existing.last_seen = last_seen_dt
+ existing.version = d.get("version")
+ existing.scraped_at = scraped_at
+ else:
+ session.add(UnifiDevice(
+ mac=mac,
+ name=d.get("name"),
+ model=d.get("model"),
+ device_type=d.get("type"),
+ ip=d.get("ip"),
+ state=d.get("state", 0),
+ uptime_seconds=d.get("uptime"),
+ last_seen=last_seen_dt,
+ version=d.get("version"),
+ scraped_at=scraped_at,
+ ))
+
+ # ── Alert metrics ─────────────────────────────────────────────────
+ if wan_stat.latency_ms is not None:
+ await record_metric(
+ session=session,
+ source_module="unifi",
+ resource_name="wan",
+ metric_name="latency_ms",
+ value=wan_stat.latency_ms,
+ )
+ await record_metric(
+ session=session,
+ source_module="unifi",
+ resource_name="wan",
+ metric_name="is_up",
+ value=1.0 if wan_stat.is_up else 0.0,
+ )
+ await record_metric(
+ session=session,
+ source_module="unifi",
+ resource_name="clients",
+ metric_name="total_clients",
+ value=float(len(clients)),
+ )
+
+ logger.debug(
+ "UniFi poll: wan=%s, clients=%d, devices=%d",
+ "up" if wan_stat.is_up else "down",
+ len(clients),
+ len(devices),
+ )
+
+ # ── Expanded data (best-effort, failures don't abort core poll) ───────────
+ try:
+ await _do_expanded(app, scraped_at)
+ except Exception as exc:
+ logger.warning("UniFi expanded poll failed: %s", exc)
+
+
+async def _do_expanded(app: "Quart", scraped_at: datetime) -> None:
+ """Fetch supplementary UniFi data: WLAN stats, events, alarms, DPI,
+ known clients, speedtest, networks, port forwards, firewall rules."""
+ from .models import (
+ UnifiAlarm, UnifiDpiSnapshot, UnifiEvent, UnifiFwRule,
+ UnifiKnownClient, UnifiNetwork, UnifiPortForward, UnifiSpeedtestResult,
+ UnifiWlanStat,
+ )
+
+ # ── Fetch from controller ─────────────────────────────────────────────────
+ try:
+ wlan_configs = await _client.get_wlan_configs()
+ except Exception:
+ wlan_configs = []
+
+ try:
+ events = await _client.get_events(limit=200)
+ except Exception:
+ events = []
+
+ try:
+ alarms = await _client.get_alarms(archived=False)
+ except Exception:
+ alarms = []
+
+ try:
+ dpi_cats = await _client.get_dpi_site_stats()
+ except Exception:
+ dpi_cats = []
+
+ try:
+ dpi_clients = await _client.get_dpi_stats()
+ except Exception:
+ dpi_clients = []
+
+ try:
+ known_clients = await _client.get_known_clients()
+ except Exception:
+ known_clients = []
+
+ try:
+ speedtest = await _client.get_speedtest_status()
+ except Exception:
+ speedtest = None
+
+ try:
+ networks = await _client.get_networks()
+ except Exception:
+ networks = []
+
+ try:
+ port_forwards = await _client.get_port_forwards()
+ except Exception:
+ port_forwards = []
+
+ try:
+ fw_rules = await _client.get_firewall_rules()
+ except Exception:
+ fw_rules = []
+
+ # ── Derive WLAN stats from device VAP table ───────────────────────────────
+ # WLAN usage stats live on the device objects' vap_table, not wlan configs.
+ # We get per-radio SSID data from the active clients grouped by essid.
+ # Store one row per SSID using wlan_configs for channel/satisfaction.
+ wlan_rows: list[UnifiWlanStat] = []
+ for wc in wlan_configs:
+ ssid = wc.get("name") or wc.get("x_passphrase", "")
+ if not ssid:
+ continue
+ wlan_rows.append(UnifiWlanStat(
+ scraped_at=scraped_at,
+ ssid=ssid,
+ radio=None,
+ num_sta=wc.get("num_sta", 0) or 0,
+ tx_bytes=wc.get("tx_bytes"),
+ rx_bytes=wc.get("rx_bytes"),
+ channel=None,
+ satisfaction=wc.get("satisfaction"),
+ ))
+
+ # ── Build DPI category summary ────────────────────────────────────────────
+ cat_summary = []
+ for entry in dpi_cats:
+ cat_id = entry.get("cat") or entry.get("cat_id")
+ cat_name = entry.get("cat_name") or str(cat_id)
+ rx = entry.get("rx_bytes", 0) or 0
+ tx = entry.get("tx_bytes", 0) or 0
+ if rx or tx:
+ cat_summary.append({"cat_id": cat_id, "cat_name": cat_name, "rx_bytes": rx, "tx_bytes": tx})
+ cat_summary.sort(key=lambda x: x["rx_bytes"] + x["tx_bytes"], reverse=True)
+
+ # Top 10 clients by DPI total
+ client_dpi: dict[str, dict] = {}
+ for entry in dpi_clients:
+ mac = entry.get("mac", "")
+ if not mac:
+ continue
+ if mac not in client_dpi:
+ client_dpi[mac] = {"mac": mac, "hostname": entry.get("hostname", mac), "bytes": 0, "top_cats": []}
+ client_dpi[mac]["bytes"] += (entry.get("rx_bytes", 0) or 0) + (entry.get("tx_bytes", 0) or 0)
+ top_dpi_clients = sorted(client_dpi.values(), key=lambda x: x["bytes"], reverse=True)[:10]
+
+ # ── Write to DB ───────────────────────────────────────────────────────────
+ async with app.db_sessionmaker() as session:
+ async with session.begin():
+ # WLAN stats
+ for row in wlan_rows:
+ session.add(row)
+
+ # DPI snapshot (only if we got data)
+ if cat_summary or top_dpi_clients:
+ session.add(UnifiDpiSnapshot(
+ scraped_at=scraped_at,
+ by_category_json=json.dumps(cat_summary),
+ by_client_json=json.dumps(top_dpi_clients),
+ ))
+
+ # Events (upsert by _id)
+ _category_map = {
+ "wlan": "wlan", "wireless": "wlan",
+ "wired": "wired",
+ "ugw": "wan", "wan": "wan",
+ "ids": "ids",
+ "system": "system",
+ "ap": "ap",
+ }
+ for ev in events:
+ uid = ev.get("_id", "")
+ if not uid:
+ continue
+ ts_raw = ev.get("datetime") or ev.get("time")
+ try:
+ if isinstance(ts_raw, (int, float)):
+ event_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc)
+ else:
+ event_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
+ except Exception:
+ event_time = scraped_at
+ subsystem = ev.get("subsystem", "system")
+ category = _category_map.get(subsystem, subsystem[:16])
+ details = {k: v for k, v in ev.items()
+ if k not in ("_id", "datetime", "time", "key", "subsystem", "msg", "user", "src_ip")}
+ existing = await session.get(UnifiEvent, uid)
+ if not existing:
+ session.add(UnifiEvent(
+ unifi_id=uid,
+ event_time=event_time,
+ event_key=ev.get("key", ""),
+ category=category,
+ message=ev.get("msg", ""),
+ source_mac=ev.get("user") or ev.get("src_mac"),
+ source_ip=ev.get("src_ip"),
+ details_json=json.dumps(details),
+ ))
+
+ # Alarms (upsert)
+ for al in alarms:
+ uid = al.get("_id", "")
+ if not uid:
+ continue
+ ts_raw = al.get("datetime") or al.get("time")
+ try:
+ if isinstance(ts_raw, (int, float)):
+ alarm_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc)
+ else:
+ alarm_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
+ except Exception:
+ alarm_time = scraped_at
+ existing = await session.get(UnifiAlarm, uid)
+ if existing:
+ existing.archived = al.get("archived", False)
+ existing.scraped_at = scraped_at
+ else:
+ session.add(UnifiAlarm(
+ unifi_id=uid,
+ alarm_time=alarm_time,
+ alarm_key=al.get("key", ""),
+ message=al.get("msg", ""),
+ archived=al.get("archived", False),
+ scraped_at=scraped_at,
+ ))
+
+ # Known clients (upsert by MAC)
+ for kc in known_clients:
+ mac = kc.get("mac", "")
+ if not mac:
+ continue
+ def _ts(val):
+ if not val:
+ return None
+ try:
+ if isinstance(val, (int, float)):
+ return datetime.fromtimestamp(val, tz=timezone.utc)
+ return datetime.fromisoformat(str(val).replace("Z", "+00:00"))
+ except Exception:
+ return None
+ existing = await session.get(UnifiKnownClient, mac)
+ if existing:
+ existing.hostname = kc.get("hostname")
+ existing.name = kc.get("name")
+ existing.ip = kc.get("ip")
+ existing.mac_vendor = kc.get("oui")
+ existing.note = kc.get("note")
+ existing.is_blocked = kc.get("blocked", False)
+ existing.first_seen = _ts(kc.get("first_seen"))
+ existing.last_seen = _ts(kc.get("last_seen"))
+ existing.scraped_at = scraped_at
+ else:
+ session.add(UnifiKnownClient(
+ mac=mac,
+ hostname=kc.get("hostname"),
+ name=kc.get("name"),
+ ip=kc.get("ip"),
+ mac_vendor=kc.get("oui"),
+ note=kc.get("note"),
+ is_blocked=kc.get("blocked", False),
+ first_seen=_ts(kc.get("first_seen")),
+ last_seen=_ts(kc.get("last_seen")),
+ scraped_at=scraped_at,
+ ))
+
+ # Speedtest (upsert by run_at)
+ if speedtest:
+ ts_raw = speedtest.get("rundate") or speedtest.get("time")
+ try:
+ if isinstance(ts_raw, (int, float)):
+ run_at = datetime.fromtimestamp(ts_raw, tz=timezone.utc)
+ else:
+ run_at = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
+ except Exception:
+ run_at = scraped_at
+ existing = await session.get(UnifiSpeedtestResult, run_at)
+ if not existing:
+ session.add(UnifiSpeedtestResult(
+ run_at=run_at,
+ download_mbps=speedtest.get("xput_download"),
+ upload_mbps=speedtest.get("xput_upload"),
+ latency_ms=speedtest.get("latency"),
+ server_name=speedtest.get("server_name"),
+ ))
+
+ # Networks (upsert by UniFi ID)
+ for net in networks:
+ uid = net.get("_id", "")
+ if not uid:
+ continue
+ existing = await session.get(UnifiNetwork, uid)
+ if existing:
+ existing.name = net.get("name", "")
+ existing.purpose = net.get("purpose")
+ existing.vlan = net.get("vlan")
+ existing.ip_subnet = net.get("ip_subnet")
+ existing.dhcp_enabled = net.get("dhcpd_enabled", False)
+ existing.is_guest = net.get("is_guest", False)
+ existing.scraped_at = scraped_at
+ else:
+ session.add(UnifiNetwork(
+ unifi_id=uid,
+ name=net.get("name", ""),
+ purpose=net.get("purpose"),
+ vlan=net.get("vlan"),
+ ip_subnet=net.get("ip_subnet"),
+ dhcp_enabled=net.get("dhcpd_enabled", False),
+ is_guest=net.get("is_guest", False),
+ scraped_at=scraped_at,
+ ))
+
+ # Port forwards (upsert by UniFi ID)
+ for pf in port_forwards:
+ uid = pf.get("_id", "")
+ if not uid:
+ continue
+ existing = await session.get(UnifiPortForward, uid)
+ if existing:
+ existing.name = pf.get("name")
+ existing.proto = pf.get("proto", "tcp")
+ existing.dst_port = pf.get("dst_port", "")
+ existing.fwd_ip = pf.get("fwd", "")
+ existing.fwd_port = pf.get("fwd_port", "")
+ existing.src_filter = pf.get("src")
+ existing.enabled = pf.get("enabled", True)
+ existing.scraped_at = scraped_at
+ else:
+ session.add(UnifiPortForward(
+ unifi_id=uid,
+ name=pf.get("name"),
+ proto=pf.get("proto", "tcp"),
+ dst_port=pf.get("dst_port", ""),
+ fwd_ip=pf.get("fwd", ""),
+ fwd_port=pf.get("fwd_port", ""),
+ src_filter=pf.get("src"),
+ enabled=pf.get("enabled", True),
+ scraped_at=scraped_at,
+ ))
+
+ # Firewall rules (upsert by UniFi ID)
+ for rule in fw_rules:
+ uid = rule.get("_id", "")
+ if not uid:
+ continue
+ existing = await session.get(UnifiFwRule, uid)
+ if existing:
+ existing.name = rule.get("name", "")
+ existing.ruleset = rule.get("ruleset", "")
+ existing.rule_index = rule.get("rule_index")
+ existing.action = rule.get("action", "")
+ existing.protocol = rule.get("protocol")
+ existing.enabled = rule.get("enabled", True)
+ existing.src_address = rule.get("src_address")
+ existing.dst_address = rule.get("dst_address")
+ existing.scraped_at = scraped_at
+ else:
+ session.add(UnifiFwRule(
+ unifi_id=uid,
+ name=rule.get("name", ""),
+ ruleset=rule.get("ruleset", ""),
+ rule_index=rule.get("rule_index"),
+ action=rule.get("action", ""),
+ protocol=rule.get("protocol"),
+ enabled=rule.get("enabled", True),
+ src_address=rule.get("src_address"),
+ dst_address=rule.get("dst_address"),
+ scraped_at=scraped_at,
+ ))
+
+ logger.debug(
+ "UniFi expanded: wlans=%d, events=%d, alarms=%d, known_clients=%d, "
+ "networks=%d, port_fwds=%d, fw_rules=%d",
+ len(wlan_rows), len(events), len(alarms), len(known_clients),
+ len(networks), len(port_forwards), len(fw_rules),
+ )
diff --git a/plugins/unifi/templates/unifi/dpi.html b/plugins/unifi/templates/unifi/dpi.html
new file mode 100644
index 0000000..6e085b7
--- /dev/null
+++ b/plugins/unifi/templates/unifi/dpi.html
@@ -0,0 +1,65 @@
+{# plugins/unifi/templates/unifi/dpi.html #}
+{% if not snapshot %}
+
+
No DPI data yet. DPI must be enabled on the UniFi controller.
+
+{% else %}
+
+
+ {# Category breakdown #}
+ {% if categories %}
+
+
+ Traffic by Category
+ {{ snapshot.scraped_at.strftime("%H:%M") }}
+
+ {% set total_bytes = categories | sum(attribute='rx_bytes') + categories | sum(attribute='tx_bytes') %}
+
+ {% for cat in categories[:20] %}
+ {% set bytes = cat.rx_bytes + cat.tx_bytes %}
+ {% set pct = (bytes / total_bytes * 100) if total_bytes else 0 %}
+
+ {{ cat.cat_name or cat.cat_id }}
+
+ {% if bytes >= 1073741824 %}{{ "%.1f"|format(bytes / 1073741824) }} GB
+ {% elif bytes >= 1048576 %}{{ "%.1f"|format(bytes / 1048576) }} MB
+ {% elif bytes >= 1024 %}{{ "%.0f"|format(bytes / 1024) }} KB
+ {% else %}{{ bytes }} B{% endif %}
+
+
+
+
+
+ {% endfor %}
+
+
+ {% endif %}
+
+ {# Top clients by DPI #}
+ {% if top_clients %}
+
+
Top Clients by Traffic
+
+ {% for c in top_clients %}
+
+
+ {{ c.hostname or c.mac }}
+ {{ c.mac }}
+
+
+ {% set bytes = c.bytes %}
+ {% if bytes >= 1073741824 %}{{ "%.1f"|format(bytes / 1073741824) }} GB
+ {% elif bytes >= 1048576 %}{{ "%.1f"|format(bytes / 1048576) }} MB
+ {% elif bytes >= 1024 %}{{ "%.0f"|format(bytes / 1024) }} KB
+ {% else %}{{ bytes }} B{% endif %}
+
+
+ {% endfor %}
+
+
+ {% endif %}
+
+
+{% endif %}
diff --git a/plugins/unifi/templates/unifi/events.html b/plugins/unifi/templates/unifi/events.html
new file mode 100644
index 0000000..79c4a21
--- /dev/null
+++ b/plugins/unifi/templates/unifi/events.html
@@ -0,0 +1,52 @@
+{# plugins/unifi/templates/unifi/events.html #}
+{% if not events %}
+
+
No events in the last {{ range_key }}.
+
+{% else %}
+
+
+ Site Events
+ {{ events|length }} in {{ range_key }}
+
+
+
+
+
+
+
+
+
+ Time
+ Category
+ Key
+ Message
+
+ {% for ev in events %}
+
+
+ {{ ev.event_time.strftime("%Y-%m-%d %H:%M:%S") }}
+
+
+
+ {{ ev.category }}
+
+
+
+ {{ ev.event_key.replace("EVT_", "") if ev.event_key else "" }}
+
+
+ {{ ev.message }}
+ {% if ev.source_mac %}
+ {{ ev.source_mac }}
+ {% endif %}
+
+
+ {% endfor %}
+
+
+{% endif %}
diff --git a/plugins/unifi/templates/unifi/index.html b/plugins/unifi/templates/unifi/index.html
new file mode 100644
index 0000000..b2113b6
--- /dev/null
+++ b/plugins/unifi/templates/unifi/index.html
@@ -0,0 +1,63 @@
+{# plugins/unifi/templates/unifi/index.html #}
+{% extends "base.html" %}
+{% block title %}UniFi — Fabled Scryer{% endblock %}
+{% block content %}
+
+
+
UniFi Network
+ polling every {{ poll_interval }}s
+
+ {% include "_time_range.html" %}
+
+
+{# Tab nav #}
+
+ {% set tabs = [
+ ("overview", "Overview", "/plugins/unifi/rows"),
+ ("events", "Events", "/plugins/unifi/events"),
+ ("dpi", "Traffic", "/plugins/unifi/dpi"),
+ ("security", "Security", "/plugins/unifi/security"),
+ ("inventory", "Inventory", "/plugins/unifi/inventory"),
+ ] %}
+ {% for tab_id, label, url in tabs %}
+
+ {{ label }}
+
+ {% endfor %}
+
+
+
+
+
+
+{% endblock %}
diff --git a/plugins/unifi/templates/unifi/inventory.html b/plugins/unifi/templates/unifi/inventory.html
new file mode 100644
index 0000000..62e99f7
--- /dev/null
+++ b/plugins/unifi/templates/unifi/inventory.html
@@ -0,0 +1,146 @@
+{# plugins/unifi/templates/unifi/inventory.html #}
+
+
+ {# Networks / VLANs #}
+ {% if networks %}
+
+
Networks / VLANs
+
+
+ Name
+ VLAN
+ Subnet
+
+ {% for net in networks %}
+
+
+ {{ net.name }}
+ {% if net.is_guest %} guest {% endif %}
+
+
+ {{ net.vlan or "—" }}
+
+
+ {{ net.ip_subnet or "—" }}
+
+
+ {% endfor %}
+
+
+ {% endif %}
+
+ {# Port forwards #}
+ {% if port_forwards %}
+
+
Port Forwards
+
+ {% for pf in port_forwards %}
+
+
+
+ {{ pf.name or "—" }}
+
+ {% if not pf.enabled %} disabled {% endif %}
+
+
+ {{ pf.proto }}:{{ pf.dst_port }} → {{ pf.fwd_ip }}:{{ pf.fwd_port }}
+
+
+ {% endfor %}
+
+
+ {% endif %}
+
+ {# Firewall rules, grouped by ruleset #}
+ {% if fw_rules %}
+ {% set ns = namespace(current_rs=None) %}
+
+
Firewall Rules
+
+ {% for rule in fw_rules %}
+ {% if rule.ruleset != ns.current_rs %}
+ {% set ns.current_rs = rule.ruleset %}
+
+
+ {{ rule.ruleset }}
+
+
+ {% endif %}
+
+
+
+ {{ rule.name }}
+
+
+
+
+ {{ rule.action }}
+
+
+
+ {% if rule.protocol %}{{ rule.protocol }}{% endif %}
+ {% if rule.src_address %} src:{{ rule.src_address }}{% endif %}
+ {% if rule.dst_address %} dst:{{ rule.dst_address }}{% endif %}
+
+
+ {% endfor %}
+
+
+ {% endif %}
+
+ {# Known clients #}
+ {% if known_clients %}
+
+
+ Known Clients
+ {{ known_clients|length }} total
+
+
+
+ Name / MAC
+ IP
+ Last Seen
+
+ {% for kc in known_clients[:50] %}
+
+
+
+ {{ kc.name or kc.hostname or kc.mac }}
+ {% if kc.is_blocked %} blocked {% endif %}
+
+ {% if kc.name or kc.hostname %}
+ {{ kc.mac }}
+ {% endif %}
+ {% if kc.mac_vendor %}
+ {{ kc.mac_vendor }}
+ {% endif %}
+
+
+ {{ kc.ip or "—" }}
+
+
+ {% if kc.last_seen %}{{ kc.last_seen.strftime("%m-%d %H:%M") }}{% else %}—{% endif %}
+
+
+ {% endfor %}
+ {% if known_clients|length > 50 %}
+
+
+ +{{ known_clients|length - 50 }} more
+
+
+ {% endif %}
+
+
+ {% endif %}
+
+ {% if not networks and not port_forwards and not fw_rules and not known_clients %}
+
+
No inventory data yet. This populates on the next poll cycle.
+
+ {% endif %}
+
+
diff --git a/plugins/unifi/templates/unifi/rows.html b/plugins/unifi/templates/unifi/rows.html
new file mode 100644
index 0000000..9de46ce
--- /dev/null
+++ b/plugins/unifi/templates/unifi/rows.html
@@ -0,0 +1,203 @@
+{# plugins/unifi/templates/unifi/rows.html #}
+
+{# ── WAN health bar ───────────────────────────────────────────────────────── #}
+{% if latest_wan %}
+
+
+ WAN
+
+ {% if latest_wan.is_up %}✓ Up{% else %}✗ Down{% endif %}
+
+ {% if latest_wan.wan_ip %}
+ {{ latest_wan.wan_ip }}
+ {% endif %}
+
+
+ {% if latest_wan.latency_ms is not none %}
+
+ Latency
+
+ {{ "%.1f"|format(latest_wan.latency_ms) }}ms
+
+ {{ sparkline_latency | safe }}
+
+ {% endif %}
+
+ {% if latest_wan.rx_bytes_rate is not none %}
+
+ ↓ RX
+
+ {% set bps = latest_wan.rx_bytes_rate %}
+ {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s
+ {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }} KB/s
+ {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %}
+
+ {{ sparkline_rx | safe }}
+
+
+ ↑ TX
+
+ {% set bps = latest_wan.tx_bytes_rate %}
+ {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s
+ {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }} KB/s
+ {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %}
+
+ {{ sparkline_tx | safe }}
+
+ {% endif %}
+
+ {% if latest_wan.uptime_seconds is not none %}
+
+ up {{ (latest_wan.uptime_seconds // 3600) }}h {{ ((latest_wan.uptime_seconds % 3600) // 60) }}m
+
+ {% endif %}
+
+{% endif %}
+
+{# ── Content columns ──────────────────────────────────────────────────────── #}
+
+
+ {# Client summary card #}
+ {% if latest_clients %}
+
+
+ Clients
+ {{ sparkline_clients | safe }}
+
+
+
+ {{ latest_clients.total_clients }}
+ total
+
+
+ {{ latest_clients.wireless_clients }}↗ wireless
+ {{ latest_clients.wired_clients }}↔ wired
+ {% if latest_clients.guest_clients %} {{ latest_clients.guest_clients }} guest{% endif %}
+
+
+
+ {% endif %}
+
+ {# Infrastructure devices card #}
+ {% if devices %}
+
+
Infrastructure Devices
+
+ {% for d in devices %}
+
+
+ ●
+ {{ d.name or d.mac }}
+
+ {{ d.model or d.device_type or "" }}
+ {{ d.ip or "" }}
+
+ {% if d.state == 1 and d.uptime_seconds %}
+
+
+ up {{ (d.uptime_seconds // 86400) }}d {{ ((d.uptime_seconds % 86400) // 3600) }}h
+
+
+ {% endif %}
+ {% endfor %}
+
+
+ {% endif %}
+
+ {# Top talkers card #}
+ {% if top_clients %}
+
+
Top Talkers
+
+ {% for c in top_clients %}
+
+
+ {{ c.hostname }}
+ {% if c.type == "wireless" and c.signal %}
+ {{ c.signal }}dBm
+ {% endif %}
+
+
+ {% set rx = c.rx_rate %}{% set tx = c.tx_rate %}
+ ↓
+ {% if rx >= 1048576 %}{{ "%.1f"|format(rx/1048576) }}M
+ {% elif rx >= 1024 %}{{ "%.0f"|format(rx/1024) }}K
+ {% else %}{{ "%.0f"|format(rx) }}B{% endif %}
+ ↑
+ {% if tx >= 1048576 %}{{ "%.1f"|format(tx/1048576) }}M
+ {% elif tx >= 1024 %}{{ "%.0f"|format(tx/1024) }}K
+ {% else %}{{ "%.0f"|format(tx) }}B{% endif %}
+
+
+ {% endfor %}
+
+
+ {% endif %}
+
+ {# WLAN SSIDs card #}
+ {% if wlan_latest %}
+
+
Wireless Networks
+
+
+ SSID
+ Clients
+ Score
+
+ {% for w in wlan_latest %}
+
+ {{ w.ssid }}
+ {{ w.num_sta }}
+
+ {% if w.satisfaction is not none %}{{ w.satisfaction }}%{% else %}—{% endif %}
+
+
+ {% endfor %}
+
+
+ {% endif %}
+
+ {# Speedtest card #}
+ {% if speedtest %}
+
+
Last Speed Test
+
+ {% if speedtest.download_mbps is not none %}
+
+ ↓ DL
+ {{ "%.1f"|format(speedtest.download_mbps) }}
+ Mbps
+
+ {% endif %}
+ {% if speedtest.upload_mbps is not none %}
+
+ ↑ UL
+ {{ "%.1f"|format(speedtest.upload_mbps) }}
+ Mbps
+
+ {% endif %}
+ {% if speedtest.latency_ms is not none %}
+
+ Latency
+ {{ "%.0f"|format(speedtest.latency_ms) }}ms
+
+ {% endif %}
+
+ {% if speedtest.server_name %}
+
via {{ speedtest.server_name }}
+ {% endif %}
+
{{ speedtest.run_at.strftime("%Y-%m-%d %H:%M") }} UTC
+
+ {% endif %}
+
+
+
+{% if not latest_wan and not devices %}
+
+
No UniFi data yet. Check your host, username, and password in the plugin config.
+
+{% endif %}
diff --git a/plugins/unifi/templates/unifi/security.html b/plugins/unifi/templates/unifi/security.html
new file mode 100644
index 0000000..f28d4c0
--- /dev/null
+++ b/plugins/unifi/templates/unifi/security.html
@@ -0,0 +1,56 @@
+{# plugins/unifi/templates/unifi/security.html #}
+
+
+ {# Active alarms #}
+
+
+ Active Alarms
+ {% if active_alarms %}
+ {{ active_alarms|length }}
+ {% endif %}
+
+ {% if not active_alarms %}
+
✓ No active alarms
+ {% else %}
+
+ {% for al in active_alarms %}
+
+
+
+ ⚠ {{ al.alarm_key }}
+
+ {{ al.message }}
+
+ {{ al.alarm_time.strftime("%Y-%m-%d %H:%M") }} UTC
+
+
+
+ {% endfor %}
+
+ {% endif %}
+
+
+ {# Recent archived alarms #}
+ {% if archived_alarms %}
+
+
+ Recent Archived Alarms
+ last {{ archived_alarms|length }}
+
+
+ {% for al in archived_alarms %}
+
+
+ {{ al.alarm_key }}
+ {{ al.message }}
+
+
+ {{ al.alarm_time.strftime("%m-%d %H:%M") }}
+
+
+ {% endfor %}
+
+
+ {% endif %}
+
+
diff --git a/plugins/unifi/templates/unifi/widget.html b/plugins/unifi/templates/unifi/widget.html
new file mode 100644
index 0000000..2993ea6
--- /dev/null
+++ b/plugins/unifi/templates/unifi/widget.html
@@ -0,0 +1,95 @@
+{# plugins/unifi/templates/unifi/widget.html #}
+
+{# Active alarms #}
+{% for al in active_alarms %}
+
+ ⚠ {{ al.alarm_key }}
+ alarm
+
+{% endfor %}
+{# Offline device alerts #}
+{% for d in offline_devices %}
+
+ ⚠ {{ d.name or d.mac }}
+ offline
+
+{% endfor %}
+{% if active_alarms or offline_devices %}
{% endif %}
+
+{% if latest_wan %}
+
+
+ WAN
+
+
+
+ {% if latest_wan.is_up %}Up{% else %}Down{% endif %}
+
+ {% if latest_wan.latency_ms is not none %}
+
+ lat
+
+ {{ "%.0f"|format(latest_wan.latency_ms) }}ms
+
+
+ {% endif %}
+ {% if latest_wan.rx_bytes_rate is not none %}
+
+ ↓
+ {% set bps = latest_wan.rx_bytes_rate %}
+ {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s
+ {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s
+ {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %}
+
+
+ ↑
+ {% set bps = latest_wan.tx_bytes_rate %}
+ {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s
+ {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s
+ {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %}
+
+ {% endif %}
+
+
+{% endif %}
+
+{% if latest_clients %}
+
+
+ Clients
+
+
+ {{ latest_clients.total_clients }}
+ {{ latest_clients.wireless_clients }}w / {{ latest_clients.wired_clients }}e
+
+
+{% endif %}
+
+{% if top_clients %}
+
+ {% for c in top_clients %}
+
+
+ {{ c.hostname }}
+
+
+ {% set total_bps = c.tx_rate + c.rx_rate %}
+ ↓
+ {% set rx = c.rx_rate %}
+ {% if rx >= 1048576 %}{{ "%.1f"|format(rx / 1048576) }}M
+ {% elif rx >= 1024 %}{{ "%.0f"|format(rx / 1024) }}K
+ {% else %}{{ "%.0f"|format(rx) }}B{% endif %}
+ ↑
+ {% set tx = c.tx_rate %}
+ {% if tx >= 1048576 %}{{ "%.1f"|format(tx / 1048576) }}M
+ {% elif tx >= 1024 %}{{ "%.0f"|format(tx / 1024) }}K
+ {% else %}{{ "%.0f"|format(tx) }}B{% endif %}
+
+
+ {% endfor %}
+
+{% endif %}
+
+{% if not latest_wan and not latest_clients and not active_alarms and not offline_devices %}
+No UniFi data yet.
+{% endif %}
diff --git a/plugins/unifi/templates/unifi/widget_clients.html b/plugins/unifi/templates/unifi/widget_clients.html
new file mode 100644
index 0000000..99a5892
--- /dev/null
+++ b/plugins/unifi/templates/unifi/widget_clients.html
@@ -0,0 +1,42 @@
+{# unifi/widget_clients.html — dashboard widget: top active clients #}
+{% if not snapshot %}
+No client data yet.
+{% else %}
+
+
+
+ {{ snapshot.total_clients }}
+ total
+
+
+ {{ snapshot.wireless_clients }}
+ wireless
+
+
+ {{ snapshot.wired_clients }}
+ wired
+
+
+
+{% if clients %}
+
+ {% for c in clients %}
+
+
+
+ {{ c.hostname or c.ip or c.mac }}
+
+ {% if c.tx_rate or c.rx_rate %}
+
+ {% if c.tx_rate %}↑{{ "%.0f" | format(c.tx_rate / 1000) }}k{% endif %}
+ {% if c.rx_rate %} ↓{{ "%.0f" | format(c.rx_rate / 1000) }}k{% endif %}
+
+ {% endif %}
+
+ {% endfor %}
+
+{% else %}
+No clients connected.
+{% endif %}
+
+{% endif %}
diff --git a/plugins/unifi/templates/unifi/widget_devices.html b/plugins/unifi/templates/unifi/widget_devices.html
new file mode 100644
index 0000000..90994ed
--- /dev/null
+++ b/plugins/unifi/templates/unifi/widget_devices.html
@@ -0,0 +1,36 @@
+{# unifi/widget_devices.html — dashboard widget: infrastructure devices #}
+{% if not devices %}
+
+ {% if type_filter != "all" %}No {{ type_filter }} devices found.{% else %}No devices found.{% endif %}
+
+{% else %}
+
+{% set online = devices | selectattr("state", "equalto", 1) | list %}
+{% set offline = devices | rejectattr("state", "equalto", 1) | list %}
+
+
+
+ {{ online | length }}
+ online
+
+ {% if offline %}
+
+ {{ offline | length }}
+ offline
+
+ {% endif %}
+
+
+
+ {% for d in devices %}
+
+
+
+ {{ d.name or d.mac }}
+
+ {{ d.model or d.device_type or "" }}
+
+ {% endfor %}
+
+
+{% endif %}
diff --git a/steward/app.py b/steward/app.py
index 676593a..b1ad710 100644
--- a/steward/app.py
+++ b/steward/app.py
@@ -20,13 +20,15 @@ def create_app(
bootstrap = {
"database_url": "postgresql+asyncpg://test/test",
"secret_key": "test-secret-key",
- "plugin_dir": "plugins",
+ "plugin_dirs": ["plugins"],
+ "plugin_install_dir": "plugins",
}
app.config.update(
SECRET_KEY=bootstrap["secret_key"],
DATABASE_URL=bootstrap["database_url"],
- PLUGIN_DIR=bootstrap["plugin_dir"],
+ PLUGIN_DIRS=bootstrap["plugin_dirs"],
+ PLUGIN_INSTALL_DIR=bootstrap["plugin_install_dir"],
TESTING=testing,
)
@@ -42,7 +44,7 @@ def create_app(
from .core.migration_runner import run_core_migrations
run_core_migrations(
app.config["DATABASE_URL"],
- plugin_dir=Path(app.config["PLUGIN_DIR"]).resolve(),
+ plugin_dirs=[Path(d).resolve() for d in app.config["PLUGIN_DIRS"]],
)
# ── 4. Load all settings from DB → populate app.config ────────────────────
@@ -81,9 +83,9 @@ def create_app(
# so this is a no-op on normal startup. We still run it with all discovered dirs so
# Alembic can resolve the full revision graph regardless of which plugins are enabled.
if not testing:
- from .core.migration_runner import run_plugin_migrations, discover_all_plugin_migration_dirs
- _plugin_dir = Path(app.config["PLUGIN_DIR"]).resolve()
- _all_plugin_dirs = discover_all_plugin_migration_dirs(_plugin_dir)
+ from .core.migration_runner import run_plugin_migrations, discover_all_in
+ _plugin_dirs = [Path(d).resolve() for d in app.config["PLUGIN_DIRS"]]
+ _all_plugin_dirs = discover_all_in(_plugin_dirs)
run_plugin_migrations(app.config["DATABASE_URL"], _all_plugin_dirs)
# ── 6. Alert pipeline ──────────────────────────────────────────────────────
diff --git a/steward/config.py b/steward/config.py
index b119a25..9c8f1fd 100644
--- a/steward/config.py
+++ b/steward/config.py
@@ -15,7 +15,7 @@ def _env(suffix: str) -> str | None:
return os.environ.get(f"STEWARD_{suffix}")
-def load_bootstrap(config_path: Path | str | None = None) -> dict[str, str]:
+def load_bootstrap(config_path: Path | str | None = None) -> dict[str, Any]:
"""Return the minimum bootstrap config: database_url and secret_key.
This is the only config read from files/env vars at startup.
@@ -48,15 +48,29 @@ def load_bootstrap(config_path: Path | str | None = None) -> dict[str, str]:
)
secret_key = _resolve_secret_key(raw)
- plugin_dir = (
+
+ # Plugin discovery spans two roots (see load_plugins / migration_runner):
+ # • bundled — first-party plugins shipped inside the image at repo-root
+ # `plugins/`; they version atomically with core and are read-only at runtime.
+ # • external — operator-mounted dir for third-party plugins, persisted in the
+ # /data volume. Downloads/installs land here, never in the bundled dir.
+ # Bundled is scanned first, so on a name collision the first-party plugin wins.
+ bundled_plugin_dir = raw.get("plugin_dir", "plugins")
+ external_plugin_dir = (
_env("PLUGIN_DIR")
- or raw.get("plugin_dir", "plugins")
+ or raw.get("external_plugin_dir")
+ or "/data/plugins"
)
+ plugin_dirs = [bundled_plugin_dir]
+ if external_plugin_dir and external_plugin_dir != bundled_plugin_dir:
+ plugin_dirs.append(external_plugin_dir)
return {
"database_url": database_url,
"secret_key": secret_key,
- "plugin_dir": plugin_dir,
+ "plugin_dirs": plugin_dirs,
+ # Installs/downloads target the external (writable, persistent) dir.
+ "plugin_install_dir": external_plugin_dir or bundled_plugin_dir,
}
diff --git a/steward/core/migration_runner.py b/steward/core/migration_runner.py
index d8e513c..c382ff5 100644
--- a/steward/core/migration_runner.py
+++ b/steward/core/migration_runner.py
@@ -24,14 +24,26 @@ def discover_all_plugin_migration_dirs(plugin_dir: Path) -> list[Path]:
return dirs
-def run_core_migrations(db_url: str, plugin_dir: Path | None = None) -> None:
+def discover_all_in(plugin_dirs: list[Path]) -> list[Path]:
+ """discover_all_plugin_migration_dirs across multiple plugin roots.
+
+ Plugins now live in more than one root (bundled + external), so the full
+ revision graph is the union of every root's migration dirs.
+ """
+ dirs: list[Path] = []
+ for pd in plugin_dirs:
+ dirs.extend(discover_all_plugin_migration_dirs(pd))
+ return dirs
+
+
+def run_core_migrations(db_url: str, plugin_dirs: list[Path] | None = None) -> None:
"""Run core Alembic migrations.
- Includes all discovered plugin migration dirs (if plugin_dir given) so
+ Includes all discovered plugin migration dirs (across every plugin root) so
Alembic can resolve any previously-applied plugin revisions in the graph.
Called first so the app_settings table exists before loading settings.
"""
- dirs = discover_all_plugin_migration_dirs(plugin_dir) if plugin_dir else []
+ dirs = discover_all_in(plugin_dirs) if plugin_dirs else []
_run(db_url, plugin_migration_dirs=dirs)
diff --git a/steward/core/plugin_manager.py b/steward/core/plugin_manager.py
index 2bc0e2c..b0c6e6d 100644
--- a/steward/core/plugin_manager.py
+++ b/steward/core/plugin_manager.py
@@ -31,6 +31,19 @@ def get_plugin_failures() -> dict[str, str]:
return dict(_FAILED_PLUGINS)
+def resolve_plugin_path(plugin_dirs: list[Path], name: str) -> Path | None:
+ """Return the first plugin root that contains `name`, else None.
+
+ Roots are searched in order, so a bundled (first-party) plugin shadows an
+ external plugin of the same name. Pure function — no app/IO beyond exists().
+ """
+ for pd in plugin_dirs:
+ candidate = pd / name
+ if candidate.exists():
+ return candidate
+ return None
+
+
def _import_plugin(name: str, plugin_path: Path):
"""Load a plugin module by file path, avoiding sys.modules stdlib collisions.
@@ -80,22 +93,24 @@ def load_plugins(app: "Quart") -> None:
"""
import steward
- plugin_dir = Path(app.config["PLUGIN_DIR"])
+ plugin_dirs = [Path(d) for d in app.config["PLUGIN_DIRS"]]
plugins_cfg: dict = app.config["PLUGINS"]
- # Ensure plugin_dir is on sys.path so plugins are importable by name
- plugin_dir_str = str(plugin_dir.resolve())
- if plugin_dir_str not in sys.path:
- sys.path.insert(0, plugin_dir_str)
+ # Ensure every plugin root is on sys.path so plugins are importable by name
+ for pd in plugin_dirs:
+ pd_str = str(pd.resolve())
+ if pd_str not in sys.path:
+ sys.path.insert(0, pd_str)
for name, cfg in list(plugins_cfg.items()):
if not cfg.get("enabled", False):
continue
- plugin_path = plugin_dir / name
- if not plugin_path.exists():
- _FAILED_PLUGINS[name] = f"Plugin directory not found: {plugin_path}"
- logger.error("Plugin %r: directory %s not found, skipping", name, plugin_path)
+ plugin_path = resolve_plugin_path(plugin_dirs, name)
+ if plugin_path is None:
+ roots = ", ".join(str(d) for d in plugin_dirs)
+ _FAILED_PLUGINS[name] = f"Plugin directory not found in: {roots}"
+ logger.error("Plugin %r: not found in any plugin root (%s), skipping", name, roots)
continue
# Load and validate plugin.yaml
@@ -224,10 +239,12 @@ async def download_and_install_plugin(
"""
import httpx
- plugin_dir = Path(app.config["PLUGIN_DIR"]).resolve()
+ # Downloads always land in the external (writable, persistent) install dir,
+ # never the read-only bundled dir.
+ plugin_dir = Path(app.config["PLUGIN_INSTALL_DIR"]).resolve()
- # Ensure plugin_dir is on sys.path (may not be set yet if no plugins were
- # enabled at startup)
+ # Ensure the install dir is on sys.path (may not be set yet if no plugins
+ # were enabled at startup)
plugin_dir_str = str(plugin_dir)
if plugin_dir_str not in sys.path:
sys.path.insert(0, plugin_dir_str)
@@ -304,9 +321,11 @@ async def download_and_install_plugin(
if mdir.exists():
from steward.core.migration_runner import (
run_plugin_migrations,
- discover_all_plugin_migration_dirs,
+ discover_all_in,
)
- all_dirs = discover_all_plugin_migration_dirs(plugin_dir)
+ # Span every plugin root so Alembic can resolve bundled-plugin
+ # revisions already stamped in alembic_version.
+ all_dirs = discover_all_in([Path(d).resolve() for d in app.config["PLUGIN_DIRS"]])
run_plugin_migrations(app.config["DATABASE_URL"], all_dirs)
except Exception:
logger.exception("Plugin %r: migration failed after install", name)
@@ -330,11 +349,11 @@ def hot_reload_plugin(app: "Quart", name: str) -> tuple[bool, str]:
if name in _LOADED_PLUGINS:
return False, "Plugin already loaded — restart required to apply updates"
- plugin_dir = Path(app.config["PLUGIN_DIR"]).resolve()
- plugin_path = plugin_dir / name
+ plugin_dirs = [Path(d).resolve() for d in app.config["PLUGIN_DIRS"]]
+ plugin_path = resolve_plugin_path(plugin_dirs, name)
- if not plugin_path.exists():
- return False, f"Plugin directory {plugin_path} not found"
+ if plugin_path is None:
+ return False, f"Plugin {name!r} not found in any plugin root"
yaml_path = plugin_path / "plugin.yaml"
if not yaml_path.exists():
@@ -383,9 +402,9 @@ def hot_reload_plugin(app: "Quart", name: str) -> tuple[bool, str]:
try:
from steward.core.migration_runner import (
run_plugin_migrations,
- discover_all_plugin_migration_dirs,
+ discover_all_in,
)
- all_dirs = discover_all_plugin_migration_dirs(plugin_dir)
+ all_dirs = discover_all_in(plugin_dirs)
run_plugin_migrations(app.config["DATABASE_URL"], all_dirs)
except Exception:
logger.exception("Plugin %r: migration failed during hot-reload", name)
diff --git a/steward/settings/routes.py b/steward/settings/routes.py
index c3e5b58..8434b70 100644
--- a/steward/settings/routes.py
+++ b/steward/settings/routes.py
@@ -19,22 +19,32 @@ logger = logging.getLogger(__name__)
def _discover_plugins() -> list[dict]:
- """Scan PLUGIN_DIR for plugin.yaml files."""
+ """Scan every plugin root for plugin.yaml files.
+
+ Roots are scanned in order (bundled first), and a plugin name found in an
+ earlier root shadows the same name in a later one — matching load_plugins.
+ """
import yaml
- plugin_dir = Path(current_app.config.get("PLUGIN_DIR", "plugins")).resolve()
- plugins = []
- if not plugin_dir.exists():
- return plugins
- for entry in sorted(plugin_dir.iterdir()):
- yaml_path = entry / "plugin.yaml"
- if entry.is_dir() and yaml_path.exists():
- try:
- with yaml_path.open() as f:
- meta = yaml.safe_load(f) or {}
- meta["_dir"] = entry.name
- plugins.append(meta)
- except Exception:
- logger.warning("Could not read %s", yaml_path)
+ plugin_dirs = [
+ Path(d).resolve()
+ for d in current_app.config.get("PLUGIN_DIRS", ["plugins"])
+ ]
+ plugins: list[dict] = []
+ seen: set[str] = set()
+ for plugin_dir in plugin_dirs:
+ if not plugin_dir.exists():
+ continue
+ for entry in sorted(plugin_dir.iterdir()):
+ yaml_path = entry / "plugin.yaml"
+ if entry.is_dir() and yaml_path.exists() and entry.name not in seen:
+ try:
+ with yaml_path.open() as f:
+ meta = yaml.safe_load(f) or {}
+ meta["_dir"] = entry.name
+ plugins.append(meta)
+ seen.add(entry.name)
+ except Exception:
+ logger.warning("Could not read %s", yaml_path)
return plugins
diff --git a/tests/test_plugin_dirs.py b/tests/test_plugin_dirs.py
new file mode 100644
index 0000000..65d7cee
--- /dev/null
+++ b/tests/test_plugin_dirs.py
@@ -0,0 +1,84 @@
+"""Unit tests for the bundled + external multi-root plugin model.
+
+Pure-function coverage (no DB, no app): path resolution, multi-root migration
+discovery, and the config split into bundled/external roots + install dir.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from steward.core.migration_runner import discover_all_in
+from steward.core.plugin_manager import resolve_plugin_path
+
+
+def _make_plugin(root: Path, name: str, with_migrations: bool = False) -> None:
+ pdir = root / name
+ pdir.mkdir(parents=True)
+ (pdir / "plugin.yaml").write_text(f"name: {name}\n")
+ if with_migrations:
+ (pdir / "migrations" / "versions").mkdir(parents=True)
+
+
+def test_resolve_plugin_path_prefers_earlier_root(tmp_path):
+ bundled = tmp_path / "bundled"
+ external = tmp_path / "external"
+ _make_plugin(bundled, "dupe")
+ _make_plugin(external, "dupe")
+ _make_plugin(external, "only_external")
+
+ # Bundled is listed first → it shadows the external copy of the same name.
+ assert resolve_plugin_path([bundled, external], "dupe") == bundled / "dupe"
+ # A name only present externally still resolves.
+ assert resolve_plugin_path([bundled, external], "only_external") == external / "only_external"
+
+
+def test_resolve_plugin_path_missing_returns_none(tmp_path):
+ assert resolve_plugin_path([tmp_path], "nope") is None
+
+
+def test_discover_all_in_unions_roots(tmp_path):
+ bundled = tmp_path / "bundled"
+ external = tmp_path / "external"
+ _make_plugin(bundled, "a", with_migrations=True)
+ _make_plugin(bundled, "no_mig") # no migrations dir → not discovered
+ _make_plugin(external, "b", with_migrations=True)
+
+ found = discover_all_in([bundled, external])
+ assert (bundled / "a" / "migrations") in found
+ assert (external / "b" / "migrations") in found
+ assert (bundled / "no_mig" / "migrations") not in found
+
+
+def test_discover_all_in_skips_missing_root(tmp_path):
+ bundled = tmp_path / "bundled"
+ _make_plugin(bundled, "a", with_migrations=True)
+ # A non-existent external root must not raise.
+ found = discover_all_in([bundled, tmp_path / "does_not_exist"])
+ assert found == [bundled / "a" / "migrations"]
+
+
+def test_bootstrap_splits_bundled_and_external(tmp_path, monkeypatch):
+ from steward.config import load_bootstrap
+
+ monkeypatch.setenv("STEWARD_DATABASE_URL", "postgresql+asyncpg://x/y")
+ monkeypatch.setenv("STEWARD_SECRET_KEY", "k" * 32)
+ monkeypatch.setenv("STEWARD_PLUGIN_DIR", "/srv/external-plugins")
+
+ boot = load_bootstrap(config_path=tmp_path / "absent.yaml")
+
+ assert boot["plugin_dirs"] == ["plugins", "/srv/external-plugins"]
+ # Installs target the external (writable) root, never the bundled one.
+ assert boot["plugin_install_dir"] == "/srv/external-plugins"
+
+
+def test_bootstrap_external_defaults_to_data_plugins(tmp_path, monkeypatch):
+ from steward.config import load_bootstrap
+
+ monkeypatch.setenv("STEWARD_DATABASE_URL", "postgresql+asyncpg://x/y")
+ monkeypatch.setenv("STEWARD_SECRET_KEY", "k" * 32)
+ monkeypatch.delenv("STEWARD_PLUGIN_DIR", raising=False)
+
+ boot = load_bootstrap(config_path=tmp_path / "absent.yaml")
+
+ assert boot["plugin_dirs"] == ["plugins", "/data/plugins"]
+ assert boot["plugin_install_dir"] == "/data/plugins"