feat(plugins): fold first-party plugins in-tree; bundled + external roots

First-party plugins (host_agent, http, snmp, traefik, unifi, docker) are now
tracked under plugins/ and baked into the image, so they version atomically
with core — ending the cross-repo import drift the roundtable->steward rename
exposed. History for these files is preserved in the archived Roundtable-plugins
repo.

Plugin discovery becomes multi-root: PLUGIN_DIR (single) -> PLUGIN_DIRS
(bundled first, then external) + PLUGIN_INSTALL_DIR. Bundled ships in the image;
third-party plugins still mount at runtime into the external root
(STEWARD_PLUGIN_DIR, default /data/plugins) and downloads/installs land there.
Bundled shadows external on a name collision.

- config.py: load_bootstrap returns plugin_dirs + plugin_install_dir
- app.py: iterate PLUGIN_DIRS at the migration + load sites
- migration_runner.py: discover_all_in() unions every plugin root
- plugin_manager.py: resolve_plugin_path() (pure, first-root-wins); load /
  install / hot-reload span all roots; installs target the external root
- settings/routes.py: _discover_plugins scans all roots, dedup bundled-first
- Dockerfile: COPY plugins/ ; docker-compose: drop host bind, document external
- tests/test_plugin_dirs.py: resolution, multi-root discovery, bootstrap split

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

steward/app.py

@@ -20,13 +20,15 @@ def create_app(
         bootstrap = {
             "database_url": "postgresql+asyncpg://test/test",
             "secret_key": "test-secret-key",
-            "plugin_dir": "plugins",
+            "plugin_dirs": ["plugins"],
+            "plugin_install_dir": "plugins",
         }
 
     app.config.update(
         SECRET_KEY=bootstrap["secret_key"],
         DATABASE_URL=bootstrap["database_url"],
-        PLUGIN_DIR=bootstrap["plugin_dir"],
+        PLUGIN_DIRS=bootstrap["plugin_dirs"],
+        PLUGIN_INSTALL_DIR=bootstrap["plugin_install_dir"],
         TESTING=testing,
     )
 
@@ -42,7 +44,7 @@ def create_app(
         from .core.migration_runner import run_core_migrations
         run_core_migrations(
             app.config["DATABASE_URL"],
-            plugin_dir=Path(app.config["PLUGIN_DIR"]).resolve(),
+            plugin_dirs=[Path(d).resolve() for d in app.config["PLUGIN_DIRS"]],
         )
 
     # ── 4. Load all settings from DB → populate app.config ────────────────────
@@ -81,9 +83,9 @@ def create_app(
     # so this is a no-op on normal startup. We still run it with all discovered dirs so
     # Alembic can resolve the full revision graph regardless of which plugins are enabled.
     if not testing:
-        from .core.migration_runner import run_plugin_migrations, discover_all_plugin_migration_dirs
-        _plugin_dir = Path(app.config["PLUGIN_DIR"]).resolve()
-        _all_plugin_dirs = discover_all_plugin_migration_dirs(_plugin_dir)
+        from .core.migration_runner import run_plugin_migrations, discover_all_in
+        _plugin_dirs = [Path(d).resolve() for d in app.config["PLUGIN_DIRS"]]
+        _all_plugin_dirs = discover_all_in(_plugin_dirs)
         run_plugin_migrations(app.config["DATABASE_URL"], _all_plugin_dirs)
 
     # ── 6. Alert pipeline ──────────────────────────────────────────────────────

steward/config.py

@@ -15,7 +15,7 @@ def _env(suffix: str) -> str | None:
     return os.environ.get(f"STEWARD_{suffix}")
 
 
-def load_bootstrap(config_path: Path | str | None = None) -> dict[str, str]:
+def load_bootstrap(config_path: Path | str | None = None) -> dict[str, Any]:
     """Return the minimum bootstrap config: database_url and secret_key.
 
     This is the only config read from files/env vars at startup.
@@ -48,15 +48,29 @@ def load_bootstrap(config_path: Path | str | None = None) -> dict[str, str]:
         )
 
     secret_key = _resolve_secret_key(raw)
-    plugin_dir = (
+
+    # Plugin discovery spans two roots (see load_plugins / migration_runner):
+    #   • bundled  — first-party plugins shipped inside the image at repo-root
+    #     `plugins/`; they version atomically with core and are read-only at runtime.
+    #   • external — operator-mounted dir for third-party plugins, persisted in the
+    #     /data volume. Downloads/installs land here, never in the bundled dir.
+    # Bundled is scanned first, so on a name collision the first-party plugin wins.
+    bundled_plugin_dir = raw.get("plugin_dir", "plugins")
+    external_plugin_dir = (
         _env("PLUGIN_DIR")
-        or raw.get("plugin_dir", "plugins")
+        or raw.get("external_plugin_dir")
+        or "/data/plugins"
     )
+    plugin_dirs = [bundled_plugin_dir]
+    if external_plugin_dir and external_plugin_dir != bundled_plugin_dir:
+        plugin_dirs.append(external_plugin_dir)
 
     return {
         "database_url": database_url,
         "secret_key": secret_key,
-        "plugin_dir": plugin_dir,
+        "plugin_dirs": plugin_dirs,
+        # Installs/downloads target the external (writable, persistent) dir.
+        "plugin_install_dir": external_plugin_dir or bundled_plugin_dir,
     }
 
 

steward/core/migration_runner.py

@@ -24,14 +24,26 @@ def discover_all_plugin_migration_dirs(plugin_dir: Path) -> list[Path]:
     return dirs
 
 
-def run_core_migrations(db_url: str, plugin_dir: Path | None = None) -> None:
+def discover_all_in(plugin_dirs: list[Path]) -> list[Path]:
+    """discover_all_plugin_migration_dirs across multiple plugin roots.
+
+    Plugins now live in more than one root (bundled + external), so the full
+    revision graph is the union of every root's migration dirs.
+    """
+    dirs: list[Path] = []
+    for pd in plugin_dirs:
+        dirs.extend(discover_all_plugin_migration_dirs(pd))
+    return dirs
+
+
+def run_core_migrations(db_url: str, plugin_dirs: list[Path] | None = None) -> None:
     """Run core Alembic migrations.
 
-    Includes all discovered plugin migration dirs (if plugin_dir given) so
+    Includes all discovered plugin migration dirs (across every plugin root) so
     Alembic can resolve any previously-applied plugin revisions in the graph.
     Called first so the app_settings table exists before loading settings.
     """
-    dirs = discover_all_plugin_migration_dirs(plugin_dir) if plugin_dir else []
+    dirs = discover_all_in(plugin_dirs) if plugin_dirs else []
     _run(db_url, plugin_migration_dirs=dirs)
 
 

steward/core/plugin_manager.py

@@ -31,6 +31,19 @@ def get_plugin_failures() -> dict[str, str]:
     return dict(_FAILED_PLUGINS)
 
 
+def resolve_plugin_path(plugin_dirs: list[Path], name: str) -> Path | None:
+    """Return the first plugin root that contains `name`, else None.
+
+    Roots are searched in order, so a bundled (first-party) plugin shadows an
+    external plugin of the same name. Pure function — no app/IO beyond exists().
+    """
+    for pd in plugin_dirs:
+        candidate = pd / name
+        if candidate.exists():
+            return candidate
+    return None
+
+
 def _import_plugin(name: str, plugin_path: Path):
     """Load a plugin module by file path, avoiding sys.modules stdlib collisions.
 
@@ -80,22 +93,24 @@ def load_plugins(app: "Quart") -> None:
     """
     import steward
 
-    plugin_dir = Path(app.config["PLUGIN_DIR"])
+    plugin_dirs = [Path(d) for d in app.config["PLUGIN_DIRS"]]
     plugins_cfg: dict = app.config["PLUGINS"]
 
-    # Ensure plugin_dir is on sys.path so plugins are importable by name
-    plugin_dir_str = str(plugin_dir.resolve())
-    if plugin_dir_str not in sys.path:
-        sys.path.insert(0, plugin_dir_str)
+    # Ensure every plugin root is on sys.path so plugins are importable by name
+    for pd in plugin_dirs:
+        pd_str = str(pd.resolve())
+        if pd_str not in sys.path:
+            sys.path.insert(0, pd_str)
 
     for name, cfg in list(plugins_cfg.items()):
         if not cfg.get("enabled", False):
             continue
 
-        plugin_path = plugin_dir / name
-        if not plugin_path.exists():
-            _FAILED_PLUGINS[name] = f"Plugin directory not found: {plugin_path}"
-            logger.error("Plugin %r: directory %s not found, skipping", name, plugin_path)
+        plugin_path = resolve_plugin_path(plugin_dirs, name)
+        if plugin_path is None:
+            roots = ", ".join(str(d) for d in plugin_dirs)
+            _FAILED_PLUGINS[name] = f"Plugin directory not found in: {roots}"
+            logger.error("Plugin %r: not found in any plugin root (%s), skipping", name, roots)
             continue
 
         # Load and validate plugin.yaml
@@ -224,10 +239,12 @@ async def download_and_install_plugin(
     """
     import httpx
 
-    plugin_dir = Path(app.config["PLUGIN_DIR"]).resolve()
+    # Downloads always land in the external (writable, persistent) install dir,
+    # never the read-only bundled dir.
+    plugin_dir = Path(app.config["PLUGIN_INSTALL_DIR"]).resolve()
 
-    # Ensure plugin_dir is on sys.path (may not be set yet if no plugins were
-    # enabled at startup)
+    # Ensure the install dir is on sys.path (may not be set yet if no plugins
+    # were enabled at startup)
     plugin_dir_str = str(plugin_dir)
     if plugin_dir_str not in sys.path:
         sys.path.insert(0, plugin_dir_str)
@@ -304,9 +321,11 @@ async def download_and_install_plugin(
         if mdir.exists():
             from steward.core.migration_runner import (
                 run_plugin_migrations,
-                discover_all_plugin_migration_dirs,
+                discover_all_in,
             )
-            all_dirs = discover_all_plugin_migration_dirs(plugin_dir)
+            # Span every plugin root so Alembic can resolve bundled-plugin
+            # revisions already stamped in alembic_version.
+            all_dirs = discover_all_in([Path(d).resolve() for d in app.config["PLUGIN_DIRS"]])
             run_plugin_migrations(app.config["DATABASE_URL"], all_dirs)
     except Exception:
         logger.exception("Plugin %r: migration failed after install", name)
@@ -330,11 +349,11 @@ def hot_reload_plugin(app: "Quart", name: str) -> tuple[bool, str]:
     if name in _LOADED_PLUGINS:
         return False, "Plugin already loaded — restart required to apply updates"
 
-    plugin_dir = Path(app.config["PLUGIN_DIR"]).resolve()
-    plugin_path = plugin_dir / name
+    plugin_dirs = [Path(d).resolve() for d in app.config["PLUGIN_DIRS"]]
+    plugin_path = resolve_plugin_path(plugin_dirs, name)
 
-    if not plugin_path.exists():
-        return False, f"Plugin directory {plugin_path} not found"
+    if plugin_path is None:
+        return False, f"Plugin {name!r} not found in any plugin root"
 
     yaml_path = plugin_path / "plugin.yaml"
     if not yaml_path.exists():
@@ -383,9 +402,9 @@ def hot_reload_plugin(app: "Quart", name: str) -> tuple[bool, str]:
         try:
             from steward.core.migration_runner import (
                 run_plugin_migrations,
-                discover_all_plugin_migration_dirs,
+                discover_all_in,
             )
-            all_dirs = discover_all_plugin_migration_dirs(plugin_dir)
+            all_dirs = discover_all_in(plugin_dirs)
             run_plugin_migrations(app.config["DATABASE_URL"], all_dirs)
         except Exception:
             logger.exception("Plugin %r: migration failed during hot-reload", name)

steward/settings/routes.py

@@ -19,22 +19,32 @@ logger = logging.getLogger(__name__)
 
 
 def _discover_plugins() -> list[dict]:
-    """Scan PLUGIN_DIR for plugin.yaml files."""
+    """Scan every plugin root for plugin.yaml files.
+
+    Roots are scanned in order (bundled first), and a plugin name found in an
+    earlier root shadows the same name in a later one — matching load_plugins.
+    """
     import yaml
-    plugin_dir = Path(current_app.config.get("PLUGIN_DIR", "plugins")).resolve()
-    plugins = []
-    if not plugin_dir.exists():
-        return plugins
-    for entry in sorted(plugin_dir.iterdir()):
-        yaml_path = entry / "plugin.yaml"
-        if entry.is_dir() and yaml_path.exists():
-            try:
-                with yaml_path.open() as f:
-                    meta = yaml.safe_load(f) or {}
-                meta["_dir"] = entry.name
-                plugins.append(meta)
-            except Exception:
-                logger.warning("Could not read %s", yaml_path)
+    plugin_dirs = [
+        Path(d).resolve()
+        for d in current_app.config.get("PLUGIN_DIRS", ["plugins"])
+    ]
+    plugins: list[dict] = []
+    seen: set[str] = set()
+    for plugin_dir in plugin_dirs:
+        if not plugin_dir.exists():
+            continue
+        for entry in sorted(plugin_dir.iterdir()):
+            yaml_path = entry / "plugin.yaml"
+            if entry.is_dir() and yaml_path.exists() and entry.name not in seen:
+                try:
+                    with yaml_path.open() as f:
+                        meta = yaml.safe_load(f) or {}
+                    meta["_dir"] = entry.name
+                    plugins.append(meta)
+                    seen.add(entry.name)
+                except Exception:
+                    logger.warning("Could not read %s", yaml_path)
     return plugins