bvandeusen/FabledSteward
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor(plugins): no plugin catalog; plugins ship in the image or are mounted (#5374)
CI / lint (push) Successful in 3s
Details
CI / integration (push) Failing after 1m12s
Details
CI / unit (push) Successful in 1m25s
Details
CI / publish (push) Skipped
Details

Browse Source 
First-party plugins have shipped inside the image since a7a281c, and the
catalog's only repository (Steward-plugins) answers 404, so Steward polled a
dead URL every minute for nothing. The operator chose to remove the catalog
entirely (#5214 "Review: readable and fast on the live instance").

Removed:
- core/plugin_index.py and the plugin_catalog_refresh startup task;
- the plugins.index_url default, and migration 0035 deletes the stored
  plugins.index_url / plugins.repositories rows;
- the Settings -> Plugins repository editor, catalog, refresh, install and
  reload routes and their templates;
- the download/checksum/extract install path in plugin_manager, and the
  PLUGIN_INSTALL_DIR config it wrote to;
- plugins/index.yaml, docs/plugins/index.yaml.example and the guide's
  "Publishing to the Catalog" section (now "Shipping a Plugin").

Kept: enabling and configuring plugins, hot-reload on enable, restart, orphan
cleanup, and the external root (/data/plugins) for operator-mounted plugins.
tests/test_no_plugin_catalog.py guards the routes, default and task.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-10-08 13:15:55 -04:00
co-authored by Claude Opus 5.5
parent 501a345aed
commit f67678998c
17 changed files with 70 additions and 1258 deletions
Download Patch File Download Diff File Expand all files Collapse all files
steward/core/plugin_index.py
-135
View File
@@ -1,135 +0,0 @@
# steward/core/plugin_index.py
"""Remote plugin catalog: fetch, parse, and cache the plugin index.yaml."""
from __future__ import annotations
import logging
import time
from typing import Any
logger = logging.getLogger(__name__)
# The catalog is fetched by the scheduled plugin_catalog_refresh task, never by
# a page load (Background Jobs ruling, 2026-10-08): a repository's copy is
# fetched when it has none or it is older than CATALOG_MAX_AGE, and a repository
# that failed is tried again after RETRY_AFTER. Refresh on Settings → Plugins and
# installing a plugin fetch at once, because the operator asked.
CATALOG_MAX_AGE = 3600 # seconds
RETRY_AFTER = 300 # seconds
# url → (raw index, fetched_at monotonic); url → failed_at monotonic
_url_cache: dict[str, tuple[dict, float]] = {}
_failed_at: dict[str, float] = {}
class CatalogPlugin:
"""One entry from the remote plugin index."""
__slots__ = (
"name", "version", "description", "author",
"min_app_version", "download_url", "checksum_sha256",
"repository_url", "homepage", "license", "tags",
)
def __init__(self, data: dict[str, Any]) -> None:
self.name: str = data.get("name", "")
self.version: str = data.get("version", "0.0.0")
self.description: str = data.get("description", "")
self.author: str = data.get("author", "")
self.min_app_version: str | None = data.get("min_app_version")
self.download_url: str = data.get("download_url", "")
self.checksum_sha256: str = data.get("checksum_sha256", "")
self.repository_url: str = data.get("repository_url", "")
self.homepage: str = data.get("homepage", "")
self.license: str = data.get("license", "")
self.tags: list[str] = data.get("tags", [])
def plugin_repos(settings: dict) -> list[dict]:
"""The configured plugin repositories, [{"name", "url"}], from settings.
Migrates the legacy plugins.index_url single-URL field to the list format
on first access if plugins.repositories is not yet set.
"""
repos = settings.get("plugins.repositories")
if repos and isinstance(repos, list):
return repos
legacy = settings.get("plugins.index_url", "").strip()
if legacy:
return [{"name": "Default", "url": legacy}]
return []
def repo_urls(settings: dict) -> list[str]:
return [r["url"].strip() for r in plugin_repos(settings) if (r.get("url") or "").strip()]
def _plugins(raw: dict) -> list[CatalogPlugin]:
return [CatalogPlugin(p) for p in raw.get("plugins", [])]
async def _fetch_one(url: str) -> bool:
"""Fetch one index URL into the cache. True when it was fetched."""
import httpx
import yaml
try:
async with httpx.AsyncClient(timeout=10) as client:
resp = await client.get(url)
resp.raise_for_status()
raw = yaml.safe_load(resp.text) or {}
except Exception as exc:
# The tick retries every RETRY_AFTER seconds, so a repository that
# answers 404 or is unreachable is one line each time, not a traceback.
logger.warning("Plugin catalog not fetched from %s: %s", url, exc)
_failed_at[url] = time.monotonic()
return False
_url_cache[url] = (raw, time.monotonic())
_failed_at.pop(url, None)
logger.info("Plugin catalog fetched from %s: %d plugin(s)", url, len(raw.get("plugins", [])))
return True
def _merge(urls: list[str]) -> list[CatalogPlugin]:
"""Plugins from the cached copies, deduplicated by name: the first
repository in the list wins."""
seen: dict[str, CatalogPlugin] = {}
for url in urls:
if url in _url_cache:
for plugin in _plugins(_url_cache[url][0]):
seen.setdefault(plugin.name, plugin)
return list(seen.values())
def cached_catalog(urls: list[str]) -> tuple[list[CatalogPlugin], list[str]]:
"""(the merged catalog from what is cached, the URLs with no copy yet).
Reads the cache only: this is what a page shows."""
return _merge(urls), [u for u in urls if u not in _url_cache]
async def fetch_catalog(urls: list[str]) -> list[CatalogPlugin]:
"""Fetch every URL now, then the merged catalog (a failed URL keeps its
last copy). For an operator's Refresh or install, not for a page load."""
for url in urls:
await _fetch_one(url)
return _merge(urls)
async def refresh_due(urls: list[str], now: float | None = None) -> int:
"""The scheduled task's tick: fetch each URL with no copy, or one older
than CATALOG_MAX_AGE, unless it failed within RETRY_AFTER. Returns how
many were fetched."""
now = time.monotonic() if now is None else now
fetched = 0
for url in urls:
cached = _url_cache.get(url)
if cached and now - cached[1] < CATALOG_MAX_AGE:
continue
if url in _failed_at and now - _failed_at[url] < RETRY_AFTER:
continue
fetched += await _fetch_one(url)
return fetched
def clear_catalog_cache() -> None:
"""Forget every cached copy and failure, so the next tick of the scheduled
task fetches the repositories again (after they are edited)."""
_url_cache.clear()
_failed_at.clear()
steward/core/plugin_manager.py
+1 -180
View File
@@ -1,11 +1,8 @@
# steward/core/plugin_manager.py
from __future__ import annotations
import hashlib
import logging
import os
import shutil
import sys
import zipfile
from pathlib import Path
from typing import TYPE_CHECKING
@@ -290,184 +287,8 @@ def load_plugins(app: "Quart") -> None:
logger.info("Plugin %r loaded (v%s)", name, meta.get("version", "?"))
# Bounds on what an install may bring in: the download itself, and what it
# unpacks to (a small zip can expand to fill the disk).
MAX_PLUGIN_DOWNLOAD_BYTES = 50 * 1024 * 1024
MAX_PLUGIN_UNPACKED_BYTES = 200 * 1024 * 1024
class PluginInstallError(Exception):
"""An install refused or failed; the message is shown to the admin."""
def check_plugin_name(name: str) -> None:
"""A plugin is imported by its folder name, so the name must be a plain
Python identifier — which also keeps it from naming a path."""
if not (name.isascii() and name.isidentifier()):
raise PluginInstallError(f"{name!r} is not a valid plugin name")
def verify_checksum(content: bytes, expected_sha256: str) -> None:
"""Refuse a download whose sha256 is missing from the catalog or differs."""
if not expected_sha256:
raise PluginInstallError(
"The catalog lists no checksum for this plugin, so Steward cannot "
"tell it is the file the catalog meant. Not installed.")
actual = hashlib.sha256(content).hexdigest()
if actual != expected_sha256.strip().lower():
raise PluginInstallError(f"Checksum mismatch: expected {expected_sha256}, got {actual}")
def extract_plugin(content: bytes, name: str, dest: Path) -> None:
"""Unpack a plugin zip into `dest`, which must not exist yet.
One top-level folder whose name starts with the plugin name is stripped:
name/ (clean plugin zip)
name-v1.0.0/ (release tag archive)
otherwise the zip is unpacked as it stands. Every member must land inside
`dest` — an absolute path or a `..` that climbs out refuses the whole zip —
and the unpacked total is capped.
"""
import io
root = dest.resolve()
with zipfile.ZipFile(io.BytesIO(content)) as zf:
infos = zf.infolist()
if sum(i.file_size for i in infos) > MAX_PLUGIN_UNPACKED_BYTES:
raise PluginInstallError(
f"Unpacks to more than {MAX_PLUGIN_UNPACKED_BYTES // (1024 * 1024)} MB")
top_level = {i.filename.split("/")[0] for i in infos}
prefix = ""
if len(top_level) == 1:
candidate = next(iter(top_level))
if candidate.lower().startswith(name.lower()) and any(
i.filename.startswith(candidate + "/") for i in infos):
prefix = candidate + "/"
planned: list[tuple[zipfile.ZipInfo, Path]] = []
for info in infos:
if prefix and not info.filename.startswith(prefix):
continue
rel = info.filename[len(prefix):]
if not rel:
continue
target = (root / rel).resolve()
if not target.is_relative_to(root) or target == root:
raise PluginInstallError(f"Refused: {info.filename!r} points outside the plugin folder")
planned.append((info, target))
root.mkdir(parents=True)
for info, target in planned:
if info.is_dir():
target.mkdir(parents=True, exist_ok=True)
continue
target.parent.mkdir(parents=True, exist_ok=True)
with zf.open(info) as src, open(target, "wb") as out:
shutil.copyfileobj(src, out)
async def _download(url: str) -> bytes:
import httpx
chunks: list[bytes] = []
size = 0
async with httpx.AsyncClient(timeout=60, follow_redirects=True) as client:
async with client.stream("GET", url) as resp:
resp.raise_for_status()
async for chunk in resp.aiter_bytes():
size += len(chunk)
if size > MAX_PLUGIN_DOWNLOAD_BYTES:
raise PluginInstallError(
f"Download is larger than {MAX_PLUGIN_DOWNLOAD_BYTES // (1024 * 1024)} MB")
chunks.append(chunk)
return b"".join(chunks)
async def download_and_install_plugin(
app: "Quart",
name: str,
download_url: str,
expected_sha256: str,
) -> tuple[bool, str]:
"""Download a plugin zip named by the catalog, verify its checksum, and
unpack it into the install dir, replacing any earlier copy whole.
The URL and checksum must come from the catalog entry, never from the
request. Returns (success, message). Does NOT hot-reload — call
hot_reload_plugin() after this to activate a new plugin without a restart.
"""
# Downloads always land in the external (writable, persistent) install dir,
# never the read-only bundled dir.
plugin_dir = Path(app.config["PLUGIN_INSTALL_DIR"]).resolve()
# Ensure the install dir is on sys.path (may not be set yet if no plugins
# were enabled at startup)
plugin_dir_str = str(plugin_dir)
if plugin_dir_str not in sys.path:
sys.path.insert(0, plugin_dir_str)
try:
check_plugin_name(name)
if not download_url:
raise PluginInstallError("The catalog lists no download URL for this plugin")
logger.info("Downloading plugin %r from %s", name, download_url)
try:
content = await _download(download_url)
except PluginInstallError:
raise
except Exception as exc:
raise PluginInstallError(f"Download failed: {exc}") from exc
verify_checksum(content, expected_sha256)
# Unpack beside the live copy, then swap: a failed or refused zip
# leaves the installed plugin untouched, and an update drops files the
# new version no longer ships.
plugin_dir.mkdir(parents=True, exist_ok=True)
dest = plugin_dir / name
staging = plugin_dir / f".{name}.installing"
old = plugin_dir / f".{name}.replaced"
for leftover in (staging, old):
shutil.rmtree(leftover, ignore_errors=True)
try:
extract_plugin(content, name, staging)
except PluginInstallError:
raise
except Exception as exc:
raise PluginInstallError(f"Extraction failed: {exc}") from exc
try:
if dest.exists():
dest.rename(old)
staging.rename(dest)
except OSError as exc:
if old.exists() and not dest.exists():
old.rename(dest) # put the previous version back
raise PluginInstallError(f"Could not move the plugin into place: {exc}") from exc
shutil.rmtree(old, ignore_errors=True)
except PluginInstallError as exc:
shutil.rmtree(plugin_dir / f".{name}.installing", ignore_errors=True)
logger.error("Plugin %r: %s", name, exc)
return False, str(exc)
# Run migrations for the newly installed plugin.
# Include all plugin migration dirs so Alembic can resolve any previously-stamped
# revisions from other plugins already in alembic_version.
try:
mdir = plugin_dir / name / "migrations"
if mdir.exists():
from steward.core.migration_runner import (
run_plugin_migrations,
discover_all_in,
)
# Span every plugin root so Alembic can resolve bundled-plugin
# revisions already stamped in alembic_version.
all_dirs = discover_all_in([Path(d).resolve() for d in app.config["PLUGIN_DIRS"]])
run_plugin_migrations(app.config["DATABASE_URL"], all_dirs)
except Exception:
logger.exception("Plugin %r: migration failed after install", name)
return False, "Plugin extracted but migrations failed — check logs"
logger.info("Plugin %r installed successfully", name)
return True, "Installed"
def hot_reload_plugin(app: "Quart", name: str) -> tuple[bool, str]:
"""Activate a newly installed plugin without restarting the app.
"""Activate a plugin that was just enabled without restarting the app.
This works for plugins that were NOT previously loaded. For plugins that
are already registered (blueprint already mounted), a full restart is
steward/core/settings.py
-1
View File
@@ -118,7 +118,6 @@ DEFAULTS: dict[str, Any] = {
# the agent's ~30s cadence, then roll up to hourly averages kept much longer.
"metrics.retention.raw_days": 7,
"metrics.retention.rollup_days": 90,
"plugins.index_url": "https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml",
# Default-enabled plugins. These are the generic, non-vendor-specific
# bundled plugins (protocols/standards, not a single product) — useful on
# almost any install, so a fresh deployment comes up monitoring rather than