refactor(plugins): no plugin catalog; plugins ship in the image or are mounted (#5374)
First-party plugins have shipped inside the image since a7a281c, and the
catalog's only repository (Steward-plugins) answers 404, so Steward polled a
dead URL every minute for nothing. The operator chose to remove the catalog
entirely (#5214 "Review: readable and fast on the live instance").
Removed:
- core/plugin_index.py and the plugin_catalog_refresh startup task;
- the plugins.index_url default, and migration 0035 deletes the stored
plugins.index_url / plugins.repositories rows;
- the Settings -> Plugins repository editor, catalog, refresh, install and
reload routes and their templates;
- the download/checksum/extract install path in plugin_manager, and the
PLUGIN_INSTALL_DIR config it wrote to;
- plugins/index.yaml, docs/plugins/index.yaml.example and the guide's
"Publishing to the Catalog" section (now "Shipping a Plugin").
Kept: enabling and configuring plugins, hot-reload on enable, restart, orphan
cleanup, and the external root (/data/plugins) for operator-mounted plugins.
tests/test_no_plugin_catalog.py guards the routes, default and task.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -1,11 +1,8 @@
|
||||
# steward/core/plugin_manager.py
|
||||
from __future__ import annotations
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -290,184 +287,8 @@ def load_plugins(app: "Quart") -> None:
|
||||
logger.info("Plugin %r loaded (v%s)", name, meta.get("version", "?"))
|
||||
|
||||
|
||||
# Bounds on what an install may bring in: the download itself, and what it
|
||||
# unpacks to (a small zip can expand to fill the disk).
|
||||
MAX_PLUGIN_DOWNLOAD_BYTES = 50 * 1024 * 1024
|
||||
MAX_PLUGIN_UNPACKED_BYTES = 200 * 1024 * 1024
|
||||
|
||||
|
||||
class PluginInstallError(Exception):
|
||||
"""An install refused or failed; the message is shown to the admin."""
|
||||
|
||||
|
||||
def check_plugin_name(name: str) -> None:
|
||||
"""A plugin is imported by its folder name, so the name must be a plain
|
||||
Python identifier — which also keeps it from naming a path."""
|
||||
if not (name.isascii() and name.isidentifier()):
|
||||
raise PluginInstallError(f"{name!r} is not a valid plugin name")
|
||||
|
||||
|
||||
def verify_checksum(content: bytes, expected_sha256: str) -> None:
|
||||
"""Refuse a download whose sha256 is missing from the catalog or differs."""
|
||||
if not expected_sha256:
|
||||
raise PluginInstallError(
|
||||
"The catalog lists no checksum for this plugin, so Steward cannot "
|
||||
"tell it is the file the catalog meant. Not installed.")
|
||||
actual = hashlib.sha256(content).hexdigest()
|
||||
if actual != expected_sha256.strip().lower():
|
||||
raise PluginInstallError(f"Checksum mismatch: expected {expected_sha256}, got {actual}")
|
||||
|
||||
|
||||
def extract_plugin(content: bytes, name: str, dest: Path) -> None:
|
||||
"""Unpack a plugin zip into `dest`, which must not exist yet.
|
||||
|
||||
One top-level folder whose name starts with the plugin name is stripped:
|
||||
name/ (clean plugin zip)
|
||||
name-v1.0.0/ (release tag archive)
|
||||
otherwise the zip is unpacked as it stands. Every member must land inside
|
||||
`dest` — an absolute path or a `..` that climbs out refuses the whole zip —
|
||||
and the unpacked total is capped.
|
||||
"""
|
||||
import io
|
||||
root = dest.resolve()
|
||||
with zipfile.ZipFile(io.BytesIO(content)) as zf:
|
||||
infos = zf.infolist()
|
||||
if sum(i.file_size for i in infos) > MAX_PLUGIN_UNPACKED_BYTES:
|
||||
raise PluginInstallError(
|
||||
f"Unpacks to more than {MAX_PLUGIN_UNPACKED_BYTES // (1024 * 1024)} MB")
|
||||
top_level = {i.filename.split("/")[0] for i in infos}
|
||||
prefix = ""
|
||||
if len(top_level) == 1:
|
||||
candidate = next(iter(top_level))
|
||||
if candidate.lower().startswith(name.lower()) and any(
|
||||
i.filename.startswith(candidate + "/") for i in infos):
|
||||
prefix = candidate + "/"
|
||||
planned: list[tuple[zipfile.ZipInfo, Path]] = []
|
||||
for info in infos:
|
||||
if prefix and not info.filename.startswith(prefix):
|
||||
continue
|
||||
rel = info.filename[len(prefix):]
|
||||
if not rel:
|
||||
continue
|
||||
target = (root / rel).resolve()
|
||||
if not target.is_relative_to(root) or target == root:
|
||||
raise PluginInstallError(f"Refused: {info.filename!r} points outside the plugin folder")
|
||||
planned.append((info, target))
|
||||
root.mkdir(parents=True)
|
||||
for info, target in planned:
|
||||
if info.is_dir():
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
continue
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(info) as src, open(target, "wb") as out:
|
||||
shutil.copyfileobj(src, out)
|
||||
|
||||
|
||||
async def _download(url: str) -> bytes:
|
||||
import httpx
|
||||
chunks: list[bytes] = []
|
||||
size = 0
|
||||
async with httpx.AsyncClient(timeout=60, follow_redirects=True) as client:
|
||||
async with client.stream("GET", url) as resp:
|
||||
resp.raise_for_status()
|
||||
async for chunk in resp.aiter_bytes():
|
||||
size += len(chunk)
|
||||
if size > MAX_PLUGIN_DOWNLOAD_BYTES:
|
||||
raise PluginInstallError(
|
||||
f"Download is larger than {MAX_PLUGIN_DOWNLOAD_BYTES // (1024 * 1024)} MB")
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
async def download_and_install_plugin(
|
||||
app: "Quart",
|
||||
name: str,
|
||||
download_url: str,
|
||||
expected_sha256: str,
|
||||
) -> tuple[bool, str]:
|
||||
"""Download a plugin zip named by the catalog, verify its checksum, and
|
||||
unpack it into the install dir, replacing any earlier copy whole.
|
||||
|
||||
The URL and checksum must come from the catalog entry, never from the
|
||||
request. Returns (success, message). Does NOT hot-reload — call
|
||||
hot_reload_plugin() after this to activate a new plugin without a restart.
|
||||
"""
|
||||
# Downloads always land in the external (writable, persistent) install dir,
|
||||
# never the read-only bundled dir.
|
||||
plugin_dir = Path(app.config["PLUGIN_INSTALL_DIR"]).resolve()
|
||||
|
||||
# Ensure the install dir is on sys.path (may not be set yet if no plugins
|
||||
# were enabled at startup)
|
||||
plugin_dir_str = str(plugin_dir)
|
||||
if plugin_dir_str not in sys.path:
|
||||
sys.path.insert(0, plugin_dir_str)
|
||||
|
||||
try:
|
||||
check_plugin_name(name)
|
||||
if not download_url:
|
||||
raise PluginInstallError("The catalog lists no download URL for this plugin")
|
||||
logger.info("Downloading plugin %r from %s", name, download_url)
|
||||
try:
|
||||
content = await _download(download_url)
|
||||
except PluginInstallError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise PluginInstallError(f"Download failed: {exc}") from exc
|
||||
verify_checksum(content, expected_sha256)
|
||||
|
||||
# Unpack beside the live copy, then swap: a failed or refused zip
|
||||
# leaves the installed plugin untouched, and an update drops files the
|
||||
# new version no longer ships.
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest = plugin_dir / name
|
||||
staging = plugin_dir / f".{name}.installing"
|
||||
old = plugin_dir / f".{name}.replaced"
|
||||
for leftover in (staging, old):
|
||||
shutil.rmtree(leftover, ignore_errors=True)
|
||||
try:
|
||||
extract_plugin(content, name, staging)
|
||||
except PluginInstallError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise PluginInstallError(f"Extraction failed: {exc}") from exc
|
||||
try:
|
||||
if dest.exists():
|
||||
dest.rename(old)
|
||||
staging.rename(dest)
|
||||
except OSError as exc:
|
||||
if old.exists() and not dest.exists():
|
||||
old.rename(dest) # put the previous version back
|
||||
raise PluginInstallError(f"Could not move the plugin into place: {exc}") from exc
|
||||
shutil.rmtree(old, ignore_errors=True)
|
||||
except PluginInstallError as exc:
|
||||
shutil.rmtree(plugin_dir / f".{name}.installing", ignore_errors=True)
|
||||
logger.error("Plugin %r: %s", name, exc)
|
||||
return False, str(exc)
|
||||
|
||||
# Run migrations for the newly installed plugin.
|
||||
# Include all plugin migration dirs so Alembic can resolve any previously-stamped
|
||||
# revisions from other plugins already in alembic_version.
|
||||
try:
|
||||
mdir = plugin_dir / name / "migrations"
|
||||
if mdir.exists():
|
||||
from steward.core.migration_runner import (
|
||||
run_plugin_migrations,
|
||||
discover_all_in,
|
||||
)
|
||||
# Span every plugin root so Alembic can resolve bundled-plugin
|
||||
# revisions already stamped in alembic_version.
|
||||
all_dirs = discover_all_in([Path(d).resolve() for d in app.config["PLUGIN_DIRS"]])
|
||||
run_plugin_migrations(app.config["DATABASE_URL"], all_dirs)
|
||||
except Exception:
|
||||
logger.exception("Plugin %r: migration failed after install", name)
|
||||
return False, "Plugin extracted but migrations failed — check logs"
|
||||
|
||||
logger.info("Plugin %r installed successfully", name)
|
||||
return True, "Installed"
|
||||
|
||||
|
||||
def hot_reload_plugin(app: "Quart", name: str) -> tuple[bool, str]:
|
||||
"""Activate a newly installed plugin without restarting the app.
|
||||
"""Activate a plugin that was just enabled without restarting the app.
|
||||
|
||||
This works for plugins that were NOT previously loaded. For plugins that
|
||||
are already registered (blueprint already mounted), a full restart is
|
||||
|
||||
Reference in New Issue
Block a user