bvandeusenandClaude Opus 5.5
de3958a362
The operator's ruling (Background Jobs, 2026-10-08): a Steward page reads stored records; the sources are contacted by scheduled collection. An audit of every GET handler, HTMX fragment, widget poll and MCP read tool found one page load that reached out: Settings -> Plugins' catalog fragment fetched each plugin repository (5 min cache, 10 s timeout each, retried on every load when unreachable). plugin_catalog_refresh ticks every minute and fetches a repository whose copy is missing or older than an hour, waiting 5 min after a failure; saving repositories clears the cache, so the next tick fetches. The fragment only reads the cache and says which repositories are not fetched yet. Refresh is now a POST that fetches at once, as does install. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
134 lines
5.0 KiB
Python
134 lines
5.0 KiB
Python
# steward/core/plugin_index.py
|
|
"""Remote plugin catalog: fetch, parse, and cache the plugin index.yaml."""
|
|
from __future__ import annotations
|
|
import logging
|
|
import time
|
|
from typing import Any
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# The catalog is fetched by the scheduled plugin_catalog_refresh task, never by
|
|
# a page load (Background Jobs ruling, 2026-10-08): a repository's copy is
|
|
# fetched when it has none or it is older than CATALOG_MAX_AGE, and a repository
|
|
# that failed is tried again after RETRY_AFTER. Refresh on Settings → Plugins and
|
|
# installing a plugin fetch at once, because the operator asked.
|
|
CATALOG_MAX_AGE = 3600 # seconds
|
|
RETRY_AFTER = 300 # seconds
|
|
# url → (raw index, fetched_at monotonic); url → failed_at monotonic
|
|
_url_cache: dict[str, tuple[dict, float]] = {}
|
|
_failed_at: dict[str, float] = {}
|
|
|
|
|
|
class CatalogPlugin:
|
|
"""One entry from the remote plugin index."""
|
|
|
|
__slots__ = (
|
|
"name", "version", "description", "author",
|
|
"min_app_version", "download_url", "checksum_sha256",
|
|
"repository_url", "homepage", "license", "tags",
|
|
)
|
|
|
|
def __init__(self, data: dict[str, Any]) -> None:
|
|
self.name: str = data.get("name", "")
|
|
self.version: str = data.get("version", "0.0.0")
|
|
self.description: str = data.get("description", "")
|
|
self.author: str = data.get("author", "")
|
|
self.min_app_version: str | None = data.get("min_app_version")
|
|
self.download_url: str = data.get("download_url", "")
|
|
self.checksum_sha256: str = data.get("checksum_sha256", "")
|
|
self.repository_url: str = data.get("repository_url", "")
|
|
self.homepage: str = data.get("homepage", "")
|
|
self.license: str = data.get("license", "")
|
|
self.tags: list[str] = data.get("tags", [])
|
|
|
|
|
|
def plugin_repos(settings: dict) -> list[dict]:
|
|
"""The configured plugin repositories, [{"name", "url"}], from settings.
|
|
|
|
Migrates the legacy plugins.index_url single-URL field to the list format
|
|
on first access if plugins.repositories is not yet set.
|
|
"""
|
|
repos = settings.get("plugins.repositories")
|
|
if repos and isinstance(repos, list):
|
|
return repos
|
|
legacy = settings.get("plugins.index_url", "").strip()
|
|
if legacy:
|
|
return [{"name": "Default", "url": legacy}]
|
|
return []
|
|
|
|
|
|
def repo_urls(settings: dict) -> list[str]:
|
|
return [r["url"].strip() for r in plugin_repos(settings) if (r.get("url") or "").strip()]
|
|
|
|
|
|
def _plugins(raw: dict) -> list[CatalogPlugin]:
|
|
return [CatalogPlugin(p) for p in raw.get("plugins", [])]
|
|
|
|
|
|
async def _fetch_one(url: str) -> bool:
|
|
"""Fetch one index URL into the cache. True when it was fetched."""
|
|
import httpx
|
|
import yaml
|
|
|
|
try:
|
|
async with httpx.AsyncClient(timeout=10) as client:
|
|
resp = await client.get(url)
|
|
resp.raise_for_status()
|
|
raw = yaml.safe_load(resp.text) or {}
|
|
except Exception:
|
|
logger.exception("Failed to fetch plugin catalog from %s", url)
|
|
_failed_at[url] = time.monotonic()
|
|
return False
|
|
_url_cache[url] = (raw, time.monotonic())
|
|
_failed_at.pop(url, None)
|
|
logger.info("Plugin catalog fetched from %s: %d plugin(s)", url, len(raw.get("plugins", [])))
|
|
return True
|
|
|
|
|
|
def _merge(urls: list[str]) -> list[CatalogPlugin]:
|
|
"""Plugins from the cached copies, deduplicated by name: the first
|
|
repository in the list wins."""
|
|
seen: dict[str, CatalogPlugin] = {}
|
|
for url in urls:
|
|
if url in _url_cache:
|
|
for plugin in _plugins(_url_cache[url][0]):
|
|
seen.setdefault(plugin.name, plugin)
|
|
return list(seen.values())
|
|
|
|
|
|
def cached_catalog(urls: list[str]) -> tuple[list[CatalogPlugin], list[str]]:
|
|
"""(the merged catalog from what is cached, the URLs with no copy yet).
|
|
Reads the cache only: this is what a page shows."""
|
|
return _merge(urls), [u for u in urls if u not in _url_cache]
|
|
|
|
|
|
async def fetch_catalog(urls: list[str]) -> list[CatalogPlugin]:
|
|
"""Fetch every URL now, then the merged catalog (a failed URL keeps its
|
|
last copy). For an operator's Refresh or install, not for a page load."""
|
|
for url in urls:
|
|
await _fetch_one(url)
|
|
return _merge(urls)
|
|
|
|
|
|
async def refresh_due(urls: list[str], now: float | None = None) -> int:
|
|
"""The scheduled task's tick: fetch each URL with no copy, or one older
|
|
than CATALOG_MAX_AGE, unless it failed within RETRY_AFTER. Returns how
|
|
many were fetched."""
|
|
now = time.monotonic() if now is None else now
|
|
fetched = 0
|
|
for url in urls:
|
|
cached = _url_cache.get(url)
|
|
if cached and now - cached[1] < CATALOG_MAX_AGE:
|
|
continue
|
|
if url in _failed_at and now - _failed_at[url] < RETRY_AFTER:
|
|
continue
|
|
fetched += await _fetch_one(url)
|
|
return fetched
|
|
|
|
|
|
def clear_catalog_cache() -> None:
|
|
"""Forget every cached copy and failure, so the next tick of the scheduled
|
|
task fetches the repositories again (after they are edited)."""
|
|
_url_cache.clear()
|
|
_failed_at.clear()
|