fix(settings): the plugin catalog is fetched by a scheduled task, never by a page load

The operator's ruling (Background Jobs, 2026-10-08): a Steward page reads
stored records; the sources are contacted by scheduled collection. An audit
of every GET handler, HTMX fragment, widget poll and MCP read tool found one
page load that reached out: Settings -> Plugins' catalog fragment fetched
each plugin repository (5 min cache, 10 s timeout each, retried on every
load when unreachable).

plugin_catalog_refresh ticks every minute and fetches a repository whose
copy is missing or older than an hour, waiting 5 min after a failure;
saving repositories clears the cache, so the next tick fetches. The
fragment only reads the cache and says which repositories are not fetched
yet. Refresh is now a POST that fetches at once, as does install.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/plugin_index.py

@@ -7,9 +7,16 @@ from typing import Any
 
 logger = logging.getLogger(__name__)
 
-_CACHE_TTL = 300  # seconds
-# Per-URL cache: url → (raw_data, fetched_at monotonic)
+# The catalog is fetched by the scheduled plugin_catalog_refresh task, never by
+# a page load (Background Jobs ruling, 2026-10-08): a repository's copy is
+# fetched when it has none or it is older than CATALOG_MAX_AGE, and a repository
+# that failed is tried again after RETRY_AFTER. Refresh on Settings → Plugins and
+# installing a plugin fetch at once, because the operator asked.
+CATALOG_MAX_AGE = 3600   # seconds
+RETRY_AFTER = 300        # seconds
+# url → (raw index, fetched_at monotonic); url → failed_at monotonic
 _url_cache: dict[str, tuple[dict, float]] = {}
+_failed_at: dict[str, float] = {}
 
 
 class CatalogPlugin:
@@ -35,59 +42,92 @@ class CatalogPlugin:
         self.tags: list[str] = data.get("tags", [])
 
 
-async def _fetch_one(url: str, force: bool = False) -> list[CatalogPlugin]:
-    """Fetch a single index URL, using per-URL cache."""
+def plugin_repos(settings: dict) -> list[dict]:
+    """The configured plugin repositories, [{"name", "url"}], from settings.
+
+    Migrates the legacy plugins.index_url single-URL field to the list format
+    on first access if plugins.repositories is not yet set.
+    """
+    repos = settings.get("plugins.repositories")
+    if repos and isinstance(repos, list):
+        return repos
+    legacy = settings.get("plugins.index_url", "").strip()
+    if legacy:
+        return [{"name": "Default", "url": legacy}]
+    return []
+
+
+def repo_urls(settings: dict) -> list[str]:
+    return [r["url"].strip() for r in plugin_repos(settings) if (r.get("url") or "").strip()]
+
+
+def _plugins(raw: dict) -> list[CatalogPlugin]:
+    return [CatalogPlugin(p) for p in raw.get("plugins", [])]
+
+
+async def _fetch_one(url: str) -> bool:
+    """Fetch one index URL into the cache. True when it was fetched."""
     import httpx
     import yaml
 
-    now = time.monotonic()
-    if not force and url in _url_cache:
-        raw, fetched_at = _url_cache[url]
-        if now - fetched_at < _CACHE_TTL:
-            return [CatalogPlugin(p) for p in raw.get("plugins", [])]
-
     try:
         async with httpx.AsyncClient(timeout=10) as client:
             resp = await client.get(url)
             resp.raise_for_status()
         raw = yaml.safe_load(resp.text) or {}
-        _url_cache[url] = (raw, now)
-        logger.info("Plugin catalog fetched from %s: %d plugin(s)", url, len(raw.get("plugins", [])))
-        return [CatalogPlugin(p) for p in raw.get("plugins", [])]
     except Exception:
         logger.exception("Failed to fetch plugin catalog from %s", url)
-        if url in _url_cache:
-            raw, _ = _url_cache[url]
-            logger.warning("Returning stale cached catalog for %s", url)
-            return [CatalogPlugin(p) for p in raw.get("plugins", [])]
-        return []
+        _failed_at[url] = time.monotonic()
+        return False
+    _url_cache[url] = (raw, time.monotonic())
+    _failed_at.pop(url, None)
+    logger.info("Plugin catalog fetched from %s: %d plugin(s)", url, len(raw.get("plugins", [])))
+    return True
 
 
-async def fetch_catalog(
-    index_urls: str | list[str],
-    force: bool = False,
-) -> list[CatalogPlugin]:
-    """Fetch and merge plugin catalogs from one or more index URLs.
-
-    Results are cached per-URL for _CACHE_TTL seconds. Plugins with the same
-    name across repos are deduplicated — the first occurrence (repo list order)
-    wins. Pass force=True to bypass all caches.
-    """
-    if isinstance(index_urls, str):
-        index_urls = [index_urls]
-
+def _merge(urls: list[str]) -> list[CatalogPlugin]:
+    """Plugins from the cached copies, deduplicated by name: the first
+    repository in the list wins."""
     seen: dict[str, CatalogPlugin] = {}
-    for url in index_urls:
-        url = url.strip()
-        if not url:
-            continue
-        for plugin in await _fetch_one(url, force=force):
-            if plugin.name not in seen:
-                seen[plugin.name] = plugin
-
+    for url in urls:
+        if url in _url_cache:
+            for plugin in _plugins(_url_cache[url][0]):
+                seen.setdefault(plugin.name, plugin)
     return list(seen.values())
 
 
+def cached_catalog(urls: list[str]) -> tuple[list[CatalogPlugin], list[str]]:
+    """(the merged catalog from what is cached, the URLs with no copy yet).
+    Reads the cache only: this is what a page shows."""
+    return _merge(urls), [u for u in urls if u not in _url_cache]
+
+
+async def fetch_catalog(urls: list[str]) -> list[CatalogPlugin]:
+    """Fetch every URL now, then the merged catalog (a failed URL keeps its
+    last copy). For an operator's Refresh or install, not for a page load."""
+    for url in urls:
+        await _fetch_one(url)
+    return _merge(urls)
+
+
+async def refresh_due(urls: list[str], now: float | None = None) -> int:
+    """The scheduled task's tick: fetch each URL with no copy, or one older
+    than CATALOG_MAX_AGE, unless it failed within RETRY_AFTER. Returns how
+    many were fetched."""
+    now = time.monotonic() if now is None else now
+    fetched = 0
+    for url in urls:
+        cached = _url_cache.get(url)
+        if cached and now - cached[1] < CATALOG_MAX_AGE:
+            continue
+        if url in _failed_at and now - _failed_at[url] < RETRY_AFTER:
+            continue
+        fetched += await _fetch_one(url)
+    return fetched
+
+
 def clear_catalog_cache() -> None:
-    """Invalidate all in-process catalog caches."""
+    """Forget every cached copy and failure, so the next tick of the scheduled
+    task fetches the repositories again (after they are edited)."""
     _url_cache.clear()
+    _failed_at.clear()