fix: serialize article fetches in lookup to avoid lxml double-free

trafilatura.extract dispatched via run_in_executor isn't safe to run
concurrently — two parallel calls can crash the process with a
libxml2-level double free. The top-level Wikipedia+SearXNG gather is
fine; only the inner per-article extraction needs to stay sequential,
matching the pre-parallelization behavior.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-04-17 23:04:05 -04:00
parent 9a252c8dde
commit 61e62a6904
+13 -10
View File
@@ -68,20 +68,23 @@ async def lookup_tool(*, user_id, arguments, **_ctx):
web_payload: list[dict] = [] web_payload: list[dict] = []
if search_results: if search_results:
# Sequential fetches: trafilatura/lxml is not safe to run concurrently
# via run_in_executor — parallel calls can trip a libxml2 double-free.
from fabledassistant.services.rss import _fetch_full_article from fabledassistant.services.rss import _fetch_full_article
top = [r for r in search_results[:2] if r.get("url")] for r in search_results[:2]:
contents = await asyncio.gather( url = r.get("url", "")
*(_fetch_full_article(r["url"]) for r in top), if not url:
return_exceptions=True, continue
) try:
for r, content in zip(top, contents, strict=False): content = await _fetch_full_article(url)
content_text = content if isinstance(content, str) else "" except Exception:
content = None
web_payload.append({ web_payload.append({
"url": r["url"], "url": url,
"title": r.get("title", r["url"]), "title": r.get("title", url),
"snippet": r.get("snippet", ""), "snippet": r.get("snippet", ""),
"content": content_text[:4000], "content": (content or "")[:4000],
}) })
if not wiki_payload and not web_payload: if not wiki_payload and not web_payload: