fix: serialize article fetches in lookup to avoid lxml double-free

trafilatura.extract dispatched via run_in_executor isn't safe to run
concurrently — two parallel calls can crash the process with a
libxml2-level double free. The top-level Wikipedia+SearXNG gather is
fine; only the inner per-article extraction needs to stay sequential,
matching the pre-parallelization behavior.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-04-17 23:04:05 -04:00
parent 9a252c8dde
commit 61e62a6904
+13 -10
View File
@@ -68,20 +68,23 @@ async def lookup_tool(*, user_id, arguments, **_ctx):
web_payload: list[dict] = []
if search_results:
# Sequential fetches: trafilatura/lxml is not safe to run concurrently
# via run_in_executor — parallel calls can trip a libxml2 double-free.
from fabledassistant.services.rss import _fetch_full_article
top = [r for r in search_results[:2] if r.get("url")]
contents = await asyncio.gather(
*(_fetch_full_article(r["url"]) for r in top),
return_exceptions=True,
)
for r, content in zip(top, contents, strict=False):
content_text = content if isinstance(content, str) else ""
for r in search_results[:2]:
url = r.get("url", "")
if not url:
continue
try:
content = await _fetch_full_article(url)
except Exception:
content = None
web_payload.append({
"url": r["url"],
"title": r.get("title", r["url"]),
"url": url,
"title": r.get("title", url),
"snippet": r.get("snippet", ""),
"content": content_text[:4000],
"content": (content or "")[:4000],
})
if not wiki_payload and not web_payload: