fix: serialize article fetches in lookup to avoid lxml double-free
trafilatura.extract dispatched via run_in_executor isn't safe to run concurrently — two parallel calls can crash the process with a libxml2-level double free. The top-level Wikipedia+SearXNG gather is fine; only the inner per-article extraction needs to stay sequential, matching the pre-parallelization behavior. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -68,20 +68,23 @@ async def lookup_tool(*, user_id, arguments, **_ctx):
|
||||
|
||||
web_payload: list[dict] = []
|
||||
if search_results:
|
||||
# Sequential fetches: trafilatura/lxml is not safe to run concurrently
|
||||
# via run_in_executor — parallel calls can trip a libxml2 double-free.
|
||||
from fabledassistant.services.rss import _fetch_full_article
|
||||
|
||||
top = [r for r in search_results[:2] if r.get("url")]
|
||||
contents = await asyncio.gather(
|
||||
*(_fetch_full_article(r["url"]) for r in top),
|
||||
return_exceptions=True,
|
||||
)
|
||||
for r, content in zip(top, contents, strict=False):
|
||||
content_text = content if isinstance(content, str) else ""
|
||||
for r in search_results[:2]:
|
||||
url = r.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
try:
|
||||
content = await _fetch_full_article(url)
|
||||
except Exception:
|
||||
content = None
|
||||
web_payload.append({
|
||||
"url": r["url"],
|
||||
"title": r.get("title", r["url"]),
|
||||
"url": url,
|
||||
"title": r.get("title", url),
|
||||
"snippet": r.get("snippet", ""),
|
||||
"content": content_text[:4000],
|
||||
"content": (content or "")[:4000],
|
||||
})
|
||||
|
||||
if not wiki_payload and not web_payload:
|
||||
|
||||
Reference in New Issue
Block a user