fix: serialize article fetches in lookup to avoid lxml double-free
trafilatura.extract dispatched via run_in_executor isn't safe to run concurrently — two parallel calls can crash the process with a libxml2-level double free. The top-level Wikipedia+SearXNG gather is fine; only the inner per-article extraction needs to stay sequential, matching the pre-parallelization behavior. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -68,20 +68,23 @@ async def lookup_tool(*, user_id, arguments, **_ctx):
|
|||||||
|
|
||||||
web_payload: list[dict] = []
|
web_payload: list[dict] = []
|
||||||
if search_results:
|
if search_results:
|
||||||
|
# Sequential fetches: trafilatura/lxml is not safe to run concurrently
|
||||||
|
# via run_in_executor — parallel calls can trip a libxml2 double-free.
|
||||||
from fabledassistant.services.rss import _fetch_full_article
|
from fabledassistant.services.rss import _fetch_full_article
|
||||||
|
|
||||||
top = [r for r in search_results[:2] if r.get("url")]
|
for r in search_results[:2]:
|
||||||
contents = await asyncio.gather(
|
url = r.get("url", "")
|
||||||
*(_fetch_full_article(r["url"]) for r in top),
|
if not url:
|
||||||
return_exceptions=True,
|
continue
|
||||||
)
|
try:
|
||||||
for r, content in zip(top, contents, strict=False):
|
content = await _fetch_full_article(url)
|
||||||
content_text = content if isinstance(content, str) else ""
|
except Exception:
|
||||||
|
content = None
|
||||||
web_payload.append({
|
web_payload.append({
|
||||||
"url": r["url"],
|
"url": url,
|
||||||
"title": r.get("title", r["url"]),
|
"title": r.get("title", url),
|
||||||
"snippet": r.get("snippet", ""),
|
"snippet": r.get("snippet", ""),
|
||||||
"content": content_text[:4000],
|
"content": (content or "")[:4000],
|
||||||
})
|
})
|
||||||
|
|
||||||
if not wiki_payload and not web_payload:
|
if not wiki_payload and not web_payload:
|
||||||
|
|||||||
Reference in New Issue
Block a user