FabledScribe/src/fabledassistant/services/article_fetcher.py

"""Generic article-text fetcher.

Fetches a URL and extracts its main body via trafilatura. The single source
of truth for article-content extraction across the codebase — used by the
``read_article`` LLM tool and the ``lookup`` tool's web-result enrichment.

Trafilatura/lxml is NOT safe to call concurrently — running it via
``run_in_executor`` from multiple coroutines can trip a libxml2 double-free.
Callers must serialize their fetches (await one before starting the next).
"""
from __future__ import annotations

import asyncio
import logging

import httpx

logger = logging.getLogger(__name__)


async def fetch_article_text(url: str) -> str | None:
    """Return the clean article body for *url*, or None on failure.

    Returns None when the HTTP fetch fails or trafilatura yields nothing
    useful. Callers should treat None as "no article content available."
    """
    try:
        async with httpx.AsyncClient(timeout=15.0, follow_redirects=True, headers={
            "User-Agent": "Mozilla/5.0 (compatible; FabledScribe/1.0; +https://fabledsword.com)",
        }) as client:
            resp = await client.get(url)
            resp.raise_for_status()
            raw_html = resp.text
    except Exception:
        logger.debug("Failed to fetch article URL %s", url)
        return None

    loop = asyncio.get_event_loop()
    try:
        import trafilatura
        text = await loop.run_in_executor(
            None,
            lambda: trafilatura.extract(
                raw_html,
                include_comments=False,
                include_tables=True,
                favor_recall=True,
            ),
        )
        return text or None
    except Exception:
        logger.debug("trafilatura extraction failed for %s", url, exc_info=True)
        return None