feat(rss): remove article content character cap
Trafilatura extracts only article body text (typically 2K–15K chars), so storing the full content is safe without an artificial ceiling. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -14,8 +14,6 @@ from fabledassistant.models.rss_feed import RssFeed, RssItem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Safety cap on stored content — effectively unlimited for typical articles
|
||||
CONTENT_MAX_CHARS = 50_000
|
||||
# Keep only items from the last N days to avoid unbounded growth
|
||||
ITEM_MAX_AGE_DAYS = 90
|
||||
|
||||
@@ -80,7 +78,7 @@ def extract_item(entry) -> dict:
|
||||
content = raw_content[0].value
|
||||
else:
|
||||
content = entry.get("summary", "")
|
||||
content = _html_to_text(content)[:CONTENT_MAX_CHARS]
|
||||
content = _html_to_text(content)
|
||||
|
||||
pub = None
|
||||
if entry.published_parsed:
|
||||
@@ -210,7 +208,7 @@ async def fetch_and_cache_feed(feed_id: int, url: str) -> int:
|
||||
async with async_session() as session:
|
||||
item = await session.get(RssItem, item_id)
|
||||
if item:
|
||||
item.content = full_text[:CONTENT_MAX_CHARS]
|
||||
item.content = full_text
|
||||
await session.commit()
|
||||
await upsert_rss_item_embedding(
|
||||
item_id, feed_user_id, item.title or "", item.content
|
||||
|
||||
@@ -16,14 +16,16 @@ def test_extract_item_fields():
|
||||
assert item["content"] == "A summary"
|
||||
|
||||
|
||||
def test_extract_item_truncates_content():
|
||||
"""extract_item() should truncate content to CONTENT_MAX_CHARS."""
|
||||
from fabledassistant.services.rss import extract_item, CONTENT_MAX_CHARS
|
||||
def test_extract_item_does_not_truncate_content():
|
||||
"""extract_item() should store content without truncation."""
|
||||
from fabledassistant.services.rss import extract_item
|
||||
long_text = "x" * 100_000
|
||||
entry = MagicMock()
|
||||
entry.get = lambda k, d="": {"summary": "x" * (CONTENT_MAX_CHARS + 1000), "title": "", "link": "", "id": "g"}.get(k, d)
|
||||
entry.get = lambda k, d="": {"summary": long_text, "title": "", "link": "", "id": "g"}.get(k, d)
|
||||
entry.content = []
|
||||
entry.published_parsed = None
|
||||
item = extract_item(entry)
|
||||
assert len(item["content"]) == CONTENT_MAX_CHARS
|
||||
assert len(item["content"]) == 100_000
|
||||
|
||||
|
||||
def test_extract_item_prefers_content_over_summary():
|
||||
|
||||
Reference in New Issue
Block a user