feat(rss): remove article content character cap

Trafilatura extracts only article body text (typically 2K–15K chars),
so storing the full content is safe without an artificial ceiling.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-05 19:32:17 -04:00
parent 9dd4178774
commit f7d54a15c0
2 changed files with 9 additions and 9 deletions
+2 -4
View File
@@ -14,8 +14,6 @@ from fabledassistant.models.rss_feed import RssFeed, RssItem
logger = logging.getLogger(__name__)
# Safety cap on stored content — effectively unlimited for typical articles
CONTENT_MAX_CHARS = 50_000
# Keep only items from the last N days to avoid unbounded growth
ITEM_MAX_AGE_DAYS = 90
@@ -80,7 +78,7 @@ def extract_item(entry) -> dict:
content = raw_content[0].value
else:
content = entry.get("summary", "")
content = _html_to_text(content)[:CONTENT_MAX_CHARS]
content = _html_to_text(content)
pub = None
if entry.published_parsed:
@@ -210,7 +208,7 @@ async def fetch_and_cache_feed(feed_id: int, url: str) -> int:
async with async_session() as session:
item = await session.get(RssItem, item_id)
if item:
item.content = full_text[:CONTENT_MAX_CHARS]
item.content = full_text
await session.commit()
await upsert_rss_item_embedding(
item_id, feed_user_id, item.title or "", item.content
+7 -5
View File
@@ -16,14 +16,16 @@ def test_extract_item_fields():
assert item["content"] == "A summary"
def test_extract_item_truncates_content():
"""extract_item() should truncate content to CONTENT_MAX_CHARS."""
from fabledassistant.services.rss import extract_item, CONTENT_MAX_CHARS
def test_extract_item_does_not_truncate_content():
"""extract_item() should store content without truncation."""
from fabledassistant.services.rss import extract_item
long_text = "x" * 100_000
entry = MagicMock()
entry.get = lambda k, d="": {"summary": "x" * (CONTENT_MAX_CHARS + 1000), "title": "", "link": "", "id": "g"}.get(k, d)
entry.get = lambda k, d="": {"summary": long_text, "title": "", "link": "", "id": "g"}.get(k, d)
entry.content = []
entry.published_parsed = None
item = extract_item(entry)
assert len(item["content"]) == CONTENT_MAX_CHARS
assert len(item["content"]) == 100_000
def test_extract_item_prefers_content_over_summary():