bvandeusen/FabledScribe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 50 Wiki Activity

feat(rss): remove article content character cap

Browse Source 
Trafilatura extracts only article body text (typically 2K–15K chars),
so storing the full content is safe without an artificial ceiling.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Bryan Van Deusen
2026-04-05 19:32:17 -04:00
parent 9dd4178774
commit f7d54a15c0
2 changed files with 9 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fabledassistant/services/rss.py
+2 -4
View File
@@ -14,8 +14,6 @@ from fabledassistant.models.rss_feed import RssFeed, RssItem
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# Safety cap on stored content — effectively unlimited for typical articles
CONTENT_MAX_CHARS = 50_000
# Keep only items from the last N days to avoid unbounded growth # Keep only items from the last N days to avoid unbounded growth
ITEM_MAX_AGE_DAYS = 90 ITEM_MAX_AGE_DAYS = 90
@@ -80,7 +78,7 @@ def extract_item(entry) -> dict:
content = raw_content[0].value content = raw_content[0].value
else: else:
content = entry.get("summary", "") content = entry.get("summary", "")
content = _html_to_text(content)[:CONTENT_MAX_CHARS] content = _html_to_text(content)
pub = None pub = None
if entry.published_parsed: if entry.published_parsed:
@@ -210,7 +208,7 @@ async def fetch_and_cache_feed(feed_id: int, url: str) -> int:
async with async_session() as session: async with async_session() as session:
item = await session.get(RssItem, item_id) item = await session.get(RssItem, item_id)
if item: if item:
item.content = full_text[:CONTENT_MAX_CHARS] item.content = full_text
await session.commit() await session.commit()
await upsert_rss_item_embedding( await upsert_rss_item_embedding(
item_id, feed_user_id, item.title or "", item.content item_id, feed_user_id, item.title or "", item.content
tests/test_rss_service.py
+7 -5
View File
@@ -16,14 +16,16 @@ def test_extract_item_fields():
assert item["content"] == "A summary" assert item["content"] == "A summary"
def test_extract_item_truncates_content(): def test_extract_item_does_not_truncate_content():
"""extract_item() should truncate content to CONTENT_MAX_CHARS.""" """extract_item() should store content without truncation."""
from fabledassistant.services.rss import extract_item, CONTENT_MAX_CHARS from fabledassistant.services.rss import extract_item
long_text = "x" * 100_000
entry = MagicMock() entry = MagicMock()
entry.get = lambda k, d="": {"summary": "x" * (CONTENT_MAX_CHARS + 1000), "title": "", "link": "", "id": "g"}.get(k, d) entry.get = lambda k, d="": {"summary": long_text, "title": "", "link": "", "id": "g"}.get(k, d)
entry.content = []
entry.published_parsed = None entry.published_parsed = None
item = extract_item(entry) item = extract_item(entry)
assert len(item["content"]) == CONTENT_MAX_CHARS assert len(item["content"]) == 100_000
def test_extract_item_prefers_content_over_summary(): def test_extract_item_prefers_content_over_summary():