refactor: hard-cut RSS infrastructure (scope C)
Removes the entire RSS feature surface — feeds, items, embeddings, reactions, discussion-note flow, briefing news context, settings, env-vars, and DB tables. Keeps the URL-generic article-reader (the read_article LLM tool) under a clean module so the LLM can still fetch arbitrary article content from URLs the user provides. Backend: - New services/article_fetcher.py — single source of trafilatura URL→text - New services/tools/article.py — read_article tool (was nested under tools/rss) - Delete services/rss.py, rss_classifier.py, rss_filtering.py, article_context.py - Delete services/tools/rss.py - Delete models/rss_feed.py (RssFeed, RssItem), models/rss_item_embedding.py - services/embeddings.py: drop upsert/semantic_search/backfill RSS helpers - services/llm.py: remove _build_briefing_article_context, briefing-conv branch, ARTICLE_DISCUSS_SEED skip-RAG branch; drop get_rss_items / add_rss_feed from the actions list - services/generation_task.py: drop _maybe_save_article_discussion_note + caller - routes/chat.py: drop /api/chat/from-article/<id> endpoint - routes/journal.py: re-import via web.py refactor (article_fetcher path) - services/tools/__init__.py: register `article`, drop `rss` - services/tools/_registry.py: drop the requires=='rss' check - app.py: drop backfill_rss_item_embeddings + backfill_rss_article_content tasks - config.py: prose-only edit (no env var change — RSS env vars were never first-class) Frontend: - stores/settings.ts: drop rssEnabled - SettingsView.vue: drop the RSS-classification mention - api/client.ts: drop openArticleInChat (the from-article endpoint is gone) Tests: - Delete tests/test_rss_service.py, test_news_api.py, test_article_reading.py Migration: - 0042_drop_rss: DROP TABLE rss_item_embeddings, rss_item_reactions, rss_items, rss_feeds; DELETE settings rows for rss_enabled / briefing_*_topics Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -1,216 +0,0 @@
|
||||
import json
|
||||
import pytest
|
||||
from unittest.mock import patch, AsyncMock
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# read_article tool tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_article_success():
|
||||
"""read_article returns success with content when _fetch_full_article succeeds."""
|
||||
from fabledassistant.services.tools import execute_tool
|
||||
|
||||
with patch(
|
||||
"fabledassistant.services.rss._fetch_full_article",
|
||||
new_callable=AsyncMock,
|
||||
return_value="Article body text here.",
|
||||
):
|
||||
result = await execute_tool(user_id=1, tool_name="read_article", arguments={"url": "https://example.com/article"})
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["type"] == "article_content"
|
||||
assert result["url"] == "https://example.com/article"
|
||||
assert result["content"] == "Article body text here."
|
||||
assert result["truncated"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_article_fetch_failure():
|
||||
"""read_article returns success=False when _fetch_full_article returns None."""
|
||||
from fabledassistant.services.tools import execute_tool
|
||||
|
||||
with patch(
|
||||
"fabledassistant.services.rss._fetch_full_article",
|
||||
new_callable=AsyncMock,
|
||||
return_value=None,
|
||||
):
|
||||
result = await execute_tool(user_id=1, tool_name="read_article", arguments={"url": "https://example.com/broken"})
|
||||
|
||||
assert result["success"] is False
|
||||
assert "Could not fetch" in result["error"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_article_truncates_at_40k():
|
||||
"""read_article truncates content at 40,000 chars and sets truncated=True."""
|
||||
from fabledassistant.services.tools import execute_tool
|
||||
|
||||
long_content = "x" * 50_000
|
||||
|
||||
with patch(
|
||||
"fabledassistant.services.rss._fetch_full_article",
|
||||
new_callable=AsyncMock,
|
||||
return_value=long_content,
|
||||
):
|
||||
result = await execute_tool(user_id=1, tool_name="read_article", arguments={"url": "https://example.com/long"})
|
||||
|
||||
assert result["success"] is True
|
||||
assert len(result["content"]) == 40_000
|
||||
assert result["truncated"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_article_empty_url():
|
||||
"""read_article returns success=False when URL is empty."""
|
||||
from fabledassistant.services.tools import execute_tool
|
||||
|
||||
result = await execute_tool(user_id=1, tool_name="read_article", arguments={"url": ""})
|
||||
assert result["success"] is False
|
||||
assert "No URL provided" in result["error"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# History builder tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_history(messages: list[dict]) -> list[dict]:
|
||||
"""Replicate the fixed history builder from routes/chat.py."""
|
||||
history = []
|
||||
for msg in messages:
|
||||
if msg["role"] == "system":
|
||||
continue
|
||||
msg_dict = {"role": msg["role"], "content": msg.get("content") or ""}
|
||||
tool_calls = msg.get("tool_calls")
|
||||
if tool_calls:
|
||||
msg_dict["tool_calls"] = [
|
||||
{"function": {"name": tc["function"], "arguments": tc["arguments"]}}
|
||||
for tc in tool_calls
|
||||
]
|
||||
history.append(msg_dict)
|
||||
for tc in tool_calls:
|
||||
history.append({"role": "tool", "content": json.dumps(tc.get("result", {}))})
|
||||
else:
|
||||
history.append(msg_dict)
|
||||
return history
|
||||
|
||||
|
||||
def test_history_builder_replays_tool_calls():
|
||||
"""History builder with tool_calls produces assistant entry + tool result entry."""
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"function": "read_article",
|
||||
"arguments": {"url": "https://example.com"},
|
||||
"result": {"success": True, "content": "Article text"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "Summarize it", "tool_calls": None},
|
||||
]
|
||||
history = _build_history(messages)
|
||||
|
||||
assert len(history) == 3
|
||||
assert history[0]["role"] == "assistant"
|
||||
assert history[0]["tool_calls"][0]["function"]["name"] == "read_article"
|
||||
assert history[1]["role"] == "tool"
|
||||
assert json.loads(history[1]["content"])["success"] is True
|
||||
assert history[2]["role"] == "user"
|
||||
assert history[2]["content"] == "Summarize it"
|
||||
|
||||
|
||||
def test_history_builder_no_tool_calls_unchanged():
|
||||
"""History builder with tool_calls=None produces same output as before."""
|
||||
messages = [
|
||||
{"role": "user", "content": "Hello", "tool_calls": None},
|
||||
{"role": "assistant", "content": "Hi there!", "tool_calls": None},
|
||||
]
|
||||
history = _build_history(messages)
|
||||
|
||||
assert len(history) == 2
|
||||
assert history[0] == {"role": "user", "content": "Hello"}
|
||||
assert history[1] == {"role": "assistant", "content": "Hi there!"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# prepare_article_context tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepare_article_context_small_passthrough():
|
||||
"""Articles under CHAR_BUDGET pass through unchanged with zero LLM calls."""
|
||||
from fabledassistant.services import article_context
|
||||
|
||||
body = "A short article.\n\nWith two paragraphs."
|
||||
with patch(
|
||||
"fabledassistant.services.article_context.generate_completion",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_gen:
|
||||
out = await article_context.prepare_article_context(
|
||||
"Title", "https://example.com", body, "test-model",
|
||||
)
|
||||
|
||||
assert out == body
|
||||
mock_gen.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepare_article_context_large_runs_map_reduce():
|
||||
"""Articles over CHAR_BUDGET are chunked and map-reduced via the background model."""
|
||||
from fabledassistant.services import article_context
|
||||
|
||||
# CHAR_BUDGET is 48_000 — build a body well over that with paragraph breaks
|
||||
# so the chunker has natural splits to work with.
|
||||
paragraph = ("Lorem ipsum dolor sit amet, consectetur adipiscing elit. " * 40).strip()
|
||||
body = "\n\n".join([paragraph] * 30) # ~70k+ chars across 30 paragraphs
|
||||
assert len(body) > article_context.CHAR_BUDGET
|
||||
|
||||
with patch(
|
||||
"fabledassistant.services.article_context.generate_completion",
|
||||
new_callable=AsyncMock,
|
||||
return_value="Summary of this section with specific claims preserved.",
|
||||
) as mock_gen:
|
||||
out = await article_context.prepare_article_context(
|
||||
"Long Article", "https://example.com/long", body, "test-model",
|
||||
)
|
||||
|
||||
# At least one LLM call fired (the map step ran)
|
||||
assert mock_gen.await_count >= 1
|
||||
# Output carries the oversized-article header and section markers
|
||||
assert "longer than the chat window" in out
|
||||
assert "## Section 1" in out
|
||||
# Map output is much smaller than the raw body
|
||||
assert len(out) < len(body)
|
||||
|
||||
|
||||
def test_chunk_by_paragraph_respects_boundaries():
|
||||
"""Chunker splits on paragraph breaks, not mid-sentence."""
|
||||
from fabledassistant.services.article_context import _chunk_by_paragraph, CHUNK_CHARS
|
||||
|
||||
paragraphs = [f"Paragraph {i}. " + ("x" * 1000) for i in range(20)]
|
||||
body = "\n\n".join(paragraphs)
|
||||
|
||||
chunks = _chunk_by_paragraph(body)
|
||||
|
||||
# Each chunk stays under the budget
|
||||
for chunk in chunks:
|
||||
assert len(chunk) <= CHUNK_CHARS
|
||||
# Total content is preserved (modulo overlap duplication, so ≥ original)
|
||||
assert sum(len(c) for c in chunks) >= len(body) * 0.9
|
||||
|
||||
|
||||
def test_chunk_by_paragraph_handles_oversized_paragraph():
|
||||
"""A single paragraph larger than CHUNK_CHARS gets split mid-paragraph."""
|
||||
from fabledassistant.services.article_context import _chunk_by_paragraph, CHUNK_CHARS
|
||||
|
||||
body = "x" * (CHUNK_CHARS * 3) # one huge paragraph, no breaks
|
||||
chunks = _chunk_by_paragraph(body)
|
||||
|
||||
assert len(chunks) >= 3
|
||||
for chunk in chunks:
|
||||
assert len(chunk) <= CHUNK_CHARS
|
||||
@@ -1,108 +0,0 @@
|
||||
"""Tests for news API — retention constant and endpoint formatting."""
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def test_rss_item_max_age_is_90_days():
|
||||
"""Retention window should be 90 days."""
|
||||
from fabledassistant.services.rss import ITEM_MAX_AGE_DAYS
|
||||
|
||||
assert ITEM_MAX_AGE_DAYS == 90
|
||||
|
||||
|
||||
def _make_mock_session(rows):
|
||||
"""Return a mock async context manager whose session.execute returns rows."""
|
||||
mock_result = MagicMock()
|
||||
mock_result.mappings.return_value.all.return_value = rows
|
||||
|
||||
mock_session = AsyncMock()
|
||||
mock_session.execute = AsyncMock(return_value=mock_result)
|
||||
|
||||
mock_cm = MagicMock()
|
||||
mock_cm.__aenter__ = AsyncMock(return_value=mock_session)
|
||||
mock_cm.__aexit__ = AsyncMock(return_value=False)
|
||||
return mock_cm
|
||||
|
||||
|
||||
def test_list_news_item_serialisation():
|
||||
"""News item serialisation should truncate content to 300 chars and include reaction."""
|
||||
pub = datetime(2026, 3, 28, 8, 0, tzinfo=timezone.utc)
|
||||
item = {
|
||||
"id": 1,
|
||||
"title": "EU AI Act deadline",
|
||||
"url": "https://example.com/ai",
|
||||
"content": "x" * 500,
|
||||
"published_at": pub,
|
||||
"topics": ["tech"],
|
||||
"feed_title": "TechCrunch",
|
||||
"reaction": "up",
|
||||
}
|
||||
|
||||
result = {
|
||||
"id": item["id"],
|
||||
"title": item["title"],
|
||||
"url": item["url"],
|
||||
"snippet": (item["content"] or "")[:300],
|
||||
"published_at": item["published_at"].isoformat() if item["published_at"] else None,
|
||||
"topics": item["topics"] or [],
|
||||
"source": item["feed_title"],
|
||||
"reaction": item["reaction"],
|
||||
}
|
||||
|
||||
assert result["snippet"] == "x" * 300
|
||||
assert result["source"] == "TechCrunch"
|
||||
assert result["reaction"] == "up"
|
||||
assert result["published_at"] == pub.isoformat()
|
||||
|
||||
|
||||
|
||||
def test_build_briefing_article_context_metadata_extraction():
|
||||
"""Verify the rss_item_ids extraction logic from message metadata."""
|
||||
import json
|
||||
|
||||
# Simulates the logic in _build_briefing_article_context
|
||||
def extract_ids(msg_metadata):
|
||||
meta = msg_metadata or {}
|
||||
if isinstance(meta, str):
|
||||
try:
|
||||
meta = json.loads(meta)
|
||||
except Exception:
|
||||
return []
|
||||
return meta.get("rss_item_ids") or []
|
||||
|
||||
assert extract_ids({}) == []
|
||||
assert extract_ids(None) == []
|
||||
assert extract_ids({"rss_item_ids": [1, 2, 3]}) == [1, 2, 3]
|
||||
assert extract_ids(json.dumps({"rss_item_ids": [4, 5]})) == [4, 5]
|
||||
assert extract_ids("not json") == []
|
||||
|
||||
|
||||
def test_list_news_null_content_serialisation():
|
||||
"""Null content should produce empty snippet without error."""
|
||||
item = {
|
||||
"id": 2,
|
||||
"title": "No content article",
|
||||
"url": "https://example.com/b",
|
||||
"content": None,
|
||||
"published_at": None,
|
||||
"topics": None,
|
||||
"feed_title": "Hacker News",
|
||||
"reaction": None,
|
||||
}
|
||||
|
||||
result = {
|
||||
"id": item["id"],
|
||||
"title": item["title"],
|
||||
"url": item["url"],
|
||||
"snippet": (item["content"] or "")[:300],
|
||||
"published_at": item["published_at"].isoformat() if item["published_at"] else None,
|
||||
"topics": item["topics"] or [],
|
||||
"source": item["feed_title"],
|
||||
"reaction": item["reaction"],
|
||||
}
|
||||
|
||||
assert result["snippet"] == ""
|
||||
assert result["topics"] == []
|
||||
assert result["published_at"] is None
|
||||
assert result["reaction"] is None
|
||||
@@ -1,132 +0,0 @@
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
def test_extract_item_fields():
|
||||
"""extract_item() should pull title, link, id, summary from a feedparser entry."""
|
||||
from fabledassistant.services.rss import extract_item
|
||||
entry = MagicMock()
|
||||
entry.get = lambda k, d="": {"title": "Test Post", "link": "https://example.com/1",
|
||||
"id": "guid-1", "summary": "A summary"}.get(k, d)
|
||||
entry.published_parsed = None
|
||||
item = extract_item(entry)
|
||||
assert item["title"] == "Test Post"
|
||||
assert item["url"] == "https://example.com/1"
|
||||
assert item["guid"] == "guid-1"
|
||||
assert item["content"] == "A summary"
|
||||
|
||||
|
||||
def test_extract_item_does_not_truncate_content():
|
||||
"""extract_item() should store content without truncation."""
|
||||
from fabledassistant.services.rss import extract_item
|
||||
long_text = "x" * 100_000
|
||||
entry = MagicMock()
|
||||
entry.get = lambda k, d="": {"summary": long_text, "title": "", "link": "", "id": "g"}.get(k, d)
|
||||
entry.content = []
|
||||
entry.published_parsed = None
|
||||
item = extract_item(entry)
|
||||
assert len(item["content"]) == 100_000
|
||||
|
||||
|
||||
def test_extract_item_prefers_content_over_summary():
|
||||
"""extract_item() should prefer 'content' field over 'summary' when present."""
|
||||
from fabledassistant.services.rss import extract_item
|
||||
entry = MagicMock()
|
||||
content_obj = MagicMock()
|
||||
content_obj.value = "Full content here"
|
||||
entry.get = lambda k, d="": {
|
||||
"title": "T", "link": "http://x.com", "id": "g",
|
||||
"summary": "Short summary",
|
||||
}.get(k, d)
|
||||
entry.content = [content_obj]
|
||||
entry.published_parsed = None
|
||||
item = extract_item(entry)
|
||||
assert item["content"] == "Full content here"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_classify_items_batch_returns_topic_map():
|
||||
"""classify_items_batch should return a dict mapping item_id to topic list."""
|
||||
from unittest.mock import AsyncMock
|
||||
from fabledassistant.services.rss_classifier import classify_items_batch
|
||||
|
||||
fake_response = '{"1": ["technology", "ai"], "2": ["politics"]}'
|
||||
|
||||
with patch(
|
||||
"fabledassistant.services.rss_classifier._llm_classify",
|
||||
new_callable=AsyncMock,
|
||||
return_value=fake_response,
|
||||
):
|
||||
items = [
|
||||
{"id": 1, "title": "OpenAI releases GPT-5", "content": "..."},
|
||||
{"id": 2, "title": "EU passes new law", "content": "..."},
|
||||
]
|
||||
result = await classify_items_batch(items, user_include_topics=[])
|
||||
assert result[1] == ["technology", "ai"]
|
||||
assert result[2] == ["politics"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_classify_items_batch_handles_llm_failure():
|
||||
"""classify_items_batch should return empty dict on LLM error."""
|
||||
from unittest.mock import AsyncMock
|
||||
from fabledassistant.services.rss_classifier import classify_items_batch
|
||||
|
||||
with patch(
|
||||
"fabledassistant.services.rss_classifier._llm_classify",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=Exception("LLM unavailable"),
|
||||
):
|
||||
items = [{"id": 5, "title": "Some news", "content": ""}]
|
||||
result = await classify_items_batch(items, user_include_topics=[])
|
||||
assert result == {}
|
||||
|
||||
|
||||
def test_score_rss_items_excludes_blacklisted_topics():
|
||||
"""Items with excluded topics should be removed."""
|
||||
from fabledassistant.services.rss_filtering import score_and_filter_items
|
||||
|
||||
items = [
|
||||
{"id": 1, "title": "Tech news", "topics": ["technology"], "published_at": "2026-03-25T08:00:00"},
|
||||
{"id": 2, "title": "Sports score", "topics": ["sports"], "published_at": "2026-03-25T08:00:00"},
|
||||
]
|
||||
result = score_and_filter_items(
|
||||
items,
|
||||
include_topics=["technology"],
|
||||
exclude_topics=["sports"],
|
||||
topic_scores={},
|
||||
max_items=10,
|
||||
)
|
||||
ids = [r["id"] for r in result]
|
||||
assert 1 in ids
|
||||
assert 2 not in ids
|
||||
|
||||
|
||||
def test_score_rss_items_boosts_included_topics():
|
||||
"""Items matching include_topics should rank higher than neutral items."""
|
||||
from fabledassistant.services.rss_filtering import score_and_filter_items
|
||||
|
||||
items = [
|
||||
{"id": 1, "title": "Random news", "topics": ["other"], "published_at": "2026-03-25T07:00:00"},
|
||||
{"id": 2, "title": "Tech news", "topics": ["technology"], "published_at": "2026-03-25T06:00:00"},
|
||||
]
|
||||
result = score_and_filter_items(
|
||||
items,
|
||||
include_topics=["technology"],
|
||||
exclude_topics=[],
|
||||
topic_scores={},
|
||||
max_items=10,
|
||||
)
|
||||
assert result[0]["id"] == 2
|
||||
|
||||
|
||||
def test_score_rss_items_no_preferences_returns_all():
|
||||
"""With no preferences, all items should be returned sorted by recency."""
|
||||
from fabledassistant.services.rss_filtering import score_and_filter_items
|
||||
|
||||
items = [
|
||||
{"id": 1, "title": "A", "topics": [], "published_at": "2026-03-24T10:00:00"},
|
||||
{"id": 2, "title": "B", "topics": [], "published_at": "2026-03-25T10:00:00"},
|
||||
]
|
||||
result = score_and_filter_items(items, [], [], {}, max_items=10)
|
||||
assert result[0]["id"] == 2 # Newer first
|
||||
Reference in New Issue
Block a user