refactor: hard-cut RSS infrastructure (scope C)
Removes the entire RSS feature surface — feeds, items, embeddings, reactions, discussion-note flow, briefing news context, settings, env-vars, and DB tables. Keeps the URL-generic article-reader (the read_article LLM tool) under a clean module so the LLM can still fetch arbitrary article content from URLs the user provides. Backend: - New services/article_fetcher.py — single source of trafilatura URL→text - New services/tools/article.py — read_article tool (was nested under tools/rss) - Delete services/rss.py, rss_classifier.py, rss_filtering.py, article_context.py - Delete services/tools/rss.py - Delete models/rss_feed.py (RssFeed, RssItem), models/rss_item_embedding.py - services/embeddings.py: drop upsert/semantic_search/backfill RSS helpers - services/llm.py: remove _build_briefing_article_context, briefing-conv branch, ARTICLE_DISCUSS_SEED skip-RAG branch; drop get_rss_items / add_rss_feed from the actions list - services/generation_task.py: drop _maybe_save_article_discussion_note + caller - routes/chat.py: drop /api/chat/from-article/<id> endpoint - routes/journal.py: re-import via web.py refactor (article_fetcher path) - services/tools/__init__.py: register `article`, drop `rss` - services/tools/_registry.py: drop the requires=='rss' check - app.py: drop backfill_rss_item_embeddings + backfill_rss_article_content tasks - config.py: prose-only edit (no env var change — RSS env vars were never first-class) Frontend: - stores/settings.ts: drop rssEnabled - SettingsView.vue: drop the RSS-classification mention - api/client.ts: drop openArticleInChat (the from-article endpoint is gone) Tests: - Delete tests/test_rss_service.py, test_news_api.py, test_article_reading.py Migration: - 0042_drop_rss: DROP TABLE rss_item_embeddings, rss_item_reactions, rss_items, rss_feeds; DELETE settings rows for rss_enabled / briefing_*_topics Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -1,7 +1,6 @@
|
||||
"""Semantic note search via Ollama embedding model (nomic-embed-text).
|
||||
|
||||
Embeddings are stored in the note_embeddings table (one row per note).
|
||||
RSS item embeddings are stored in rss_item_embeddings (one row per item).
|
||||
All search operations degrade gracefully — if the embedding model is
|
||||
unavailable the callers fall back to keyword search.
|
||||
"""
|
||||
@@ -9,7 +8,6 @@ unavailable the callers fall back to keyword search.
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import delete, select
|
||||
@@ -18,8 +16,6 @@ from fabledassistant.config import Config
|
||||
from fabledassistant.models import async_session
|
||||
from fabledassistant.models.embedding import NoteEmbedding
|
||||
from fabledassistant.models.note import Note
|
||||
from fabledassistant.models.rss_feed import RssItem
|
||||
from fabledassistant.models.rss_item_embedding import RssItemEmbedding
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -28,10 +24,6 @@ logger = logging.getLogger(__name__)
|
||||
# 0.45 keeps only genuinely relevant notes; lower values like 0.30 let in
|
||||
# loosely-related results that pad the sidebar without adding real value.
|
||||
_SIMILARITY_THRESHOLD = 0.45
|
||||
_RSS_SIMILARITY_THRESHOLD = 0.55
|
||||
_RSS_SEARCH_LIMIT = 3
|
||||
_RSS_SEARCH_DAYS = 30
|
||||
_RSS_SNIPPET_CHARS = 500
|
||||
|
||||
|
||||
async def get_embedding(text: str, model: str | None = None) -> list[float]:
|
||||
@@ -186,174 +178,3 @@ async def backfill_note_embeddings() -> None:
|
||||
logger.info("Embedding backfill complete: %d/%d notes embedded", success, len(notes_to_embed))
|
||||
|
||||
|
||||
# ── RSS item embeddings ───────────────────────────────────────────────────────
|
||||
|
||||
async def upsert_rss_item_embedding(item_id: int, user_id: int, title: str, content: str) -> None:
|
||||
"""Generate and persist an embedding for an RSS item. Safe to fire-and-forget."""
|
||||
text = f"{title}\n{content}".strip()
|
||||
if not text:
|
||||
return
|
||||
try:
|
||||
embedding = await get_embedding(text)
|
||||
except Exception:
|
||||
logger.debug("Skipping embedding for RSS item %d — model unavailable", item_id)
|
||||
return
|
||||
|
||||
try:
|
||||
async with async_session() as session:
|
||||
await session.execute(
|
||||
delete(RssItemEmbedding).where(RssItemEmbedding.rss_item_id == item_id)
|
||||
)
|
||||
session.add(RssItemEmbedding(rss_item_id=item_id, user_id=user_id, embedding=embedding))
|
||||
await session.commit()
|
||||
logger.debug("Upserted embedding for RSS item %d", item_id)
|
||||
except Exception:
|
||||
logger.warning("Failed to persist embedding for RSS item %d", item_id, exc_info=True)
|
||||
|
||||
|
||||
async def semantic_search_rss_items(
|
||||
user_id: int,
|
||||
query_vector: list[float],
|
||||
limit: int = _RSS_SEARCH_LIMIT,
|
||||
days: int = _RSS_SEARCH_DAYS,
|
||||
) -> list[tuple[float, RssItem]]:
|
||||
"""Return up to *limit* (score, RssItem) pairs most relevant to *query_vector*.
|
||||
|
||||
Only considers items fetched within the last *days* days.
|
||||
Returns an empty list on any error.
|
||||
"""
|
||||
since = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
try:
|
||||
async with async_session() as session:
|
||||
stmt = (
|
||||
select(RssItemEmbedding, RssItem)
|
||||
.join(RssItem, RssItemEmbedding.rss_item_id == RssItem.id)
|
||||
.where(
|
||||
RssItemEmbedding.user_id == user_id,
|
||||
RssItem.fetched_at >= since,
|
||||
)
|
||||
)
|
||||
rows = list((await session.execute(stmt)).all())
|
||||
except Exception:
|
||||
logger.warning("Failed to query RSS item embeddings", exc_info=True)
|
||||
return []
|
||||
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
scored: list[tuple[float, RssItem]] = []
|
||||
for rie, item in rows:
|
||||
try:
|
||||
sim = _cosine_similarity(query_vector, rie.embedding)
|
||||
except Exception:
|
||||
continue
|
||||
if sim >= _RSS_SIMILARITY_THRESHOLD:
|
||||
scored.append((sim, item))
|
||||
|
||||
scored.sort(key=lambda x: x[0], reverse=True)
|
||||
return scored[:limit]
|
||||
|
||||
|
||||
async def backfill_rss_item_embeddings() -> None:
|
||||
"""Generate embeddings for all RSS items that don't have one yet.
|
||||
|
||||
Runs as a background task at startup. Adds a small sleep between items
|
||||
to avoid overwhelming Ollama.
|
||||
"""
|
||||
try:
|
||||
async with async_session() as session:
|
||||
existing = {
|
||||
row[0]
|
||||
for row in (
|
||||
await session.execute(select(RssItemEmbedding.rss_item_id))
|
||||
).fetchall()
|
||||
}
|
||||
result = await session.execute(
|
||||
select(RssItem.id, RssItem.feed_id, RssItem.title, RssItem.content)
|
||||
)
|
||||
items_to_embed = [row for row in result.fetchall() if row[0] not in existing]
|
||||
except Exception:
|
||||
logger.warning("RSS embedding backfill: failed to query items", exc_info=True)
|
||||
return
|
||||
|
||||
if not items_to_embed:
|
||||
logger.info("RSS embedding backfill: all items already have embeddings")
|
||||
return
|
||||
|
||||
# Resolve user_id per feed_id
|
||||
try:
|
||||
from fabledassistant.models.rss_feed import RssFeed
|
||||
async with async_session() as session:
|
||||
result = await session.execute(select(RssFeed.id, RssFeed.user_id))
|
||||
feed_user_map = {fid: uid for fid, uid in result.fetchall()}
|
||||
except Exception:
|
||||
logger.warning("RSS embedding backfill: failed to load feed user map", exc_info=True)
|
||||
return
|
||||
|
||||
logger.info("RSS embedding backfill: generating embeddings for %d items", len(items_to_embed))
|
||||
success = 0
|
||||
for item_id, feed_id, title, content in items_to_embed:
|
||||
user_id = feed_user_map.get(feed_id)
|
||||
if user_id is None:
|
||||
continue
|
||||
await upsert_rss_item_embedding(item_id, user_id, title or "", content or "")
|
||||
success += 1
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
logger.info("RSS embedding backfill complete: %d/%d items embedded", success, len(items_to_embed))
|
||||
|
||||
|
||||
async def backfill_rss_article_content() -> None:
|
||||
"""Fetch full article text for RSS items that only have short feed-provided content.
|
||||
|
||||
An item is considered unenriched if its content is shorter than 1000 chars —
|
||||
typical of feed summaries/teasers rather than full articles.
|
||||
Runs at startup after the embedding backfill.
|
||||
"""
|
||||
from fabledassistant.services.rss import _fetch_full_article
|
||||
from fabledassistant.models.rss_feed import RssFeed
|
||||
|
||||
SHORT_THRESHOLD = 1000
|
||||
|
||||
try:
|
||||
async with async_session() as session:
|
||||
feed_result = await session.execute(select(RssFeed.id, RssFeed.user_id))
|
||||
feed_user_map = {fid: uid for fid, uid in feed_result.fetchall()}
|
||||
|
||||
item_result = await session.execute(
|
||||
select(RssItem.id, RssItem.feed_id, RssItem.url, RssItem.title, RssItem.content)
|
||||
.where(RssItem.url != "")
|
||||
)
|
||||
candidates = [
|
||||
row for row in item_result.fetchall()
|
||||
if len(row[4] or "") < SHORT_THRESHOLD
|
||||
]
|
||||
except Exception:
|
||||
logger.warning("Article content backfill: failed to query items", exc_info=True)
|
||||
return
|
||||
|
||||
if not candidates:
|
||||
logger.info("Article content backfill: no unenriched items found")
|
||||
return
|
||||
|
||||
logger.info("Article content backfill: enriching %d items", len(candidates))
|
||||
enriched = 0
|
||||
for item_id, feed_id, url, title, _ in candidates:
|
||||
user_id = feed_user_map.get(feed_id)
|
||||
if user_id is None:
|
||||
continue
|
||||
full_text = await _fetch_full_article(url)
|
||||
if full_text and len(full_text) > SHORT_THRESHOLD:
|
||||
try:
|
||||
async with async_session() as session:
|
||||
item = await session.get(RssItem, item_id)
|
||||
if item:
|
||||
item.content = full_text
|
||||
await session.commit()
|
||||
await upsert_rss_item_embedding(item_id, user_id, title or "", full_text)
|
||||
enriched += 1
|
||||
except Exception:
|
||||
logger.debug("Failed to store enriched content for item %d", item_id, exc_info=True)
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
logger.info("Article content backfill complete: %d/%d items enriched", enriched, len(candidates))
|
||||
|
||||
Reference in New Issue
Block a user