diff --git a/alembic/versions/0042_drop_rss.py b/alembic/versions/0042_drop_rss.py new file mode 100644 index 0000000..7266d74 --- /dev/null +++ b/alembic/versions/0042_drop_rss.py @@ -0,0 +1,36 @@ +"""drop RSS infrastructure (tables + leftover briefing settings) + +Revision ID: 0042 +Revises: 0041 +Create Date: 2026-04-26 + +Hard-cut removal of the RSS feature. Drops rss_feeds, rss_items, +rss_item_reactions, rss_item_embeddings, and clears the leftover +briefing-era settings rows that referenced RSS topic preferences. +""" +from alembic import op + + +revision = "0042" +down_revision = "0041" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Drop in dependency order — children first. + op.execute("DROP TABLE IF EXISTS rss_item_embeddings CASCADE") + op.execute("DROP TABLE IF EXISTS rss_item_reactions CASCADE") + op.execute("DROP TABLE IF EXISTS rss_items CASCADE") + op.execute("DROP TABLE IF EXISTS rss_feeds CASCADE") + op.execute( + "DELETE FROM settings WHERE key IN " + "('rss_enabled', 'briefing_include_topics', 'briefing_exclude_topics')" + ) + + +def downgrade() -> None: + # No downgrade — this is a destructive cleanup. The original RSS tables + # would have to be recreated by replaying migrations 0026, 0028, 0035, + # 0038, 0039 manually (and the data would not come back). + raise NotImplementedError("RSS feature is permanently removed in 0042") diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index ba582d6..29bf322 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -415,12 +415,6 @@ export async function deleteJournalMoment(id: number): Promise { await apiDelete(`/api/journal/moments/${id}`); } -export async function openArticleInChat( - itemId: number, -): Promise<{ conversation_id: number; assistant_message_id?: number; status?: string }> { - return apiPost(`/api/chat/from-article/${itemId}`, {}); -} - export async function geocodeAddress(address: string): Promise<{ lat: number; lon: number; display_name: string } | null> { try { const r = await apiPost<{ lat: number; lon: number; label: string }>('/api/journal/weather/geocode', { query: address }); diff --git a/frontend/src/stores/settings.ts b/frontend/src/stores/settings.ts index 7d41ffb..4d07108 100644 --- a/frontend/src/stores/settings.ts +++ b/frontend/src/stores/settings.ts @@ -16,10 +16,6 @@ export const useSettingsStore = defineStore("settings", () => { () => settings.value.default_model || "" ); - const rssEnabled = computed( - () => settings.value.rss_enabled === "true" - ); - // Voice status — checked once on login, refreshable from Settings const voiceEnabled = ref(false); const voiceSttReady = ref(false); @@ -66,7 +62,6 @@ export const useSettingsStore = defineStore("settings", () => { loading, assistantName, defaultModel, - rssEnabled, voiceEnabled, voiceSttReady, voiceTtsReady, diff --git a/frontend/src/views/SettingsView.vue b/frontend/src/views/SettingsView.vue index 87f136b..ade0c05 100644 --- a/frontend/src/views/SettingsView.vue +++ b/frontend/src/views/SettingsView.vue @@ -1368,7 +1368,7 @@ function formatUserDate(iso: string): string {

- Model used for background tasks: title generation, tag suggestions, project summaries, and RSS classification. + Model used for background tasks: title generation, tag suggestions, and project summaries. Using a small dedicated model (e.g. qwen2.5:0.5b) keeps the chat model's KV cache warm between messages, significantly reducing response time. ⚠ Setting this to the same model as Chat Model will wipe the KV cache after every background task, increasing response latency. diff --git a/src/fabledassistant/app.py b/src/fabledassistant/app.py index 70fbb86..fce073b 100644 --- a/src/fabledassistant/app.py +++ b/src/fabledassistant/app.py @@ -320,16 +320,6 @@ def create_app() -> Quart: await backfill_project_summaries() except Exception: logger.warning("Project summary backfill failed", exc_info=True) - try: - from fabledassistant.services.embeddings import backfill_rss_item_embeddings - await backfill_rss_item_embeddings() - except Exception: - logger.warning("RSS embedding backfill failed", exc_info=True) - try: - from fabledassistant.services.embeddings import backfill_rss_article_content - await backfill_rss_article_content() - except Exception: - logger.warning("RSS article content backfill failed", exc_info=True) asyncio.create_task(_delayed_backfill()) diff --git a/src/fabledassistant/config.py b/src/fabledassistant/config.py index 09740e9..87a00ba 100644 --- a/src/fabledassistant/config.py +++ b/src/fabledassistant/config.py @@ -25,7 +25,7 @@ class Config: OLLAMA_URL: str = os.environ.get("OLLAMA_URL", "http://localhost:11434") OLLAMA_MODEL: str = os.environ.get("OLLAMA_MODEL", "qwen3:latest") # Lightweight model for background tasks (title generation, tag suggestions, - # project summaries, RSS classification). Using a separate model keeps the + # project summaries). Using a separate model keeps the # main model's KV cache intact between user messages, enabling prefix cache hits. OLLAMA_BACKGROUND_MODEL: str = os.environ.get("OLLAMA_BACKGROUND_MODEL", "gemma3:4b") # Ollama keep_alive — how long a model stays resident in VRAM after its last diff --git a/src/fabledassistant/models/__init__.py b/src/fabledassistant/models/__init__.py index 2eda686..a8b9c71 100644 --- a/src/fabledassistant/models/__init__.py +++ b/src/fabledassistant/models/__init__.py @@ -38,11 +38,9 @@ from fabledassistant.models.note_version import NoteVersion # noqa: E402, F401 from fabledassistant.models.group import Group, GroupMembership # noqa: E402, F401 from fabledassistant.models.share import NoteShare, ProjectShare # noqa: E402, F401 from fabledassistant.models.notification import Notification # noqa: E402, F401 -from fabledassistant.models.rss_feed import RssFeed, RssItem # noqa: E402, F401 from fabledassistant.models.weather_cache import WeatherCache # noqa: E402, F401 from fabledassistant.models.api_key import ApiKey # noqa: E402, F401 from fabledassistant.models.user_profile import UserProfile # noqa: E402, F401 -from fabledassistant.models.rss_item_embedding import RssItemEmbedding # noqa: E402, F401 from fabledassistant.models.moment import ( # noqa: E402, F401 Moment, MomentEmbedding, diff --git a/src/fabledassistant/models/rss_feed.py b/src/fabledassistant/models/rss_feed.py deleted file mode 100644 index 0df5f49..0000000 --- a/src/fabledassistant/models/rss_feed.py +++ /dev/null @@ -1,96 +0,0 @@ -from datetime import datetime, timezone - -from sqlalchemy import ARRAY, BigInteger, DateTime, ForeignKey, Index, Integer, Text, UniqueConstraint -from sqlalchemy.orm import Mapped, mapped_column, relationship - -from fabledassistant.models import Base - - -class RssFeed(Base): - __tablename__ = "rss_feeds" - - id: Mapped[int] = mapped_column(primary_key=True) - user_id: Mapped[int] = mapped_column(Integer, ForeignKey("users.id", ondelete="CASCADE")) - url: Mapped[str] = mapped_column(Text) - title: Mapped[str] = mapped_column(Text, default="") - category: Mapped[str | None] = mapped_column(Text, nullable=True) - last_fetched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) - - items: Mapped[list["RssItem"]] = relationship( - back_populates="feed", cascade="all, delete-orphan" - ) - - __table_args__ = ( - UniqueConstraint("user_id", "url", name="uq_rss_feeds_user_url"), - Index("ix_rss_feeds_user_id", "user_id"), - ) - - def to_dict(self) -> dict: - return { - "id": self.id, - "url": self.url, - "title": self.title, - "category": self.category, - "last_fetched_at": self.last_fetched_at.isoformat() if self.last_fetched_at else None, - } - - -class RssItem(Base): - __tablename__ = "rss_items" - - id: Mapped[int] = mapped_column(primary_key=True) - feed_id: Mapped[int] = mapped_column(Integer, ForeignKey("rss_feeds.id", ondelete="CASCADE")) - guid: Mapped[str] = mapped_column(Text) - title: Mapped[str] = mapped_column(Text, default="") - url: Mapped[str] = mapped_column(Text, default="") - published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) - # Truncated to 2000 chars to keep DB size reasonable - content: Mapped[str] = mapped_column(Text, default="") - # Full trafilatura-extracted article body, populated lazily on first - # discuss-click / enrichment pass. Nullable — most items never get this - # cached. Expires naturally with the item (90-day retention). - content_full: Mapped[str | None] = mapped_column(Text, nullable=True) - # Map-reduced conversation-ready context derived from content_full. See - # services/article_context.py — populated on first discuss click so - # repeat clicks skip both the fetch and the LLM map step. - context_prepared: Mapped[str | None] = mapped_column(Text, nullable=True) - content_fetched_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True - ) - fetched_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) - ) - topics: Mapped[list[str]] = mapped_column( - ARRAY(Text), nullable=False, default=list, server_default="{}" - ) - classified_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True - ) - # Note persisting the first-click discussion summary. Set by the article - # discussion pipeline once the seeded chat completes its first assistant - # reply; links back into RAG so re-discussing the same article lands the - # prior summary in context. - discussion_note_id: Mapped[int | None] = mapped_column( - BigInteger, ForeignKey("notes.id", ondelete="SET NULL"), nullable=True - ) - - feed: Mapped["RssFeed"] = relationship(back_populates="items") - - __table_args__ = ( - UniqueConstraint("feed_id", "guid", name="uq_rss_items_feed_guid"), - Index("ix_rss_items_feed_id", "feed_id"), - Index("ix_rss_items_published_at", "published_at"), - ) - - def to_dict(self) -> dict: - return { - "id": self.id, - "feed_id": self.feed_id, - "guid": self.guid, - "title": self.title, - "url": self.url, - "published_at": self.published_at.isoformat() if self.published_at else None, - "content": self.content, - "topics": self.topics or [], - "classified_at": self.classified_at.isoformat() if self.classified_at else None, - } diff --git a/src/fabledassistant/models/rss_item_embedding.py b/src/fabledassistant/models/rss_item_embedding.py deleted file mode 100644 index 0a1cf83..0000000 --- a/src/fabledassistant/models/rss_item_embedding.py +++ /dev/null @@ -1,25 +0,0 @@ -from datetime import datetime, timezone - -from sqlalchemy import DateTime, ForeignKey, Integer -from sqlalchemy.dialects.postgresql import JSONB -from sqlalchemy.orm import Mapped, mapped_column - -from fabledassistant.models import Base - - -class RssItemEmbedding(Base): - """Stores the embedding vector for an RSS item, used for semantic news search.""" - - __tablename__ = "rss_item_embeddings" - - rss_item_id: Mapped[int] = mapped_column( - Integer, - ForeignKey("rss_items.id", ondelete="CASCADE"), - primary_key=True, - ) - user_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True) - embedding: Mapped[list] = mapped_column(JSONB, nullable=False) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), - default=lambda: datetime.now(timezone.utc), - ) diff --git a/src/fabledassistant/routes/chat.py b/src/fabledassistant/routes/chat.py index 6212d57..55f6c2a 100644 --- a/src/fabledassistant/routes/chat.py +++ b/src/fabledassistant/routes/chat.py @@ -507,81 +507,3 @@ async def delete_model_route(): return jsonify({"error": str(e)}), 500 -@chat_bp.route("/from-article/", methods=["POST"]) -@login_required -async def create_conversation_from_article(item_id: int): - """Create a chat conversation seeded for article discussion and auto-run. - - Mirrors the briefing ``discuss_article`` route: creates a fresh - conversation, stages the shared synthetic read_article exchange + seed - prompt, then kicks off generation so the client lands on an in-flight - stream. The Flutter and web chat screens reconnect to the running buffer - on mount. - """ - from sqlalchemy import select as _select - from fabledassistant.models import async_session as _async_session - from fabledassistant.models.rss_feed import RssItem, RssFeed - from fabledassistant.services.article_context import ( - EmptyArticleError, - seed_article_discussion, - ) - - uid = get_current_user_id() - - async with _async_session() as session: - result = await session.execute( - _select(RssItem) - .join(RssFeed, RssItem.feed_id == RssFeed.id) - .where(RssItem.id == item_id, RssFeed.user_id == uid) - ) - item = result.scalars().first() - - if item is None: - return jsonify({"error": "Article not found"}), 404 - - conv_title = (item.title or "Article discussion")[:80] - conv = await create_conversation(uid, title=conv_title, conversation_type="chat") - - model = await get_setting(uid, "default_model", "") or Config.OLLAMA_MODEL - try: - discuss_prompt = await seed_article_discussion(conv.id, item, model) - except EmptyArticleError as e: - # Roll back the empty conversation so the user doesn't end up with a - # phantom entry in their chat list. - try: - await delete_conversation(uid, conv.id) - except Exception: - logger.warning("Failed to clean up empty article conversation %s", conv.id) - return jsonify({"error": str(e)}), 422 - - # Reload conversation so we see the two messages the helper just added. - conv = await get_conversation(uid, conv.id) - assert conv is not None - - history: list[dict] = [] - for msg in conv.messages: - if msg.role == "system": - continue - msg_dict: dict = {"role": msg.role, "content": msg.content or ""} - if msg.tool_calls: - msg_dict["tool_calls"] = [ - {"function": {"name": tc["function"], "arguments": tc["arguments"]}} - for tc in msg.tool_calls - ] - history.append(msg_dict) - for tc in msg.tool_calls: - history.append({"role": "tool", "content": json.dumps(tc.get("result", {}))}) - else: - history.append(msg_dict) - - assistant_msg = await add_message(conv.id, "assistant", "", status="generating") - buf = create_buffer(conv.id, assistant_msg.id) - asyncio.create_task(run_generation( - buf, history, model, uid, conv.id, conv.title or "", discuss_prompt, - )) - - return jsonify({ - "conversation_id": conv.id, - "assistant_message_id": assistant_msg.id, - "status": "generating", - }), 202 diff --git a/src/fabledassistant/services/article_context.py b/src/fabledassistant/services/article_context.py deleted file mode 100644 index 79bd685..0000000 --- a/src/fabledassistant/services/article_context.py +++ /dev/null @@ -1,270 +0,0 @@ -"""Prepare article bodies as conversation-ready context. - -Used by the briefing ``discuss-article`` flow and the ``/news`` discuss button. -A raw trafilatura extraction is often too large to drop whole into a chat -history without eating the context window, so this module runs a map-reduce -step over oversized articles and returns a compact, structured context that -still preserves the article's meaning across sections. - -Small articles pass through unchanged — map-reduce only fires when the raw -body exceeds CHAR_BUDGET. The output is cached on ``rss_items.context_prepared`` -by the caller, so repeat discuss-clicks on the same article skip this work -entirely. - -The module also owns ``seed_article_discussion``, the shared routine that -stages a synthetic ``read_article`` tool exchange plus a conversational seed -prompt into a conversation. Both the briefing and ``/news`` entry points call -it so the two flows stay byte-identical — the only thing that differs between -them is whether the conversation already existed or was freshly created. -""" - -from __future__ import annotations - -import asyncio -import logging -import re - -from fabledassistant.models import async_session -from fabledassistant.models.rss_feed import RssItem -from fabledassistant.services.chat import add_message -from fabledassistant.services.llm import generate_completion - -logger = logging.getLogger(__name__) - -# ~12k tokens at 4 chars/token. Comfortably under OLLAMA_NUM_CTX=16384 -# with room left for system prompt, chat history, and the assistant reply. -CHAR_BUDGET = 48_000 - -# Chunk size for the map step on oversized articles. Overlap preserves -# context across paragraph boundaries that happen to land mid-sentence. -CHUNK_CHARS = 8_000 -CHUNK_OVERLAP = 400 - -_PARA_SPLIT = re.compile(r"\n\s*\n") - - -def _chunk_by_paragraph(body: str) -> list[str]: - """Split ``body`` into chunks of up to CHUNK_CHARS, respecting paragraphs. - - Paragraphs longer than CHUNK_CHARS are split mid-paragraph as a last - resort. Adjacent chunks share CHUNK_OVERLAP chars of trailing text so - a sentence straddling the boundary stays readable on both sides. - """ - paragraphs = [p.strip() for p in _PARA_SPLIT.split(body) if p.strip()] - chunks: list[str] = [] - current: list[str] = [] - current_len = 0 - for para in paragraphs: - para_len = len(para) - if para_len > CHUNK_CHARS: - if current: - chunks.append("\n\n".join(current)) - current, current_len = [], 0 - for i in range(0, para_len, CHUNK_CHARS - CHUNK_OVERLAP): - chunks.append(para[i : i + CHUNK_CHARS]) - continue - if current_len + para_len + 2 > CHUNK_CHARS and current: - chunks.append("\n\n".join(current)) - tail = current[-1][-CHUNK_OVERLAP:] if current else "" - current = [tail, para] if tail else [para] - current_len = len(tail) + para_len + (2 if tail else 0) - else: - current.append(para) - current_len += para_len + 2 - if current: - chunks.append("\n\n".join(current)) - return chunks - - -async def _summarize_chunk(title: str, chunk: str, index: int, total: int, model: str) -> str: - """Map-step summary of one article chunk. - - Aims for ~300 words of dense, factual prose — not bullet points — so the - downstream chat model can quote from it naturally. - """ - messages = [ - { - "role": "system", - "content": ( - "You are summarizing one section of a larger article so a downstream " - "conversation model can discuss the full article without having to read " - "every word.\n\n" - "Requirements:\n" - "- 250–350 words of dense factual prose\n" - "- Preserve specific claims, numbers, names, and quotes\n" - "- Do NOT editorialize or add analysis\n" - "- Do NOT use bullet points or headings\n" - "- Do NOT say 'this section' or 'this article' — write content, not meta" - ), - }, - { - "role": "user", - "content": ( - f"Article: {title}\n" - f"Section {index + 1} of {total}:\n\n{chunk}" - ), - }, - ] - try: - # Pin num_ctx — same rationale as services/research.py:66. A large - # chunk plus system prompt can push well past the default window; - # silent truncation here would drop the tail of the chunk without - # any error, producing a misleading summary. - raw = await generate_completion( - messages, model, max_tokens=600, num_ctx=16384 - ) - return raw.strip() - except Exception: - logger.warning( - "Article chunk summary failed for section %d/%d of '%s'", - index + 1, total, title, exc_info=True, - ) - # Fall back to the raw chunk truncated to ~1500 chars so the overall - # pipeline still delivers something rather than dropping the section. - return chunk[:1500] - - -async def prepare_article_context( - title: str, - url: str, - body: str, - model: str, -) -> str: - """Return a conversation-ready context block for ``body``. - - - Small article (≤ CHAR_BUDGET): returns ``body`` unchanged. - - Oversized article: runs a parallel map step over paragraph-aware - chunks and concatenates the summaries under section headers. - - The returned string is what should go into the ``read_article`` synthetic - tool-result in chat history. Callers are responsible for caching it to - ``rss_items.context_prepared``. - """ - body = body or "" - if len(body) <= CHAR_BUDGET: - return body - - chunks = _chunk_by_paragraph(body) - logger.info( - "Article '%s' is %d chars, map-reducing into %d chunks", - title, len(body), len(chunks), - ) - - summaries = await asyncio.gather( - *[ - _summarize_chunk(title, chunk, i, len(chunks), model) - for i, chunk in enumerate(chunks) - ] - ) - - header = ( - f"(This article was longer than the chat window could hold verbatim, " - f"so the full text was split into {len(chunks)} sections and each was " - "summarized below. Each section preserves specific claims, numbers, " - "and quotes from the original.)\n\n" - ) - parts = [ - f"## Section {i + 1}\n\n{summary}" - for i, summary in enumerate(summaries) - ] - return header + "\n\n".join(parts) - - -# Conversational seed prompt for article discussions. Kept here so both the -# briefing and /news entry points use the exact same wording. See -# feedback_discuss_prompt_style memory: numbered checklists produce -# assignment-completion responses; this conversational seed opens a dialogue. -ARTICLE_DISCUSS_SEED = ( - "I want to talk about this article. Start with a substantive summary " - "of what it's arguing and the key evidence it uses, then tell me what " - "stood out to you or seems worth pushing back on. I'll ask follow-ups " - "from there." -) - - -class EmptyArticleError(Exception): - """Raised when an article has no extractable body text. - - Callers (the briefing and /news discuss routes) map this to a 422 so the - user sees a clear error instead of a hallucinated summary built from an - empty synthetic tool result. - """ - - -async def seed_article_discussion( - conv_id: int, - item: RssItem, - model: str, -) -> str: - """Stage the synthetic read_article tool exchange + conversational seed. - - Used by both the briefing ``discuss_article`` route and the ``/news`` - ``from-article`` conversation creator. Handles the three-layer cache - (``context_prepared`` → ``content_full`` → fresh fetch) and inserts two - messages into ``conv_id``: - - 1. An assistant message with a synthetic ``read_article`` tool_call whose - ``result.content`` carries the prepared article context. The message - also carries ``msg_metadata={"rss_item_id": ...}`` so the post-generation - hook in ``generation_task.py`` can locate it and persist the first - reply as a discussion-summary Note. - 2. A user message with the shared conversational seed prompt. - - Returns the seed prompt string so callers can pass it to ``run_generation`` - as ``user_content``. - """ - # Avoid circulars: rss helper imports article_context indirectly nowhere, - # but keep this local for symmetry with the route-level imports it - # replaces. - from fabledassistant.services.rss import get_or_fetch_full_article - - if item.context_prepared: - article_content = item.context_prepared - else: - raw_body = await get_or_fetch_full_article(item) or item.content or "" - if not raw_body.strip(): - # Hard-fail rather than stage an empty synthetic tool result. - # An empty `content` field silently tells the model "the article - # has nothing in it" and it confabulates from RAG/history. Better - # to surface a clean error to the user. - logger.warning( - "Article discussion aborted: empty body for rss_item %s (%s)", - item.id, item.url, - ) - raise EmptyArticleError( - "Couldn't extract any readable text from this article." - ) - article_content = await prepare_article_context( - item.title or "", item.url, raw_body, model, - ) - if not article_content.strip(): - raise EmptyArticleError( - "Couldn't extract any readable text from this article." - ) - async with async_session() as session: - fresh = await session.get(RssItem, item.id) - if fresh is not None: - fresh.context_prepared = article_content - await session.commit() - - synthetic_tool_calls = [{ - "function": "read_article", - "arguments": {"url": item.url}, - "result": { - "success": True, - "type": "article_content", - "url": item.url, - "content": article_content, - "truncated": False, - }, - }] - await add_message( - conv_id, - "assistant", - "", - status="complete", - tool_calls=synthetic_tool_calls, - msg_metadata={"rss_item_id": item.id, "article_seed": True}, - ) - await add_message(conv_id, "user", ARTICLE_DISCUSS_SEED) - return ARTICLE_DISCUSS_SEED diff --git a/src/fabledassistant/services/article_fetcher.py b/src/fabledassistant/services/article_fetcher.py new file mode 100644 index 0000000..5f3f717 --- /dev/null +++ b/src/fabledassistant/services/article_fetcher.py @@ -0,0 +1,53 @@ +"""Generic article-text fetcher. + +Fetches a URL and extracts its main body via trafilatura. The single source +of truth for article-content extraction across the codebase — used by the +``read_article`` LLM tool and the ``lookup`` tool's web-result enrichment. + +Trafilatura/lxml is NOT safe to call concurrently — running it via +``run_in_executor`` from multiple coroutines can trip a libxml2 double-free. +Callers must serialize their fetches (await one before starting the next). +""" +from __future__ import annotations + +import asyncio +import logging + +import httpx + +logger = logging.getLogger(__name__) + + +async def fetch_article_text(url: str) -> str | None: + """Return the clean article body for *url*, or None on failure. + + Returns None when the HTTP fetch fails or trafilatura yields nothing + useful. Callers should treat None as "no article content available." + """ + try: + async with httpx.AsyncClient(timeout=15.0, follow_redirects=True, headers={ + "User-Agent": "Mozilla/5.0 (compatible; FabledScribe/1.0; +https://fabledsword.com)", + }) as client: + resp = await client.get(url) + resp.raise_for_status() + raw_html = resp.text + except Exception: + logger.debug("Failed to fetch article URL %s", url) + return None + + loop = asyncio.get_event_loop() + try: + import trafilatura + text = await loop.run_in_executor( + None, + lambda: trafilatura.extract( + raw_html, + include_comments=False, + include_tables=True, + favor_recall=True, + ), + ) + return text or None + except Exception: + logger.debug("trafilatura extraction failed for %s", url, exc_info=True) + return None diff --git a/src/fabledassistant/services/embeddings.py b/src/fabledassistant/services/embeddings.py index 8c59101..69643ef 100644 --- a/src/fabledassistant/services/embeddings.py +++ b/src/fabledassistant/services/embeddings.py @@ -1,7 +1,6 @@ """Semantic note search via Ollama embedding model (nomic-embed-text). Embeddings are stored in the note_embeddings table (one row per note). -RSS item embeddings are stored in rss_item_embeddings (one row per item). All search operations degrade gracefully — if the embedding model is unavailable the callers fall back to keyword search. """ @@ -9,7 +8,6 @@ unavailable the callers fall back to keyword search. import asyncio import logging import math -from datetime import datetime, timedelta, timezone import httpx from sqlalchemy import delete, select @@ -18,8 +16,6 @@ from fabledassistant.config import Config from fabledassistant.models import async_session from fabledassistant.models.embedding import NoteEmbedding from fabledassistant.models.note import Note -from fabledassistant.models.rss_feed import RssItem -from fabledassistant.models.rss_item_embedding import RssItemEmbedding logger = logging.getLogger(__name__) @@ -28,10 +24,6 @@ logger = logging.getLogger(__name__) # 0.45 keeps only genuinely relevant notes; lower values like 0.30 let in # loosely-related results that pad the sidebar without adding real value. _SIMILARITY_THRESHOLD = 0.45 -_RSS_SIMILARITY_THRESHOLD = 0.55 -_RSS_SEARCH_LIMIT = 3 -_RSS_SEARCH_DAYS = 30 -_RSS_SNIPPET_CHARS = 500 async def get_embedding(text: str, model: str | None = None) -> list[float]: @@ -186,174 +178,3 @@ async def backfill_note_embeddings() -> None: logger.info("Embedding backfill complete: %d/%d notes embedded", success, len(notes_to_embed)) -# ── RSS item embeddings ─────────────────────────────────────────────────────── - -async def upsert_rss_item_embedding(item_id: int, user_id: int, title: str, content: str) -> None: - """Generate and persist an embedding for an RSS item. Safe to fire-and-forget.""" - text = f"{title}\n{content}".strip() - if not text: - return - try: - embedding = await get_embedding(text) - except Exception: - logger.debug("Skipping embedding for RSS item %d — model unavailable", item_id) - return - - try: - async with async_session() as session: - await session.execute( - delete(RssItemEmbedding).where(RssItemEmbedding.rss_item_id == item_id) - ) - session.add(RssItemEmbedding(rss_item_id=item_id, user_id=user_id, embedding=embedding)) - await session.commit() - logger.debug("Upserted embedding for RSS item %d", item_id) - except Exception: - logger.warning("Failed to persist embedding for RSS item %d", item_id, exc_info=True) - - -async def semantic_search_rss_items( - user_id: int, - query_vector: list[float], - limit: int = _RSS_SEARCH_LIMIT, - days: int = _RSS_SEARCH_DAYS, -) -> list[tuple[float, RssItem]]: - """Return up to *limit* (score, RssItem) pairs most relevant to *query_vector*. - - Only considers items fetched within the last *days* days. - Returns an empty list on any error. - """ - since = datetime.now(timezone.utc) - timedelta(days=days) - try: - async with async_session() as session: - stmt = ( - select(RssItemEmbedding, RssItem) - .join(RssItem, RssItemEmbedding.rss_item_id == RssItem.id) - .where( - RssItemEmbedding.user_id == user_id, - RssItem.fetched_at >= since, - ) - ) - rows = list((await session.execute(stmt)).all()) - except Exception: - logger.warning("Failed to query RSS item embeddings", exc_info=True) - return [] - - if not rows: - return [] - - scored: list[tuple[float, RssItem]] = [] - for rie, item in rows: - try: - sim = _cosine_similarity(query_vector, rie.embedding) - except Exception: - continue - if sim >= _RSS_SIMILARITY_THRESHOLD: - scored.append((sim, item)) - - scored.sort(key=lambda x: x[0], reverse=True) - return scored[:limit] - - -async def backfill_rss_item_embeddings() -> None: - """Generate embeddings for all RSS items that don't have one yet. - - Runs as a background task at startup. Adds a small sleep between items - to avoid overwhelming Ollama. - """ - try: - async with async_session() as session: - existing = { - row[0] - for row in ( - await session.execute(select(RssItemEmbedding.rss_item_id)) - ).fetchall() - } - result = await session.execute( - select(RssItem.id, RssItem.feed_id, RssItem.title, RssItem.content) - ) - items_to_embed = [row for row in result.fetchall() if row[0] not in existing] - except Exception: - logger.warning("RSS embedding backfill: failed to query items", exc_info=True) - return - - if not items_to_embed: - logger.info("RSS embedding backfill: all items already have embeddings") - return - - # Resolve user_id per feed_id - try: - from fabledassistant.models.rss_feed import RssFeed - async with async_session() as session: - result = await session.execute(select(RssFeed.id, RssFeed.user_id)) - feed_user_map = {fid: uid for fid, uid in result.fetchall()} - except Exception: - logger.warning("RSS embedding backfill: failed to load feed user map", exc_info=True) - return - - logger.info("RSS embedding backfill: generating embeddings for %d items", len(items_to_embed)) - success = 0 - for item_id, feed_id, title, content in items_to_embed: - user_id = feed_user_map.get(feed_id) - if user_id is None: - continue - await upsert_rss_item_embedding(item_id, user_id, title or "", content or "") - success += 1 - await asyncio.sleep(0.05) - - logger.info("RSS embedding backfill complete: %d/%d items embedded", success, len(items_to_embed)) - - -async def backfill_rss_article_content() -> None: - """Fetch full article text for RSS items that only have short feed-provided content. - - An item is considered unenriched if its content is shorter than 1000 chars — - typical of feed summaries/teasers rather than full articles. - Runs at startup after the embedding backfill. - """ - from fabledassistant.services.rss import _fetch_full_article - from fabledassistant.models.rss_feed import RssFeed - - SHORT_THRESHOLD = 1000 - - try: - async with async_session() as session: - feed_result = await session.execute(select(RssFeed.id, RssFeed.user_id)) - feed_user_map = {fid: uid for fid, uid in feed_result.fetchall()} - - item_result = await session.execute( - select(RssItem.id, RssItem.feed_id, RssItem.url, RssItem.title, RssItem.content) - .where(RssItem.url != "") - ) - candidates = [ - row for row in item_result.fetchall() - if len(row[4] or "") < SHORT_THRESHOLD - ] - except Exception: - logger.warning("Article content backfill: failed to query items", exc_info=True) - return - - if not candidates: - logger.info("Article content backfill: no unenriched items found") - return - - logger.info("Article content backfill: enriching %d items", len(candidates)) - enriched = 0 - for item_id, feed_id, url, title, _ in candidates: - user_id = feed_user_map.get(feed_id) - if user_id is None: - continue - full_text = await _fetch_full_article(url) - if full_text and len(full_text) > SHORT_THRESHOLD: - try: - async with async_session() as session: - item = await session.get(RssItem, item_id) - if item: - item.content = full_text - await session.commit() - await upsert_rss_item_embedding(item_id, user_id, title or "", full_text) - enriched += 1 - except Exception: - logger.debug("Failed to store enriched content for item %d", item_id, exc_info=True) - await asyncio.sleep(0.5) - - logger.info("Article content backfill complete: %d/%d items enriched", enriched, len(candidates)) diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index 31455c4..057fe43 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -37,84 +37,6 @@ _TOOL_CALL_MARKER = re.compile(r"^\s*\[TOOL_CALLS\]\s*", re.IGNORECASE) DB_FLUSH_INTERVAL = 5.0 # seconds between partial DB flushes -async def _maybe_save_article_discussion_note( - user_id: int, conv_id: int, reply_content: str, -) -> None: - """Persist a seeded article-discussion's first reply as a Note. - - Fires after ``run_generation`` completes. Looks for a synthetic - read_article seed message on the conversation; if found AND the linked - ``rss_items`` row has no ``discussion_note_id`` yet, saves ``reply_content`` - as a Note, tags it, and writes the backlink. Subsequent discuss clicks on - the same article are a no-op (already linked). - - Failures are logged and swallowed — the chat UI should never break because - Note persistence hit a snag. - """ - try: - if not reply_content or not reply_content.strip(): - return - from sqlalchemy import select as _select - from fabledassistant.models.conversation import Message as _Message - from fabledassistant.models.rss_feed import RssItem as _RssItem - from fabledassistant.services.notes import create_note - - async with async_session() as session: - result = await session.execute( - _select(_Message) - .where(_Message.conversation_id == conv_id) - .order_by(_Message.id.asc()) - ) - messages = result.scalars().all() - seed_meta = None - for m in messages: - meta = m.msg_metadata or {} - if meta.get("article_seed") and meta.get("rss_item_id"): - seed_meta = meta - break - if seed_meta is None: - return - item_id = int(seed_meta["rss_item_id"]) - item = await session.get(_RssItem, item_id) - if item is None or item.discussion_note_id is not None: - return - article_title = (item.title or "Untitled article").strip() - article_url = item.url - article_topics = list(item.topics or []) - - note_title = f"Article: {article_title}"[:200] - body_parts = [f"**Source:** {article_url}"] if article_url else [] - body_parts.append(reply_content.strip()) - note_body = "\n\n".join(body_parts) - tags = ["article-summary"] + [t for t in article_topics if t] - note = await create_note( - user_id=user_id, - title=note_title, - body=note_body, - tags=tags, - entity_meta={ - "source": "article_discussion", - "rss_item_id": item_id, - "url": article_url, - "conversation_id": conv_id, - }, - ) - - async with async_session() as session: - fresh = await session.get(_RssItem, item_id) - if fresh is not None and fresh.discussion_note_id is None: - fresh.discussion_note_id = note.id - await session.commit() - logger.info( - "Saved article-discussion summary as note %d for rss_item %d (conv %d)", - note.id, item_id, conv_id, - ) - except Exception: - logger.warning( - "Failed to persist article-discussion note for conv %d", - conv_id, exc_info=True, - ) - # Human-readable labels for each tool, shown in the status indicator _TOOL_LABELS: dict[str, str] = { "create_note": "Creating note/task", @@ -593,28 +515,16 @@ async def run_generation( msg_count = len(non_system) should_gen_title = not conv_title or (msg_count > 0 and msg_count % 10 == 0) - # Persist article-discussion seed conversations as a Note on their - # first assistant reply. This makes "Discuss" summaries part of RAG - # so the knowledge base stops being amnesiac about articles the user - # has already engaged with. The hook detects a seeded conversation by - # finding a synthetic read_article assistant message whose - # msg_metadata carries ``article_seed: True`` and whose rss_items row - # has no discussion_note_id yet. Fire-and-forget so the done event - # lands immediately. - asyncio.create_task(_maybe_save_article_discussion_note( - user_id, conv_id, buf.content_so_far, - )) - if should_gen_title: # Feed the title model the *raw* conversation turns only — never # the post-build_context ``messages`` list. ``build_context`` - # prepends RAG snippets, RSS excerpts, URL content, and briefing - # article dumps INTO the user message string itself, so filtering - # by role="user" downstream still surfaces that noise as the - # "user's message". That pollution caused wildly-wrong titles - # (bug #109) — the small background model was staring at article - # excerpts instead of what the user actually typed. Pass the - # original history + the raw user_content + the assistant reply. + # prepends RAG snippets and URL content INTO the user message + # string itself, so filtering by role="user" downstream still + # surfaces that noise as the "user's message". That pollution + # caused wildly-wrong titles (bug #109) — the small background + # model was staring at article excerpts instead of what the user + # actually typed. Pass the original history + the raw user_content + # + the assistant reply. title_messages: list[dict] = [ {"role": m["role"], "content": m.get("content") or ""} for m in history diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index a5c2bd6..21238c6 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -623,7 +623,7 @@ async def build_context( "search_projects", "create_milestone", "update_milestone", "list_milestones", "save_person", "save_place", "create_list", "add_to_list", "clear_checked_items", "set_rag_scope", "get_profile", "update_profile", "get_weather", "calculate", - "get_rss_items", "add_rss_feed", "read_article", + "read_article", ] if has_caldav: actions.extend(["create_event", "list_events", "search_events", "update_event", "delete_event", "list_calendars"]) @@ -683,8 +683,8 @@ async def build_context( # --- System message: stable content only --- # Workspace context and history summary stay here because they carry # behavioural instructions / conversational state, not retrieved content. - # Everything retrieval-based (RAG notes, RSS, URL content, current note, - # briefing articles) goes into the user turn below so the system message + # Everything retrieval-based (RAG notes, URL content, current note) + # goes into the user turn below so the system message # prefix stays byte-for-byte identical across requests, enabling Ollama's # KV prefix cache to fire reliably. @@ -726,25 +726,6 @@ async def build_context( f"\n\n--- Earlier Conversation ---\n{history_summary}\n--- End Earlier Conversation ---" ) - # Detect briefing conversation — used for both system prompt instruction and article injection - _is_briefing_conv = False - if conv_id is not None: - from fabledassistant.models import async_session as _async_session - from fabledassistant.models.conversation import Conversation as _Conversation - async with _async_session() as _sess: - _conv = await _sess.get(_Conversation, conv_id) - if _conv and getattr(_conv, "conversation_type", None) == "briefing": - _is_briefing_conv = True - - if _is_briefing_conv: - system_content += ( - "\n\nYou are in a briefing conversation. " - "The conversation history contains today's briefing — news stories, weather, and tasks. " - "When the user asks about a topic, person, or event from the briefing, answer directly " - "from the conversation history and the article context that follows. " - "Do NOT search the web for information that is already present in the briefing." - ) - context_meta: dict = { "context_note_id": None, "context_note_title": None, @@ -780,27 +761,18 @@ async def build_context( orphan_only = rag_project_id is None effective_project_id = rag_project_id if (rag_project_id is not None and rag_project_id != -1) else None - # Skip RAG auto-injection on the first turn of a seeded article discussion. - # The article body is already the sole context the user wants — pulling in - # unrelated orphan notes tricks the model into summarizing those instead. - # Follow-up turns keep RAG on because by then the user's own messages drive - # the query rather than the generic seed prompt. - from fabledassistant.services.article_context import ARTICLE_DISCUSS_SEED - _skip_rag_for_article_seed = user_message.strip() == ARTICLE_DISCUSS_SEED + try: + from fabledassistant.services.embeddings import semantic_search_notes + for score, note in await semantic_search_notes( + user_id, user_message, exclude_ids=search_exclude or None, limit=8, + project_id=effective_project_id, + orphan_only=orphan_only, + ): + found_scored.append((score, note)) + except Exception: + logger.warning("Semantic note search failed, falling back to keyword search", exc_info=True) - if not _skip_rag_for_article_seed: - try: - from fabledassistant.services.embeddings import semantic_search_notes - for score, note in await semantic_search_notes( - user_id, user_message, exclude_ids=search_exclude or None, limit=8, - project_id=effective_project_id, - orphan_only=orphan_only, - ): - found_scored.append((score, note)) - except Exception: - logger.warning("Semantic note search failed, falling back to keyword search", exc_info=True) - - if not found_scored and not _skip_rag_for_article_seed: + if not found_scored: keywords = _extract_keywords(user_message) if keywords: try: @@ -890,12 +862,6 @@ async def build_context( f"--- Content from {url} ---\n{content}\n--- End URL Content ---" ) - # Briefing article context for follow-up Q&A - if _is_briefing_conv: - article_context = await _build_briefing_article_context(conv_id) # type: ignore[arg-type] - if article_context: - user_context_parts.append(article_context.strip()) - # Build final user message — context prefix (if any) followed by the actual message if user_context_parts: user_turn = "\n\n".join(user_context_parts) + "\n\n" + user_message @@ -906,72 +872,3 @@ async def build_context( messages.extend(history) messages.append({"role": "user", "content": user_turn}) return messages, context_meta - - -async def _build_briefing_article_context(conv_id: int) -> str: - """Fetch article content from today's briefing message and return a - formatted context block for injection into the system prompt. - - Looks at the most recent assistant briefing messages for rss_item_ids - in their metadata, then loads those items from the DB. - Capped at 10 articles × 500 chars to keep token use reasonable. - """ - import json as _json - - from sqlalchemy import select, text as _text - - from fabledassistant.models import async_session as _async_session - from fabledassistant.models.conversation import Message - - async with _async_session() as session: - result = await session.execute( - select(Message) - .where( - Message.conversation_id == conv_id, - Message.role == "assistant", - ) - .order_by(Message.created_at.desc()) - .limit(10) - ) - messages = result.scalars().all() - - rss_item_ids: list[int] = [] - for msg in messages: - meta = msg.msg_metadata or {} - if isinstance(meta, str): - try: - meta = _json.loads(meta) - except Exception: - continue - ids = meta.get("rss_item_ids") or [] - if ids: - rss_item_ids = ids - break - - if not rss_item_ids: - return "" - - async with _async_session() as session: - result = await session.execute( - _text(""" - SELECT i.title, i.url, i.content, f.title AS feed_title - FROM rss_items i - JOIN rss_feeds f ON f.id = i.feed_id - WHERE i.id = ANY(:ids) - ORDER BY i.published_at DESC NULLS LAST - LIMIT 10 - """).bindparams(ids=rss_item_ids[:10]) - ) - rows = result.mappings().all() - - if not rows: - return "" - - lines = ["\n\nARTICLE CONTEXT (source articles from today's briefing):"] - for row in rows: - lines.append(f"\n[{row['feed_title']}] {row['title']}") - if row["url"]: - lines.append(f"URL: {row['url']}") - if row["content"]: - lines.append(row["content"][:500]) - return "\n".join(lines) diff --git a/src/fabledassistant/services/rss.py b/src/fabledassistant/services/rss.py deleted file mode 100644 index 1785709..0000000 --- a/src/fabledassistant/services/rss.py +++ /dev/null @@ -1,317 +0,0 @@ -"""RSS feed service: fetch, parse with feedparser, and cache items to DB.""" - -import asyncio -import logging -from datetime import datetime, timezone - -import feedparser -import html2text -import httpx -from sqlalchemy import select, text - -from fabledassistant.models import async_session -from fabledassistant.models.rss_feed import RssFeed, RssItem - -logger = logging.getLogger(__name__) - -# Keep only items from the last N days to avoid unbounded growth -ITEM_MAX_AGE_DAYS = 90 - -_h2t = html2text.HTML2Text() -_h2t.ignore_links = True -_h2t.ignore_images = True -_h2t.ignore_emphasis = False -_h2t.body_width = 0 # No line-wrapping - - -def _html_to_text(html: str) -> str: - """Convert HTML to clean plain text via html2text.""" - if not html or "<" not in html: - return html - try: - return _h2t.handle(html).strip() - except Exception: - return html - - -async def get_or_fetch_full_article(item: RssItem) -> str | None: - """Return the full article body, fetching+caching on miss. - - Checks ``item.content_full`` first — populated either by the enrichment - pass at feed-ingest time or by a previous discuss-click. On miss, fetches - via ``_fetch_full_article`` and writes through. Returns ``None`` only if - the fetch itself fails; ``item.content_full == ""`` is still a cache hit. - - Callers must pass an RssItem attached to an open session if they want - the write-through to persist — otherwise the fetched text is returned - but the cache stays empty and the next click will re-fetch. - """ - if item.content_full is not None: - return item.content_full - if not item.url: - return None - text = await _fetch_full_article(item.url) - if text is None: - return None - async with async_session() as session: - fresh = await session.get(RssItem, item.id) - if fresh is not None: - fresh.content_full = text - fresh.content_fetched_at = datetime.now(timezone.utc) - await session.commit() - return text - - -async def _fetch_full_article(url: str) -> str | None: - """Fetch a URL and extract its main article text via trafilatura. - - Returns clean plain text, or None if extraction fails or yields nothing useful. - Runs trafilatura in a thread executor since it does synchronous HTML parsing. - """ - try: - async with httpx.AsyncClient(timeout=15.0, follow_redirects=True, headers={ - "User-Agent": "Mozilla/5.0 (compatible; FabledAssistant/1.0; +https://fabledsword.com)", - }) as client: - resp = await client.get(url) - resp.raise_for_status() - raw_html = resp.text - except Exception: - logger.debug("Failed to fetch article URL %s", url) - return None - - loop = asyncio.get_event_loop() - try: - import trafilatura - text = await loop.run_in_executor( - None, - lambda: trafilatura.extract( - raw_html, - include_comments=False, - include_tables=True, - favor_recall=True, - ), - ) - return text or None - except Exception: - logger.debug("trafilatura extraction failed for %s", url, exc_info=True) - return None - - -def extract_item(entry) -> dict: - """Extract a clean item dict from a feedparser entry object.""" - # Prefer full content over summary (feedparser uses a list of Content objects) - content = "" - raw_content = getattr(entry, "content", None) - if isinstance(raw_content, list) and raw_content: - content = raw_content[0].value - else: - content = entry.get("summary", "") - content = _html_to_text(content) - - pub = None - if entry.published_parsed: - try: - pub = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc) - except Exception: - pass - - return { - "guid": entry.get("id", entry.get("link", "")), - "title": entry.get("title", ""), - "url": entry.get("link", ""), - "content": content, - "published_at": pub, - } - - -async def _parse_feed(url: str) -> feedparser.FeedParserDict: - """Run feedparser in a thread executor so we don't block the event loop.""" - loop = asyncio.get_event_loop() - return await loop.run_in_executor(None, feedparser.parse, url) - - -async def fetch_and_cache_feed(feed_id: int, url: str) -> int: - """ - Fetch a feed URL, parse it, and upsert new items into rss_items. - Returns the number of new items stored. - """ - scheme = url.split("://")[0].lower() if "://" in url else "" - if scheme not in ("http", "https"): - logger.warning("Blocked RSS fetch with non-http(s) scheme: %s", url[:80]) - return 0 - try: - parsed = await _parse_feed(url) - except Exception: - logger.warning("Failed to fetch RSS feed %s", url, exc_info=True) - return 0 - - if parsed.bozo and not parsed.entries: - logger.warning("Malformed RSS feed %s: %s", url, parsed.bozo_exception) - return 0 - - new_count = 0 - feed_user_id: int | None = None - - async with async_session() as session: - for entry in parsed.entries: - item_data = extract_item(entry) - if not item_data["guid"]: - continue - # Check if already stored - existing = await session.execute( - select(RssItem).where( - RssItem.feed_id == feed_id, - RssItem.guid == item_data["guid"], - ) - ) - if existing.scalars().first() is not None: - continue - item = RssItem( - feed_id=feed_id, - **item_data, - ) - session.add(item) - new_count += 1 - - # Update last_fetched_at on the feed - feed_row = await session.get(RssFeed, feed_id) - if feed_row: - feed_row.last_fetched_at = datetime.now(timezone.utc) - feed_user_id = feed_row.user_id - # Auto-populate title from feed metadata if blank - if not feed_row.title and parsed.feed.get("title"): - feed_row.title = parsed.feed.title[:200] - - await session.commit() - - # Collect IDs of unclassified items after commit. - # We query classified_at IS NULL (not just the items inserted above) because - # classification is best-effort and may have failed on previous fetches. - # Re-queuing all unclassified items for this feed on each fetch is intentional: - # it provides automatic retry without a separate retry loop. The classifier - # only writes to items it successfully classifies, so already-classified items - # are not re-processed (they have classified_at set). - unclassified_ids: list[int] = [] - new_item_data: list[tuple[int, str, str]] = [] # (id, title, content) for embedding - if new_count > 0: - result = await session.execute( - select(RssItem.id, RssItem.title, RssItem.content, RssItem.classified_at).where( - RssItem.feed_id == feed_id, - ) - ) - for row in result.fetchall(): - item_id, title, content, classified_at = row - if classified_at is None: - unclassified_ids.append(item_id) - new_item_data.append((item_id, title or "", content or "")) - - # Prune old items to keep DB tidy - await _prune_old_items(feed_id) - - # Fire-and-forget classification for unclassified items - if unclassified_ids and feed_user_id is not None: - from fabledassistant.services.rss_classifier import classify_and_store - asyncio.create_task(classify_and_store(unclassified_ids, feed_user_id)) - - # Collect (id, url) for newly inserted items to enrich with full article text - new_items_for_enrichment: list[tuple[int, str]] = [] - if new_count > 0: - async with async_session() as session: - result = await session.execute( - select(RssItem.id, RssItem.url).where( - RssItem.feed_id == feed_id, - RssItem.url != "", - ).order_by(RssItem.fetched_at.desc()).limit(new_count) - ) - new_items_for_enrichment = list(result.fetchall()) - - # Fire-and-forget: fetch full article text, then re-embed with richer content - if new_items_for_enrichment and feed_user_id is not None: - from fabledassistant.services.embeddings import upsert_rss_item_embedding - - async def _enrich_and_embed() -> None: - for item_id, article_url in new_items_for_enrichment: - full_text = await _fetch_full_article(article_url) - if full_text and len(full_text) > 200: - async with async_session() as session: - item = await session.get(RssItem, item_id) - if item: - item.content = full_text - # Populate the discuss-click cache too so the - # first click skips straight to the map-reduce - # step without re-fetching. - item.content_full = full_text - item.content_fetched_at = datetime.now(timezone.utc) - await session.commit() - await upsert_rss_item_embedding( - item_id, feed_user_id, item.title or "", item.content - ) - else: - # Enrich failed — still embed with RSS-provided content - async with async_session() as session: - item = await session.get(RssItem, item_id) - if item: - await upsert_rss_item_embedding( - item_id, feed_user_id, item.title or "", item.content or "" - ) - await asyncio.sleep(0.5) # Polite pacing between article fetches - - asyncio.create_task(_enrich_and_embed()) - elif new_item_data and feed_user_id is not None: - # No URLs to enrich — embed with RSS content only - from fabledassistant.services.embeddings import upsert_rss_item_embedding - - async def _embed_only() -> None: - for item_id, title, content in new_item_data: - await upsert_rss_item_embedding(item_id, feed_user_id, title, content) - await asyncio.sleep(0.05) - - asyncio.create_task(_embed_only()) - - return new_count - - -async def _prune_old_items(feed_id: int) -> None: - """Delete items older than ITEM_MAX_AGE_DAYS from a feed.""" - async with async_session() as session: - await session.execute( - text(""" - DELETE FROM rss_items - WHERE feed_id = :feed_id - AND published_at < NOW() - INTERVAL '90 days' - """).bindparams(feed_id=feed_id) - ) - await session.commit() - - -async def get_recent_items(user_id: int, limit: int = 20) -> list[dict]: - """Return the most recent RSS items across all of a user's feeds.""" - async with async_session() as session: - result = await session.execute( - select(RssItem, RssFeed.title.label("feed_title")) - .join(RssFeed, RssItem.feed_id == RssFeed.id) - .where(RssFeed.user_id == user_id) - .order_by(RssItem.published_at.desc().nullslast()) - .limit(limit) - ) - rows = result.all() - - return [ - {**item.to_dict(), "feed_title": feed_title} - for item, feed_title in rows - ] - - -async def refresh_all_feeds(user_id: int) -> dict[int, int]: - """Fetch all feeds for a user. Returns {feed_id: new_items_count}.""" - async with async_session() as session: - result = await session.execute( - select(RssFeed).where(RssFeed.user_id == user_id) - ) - feeds = list(result.scalars().all()) - - results = {} - for feed in feeds: - count = await fetch_and_cache_feed(feed.id, feed.url) - results[feed.id] = count - return results diff --git a/src/fabledassistant/services/rss_classifier.py b/src/fabledassistant/services/rss_classifier.py deleted file mode 100644 index 6ddbb9b..0000000 --- a/src/fabledassistant/services/rss_classifier.py +++ /dev/null @@ -1,152 +0,0 @@ -""" -RSS item topic classifier. - -Classifies RSS items into topic tags using a fast non-streaming LLM call. -Called from rss.py after new items are stored — fire-and-forget. -""" - -import json -import logging -import re -from datetime import datetime, timezone - -import httpx - -from fabledassistant.config import Config - -logger = logging.getLogger(__name__) - -STANDARD_TOPICS = [ - "technology", "science", "politics", "business", - "health", "environment", "local", "entertainment", "sports", "other", -] - -_CLASSIFY_PROMPT = """\ -Classify each news item into 1-3 topics. Use only topics from this list: {vocab}. -Return ONLY a JSON object mapping item_id (as string) to a list of topics. -Example: {{"1": ["technology", "ai"], "2": ["politics"]}} - -Items: -{items_block}""" - - -async def _llm_classify(prompt: str, model: str) -> str: - """Make a fast non-streaming LLM call and return the raw text response.""" - payload = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "stream": False, - "options": {"num_ctx": 2048, "temperature": 0.0}, - } - async with httpx.AsyncClient(timeout=120.0) as client: - resp = await client.post(f"{Config.OLLAMA_URL}/api/chat", json=payload) - resp.raise_for_status() - return resp.json().get("message", {}).get("content", "") - - -async def classify_items_batch( - items: list[dict], - user_include_topics: list[str], - model: str | None = None, -) -> dict[int, list[str]]: - """ - Classify a batch of RSS items into topic tags. - - Args: - items: list of dicts with 'id', 'title', 'content' - user_include_topics: extra topics from user preferences to add to vocabulary - model: Ollama model name; defaults to Config.OLLAMA_MODEL - - Returns: - dict mapping item_id (int) -> list of topic strings. - Items not returned had classification fail; callers should leave classified_at=NULL. - """ - if not items: - return {} - - if model is None: - model = Config.OLLAMA_BACKGROUND_MODEL - - vocab = STANDARD_TOPICS + [t for t in user_include_topics if t not in STANDARD_TOPICS] - items_block = "\n".join( - f"[{item['id']}] {item['title']} — {item.get('content', '')[:300]}" - for item in items - ) - prompt = _CLASSIFY_PROMPT.format(vocab=", ".join(vocab), items_block=items_block) - - try: - raw = await _llm_classify(prompt, model) - # Strip ... blocks emitted by reasoning models (e.g. qwen3) - raw = re.sub(r".*?", "", raw, flags=re.DOTALL) - # Extract JSON from response (LLM may wrap it in markdown) - raw = raw.strip() - if raw.startswith("```"): - raw = raw.split("```")[1] - if raw.startswith("json"): - raw = raw[4:] - raw = raw.strip() - # Allow control characters that may appear in LLM-generated JSON strings - parsed = json.loads(raw, strict=False) - return {int(k): v for k, v in parsed.items() if isinstance(v, list)} - except Exception: - logger.warning("RSS classification failed", exc_info=True) - return {} - - -async def classify_and_store( - item_ids: list[int], - user_id: int, -) -> None: - """ - Classify unclassified RSS items and write results to DB. - Called as a fire-and-forget task from rss.py. - """ - from sqlalchemy import select - - from fabledassistant.models import async_session - from fabledassistant.models.rss_feed import RssItem - from fabledassistant.services.settings import get_setting - - if not item_ids: - return - - # Load the items - async with async_session() as session: - result = await session.execute( - select(RssItem).where(RssItem.id.in_(item_ids)) - ) - items = list(result.scalars().all()) - - if not items: - return - - # Get user's include topics to extend vocabulary - raw_include = await get_setting(user_id, "briefing_include_topics", "[]") - try: - include_topics = json.loads(raw_include) if isinstance(raw_include, str) else [] - except Exception: - include_topics = [] - - model = await get_setting(user_id, "background_model", Config.OLLAMA_BACKGROUND_MODEL) - - # Classify in batches of 10 - batch_size = 10 - all_results: dict[int, list[str]] = {} - for i in range(0, len(items), batch_size): - batch = items[i: i + batch_size] - batch_dicts = [{"id": it.id, "title": it.title, "content": it.content} for it in batch] - results = await classify_items_batch(batch_dicts, include_topics, model=model) - all_results.update(results) - - # Write back to DB - now = datetime.now(timezone.utc) - async with async_session() as session: - for item in items: - item_db = await session.get(RssItem, item.id) - if item_db is None: - continue - topics = all_results.get(item.id) - if topics is not None: - item_db.topics = topics - item_db.classified_at = now - await session.commit() diff --git a/src/fabledassistant/services/rss_filtering.py b/src/fabledassistant/services/rss_filtering.py deleted file mode 100644 index 9a142ac..0000000 --- a/src/fabledassistant/services/rss_filtering.py +++ /dev/null @@ -1,110 +0,0 @@ -""" -Briefing preferences: load topic settings, aggregate reaction scores, -filter and rank RSS items for briefing inclusion. -""" - -import json -import logging -from datetime import datetime, timezone - -from fabledassistant.models import async_session - -logger = logging.getLogger(__name__) - - -async def load_topic_preferences(user_id: int) -> tuple[list[str], list[str]]: - """ - Return (include_topics, exclude_topics) from user settings. - """ - from fabledassistant.services.settings import get_setting - - raw_include = await get_setting(user_id, "briefing_include_topics", "[]") - raw_exclude = await get_setting(user_id, "briefing_exclude_topics", "[]") - - def _parse(raw) -> list[str]: - try: - val = json.loads(raw) if isinstance(raw, str) else raw - return [str(t) for t in val] if isinstance(val, list) else [] - except Exception: - return [] - - return _parse(raw_include), _parse(raw_exclude) - - -async def load_topic_reaction_scores(user_id: int) -> dict[str, float]: - """ - Aggregate per-topic reaction scores from the last 30 days. - Returns a dict of topic -> net_score (positive = liked, negative = disliked). - Uses rss_item_reactions joined to rss_items.topics. - """ - try: - from sqlalchemy import text as _text - - async with async_session() as session: - result = await session.execute( - _text(""" - SELECT unnest(i.topics) AS topic, - SUM(CASE r.reaction WHEN 'up' THEN 1 ELSE -1 END) AS score - FROM rss_item_reactions r - JOIN rss_items i ON i.id = r.rss_item_id - WHERE r.user_id = :uid - AND r.created_at > NOW() - INTERVAL '30 days' - GROUP BY topic - """).bindparams(uid=user_id) - ) - return {row.topic: float(row.score) for row in result} - except Exception: - logger.warning("Failed to load topic reaction scores", exc_info=True) - return {} - - -def score_and_filter_items( - items: list[dict], - include_topics: list[str], - exclude_topics: list[str], - topic_scores: dict[str, float], - max_items: int = 10, -) -> list[dict]: - """ - Score, filter, and rank RSS items for briefing inclusion. - - Scoring: - - Hard-exclude: any item tagged with an excluded topic is removed. - - Base score: 0.0 - - +2.0 per topic that appears in include_topics - - +1.0 / -1.0 per topic based on reaction score (clamped per topic) - - Tiebreak: newer published_at wins - - Returns up to max_items items, highest score first. - Items with classified_at=None (unclassified) pass through with score=0. - """ - include_set = set(include_topics) - exclude_set = set(exclude_topics) - scored = [] - - for item in items: - item_topics = item.get("topics") or [] - - # Hard exclude - if exclude_set and any(t in exclude_set for t in item_topics): - continue - - score = 0.0 - for topic in item_topics: - if topic in include_set: - score += 2.0 - if topic in topic_scores: - score += max(-1.0, min(1.0, topic_scores[topic])) - - # Parse published_at for tiebreak - pub_str = item.get("published_at") or "" - try: - pub_ts = datetime.fromisoformat(pub_str).timestamp() if pub_str else 0.0 - except ValueError: - pub_ts = 0.0 - - scored.append((score, pub_ts, item)) - - # Sort: highest score first, then newest first - scored.sort(key=lambda x: (x[0], x[1]), reverse=True) - return [item for _, _, item in scored[:max_items]] diff --git a/src/fabledassistant/services/tools/__init__.py b/src/fabledassistant/services/tools/__init__.py index b5a32ae..6d6b4e4 100644 --- a/src/fabledassistant/services/tools/__init__.py +++ b/src/fabledassistant/services/tools/__init__.py @@ -8,6 +8,7 @@ of the app depends on. # Import every tool module so their @tool decorators run at import time. # Order does not matter — registration is additive. from fabledassistant.services.tools import ( # noqa: F401 + article, calendar, entities, journal, @@ -15,7 +16,6 @@ from fabledassistant.services.tools import ( # noqa: F401 profile, projects, rag, - rss, tasks, utility, weather, diff --git a/src/fabledassistant/services/tools/_registry.py b/src/fabledassistant/services/tools/_registry.py index 021cca6..1c4531e 100644 --- a/src/fabledassistant/services/tools/_registry.py +++ b/src/fabledassistant/services/tools/_registry.py @@ -87,9 +87,6 @@ async def _check_requires(user_id: int, requires: str) -> bool: return await is_caldav_configured(user_id) if requires == "searxng": return Config.searxng_enabled() - if requires == "rss": - from fabledassistant.services.settings import get_setting - return (await get_setting(user_id, "rss_enabled", "false")).lower() == "true" return True diff --git a/src/fabledassistant/services/tools/article.py b/src/fabledassistant/services/tools/article.py new file mode 100644 index 0000000..9a97f2d --- /dev/null +++ b/src/fabledassistant/services/tools/article.py @@ -0,0 +1,35 @@ +"""Generic article-reading LLM tool. + +The ``read_article`` tool fetches any URL and returns its main body text via +trafilatura. URL-generic — not coupled to any feed system. +""" +from __future__ import annotations + +from fabledassistant.services.article_fetcher import fetch_article_text +from fabledassistant.services.tools._registry import tool + + +@tool( + name="read_article", + description=( + "Fetch the main body text of an article at a URL. Use when the user asks " + "to read, summarize, or discuss a specific article they've linked. " + "Returns the cleaned article text or an empty result if extraction fails." + ), + parameters={ + "url": { + "type": "string", + "description": "The article URL to fetch.", + }, + }, + required=["url"], + read_only=True, +) +async def read_article_tool(*, user_id, arguments, **_ctx): + url = (arguments.get("url") or "").strip() + if not url: + return {"success": False, "error": "url is required"} + content = await fetch_article_text(url) + if not content: + return {"success": True, "type": "article", "data": {"url": url, "content": None, "note": "no content extracted"}} + return {"success": True, "type": "article", "data": {"url": url, "content": content[:6000]}} diff --git a/src/fabledassistant/services/tools/rss.py b/src/fabledassistant/services/tools/rss.py deleted file mode 100644 index 292a4be..0000000 --- a/src/fabledassistant/services/tools/rss.py +++ /dev/null @@ -1,101 +0,0 @@ -"""RSS and article tools.""" - -from __future__ import annotations - -import logging - -from fabledassistant.services.tools._registry import tool - -logger = logging.getLogger(__name__) - - -@tool( - name="get_rss_items", - description="Get recent items from the user's RSS feeds (news, blogs, Reddit, podcasts). Returns titles, URLs, and summaries of recent posts.", - parameters={ - "limit": {"type": "integer", "description": "Number of items to return (default 15, max 50)"}, - "category": {"type": "string", "description": "Filter by feed category (e.g. 'news', 'tech'). Omit for all."}, - }, - read_only=True, - briefing=True, - requires="rss", -) -async def get_rss_items_tool(*, user_id, arguments, **_ctx): - from fabledassistant.services.rss import get_recent_items - - limit = min(int(arguments.get("limit", 15)), 50) - items = await get_recent_items(user_id, limit=limit) - return {"data": {"items": items, "count": len(items)}} - - -@tool( - name="add_rss_feed", - description="Add an RSS/Atom feed. Use when user asks to subscribe to or track a feed, blog, subreddit, or podcast.", - parameters={ - "url": {"type": "string", "description": "The RSS/Atom feed URL to add."}, - "category": {"type": "string", "description": "Optional category label (e.g. 'news', 'tech', 'reddit'). Omit if unsure."}, - }, - required=["url"], - requires="rss", -) -async def add_rss_feed_tool(*, user_id, arguments, **_ctx): - import asyncio as _asyncio - - from sqlalchemy import select as _select - - from fabledassistant.models import async_session as _async_session - from fabledassistant.models.rss_feed import RssFeed - from fabledassistant.services.rss import fetch_and_cache_feed - - url = str(arguments.get("url", "")).strip() - if not url: - return {"error": "url is required"} - category = arguments.get("category") or None - async with _async_session() as session: - existing = await session.execute( - _select(RssFeed).where(RssFeed.user_id == user_id, RssFeed.url == url) - ) - if existing.scalars().first(): - return {"error": "Feed already added", "url": url} - feed = RssFeed(user_id=user_id, url=url, title="", category=category) - session.add(feed) - await session.commit() - await session.refresh(feed) - feed_id = feed.id - _asyncio.create_task(fetch_and_cache_feed(feed_id, url)) - return {"data": {"id": feed_id, "url": url, "message": "Feed added and fetching started."}} - - -@tool( - name="read_article", - description=( - "Fetch and read the full text of a web page or article from a URL. " - "Use when the user shares a URL and wants you to read it, " - "or to get the full content of a linked page. " - "Do NOT use lookup for URLs — use this tool instead." - ), - parameters={ - "url": {"type": "string", "description": "The URL to fetch and read"}, - }, - required=["url"], - read_only=True, - briefing=True, -) -async def read_article_tool(*, user_id, arguments, **_ctx): - from fabledassistant.services.rss import _fetch_full_article - - url = arguments.get("url", "").strip() - if not url: - return {"success": False, "error": "No URL provided"} - content = await _fetch_full_article(url) - if not content: - return {"success": False, "error": f"Could not fetch article content from {url}"} - _TOOL_CONTENT_CAP = 40_000 - truncated = len(content) > _TOOL_CONTENT_CAP - return { - "success": True, - "type": "article_content", - "url": url, - "content": content[:_TOOL_CONTENT_CAP], - "truncated": truncated, - } diff --git a/src/fabledassistant/services/tools/web.py b/src/fabledassistant/services/tools/web.py index 3b9c249..8a2652e 100644 --- a/src/fabledassistant/services/tools/web.py +++ b/src/fabledassistant/services/tools/web.py @@ -70,14 +70,14 @@ async def lookup_tool(*, user_id, arguments, **_ctx): if search_results: # Sequential fetches: trafilatura/lxml is not safe to run concurrently # via run_in_executor — parallel calls can trip a libxml2 double-free. - from fabledassistant.services.rss import _fetch_full_article + from fabledassistant.services.article_fetcher import fetch_article_text for r in search_results[:2]: url = r.get("url", "") if not url: continue try: - content = await _fetch_full_article(url) + content = await fetch_article_text(url) except Exception: content = None web_payload.append({ diff --git a/tests/test_article_reading.py b/tests/test_article_reading.py deleted file mode 100644 index 715013b..0000000 --- a/tests/test_article_reading.py +++ /dev/null @@ -1,216 +0,0 @@ -import json -import pytest -from unittest.mock import patch, AsyncMock - - -# --------------------------------------------------------------------------- -# read_article tool tests -# --------------------------------------------------------------------------- - -@pytest.mark.asyncio -async def test_read_article_success(): - """read_article returns success with content when _fetch_full_article succeeds.""" - from fabledassistant.services.tools import execute_tool - - with patch( - "fabledassistant.services.rss._fetch_full_article", - new_callable=AsyncMock, - return_value="Article body text here.", - ): - result = await execute_tool(user_id=1, tool_name="read_article", arguments={"url": "https://example.com/article"}) - - assert result["success"] is True - assert result["type"] == "article_content" - assert result["url"] == "https://example.com/article" - assert result["content"] == "Article body text here." - assert result["truncated"] is False - - -@pytest.mark.asyncio -async def test_read_article_fetch_failure(): - """read_article returns success=False when _fetch_full_article returns None.""" - from fabledassistant.services.tools import execute_tool - - with patch( - "fabledassistant.services.rss._fetch_full_article", - new_callable=AsyncMock, - return_value=None, - ): - result = await execute_tool(user_id=1, tool_name="read_article", arguments={"url": "https://example.com/broken"}) - - assert result["success"] is False - assert "Could not fetch" in result["error"] - - -@pytest.mark.asyncio -async def test_read_article_truncates_at_40k(): - """read_article truncates content at 40,000 chars and sets truncated=True.""" - from fabledassistant.services.tools import execute_tool - - long_content = "x" * 50_000 - - with patch( - "fabledassistant.services.rss._fetch_full_article", - new_callable=AsyncMock, - return_value=long_content, - ): - result = await execute_tool(user_id=1, tool_name="read_article", arguments={"url": "https://example.com/long"}) - - assert result["success"] is True - assert len(result["content"]) == 40_000 - assert result["truncated"] is True - - -@pytest.mark.asyncio -async def test_read_article_empty_url(): - """read_article returns success=False when URL is empty.""" - from fabledassistant.services.tools import execute_tool - - result = await execute_tool(user_id=1, tool_name="read_article", arguments={"url": ""}) - assert result["success"] is False - assert "No URL provided" in result["error"] - - -# --------------------------------------------------------------------------- -# History builder tests -# --------------------------------------------------------------------------- - -def _build_history(messages: list[dict]) -> list[dict]: - """Replicate the fixed history builder from routes/chat.py.""" - history = [] - for msg in messages: - if msg["role"] == "system": - continue - msg_dict = {"role": msg["role"], "content": msg.get("content") or ""} - tool_calls = msg.get("tool_calls") - if tool_calls: - msg_dict["tool_calls"] = [ - {"function": {"name": tc["function"], "arguments": tc["arguments"]}} - for tc in tool_calls - ] - history.append(msg_dict) - for tc in tool_calls: - history.append({"role": "tool", "content": json.dumps(tc.get("result", {}))}) - else: - history.append(msg_dict) - return history - - -def test_history_builder_replays_tool_calls(): - """History builder with tool_calls produces assistant entry + tool result entry.""" - messages = [ - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "function": "read_article", - "arguments": {"url": "https://example.com"}, - "result": {"success": True, "content": "Article text"}, - } - ], - }, - {"role": "user", "content": "Summarize it", "tool_calls": None}, - ] - history = _build_history(messages) - - assert len(history) == 3 - assert history[0]["role"] == "assistant" - assert history[0]["tool_calls"][0]["function"]["name"] == "read_article" - assert history[1]["role"] == "tool" - assert json.loads(history[1]["content"])["success"] is True - assert history[2]["role"] == "user" - assert history[2]["content"] == "Summarize it" - - -def test_history_builder_no_tool_calls_unchanged(): - """History builder with tool_calls=None produces same output as before.""" - messages = [ - {"role": "user", "content": "Hello", "tool_calls": None}, - {"role": "assistant", "content": "Hi there!", "tool_calls": None}, - ] - history = _build_history(messages) - - assert len(history) == 2 - assert history[0] == {"role": "user", "content": "Hello"} - assert history[1] == {"role": "assistant", "content": "Hi there!"} - - -# --------------------------------------------------------------------------- -# prepare_article_context tests -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_prepare_article_context_small_passthrough(): - """Articles under CHAR_BUDGET pass through unchanged with zero LLM calls.""" - from fabledassistant.services import article_context - - body = "A short article.\n\nWith two paragraphs." - with patch( - "fabledassistant.services.article_context.generate_completion", - new_callable=AsyncMock, - ) as mock_gen: - out = await article_context.prepare_article_context( - "Title", "https://example.com", body, "test-model", - ) - - assert out == body - mock_gen.assert_not_called() - - -@pytest.mark.asyncio -async def test_prepare_article_context_large_runs_map_reduce(): - """Articles over CHAR_BUDGET are chunked and map-reduced via the background model.""" - from fabledassistant.services import article_context - - # CHAR_BUDGET is 48_000 — build a body well over that with paragraph breaks - # so the chunker has natural splits to work with. - paragraph = ("Lorem ipsum dolor sit amet, consectetur adipiscing elit. " * 40).strip() - body = "\n\n".join([paragraph] * 30) # ~70k+ chars across 30 paragraphs - assert len(body) > article_context.CHAR_BUDGET - - with patch( - "fabledassistant.services.article_context.generate_completion", - new_callable=AsyncMock, - return_value="Summary of this section with specific claims preserved.", - ) as mock_gen: - out = await article_context.prepare_article_context( - "Long Article", "https://example.com/long", body, "test-model", - ) - - # At least one LLM call fired (the map step ran) - assert mock_gen.await_count >= 1 - # Output carries the oversized-article header and section markers - assert "longer than the chat window" in out - assert "## Section 1" in out - # Map output is much smaller than the raw body - assert len(out) < len(body) - - -def test_chunk_by_paragraph_respects_boundaries(): - """Chunker splits on paragraph breaks, not mid-sentence.""" - from fabledassistant.services.article_context import _chunk_by_paragraph, CHUNK_CHARS - - paragraphs = [f"Paragraph {i}. " + ("x" * 1000) for i in range(20)] - body = "\n\n".join(paragraphs) - - chunks = _chunk_by_paragraph(body) - - # Each chunk stays under the budget - for chunk in chunks: - assert len(chunk) <= CHUNK_CHARS - # Total content is preserved (modulo overlap duplication, so ≥ original) - assert sum(len(c) for c in chunks) >= len(body) * 0.9 - - -def test_chunk_by_paragraph_handles_oversized_paragraph(): - """A single paragraph larger than CHUNK_CHARS gets split mid-paragraph.""" - from fabledassistant.services.article_context import _chunk_by_paragraph, CHUNK_CHARS - - body = "x" * (CHUNK_CHARS * 3) # one huge paragraph, no breaks - chunks = _chunk_by_paragraph(body) - - assert len(chunks) >= 3 - for chunk in chunks: - assert len(chunk) <= CHUNK_CHARS diff --git a/tests/test_news_api.py b/tests/test_news_api.py deleted file mode 100644 index 533fc64..0000000 --- a/tests/test_news_api.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Tests for news API — retention constant and endpoint formatting.""" -import pytest -from unittest.mock import AsyncMock, MagicMock -from datetime import datetime, timezone - - -def test_rss_item_max_age_is_90_days(): - """Retention window should be 90 days.""" - from fabledassistant.services.rss import ITEM_MAX_AGE_DAYS - - assert ITEM_MAX_AGE_DAYS == 90 - - -def _make_mock_session(rows): - """Return a mock async context manager whose session.execute returns rows.""" - mock_result = MagicMock() - mock_result.mappings.return_value.all.return_value = rows - - mock_session = AsyncMock() - mock_session.execute = AsyncMock(return_value=mock_result) - - mock_cm = MagicMock() - mock_cm.__aenter__ = AsyncMock(return_value=mock_session) - mock_cm.__aexit__ = AsyncMock(return_value=False) - return mock_cm - - -def test_list_news_item_serialisation(): - """News item serialisation should truncate content to 300 chars and include reaction.""" - pub = datetime(2026, 3, 28, 8, 0, tzinfo=timezone.utc) - item = { - "id": 1, - "title": "EU AI Act deadline", - "url": "https://example.com/ai", - "content": "x" * 500, - "published_at": pub, - "topics": ["tech"], - "feed_title": "TechCrunch", - "reaction": "up", - } - - result = { - "id": item["id"], - "title": item["title"], - "url": item["url"], - "snippet": (item["content"] or "")[:300], - "published_at": item["published_at"].isoformat() if item["published_at"] else None, - "topics": item["topics"] or [], - "source": item["feed_title"], - "reaction": item["reaction"], - } - - assert result["snippet"] == "x" * 300 - assert result["source"] == "TechCrunch" - assert result["reaction"] == "up" - assert result["published_at"] == pub.isoformat() - - - -def test_build_briefing_article_context_metadata_extraction(): - """Verify the rss_item_ids extraction logic from message metadata.""" - import json - - # Simulates the logic in _build_briefing_article_context - def extract_ids(msg_metadata): - meta = msg_metadata or {} - if isinstance(meta, str): - try: - meta = json.loads(meta) - except Exception: - return [] - return meta.get("rss_item_ids") or [] - - assert extract_ids({}) == [] - assert extract_ids(None) == [] - assert extract_ids({"rss_item_ids": [1, 2, 3]}) == [1, 2, 3] - assert extract_ids(json.dumps({"rss_item_ids": [4, 5]})) == [4, 5] - assert extract_ids("not json") == [] - - -def test_list_news_null_content_serialisation(): - """Null content should produce empty snippet without error.""" - item = { - "id": 2, - "title": "No content article", - "url": "https://example.com/b", - "content": None, - "published_at": None, - "topics": None, - "feed_title": "Hacker News", - "reaction": None, - } - - result = { - "id": item["id"], - "title": item["title"], - "url": item["url"], - "snippet": (item["content"] or "")[:300], - "published_at": item["published_at"].isoformat() if item["published_at"] else None, - "topics": item["topics"] or [], - "source": item["feed_title"], - "reaction": item["reaction"], - } - - assert result["snippet"] == "" - assert result["topics"] == [] - assert result["published_at"] is None - assert result["reaction"] is None diff --git a/tests/test_rss_service.py b/tests/test_rss_service.py deleted file mode 100644 index b6e7f3f..0000000 --- a/tests/test_rss_service.py +++ /dev/null @@ -1,132 +0,0 @@ -import pytest -from unittest.mock import patch, MagicMock - - -def test_extract_item_fields(): - """extract_item() should pull title, link, id, summary from a feedparser entry.""" - from fabledassistant.services.rss import extract_item - entry = MagicMock() - entry.get = lambda k, d="": {"title": "Test Post", "link": "https://example.com/1", - "id": "guid-1", "summary": "A summary"}.get(k, d) - entry.published_parsed = None - item = extract_item(entry) - assert item["title"] == "Test Post" - assert item["url"] == "https://example.com/1" - assert item["guid"] == "guid-1" - assert item["content"] == "A summary" - - -def test_extract_item_does_not_truncate_content(): - """extract_item() should store content without truncation.""" - from fabledassistant.services.rss import extract_item - long_text = "x" * 100_000 - entry = MagicMock() - entry.get = lambda k, d="": {"summary": long_text, "title": "", "link": "", "id": "g"}.get(k, d) - entry.content = [] - entry.published_parsed = None - item = extract_item(entry) - assert len(item["content"]) == 100_000 - - -def test_extract_item_prefers_content_over_summary(): - """extract_item() should prefer 'content' field over 'summary' when present.""" - from fabledassistant.services.rss import extract_item - entry = MagicMock() - content_obj = MagicMock() - content_obj.value = "Full content here" - entry.get = lambda k, d="": { - "title": "T", "link": "http://x.com", "id": "g", - "summary": "Short summary", - }.get(k, d) - entry.content = [content_obj] - entry.published_parsed = None - item = extract_item(entry) - assert item["content"] == "Full content here" - - -@pytest.mark.asyncio -async def test_classify_items_batch_returns_topic_map(): - """classify_items_batch should return a dict mapping item_id to topic list.""" - from unittest.mock import AsyncMock - from fabledassistant.services.rss_classifier import classify_items_batch - - fake_response = '{"1": ["technology", "ai"], "2": ["politics"]}' - - with patch( - "fabledassistant.services.rss_classifier._llm_classify", - new_callable=AsyncMock, - return_value=fake_response, - ): - items = [ - {"id": 1, "title": "OpenAI releases GPT-5", "content": "..."}, - {"id": 2, "title": "EU passes new law", "content": "..."}, - ] - result = await classify_items_batch(items, user_include_topics=[]) - assert result[1] == ["technology", "ai"] - assert result[2] == ["politics"] - - -@pytest.mark.asyncio -async def test_classify_items_batch_handles_llm_failure(): - """classify_items_batch should return empty dict on LLM error.""" - from unittest.mock import AsyncMock - from fabledassistant.services.rss_classifier import classify_items_batch - - with patch( - "fabledassistant.services.rss_classifier._llm_classify", - new_callable=AsyncMock, - side_effect=Exception("LLM unavailable"), - ): - items = [{"id": 5, "title": "Some news", "content": ""}] - result = await classify_items_batch(items, user_include_topics=[]) - assert result == {} - - -def test_score_rss_items_excludes_blacklisted_topics(): - """Items with excluded topics should be removed.""" - from fabledassistant.services.rss_filtering import score_and_filter_items - - items = [ - {"id": 1, "title": "Tech news", "topics": ["technology"], "published_at": "2026-03-25T08:00:00"}, - {"id": 2, "title": "Sports score", "topics": ["sports"], "published_at": "2026-03-25T08:00:00"}, - ] - result = score_and_filter_items( - items, - include_topics=["technology"], - exclude_topics=["sports"], - topic_scores={}, - max_items=10, - ) - ids = [r["id"] for r in result] - assert 1 in ids - assert 2 not in ids - - -def test_score_rss_items_boosts_included_topics(): - """Items matching include_topics should rank higher than neutral items.""" - from fabledassistant.services.rss_filtering import score_and_filter_items - - items = [ - {"id": 1, "title": "Random news", "topics": ["other"], "published_at": "2026-03-25T07:00:00"}, - {"id": 2, "title": "Tech news", "topics": ["technology"], "published_at": "2026-03-25T06:00:00"}, - ] - result = score_and_filter_items( - items, - include_topics=["technology"], - exclude_topics=[], - topic_scores={}, - max_items=10, - ) - assert result[0]["id"] == 2 - - -def test_score_rss_items_no_preferences_returns_all(): - """With no preferences, all items should be returned sorted by recency.""" - from fabledassistant.services.rss_filtering import score_and_filter_items - - items = [ - {"id": 1, "title": "A", "topics": [], "published_at": "2026-03-24T10:00:00"}, - {"id": 2, "title": "B", "topics": [], "published_at": "2026-03-25T10:00:00"}, - ] - result = score_and_filter_items(items, [], [], {}, max_items=10) - assert result[0]["id"] == 2 # Newer first