refactor: hard-cut RSS infrastructure (scope C)
Removes the entire RSS feature surface — feeds, items, embeddings, reactions, discussion-note flow, briefing news context, settings, env-vars, and DB tables. Keeps the URL-generic article-reader (the read_article LLM tool) under a clean module so the LLM can still fetch arbitrary article content from URLs the user provides. Backend: - New services/article_fetcher.py — single source of trafilatura URL→text - New services/tools/article.py — read_article tool (was nested under tools/rss) - Delete services/rss.py, rss_classifier.py, rss_filtering.py, article_context.py - Delete services/tools/rss.py - Delete models/rss_feed.py (RssFeed, RssItem), models/rss_item_embedding.py - services/embeddings.py: drop upsert/semantic_search/backfill RSS helpers - services/llm.py: remove _build_briefing_article_context, briefing-conv branch, ARTICLE_DISCUSS_SEED skip-RAG branch; drop get_rss_items / add_rss_feed from the actions list - services/generation_task.py: drop _maybe_save_article_discussion_note + caller - routes/chat.py: drop /api/chat/from-article/<id> endpoint - routes/journal.py: re-import via web.py refactor (article_fetcher path) - services/tools/__init__.py: register `article`, drop `rss` - services/tools/_registry.py: drop the requires=='rss' check - app.py: drop backfill_rss_item_embeddings + backfill_rss_article_content tasks - config.py: prose-only edit (no env var change — RSS env vars were never first-class) Frontend: - stores/settings.ts: drop rssEnabled - SettingsView.vue: drop the RSS-classification mention - api/client.ts: drop openArticleInChat (the from-article endpoint is gone) Tests: - Delete tests/test_rss_service.py, test_news_api.py, test_article_reading.py Migration: - 0042_drop_rss: DROP TABLE rss_item_embeddings, rss_item_reactions, rss_items, rss_feeds; DELETE settings rows for rss_enabled / briefing_*_topics Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -320,16 +320,6 @@ def create_app() -> Quart:
|
||||
await backfill_project_summaries()
|
||||
except Exception:
|
||||
logger.warning("Project summary backfill failed", exc_info=True)
|
||||
try:
|
||||
from fabledassistant.services.embeddings import backfill_rss_item_embeddings
|
||||
await backfill_rss_item_embeddings()
|
||||
except Exception:
|
||||
logger.warning("RSS embedding backfill failed", exc_info=True)
|
||||
try:
|
||||
from fabledassistant.services.embeddings import backfill_rss_article_content
|
||||
await backfill_rss_article_content()
|
||||
except Exception:
|
||||
logger.warning("RSS article content backfill failed", exc_info=True)
|
||||
|
||||
asyncio.create_task(_delayed_backfill())
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ class Config:
|
||||
OLLAMA_URL: str = os.environ.get("OLLAMA_URL", "http://localhost:11434")
|
||||
OLLAMA_MODEL: str = os.environ.get("OLLAMA_MODEL", "qwen3:latest")
|
||||
# Lightweight model for background tasks (title generation, tag suggestions,
|
||||
# project summaries, RSS classification). Using a separate model keeps the
|
||||
# project summaries). Using a separate model keeps the
|
||||
# main model's KV cache intact between user messages, enabling prefix cache hits.
|
||||
OLLAMA_BACKGROUND_MODEL: str = os.environ.get("OLLAMA_BACKGROUND_MODEL", "gemma3:4b")
|
||||
# Ollama keep_alive — how long a model stays resident in VRAM after its last
|
||||
|
||||
@@ -38,11 +38,9 @@ from fabledassistant.models.note_version import NoteVersion # noqa: E402, F401
|
||||
from fabledassistant.models.group import Group, GroupMembership # noqa: E402, F401
|
||||
from fabledassistant.models.share import NoteShare, ProjectShare # noqa: E402, F401
|
||||
from fabledassistant.models.notification import Notification # noqa: E402, F401
|
||||
from fabledassistant.models.rss_feed import RssFeed, RssItem # noqa: E402, F401
|
||||
from fabledassistant.models.weather_cache import WeatherCache # noqa: E402, F401
|
||||
from fabledassistant.models.api_key import ApiKey # noqa: E402, F401
|
||||
from fabledassistant.models.user_profile import UserProfile # noqa: E402, F401
|
||||
from fabledassistant.models.rss_item_embedding import RssItemEmbedding # noqa: E402, F401
|
||||
from fabledassistant.models.moment import ( # noqa: E402, F401
|
||||
Moment,
|
||||
MomentEmbedding,
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import ARRAY, BigInteger, DateTime, ForeignKey, Index, Integer, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from fabledassistant.models import Base
|
||||
|
||||
|
||||
class RssFeed(Base):
|
||||
__tablename__ = "rss_feeds"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(Integer, ForeignKey("users.id", ondelete="CASCADE"))
|
||||
url: Mapped[str] = mapped_column(Text)
|
||||
title: Mapped[str] = mapped_column(Text, default="")
|
||||
category: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
last_fetched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
items: Mapped[list["RssItem"]] = relationship(
|
||||
back_populates="feed", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "url", name="uq_rss_feeds_user_url"),
|
||||
Index("ix_rss_feeds_user_id", "user_id"),
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"url": self.url,
|
||||
"title": self.title,
|
||||
"category": self.category,
|
||||
"last_fetched_at": self.last_fetched_at.isoformat() if self.last_fetched_at else None,
|
||||
}
|
||||
|
||||
|
||||
class RssItem(Base):
|
||||
__tablename__ = "rss_items"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
feed_id: Mapped[int] = mapped_column(Integer, ForeignKey("rss_feeds.id", ondelete="CASCADE"))
|
||||
guid: Mapped[str] = mapped_column(Text)
|
||||
title: Mapped[str] = mapped_column(Text, default="")
|
||||
url: Mapped[str] = mapped_column(Text, default="")
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
# Truncated to 2000 chars to keep DB size reasonable
|
||||
content: Mapped[str] = mapped_column(Text, default="")
|
||||
# Full trafilatura-extracted article body, populated lazily on first
|
||||
# discuss-click / enrichment pass. Nullable — most items never get this
|
||||
# cached. Expires naturally with the item (90-day retention).
|
||||
content_full: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
# Map-reduced conversation-ready context derived from content_full. See
|
||||
# services/article_context.py — populated on first discuss click so
|
||||
# repeat clicks skip both the fetch and the LLM map step.
|
||||
context_prepared: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
content_fetched_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
fetched_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
topics: Mapped[list[str]] = mapped_column(
|
||||
ARRAY(Text), nullable=False, default=list, server_default="{}"
|
||||
)
|
||||
classified_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
# Note persisting the first-click discussion summary. Set by the article
|
||||
# discussion pipeline once the seeded chat completes its first assistant
|
||||
# reply; links back into RAG so re-discussing the same article lands the
|
||||
# prior summary in context.
|
||||
discussion_note_id: Mapped[int | None] = mapped_column(
|
||||
BigInteger, ForeignKey("notes.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
|
||||
feed: Mapped["RssFeed"] = relationship(back_populates="items")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("feed_id", "guid", name="uq_rss_items_feed_guid"),
|
||||
Index("ix_rss_items_feed_id", "feed_id"),
|
||||
Index("ix_rss_items_published_at", "published_at"),
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"feed_id": self.feed_id,
|
||||
"guid": self.guid,
|
||||
"title": self.title,
|
||||
"url": self.url,
|
||||
"published_at": self.published_at.isoformat() if self.published_at else None,
|
||||
"content": self.content,
|
||||
"topics": self.topics or [],
|
||||
"classified_at": self.classified_at.isoformat() if self.classified_at else None,
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, Integer
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fabledassistant.models import Base
|
||||
|
||||
|
||||
class RssItemEmbedding(Base):
|
||||
"""Stores the embedding vector for an RSS item, used for semantic news search."""
|
||||
|
||||
__tablename__ = "rss_item_embeddings"
|
||||
|
||||
rss_item_id: Mapped[int] = mapped_column(
|
||||
Integer,
|
||||
ForeignKey("rss_items.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
)
|
||||
user_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True)
|
||||
embedding: Mapped[list] = mapped_column(JSONB, nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
@@ -507,81 +507,3 @@ async def delete_model_route():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
|
||||
@chat_bp.route("/from-article/<int:item_id>", methods=["POST"])
|
||||
@login_required
|
||||
async def create_conversation_from_article(item_id: int):
|
||||
"""Create a chat conversation seeded for article discussion and auto-run.
|
||||
|
||||
Mirrors the briefing ``discuss_article`` route: creates a fresh
|
||||
conversation, stages the shared synthetic read_article exchange + seed
|
||||
prompt, then kicks off generation so the client lands on an in-flight
|
||||
stream. The Flutter and web chat screens reconnect to the running buffer
|
||||
on mount.
|
||||
"""
|
||||
from sqlalchemy import select as _select
|
||||
from fabledassistant.models import async_session as _async_session
|
||||
from fabledassistant.models.rss_feed import RssItem, RssFeed
|
||||
from fabledassistant.services.article_context import (
|
||||
EmptyArticleError,
|
||||
seed_article_discussion,
|
||||
)
|
||||
|
||||
uid = get_current_user_id()
|
||||
|
||||
async with _async_session() as session:
|
||||
result = await session.execute(
|
||||
_select(RssItem)
|
||||
.join(RssFeed, RssItem.feed_id == RssFeed.id)
|
||||
.where(RssItem.id == item_id, RssFeed.user_id == uid)
|
||||
)
|
||||
item = result.scalars().first()
|
||||
|
||||
if item is None:
|
||||
return jsonify({"error": "Article not found"}), 404
|
||||
|
||||
conv_title = (item.title or "Article discussion")[:80]
|
||||
conv = await create_conversation(uid, title=conv_title, conversation_type="chat")
|
||||
|
||||
model = await get_setting(uid, "default_model", "") or Config.OLLAMA_MODEL
|
||||
try:
|
||||
discuss_prompt = await seed_article_discussion(conv.id, item, model)
|
||||
except EmptyArticleError as e:
|
||||
# Roll back the empty conversation so the user doesn't end up with a
|
||||
# phantom entry in their chat list.
|
||||
try:
|
||||
await delete_conversation(uid, conv.id)
|
||||
except Exception:
|
||||
logger.warning("Failed to clean up empty article conversation %s", conv.id)
|
||||
return jsonify({"error": str(e)}), 422
|
||||
|
||||
# Reload conversation so we see the two messages the helper just added.
|
||||
conv = await get_conversation(uid, conv.id)
|
||||
assert conv is not None
|
||||
|
||||
history: list[dict] = []
|
||||
for msg in conv.messages:
|
||||
if msg.role == "system":
|
||||
continue
|
||||
msg_dict: dict = {"role": msg.role, "content": msg.content or ""}
|
||||
if msg.tool_calls:
|
||||
msg_dict["tool_calls"] = [
|
||||
{"function": {"name": tc["function"], "arguments": tc["arguments"]}}
|
||||
for tc in msg.tool_calls
|
||||
]
|
||||
history.append(msg_dict)
|
||||
for tc in msg.tool_calls:
|
||||
history.append({"role": "tool", "content": json.dumps(tc.get("result", {}))})
|
||||
else:
|
||||
history.append(msg_dict)
|
||||
|
||||
assistant_msg = await add_message(conv.id, "assistant", "", status="generating")
|
||||
buf = create_buffer(conv.id, assistant_msg.id)
|
||||
asyncio.create_task(run_generation(
|
||||
buf, history, model, uid, conv.id, conv.title or "", discuss_prompt,
|
||||
))
|
||||
|
||||
return jsonify({
|
||||
"conversation_id": conv.id,
|
||||
"assistant_message_id": assistant_msg.id,
|
||||
"status": "generating",
|
||||
}), 202
|
||||
|
||||
@@ -1,270 +0,0 @@
|
||||
"""Prepare article bodies as conversation-ready context.
|
||||
|
||||
Used by the briefing ``discuss-article`` flow and the ``/news`` discuss button.
|
||||
A raw trafilatura extraction is often too large to drop whole into a chat
|
||||
history without eating the context window, so this module runs a map-reduce
|
||||
step over oversized articles and returns a compact, structured context that
|
||||
still preserves the article's meaning across sections.
|
||||
|
||||
Small articles pass through unchanged — map-reduce only fires when the raw
|
||||
body exceeds CHAR_BUDGET. The output is cached on ``rss_items.context_prepared``
|
||||
by the caller, so repeat discuss-clicks on the same article skip this work
|
||||
entirely.
|
||||
|
||||
The module also owns ``seed_article_discussion``, the shared routine that
|
||||
stages a synthetic ``read_article`` tool exchange plus a conversational seed
|
||||
prompt into a conversation. Both the briefing and ``/news`` entry points call
|
||||
it so the two flows stay byte-identical — the only thing that differs between
|
||||
them is whether the conversation already existed or was freshly created.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
|
||||
from fabledassistant.models import async_session
|
||||
from fabledassistant.models.rss_feed import RssItem
|
||||
from fabledassistant.services.chat import add_message
|
||||
from fabledassistant.services.llm import generate_completion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ~12k tokens at 4 chars/token. Comfortably under OLLAMA_NUM_CTX=16384
|
||||
# with room left for system prompt, chat history, and the assistant reply.
|
||||
CHAR_BUDGET = 48_000
|
||||
|
||||
# Chunk size for the map step on oversized articles. Overlap preserves
|
||||
# context across paragraph boundaries that happen to land mid-sentence.
|
||||
CHUNK_CHARS = 8_000
|
||||
CHUNK_OVERLAP = 400
|
||||
|
||||
_PARA_SPLIT = re.compile(r"\n\s*\n")
|
||||
|
||||
|
||||
def _chunk_by_paragraph(body: str) -> list[str]:
|
||||
"""Split ``body`` into chunks of up to CHUNK_CHARS, respecting paragraphs.
|
||||
|
||||
Paragraphs longer than CHUNK_CHARS are split mid-paragraph as a last
|
||||
resort. Adjacent chunks share CHUNK_OVERLAP chars of trailing text so
|
||||
a sentence straddling the boundary stays readable on both sides.
|
||||
"""
|
||||
paragraphs = [p.strip() for p in _PARA_SPLIT.split(body) if p.strip()]
|
||||
chunks: list[str] = []
|
||||
current: list[str] = []
|
||||
current_len = 0
|
||||
for para in paragraphs:
|
||||
para_len = len(para)
|
||||
if para_len > CHUNK_CHARS:
|
||||
if current:
|
||||
chunks.append("\n\n".join(current))
|
||||
current, current_len = [], 0
|
||||
for i in range(0, para_len, CHUNK_CHARS - CHUNK_OVERLAP):
|
||||
chunks.append(para[i : i + CHUNK_CHARS])
|
||||
continue
|
||||
if current_len + para_len + 2 > CHUNK_CHARS and current:
|
||||
chunks.append("\n\n".join(current))
|
||||
tail = current[-1][-CHUNK_OVERLAP:] if current else ""
|
||||
current = [tail, para] if tail else [para]
|
||||
current_len = len(tail) + para_len + (2 if tail else 0)
|
||||
else:
|
||||
current.append(para)
|
||||
current_len += para_len + 2
|
||||
if current:
|
||||
chunks.append("\n\n".join(current))
|
||||
return chunks
|
||||
|
||||
|
||||
async def _summarize_chunk(title: str, chunk: str, index: int, total: int, model: str) -> str:
|
||||
"""Map-step summary of one article chunk.
|
||||
|
||||
Aims for ~300 words of dense, factual prose — not bullet points — so the
|
||||
downstream chat model can quote from it naturally.
|
||||
"""
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You are summarizing one section of a larger article so a downstream "
|
||||
"conversation model can discuss the full article without having to read "
|
||||
"every word.\n\n"
|
||||
"Requirements:\n"
|
||||
"- 250–350 words of dense factual prose\n"
|
||||
"- Preserve specific claims, numbers, names, and quotes\n"
|
||||
"- Do NOT editorialize or add analysis\n"
|
||||
"- Do NOT use bullet points or headings\n"
|
||||
"- Do NOT say 'this section' or 'this article' — write content, not meta"
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"Article: {title}\n"
|
||||
f"Section {index + 1} of {total}:\n\n{chunk}"
|
||||
),
|
||||
},
|
||||
]
|
||||
try:
|
||||
# Pin num_ctx — same rationale as services/research.py:66. A large
|
||||
# chunk plus system prompt can push well past the default window;
|
||||
# silent truncation here would drop the tail of the chunk without
|
||||
# any error, producing a misleading summary.
|
||||
raw = await generate_completion(
|
||||
messages, model, max_tokens=600, num_ctx=16384
|
||||
)
|
||||
return raw.strip()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Article chunk summary failed for section %d/%d of '%s'",
|
||||
index + 1, total, title, exc_info=True,
|
||||
)
|
||||
# Fall back to the raw chunk truncated to ~1500 chars so the overall
|
||||
# pipeline still delivers something rather than dropping the section.
|
||||
return chunk[:1500]
|
||||
|
||||
|
||||
async def prepare_article_context(
|
||||
title: str,
|
||||
url: str,
|
||||
body: str,
|
||||
model: str,
|
||||
) -> str:
|
||||
"""Return a conversation-ready context block for ``body``.
|
||||
|
||||
- Small article (≤ CHAR_BUDGET): returns ``body`` unchanged.
|
||||
- Oversized article: runs a parallel map step over paragraph-aware
|
||||
chunks and concatenates the summaries under section headers.
|
||||
|
||||
The returned string is what should go into the ``read_article`` synthetic
|
||||
tool-result in chat history. Callers are responsible for caching it to
|
||||
``rss_items.context_prepared``.
|
||||
"""
|
||||
body = body or ""
|
||||
if len(body) <= CHAR_BUDGET:
|
||||
return body
|
||||
|
||||
chunks = _chunk_by_paragraph(body)
|
||||
logger.info(
|
||||
"Article '%s' is %d chars, map-reducing into %d chunks",
|
||||
title, len(body), len(chunks),
|
||||
)
|
||||
|
||||
summaries = await asyncio.gather(
|
||||
*[
|
||||
_summarize_chunk(title, chunk, i, len(chunks), model)
|
||||
for i, chunk in enumerate(chunks)
|
||||
]
|
||||
)
|
||||
|
||||
header = (
|
||||
f"(This article was longer than the chat window could hold verbatim, "
|
||||
f"so the full text was split into {len(chunks)} sections and each was "
|
||||
"summarized below. Each section preserves specific claims, numbers, "
|
||||
"and quotes from the original.)\n\n"
|
||||
)
|
||||
parts = [
|
||||
f"## Section {i + 1}\n\n{summary}"
|
||||
for i, summary in enumerate(summaries)
|
||||
]
|
||||
return header + "\n\n".join(parts)
|
||||
|
||||
|
||||
# Conversational seed prompt for article discussions. Kept here so both the
|
||||
# briefing and /news entry points use the exact same wording. See
|
||||
# feedback_discuss_prompt_style memory: numbered checklists produce
|
||||
# assignment-completion responses; this conversational seed opens a dialogue.
|
||||
ARTICLE_DISCUSS_SEED = (
|
||||
"I want to talk about this article. Start with a substantive summary "
|
||||
"of what it's arguing and the key evidence it uses, then tell me what "
|
||||
"stood out to you or seems worth pushing back on. I'll ask follow-ups "
|
||||
"from there."
|
||||
)
|
||||
|
||||
|
||||
class EmptyArticleError(Exception):
|
||||
"""Raised when an article has no extractable body text.
|
||||
|
||||
Callers (the briefing and /news discuss routes) map this to a 422 so the
|
||||
user sees a clear error instead of a hallucinated summary built from an
|
||||
empty synthetic tool result.
|
||||
"""
|
||||
|
||||
|
||||
async def seed_article_discussion(
|
||||
conv_id: int,
|
||||
item: RssItem,
|
||||
model: str,
|
||||
) -> str:
|
||||
"""Stage the synthetic read_article tool exchange + conversational seed.
|
||||
|
||||
Used by both the briefing ``discuss_article`` route and the ``/news``
|
||||
``from-article`` conversation creator. Handles the three-layer cache
|
||||
(``context_prepared`` → ``content_full`` → fresh fetch) and inserts two
|
||||
messages into ``conv_id``:
|
||||
|
||||
1. An assistant message with a synthetic ``read_article`` tool_call whose
|
||||
``result.content`` carries the prepared article context. The message
|
||||
also carries ``msg_metadata={"rss_item_id": ...}`` so the post-generation
|
||||
hook in ``generation_task.py`` can locate it and persist the first
|
||||
reply as a discussion-summary Note.
|
||||
2. A user message with the shared conversational seed prompt.
|
||||
|
||||
Returns the seed prompt string so callers can pass it to ``run_generation``
|
||||
as ``user_content``.
|
||||
"""
|
||||
# Avoid circulars: rss helper imports article_context indirectly nowhere,
|
||||
# but keep this local for symmetry with the route-level imports it
|
||||
# replaces.
|
||||
from fabledassistant.services.rss import get_or_fetch_full_article
|
||||
|
||||
if item.context_prepared:
|
||||
article_content = item.context_prepared
|
||||
else:
|
||||
raw_body = await get_or_fetch_full_article(item) or item.content or ""
|
||||
if not raw_body.strip():
|
||||
# Hard-fail rather than stage an empty synthetic tool result.
|
||||
# An empty `content` field silently tells the model "the article
|
||||
# has nothing in it" and it confabulates from RAG/history. Better
|
||||
# to surface a clean error to the user.
|
||||
logger.warning(
|
||||
"Article discussion aborted: empty body for rss_item %s (%s)",
|
||||
item.id, item.url,
|
||||
)
|
||||
raise EmptyArticleError(
|
||||
"Couldn't extract any readable text from this article."
|
||||
)
|
||||
article_content = await prepare_article_context(
|
||||
item.title or "", item.url, raw_body, model,
|
||||
)
|
||||
if not article_content.strip():
|
||||
raise EmptyArticleError(
|
||||
"Couldn't extract any readable text from this article."
|
||||
)
|
||||
async with async_session() as session:
|
||||
fresh = await session.get(RssItem, item.id)
|
||||
if fresh is not None:
|
||||
fresh.context_prepared = article_content
|
||||
await session.commit()
|
||||
|
||||
synthetic_tool_calls = [{
|
||||
"function": "read_article",
|
||||
"arguments": {"url": item.url},
|
||||
"result": {
|
||||
"success": True,
|
||||
"type": "article_content",
|
||||
"url": item.url,
|
||||
"content": article_content,
|
||||
"truncated": False,
|
||||
},
|
||||
}]
|
||||
await add_message(
|
||||
conv_id,
|
||||
"assistant",
|
||||
"",
|
||||
status="complete",
|
||||
tool_calls=synthetic_tool_calls,
|
||||
msg_metadata={"rss_item_id": item.id, "article_seed": True},
|
||||
)
|
||||
await add_message(conv_id, "user", ARTICLE_DISCUSS_SEED)
|
||||
return ARTICLE_DISCUSS_SEED
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Generic article-text fetcher.
|
||||
|
||||
Fetches a URL and extracts its main body via trafilatura. The single source
|
||||
of truth for article-content extraction across the codebase — used by the
|
||||
``read_article`` LLM tool and the ``lookup`` tool's web-result enrichment.
|
||||
|
||||
Trafilatura/lxml is NOT safe to call concurrently — running it via
|
||||
``run_in_executor`` from multiple coroutines can trip a libxml2 double-free.
|
||||
Callers must serialize their fetches (await one before starting the next).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def fetch_article_text(url: str) -> str | None:
|
||||
"""Return the clean article body for *url*, or None on failure.
|
||||
|
||||
Returns None when the HTTP fetch fails or trafilatura yields nothing
|
||||
useful. Callers should treat None as "no article content available."
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15.0, follow_redirects=True, headers={
|
||||
"User-Agent": "Mozilla/5.0 (compatible; FabledScribe/1.0; +https://fabledsword.com)",
|
||||
}) as client:
|
||||
resp = await client.get(url)
|
||||
resp.raise_for_status()
|
||||
raw_html = resp.text
|
||||
except Exception:
|
||||
logger.debug("Failed to fetch article URL %s", url)
|
||||
return None
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
import trafilatura
|
||||
text = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: trafilatura.extract(
|
||||
raw_html,
|
||||
include_comments=False,
|
||||
include_tables=True,
|
||||
favor_recall=True,
|
||||
),
|
||||
)
|
||||
return text or None
|
||||
except Exception:
|
||||
logger.debug("trafilatura extraction failed for %s", url, exc_info=True)
|
||||
return None
|
||||
@@ -1,7 +1,6 @@
|
||||
"""Semantic note search via Ollama embedding model (nomic-embed-text).
|
||||
|
||||
Embeddings are stored in the note_embeddings table (one row per note).
|
||||
RSS item embeddings are stored in rss_item_embeddings (one row per item).
|
||||
All search operations degrade gracefully — if the embedding model is
|
||||
unavailable the callers fall back to keyword search.
|
||||
"""
|
||||
@@ -9,7 +8,6 @@ unavailable the callers fall back to keyword search.
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import delete, select
|
||||
@@ -18,8 +16,6 @@ from fabledassistant.config import Config
|
||||
from fabledassistant.models import async_session
|
||||
from fabledassistant.models.embedding import NoteEmbedding
|
||||
from fabledassistant.models.note import Note
|
||||
from fabledassistant.models.rss_feed import RssItem
|
||||
from fabledassistant.models.rss_item_embedding import RssItemEmbedding
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -28,10 +24,6 @@ logger = logging.getLogger(__name__)
|
||||
# 0.45 keeps only genuinely relevant notes; lower values like 0.30 let in
|
||||
# loosely-related results that pad the sidebar without adding real value.
|
||||
_SIMILARITY_THRESHOLD = 0.45
|
||||
_RSS_SIMILARITY_THRESHOLD = 0.55
|
||||
_RSS_SEARCH_LIMIT = 3
|
||||
_RSS_SEARCH_DAYS = 30
|
||||
_RSS_SNIPPET_CHARS = 500
|
||||
|
||||
|
||||
async def get_embedding(text: str, model: str | None = None) -> list[float]:
|
||||
@@ -186,174 +178,3 @@ async def backfill_note_embeddings() -> None:
|
||||
logger.info("Embedding backfill complete: %d/%d notes embedded", success, len(notes_to_embed))
|
||||
|
||||
|
||||
# ── RSS item embeddings ───────────────────────────────────────────────────────
|
||||
|
||||
async def upsert_rss_item_embedding(item_id: int, user_id: int, title: str, content: str) -> None:
|
||||
"""Generate and persist an embedding for an RSS item. Safe to fire-and-forget."""
|
||||
text = f"{title}\n{content}".strip()
|
||||
if not text:
|
||||
return
|
||||
try:
|
||||
embedding = await get_embedding(text)
|
||||
except Exception:
|
||||
logger.debug("Skipping embedding for RSS item %d — model unavailable", item_id)
|
||||
return
|
||||
|
||||
try:
|
||||
async with async_session() as session:
|
||||
await session.execute(
|
||||
delete(RssItemEmbedding).where(RssItemEmbedding.rss_item_id == item_id)
|
||||
)
|
||||
session.add(RssItemEmbedding(rss_item_id=item_id, user_id=user_id, embedding=embedding))
|
||||
await session.commit()
|
||||
logger.debug("Upserted embedding for RSS item %d", item_id)
|
||||
except Exception:
|
||||
logger.warning("Failed to persist embedding for RSS item %d", item_id, exc_info=True)
|
||||
|
||||
|
||||
async def semantic_search_rss_items(
|
||||
user_id: int,
|
||||
query_vector: list[float],
|
||||
limit: int = _RSS_SEARCH_LIMIT,
|
||||
days: int = _RSS_SEARCH_DAYS,
|
||||
) -> list[tuple[float, RssItem]]:
|
||||
"""Return up to *limit* (score, RssItem) pairs most relevant to *query_vector*.
|
||||
|
||||
Only considers items fetched within the last *days* days.
|
||||
Returns an empty list on any error.
|
||||
"""
|
||||
since = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
try:
|
||||
async with async_session() as session:
|
||||
stmt = (
|
||||
select(RssItemEmbedding, RssItem)
|
||||
.join(RssItem, RssItemEmbedding.rss_item_id == RssItem.id)
|
||||
.where(
|
||||
RssItemEmbedding.user_id == user_id,
|
||||
RssItem.fetched_at >= since,
|
||||
)
|
||||
)
|
||||
rows = list((await session.execute(stmt)).all())
|
||||
except Exception:
|
||||
logger.warning("Failed to query RSS item embeddings", exc_info=True)
|
||||
return []
|
||||
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
scored: list[tuple[float, RssItem]] = []
|
||||
for rie, item in rows:
|
||||
try:
|
||||
sim = _cosine_similarity(query_vector, rie.embedding)
|
||||
except Exception:
|
||||
continue
|
||||
if sim >= _RSS_SIMILARITY_THRESHOLD:
|
||||
scored.append((sim, item))
|
||||
|
||||
scored.sort(key=lambda x: x[0], reverse=True)
|
||||
return scored[:limit]
|
||||
|
||||
|
||||
async def backfill_rss_item_embeddings() -> None:
|
||||
"""Generate embeddings for all RSS items that don't have one yet.
|
||||
|
||||
Runs as a background task at startup. Adds a small sleep between items
|
||||
to avoid overwhelming Ollama.
|
||||
"""
|
||||
try:
|
||||
async with async_session() as session:
|
||||
existing = {
|
||||
row[0]
|
||||
for row in (
|
||||
await session.execute(select(RssItemEmbedding.rss_item_id))
|
||||
).fetchall()
|
||||
}
|
||||
result = await session.execute(
|
||||
select(RssItem.id, RssItem.feed_id, RssItem.title, RssItem.content)
|
||||
)
|
||||
items_to_embed = [row for row in result.fetchall() if row[0] not in existing]
|
||||
except Exception:
|
||||
logger.warning("RSS embedding backfill: failed to query items", exc_info=True)
|
||||
return
|
||||
|
||||
if not items_to_embed:
|
||||
logger.info("RSS embedding backfill: all items already have embeddings")
|
||||
return
|
||||
|
||||
# Resolve user_id per feed_id
|
||||
try:
|
||||
from fabledassistant.models.rss_feed import RssFeed
|
||||
async with async_session() as session:
|
||||
result = await session.execute(select(RssFeed.id, RssFeed.user_id))
|
||||
feed_user_map = {fid: uid for fid, uid in result.fetchall()}
|
||||
except Exception:
|
||||
logger.warning("RSS embedding backfill: failed to load feed user map", exc_info=True)
|
||||
return
|
||||
|
||||
logger.info("RSS embedding backfill: generating embeddings for %d items", len(items_to_embed))
|
||||
success = 0
|
||||
for item_id, feed_id, title, content in items_to_embed:
|
||||
user_id = feed_user_map.get(feed_id)
|
||||
if user_id is None:
|
||||
continue
|
||||
await upsert_rss_item_embedding(item_id, user_id, title or "", content or "")
|
||||
success += 1
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
logger.info("RSS embedding backfill complete: %d/%d items embedded", success, len(items_to_embed))
|
||||
|
||||
|
||||
async def backfill_rss_article_content() -> None:
|
||||
"""Fetch full article text for RSS items that only have short feed-provided content.
|
||||
|
||||
An item is considered unenriched if its content is shorter than 1000 chars —
|
||||
typical of feed summaries/teasers rather than full articles.
|
||||
Runs at startup after the embedding backfill.
|
||||
"""
|
||||
from fabledassistant.services.rss import _fetch_full_article
|
||||
from fabledassistant.models.rss_feed import RssFeed
|
||||
|
||||
SHORT_THRESHOLD = 1000
|
||||
|
||||
try:
|
||||
async with async_session() as session:
|
||||
feed_result = await session.execute(select(RssFeed.id, RssFeed.user_id))
|
||||
feed_user_map = {fid: uid for fid, uid in feed_result.fetchall()}
|
||||
|
||||
item_result = await session.execute(
|
||||
select(RssItem.id, RssItem.feed_id, RssItem.url, RssItem.title, RssItem.content)
|
||||
.where(RssItem.url != "")
|
||||
)
|
||||
candidates = [
|
||||
row for row in item_result.fetchall()
|
||||
if len(row[4] or "") < SHORT_THRESHOLD
|
||||
]
|
||||
except Exception:
|
||||
logger.warning("Article content backfill: failed to query items", exc_info=True)
|
||||
return
|
||||
|
||||
if not candidates:
|
||||
logger.info("Article content backfill: no unenriched items found")
|
||||
return
|
||||
|
||||
logger.info("Article content backfill: enriching %d items", len(candidates))
|
||||
enriched = 0
|
||||
for item_id, feed_id, url, title, _ in candidates:
|
||||
user_id = feed_user_map.get(feed_id)
|
||||
if user_id is None:
|
||||
continue
|
||||
full_text = await _fetch_full_article(url)
|
||||
if full_text and len(full_text) > SHORT_THRESHOLD:
|
||||
try:
|
||||
async with async_session() as session:
|
||||
item = await session.get(RssItem, item_id)
|
||||
if item:
|
||||
item.content = full_text
|
||||
await session.commit()
|
||||
await upsert_rss_item_embedding(item_id, user_id, title or "", full_text)
|
||||
enriched += 1
|
||||
except Exception:
|
||||
logger.debug("Failed to store enriched content for item %d", item_id, exc_info=True)
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
logger.info("Article content backfill complete: %d/%d items enriched", enriched, len(candidates))
|
||||
|
||||
@@ -37,84 +37,6 @@ _TOOL_CALL_MARKER = re.compile(r"^\s*\[TOOL_CALLS\]\s*", re.IGNORECASE)
|
||||
DB_FLUSH_INTERVAL = 5.0 # seconds between partial DB flushes
|
||||
|
||||
|
||||
async def _maybe_save_article_discussion_note(
|
||||
user_id: int, conv_id: int, reply_content: str,
|
||||
) -> None:
|
||||
"""Persist a seeded article-discussion's first reply as a Note.
|
||||
|
||||
Fires after ``run_generation`` completes. Looks for a synthetic
|
||||
read_article seed message on the conversation; if found AND the linked
|
||||
``rss_items`` row has no ``discussion_note_id`` yet, saves ``reply_content``
|
||||
as a Note, tags it, and writes the backlink. Subsequent discuss clicks on
|
||||
the same article are a no-op (already linked).
|
||||
|
||||
Failures are logged and swallowed — the chat UI should never break because
|
||||
Note persistence hit a snag.
|
||||
"""
|
||||
try:
|
||||
if not reply_content or not reply_content.strip():
|
||||
return
|
||||
from sqlalchemy import select as _select
|
||||
from fabledassistant.models.conversation import Message as _Message
|
||||
from fabledassistant.models.rss_feed import RssItem as _RssItem
|
||||
from fabledassistant.services.notes import create_note
|
||||
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
_select(_Message)
|
||||
.where(_Message.conversation_id == conv_id)
|
||||
.order_by(_Message.id.asc())
|
||||
)
|
||||
messages = result.scalars().all()
|
||||
seed_meta = None
|
||||
for m in messages:
|
||||
meta = m.msg_metadata or {}
|
||||
if meta.get("article_seed") and meta.get("rss_item_id"):
|
||||
seed_meta = meta
|
||||
break
|
||||
if seed_meta is None:
|
||||
return
|
||||
item_id = int(seed_meta["rss_item_id"])
|
||||
item = await session.get(_RssItem, item_id)
|
||||
if item is None or item.discussion_note_id is not None:
|
||||
return
|
||||
article_title = (item.title or "Untitled article").strip()
|
||||
article_url = item.url
|
||||
article_topics = list(item.topics or [])
|
||||
|
||||
note_title = f"Article: {article_title}"[:200]
|
||||
body_parts = [f"**Source:** {article_url}"] if article_url else []
|
||||
body_parts.append(reply_content.strip())
|
||||
note_body = "\n\n".join(body_parts)
|
||||
tags = ["article-summary"] + [t for t in article_topics if t]
|
||||
note = await create_note(
|
||||
user_id=user_id,
|
||||
title=note_title,
|
||||
body=note_body,
|
||||
tags=tags,
|
||||
entity_meta={
|
||||
"source": "article_discussion",
|
||||
"rss_item_id": item_id,
|
||||
"url": article_url,
|
||||
"conversation_id": conv_id,
|
||||
},
|
||||
)
|
||||
|
||||
async with async_session() as session:
|
||||
fresh = await session.get(_RssItem, item_id)
|
||||
if fresh is not None and fresh.discussion_note_id is None:
|
||||
fresh.discussion_note_id = note.id
|
||||
await session.commit()
|
||||
logger.info(
|
||||
"Saved article-discussion summary as note %d for rss_item %d (conv %d)",
|
||||
note.id, item_id, conv_id,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to persist article-discussion note for conv %d",
|
||||
conv_id, exc_info=True,
|
||||
)
|
||||
|
||||
# Human-readable labels for each tool, shown in the status indicator
|
||||
_TOOL_LABELS: dict[str, str] = {
|
||||
"create_note": "Creating note/task",
|
||||
@@ -593,28 +515,16 @@ async def run_generation(
|
||||
msg_count = len(non_system)
|
||||
should_gen_title = not conv_title or (msg_count > 0 and msg_count % 10 == 0)
|
||||
|
||||
# Persist article-discussion seed conversations as a Note on their
|
||||
# first assistant reply. This makes "Discuss" summaries part of RAG
|
||||
# so the knowledge base stops being amnesiac about articles the user
|
||||
# has already engaged with. The hook detects a seeded conversation by
|
||||
# finding a synthetic read_article assistant message whose
|
||||
# msg_metadata carries ``article_seed: True`` and whose rss_items row
|
||||
# has no discussion_note_id yet. Fire-and-forget so the done event
|
||||
# lands immediately.
|
||||
asyncio.create_task(_maybe_save_article_discussion_note(
|
||||
user_id, conv_id, buf.content_so_far,
|
||||
))
|
||||
|
||||
if should_gen_title:
|
||||
# Feed the title model the *raw* conversation turns only — never
|
||||
# the post-build_context ``messages`` list. ``build_context``
|
||||
# prepends RAG snippets, RSS excerpts, URL content, and briefing
|
||||
# article dumps INTO the user message string itself, so filtering
|
||||
# by role="user" downstream still surfaces that noise as the
|
||||
# "user's message". That pollution caused wildly-wrong titles
|
||||
# (bug #109) — the small background model was staring at article
|
||||
# excerpts instead of what the user actually typed. Pass the
|
||||
# original history + the raw user_content + the assistant reply.
|
||||
# prepends RAG snippets and URL content INTO the user message
|
||||
# string itself, so filtering by role="user" downstream still
|
||||
# surfaces that noise as the "user's message". That pollution
|
||||
# caused wildly-wrong titles (bug #109) — the small background
|
||||
# model was staring at article excerpts instead of what the user
|
||||
# actually typed. Pass the original history + the raw user_content
|
||||
# + the assistant reply.
|
||||
title_messages: list[dict] = [
|
||||
{"role": m["role"], "content": m.get("content") or ""}
|
||||
for m in history
|
||||
|
||||
@@ -623,7 +623,7 @@ async def build_context(
|
||||
"search_projects", "create_milestone", "update_milestone", "list_milestones",
|
||||
"save_person", "save_place", "create_list", "add_to_list", "clear_checked_items",
|
||||
"set_rag_scope", "get_profile", "update_profile", "get_weather", "calculate",
|
||||
"get_rss_items", "add_rss_feed", "read_article",
|
||||
"read_article",
|
||||
]
|
||||
if has_caldav:
|
||||
actions.extend(["create_event", "list_events", "search_events", "update_event", "delete_event", "list_calendars"])
|
||||
@@ -683,8 +683,8 @@ async def build_context(
|
||||
# --- System message: stable content only ---
|
||||
# Workspace context and history summary stay here because they carry
|
||||
# behavioural instructions / conversational state, not retrieved content.
|
||||
# Everything retrieval-based (RAG notes, RSS, URL content, current note,
|
||||
# briefing articles) goes into the user turn below so the system message
|
||||
# Everything retrieval-based (RAG notes, URL content, current note)
|
||||
# goes into the user turn below so the system message
|
||||
# prefix stays byte-for-byte identical across requests, enabling Ollama's
|
||||
# KV prefix cache to fire reliably.
|
||||
|
||||
@@ -726,25 +726,6 @@ async def build_context(
|
||||
f"\n\n--- Earlier Conversation ---\n{history_summary}\n--- End Earlier Conversation ---"
|
||||
)
|
||||
|
||||
# Detect briefing conversation — used for both system prompt instruction and article injection
|
||||
_is_briefing_conv = False
|
||||
if conv_id is not None:
|
||||
from fabledassistant.models import async_session as _async_session
|
||||
from fabledassistant.models.conversation import Conversation as _Conversation
|
||||
async with _async_session() as _sess:
|
||||
_conv = await _sess.get(_Conversation, conv_id)
|
||||
if _conv and getattr(_conv, "conversation_type", None) == "briefing":
|
||||
_is_briefing_conv = True
|
||||
|
||||
if _is_briefing_conv:
|
||||
system_content += (
|
||||
"\n\nYou are in a briefing conversation. "
|
||||
"The conversation history contains today's briefing — news stories, weather, and tasks. "
|
||||
"When the user asks about a topic, person, or event from the briefing, answer directly "
|
||||
"from the conversation history and the article context that follows. "
|
||||
"Do NOT search the web for information that is already present in the briefing."
|
||||
)
|
||||
|
||||
context_meta: dict = {
|
||||
"context_note_id": None,
|
||||
"context_note_title": None,
|
||||
@@ -780,27 +761,18 @@ async def build_context(
|
||||
orphan_only = rag_project_id is None
|
||||
effective_project_id = rag_project_id if (rag_project_id is not None and rag_project_id != -1) else None
|
||||
|
||||
# Skip RAG auto-injection on the first turn of a seeded article discussion.
|
||||
# The article body is already the sole context the user wants — pulling in
|
||||
# unrelated orphan notes tricks the model into summarizing those instead.
|
||||
# Follow-up turns keep RAG on because by then the user's own messages drive
|
||||
# the query rather than the generic seed prompt.
|
||||
from fabledassistant.services.article_context import ARTICLE_DISCUSS_SEED
|
||||
_skip_rag_for_article_seed = user_message.strip() == ARTICLE_DISCUSS_SEED
|
||||
try:
|
||||
from fabledassistant.services.embeddings import semantic_search_notes
|
||||
for score, note in await semantic_search_notes(
|
||||
user_id, user_message, exclude_ids=search_exclude or None, limit=8,
|
||||
project_id=effective_project_id,
|
||||
orphan_only=orphan_only,
|
||||
):
|
||||
found_scored.append((score, note))
|
||||
except Exception:
|
||||
logger.warning("Semantic note search failed, falling back to keyword search", exc_info=True)
|
||||
|
||||
if not _skip_rag_for_article_seed:
|
||||
try:
|
||||
from fabledassistant.services.embeddings import semantic_search_notes
|
||||
for score, note in await semantic_search_notes(
|
||||
user_id, user_message, exclude_ids=search_exclude or None, limit=8,
|
||||
project_id=effective_project_id,
|
||||
orphan_only=orphan_only,
|
||||
):
|
||||
found_scored.append((score, note))
|
||||
except Exception:
|
||||
logger.warning("Semantic note search failed, falling back to keyword search", exc_info=True)
|
||||
|
||||
if not found_scored and not _skip_rag_for_article_seed:
|
||||
if not found_scored:
|
||||
keywords = _extract_keywords(user_message)
|
||||
if keywords:
|
||||
try:
|
||||
@@ -890,12 +862,6 @@ async def build_context(
|
||||
f"--- Content from {url} ---\n{content}\n--- End URL Content ---"
|
||||
)
|
||||
|
||||
# Briefing article context for follow-up Q&A
|
||||
if _is_briefing_conv:
|
||||
article_context = await _build_briefing_article_context(conv_id) # type: ignore[arg-type]
|
||||
if article_context:
|
||||
user_context_parts.append(article_context.strip())
|
||||
|
||||
# Build final user message — context prefix (if any) followed by the actual message
|
||||
if user_context_parts:
|
||||
user_turn = "\n\n".join(user_context_parts) + "\n\n" + user_message
|
||||
@@ -906,72 +872,3 @@ async def build_context(
|
||||
messages.extend(history)
|
||||
messages.append({"role": "user", "content": user_turn})
|
||||
return messages, context_meta
|
||||
|
||||
|
||||
async def _build_briefing_article_context(conv_id: int) -> str:
|
||||
"""Fetch article content from today's briefing message and return a
|
||||
formatted context block for injection into the system prompt.
|
||||
|
||||
Looks at the most recent assistant briefing messages for rss_item_ids
|
||||
in their metadata, then loads those items from the DB.
|
||||
Capped at 10 articles × 500 chars to keep token use reasonable.
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
from sqlalchemy import select, text as _text
|
||||
|
||||
from fabledassistant.models import async_session as _async_session
|
||||
from fabledassistant.models.conversation import Message
|
||||
|
||||
async with _async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Message)
|
||||
.where(
|
||||
Message.conversation_id == conv_id,
|
||||
Message.role == "assistant",
|
||||
)
|
||||
.order_by(Message.created_at.desc())
|
||||
.limit(10)
|
||||
)
|
||||
messages = result.scalars().all()
|
||||
|
||||
rss_item_ids: list[int] = []
|
||||
for msg in messages:
|
||||
meta = msg.msg_metadata or {}
|
||||
if isinstance(meta, str):
|
||||
try:
|
||||
meta = _json.loads(meta)
|
||||
except Exception:
|
||||
continue
|
||||
ids = meta.get("rss_item_ids") or []
|
||||
if ids:
|
||||
rss_item_ids = ids
|
||||
break
|
||||
|
||||
if not rss_item_ids:
|
||||
return ""
|
||||
|
||||
async with _async_session() as session:
|
||||
result = await session.execute(
|
||||
_text("""
|
||||
SELECT i.title, i.url, i.content, f.title AS feed_title
|
||||
FROM rss_items i
|
||||
JOIN rss_feeds f ON f.id = i.feed_id
|
||||
WHERE i.id = ANY(:ids)
|
||||
ORDER BY i.published_at DESC NULLS LAST
|
||||
LIMIT 10
|
||||
""").bindparams(ids=rss_item_ids[:10])
|
||||
)
|
||||
rows = result.mappings().all()
|
||||
|
||||
if not rows:
|
||||
return ""
|
||||
|
||||
lines = ["\n\nARTICLE CONTEXT (source articles from today's briefing):"]
|
||||
for row in rows:
|
||||
lines.append(f"\n[{row['feed_title']}] {row['title']}")
|
||||
if row["url"]:
|
||||
lines.append(f"URL: {row['url']}")
|
||||
if row["content"]:
|
||||
lines.append(row["content"][:500])
|
||||
return "\n".join(lines)
|
||||
|
||||
@@ -1,317 +0,0 @@
|
||||
"""RSS feed service: fetch, parse with feedparser, and cache items to DB."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import feedparser
|
||||
import html2text
|
||||
import httpx
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from fabledassistant.models import async_session
|
||||
from fabledassistant.models.rss_feed import RssFeed, RssItem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Keep only items from the last N days to avoid unbounded growth
|
||||
ITEM_MAX_AGE_DAYS = 90
|
||||
|
||||
_h2t = html2text.HTML2Text()
|
||||
_h2t.ignore_links = True
|
||||
_h2t.ignore_images = True
|
||||
_h2t.ignore_emphasis = False
|
||||
_h2t.body_width = 0 # No line-wrapping
|
||||
|
||||
|
||||
def _html_to_text(html: str) -> str:
|
||||
"""Convert HTML to clean plain text via html2text."""
|
||||
if not html or "<" not in html:
|
||||
return html
|
||||
try:
|
||||
return _h2t.handle(html).strip()
|
||||
except Exception:
|
||||
return html
|
||||
|
||||
|
||||
async def get_or_fetch_full_article(item: RssItem) -> str | None:
|
||||
"""Return the full article body, fetching+caching on miss.
|
||||
|
||||
Checks ``item.content_full`` first — populated either by the enrichment
|
||||
pass at feed-ingest time or by a previous discuss-click. On miss, fetches
|
||||
via ``_fetch_full_article`` and writes through. Returns ``None`` only if
|
||||
the fetch itself fails; ``item.content_full == ""`` is still a cache hit.
|
||||
|
||||
Callers must pass an RssItem attached to an open session if they want
|
||||
the write-through to persist — otherwise the fetched text is returned
|
||||
but the cache stays empty and the next click will re-fetch.
|
||||
"""
|
||||
if item.content_full is not None:
|
||||
return item.content_full
|
||||
if not item.url:
|
||||
return None
|
||||
text = await _fetch_full_article(item.url)
|
||||
if text is None:
|
||||
return None
|
||||
async with async_session() as session:
|
||||
fresh = await session.get(RssItem, item.id)
|
||||
if fresh is not None:
|
||||
fresh.content_full = text
|
||||
fresh.content_fetched_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
return text
|
||||
|
||||
|
||||
async def _fetch_full_article(url: str) -> str | None:
|
||||
"""Fetch a URL and extract its main article text via trafilatura.
|
||||
|
||||
Returns clean plain text, or None if extraction fails or yields nothing useful.
|
||||
Runs trafilatura in a thread executor since it does synchronous HTML parsing.
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15.0, follow_redirects=True, headers={
|
||||
"User-Agent": "Mozilla/5.0 (compatible; FabledAssistant/1.0; +https://fabledsword.com)",
|
||||
}) as client:
|
||||
resp = await client.get(url)
|
||||
resp.raise_for_status()
|
||||
raw_html = resp.text
|
||||
except Exception:
|
||||
logger.debug("Failed to fetch article URL %s", url)
|
||||
return None
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
import trafilatura
|
||||
text = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: trafilatura.extract(
|
||||
raw_html,
|
||||
include_comments=False,
|
||||
include_tables=True,
|
||||
favor_recall=True,
|
||||
),
|
||||
)
|
||||
return text or None
|
||||
except Exception:
|
||||
logger.debug("trafilatura extraction failed for %s", url, exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def extract_item(entry) -> dict:
|
||||
"""Extract a clean item dict from a feedparser entry object."""
|
||||
# Prefer full content over summary (feedparser uses a list of Content objects)
|
||||
content = ""
|
||||
raw_content = getattr(entry, "content", None)
|
||||
if isinstance(raw_content, list) and raw_content:
|
||||
content = raw_content[0].value
|
||||
else:
|
||||
content = entry.get("summary", "")
|
||||
content = _html_to_text(content)
|
||||
|
||||
pub = None
|
||||
if entry.published_parsed:
|
||||
try:
|
||||
pub = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
"guid": entry.get("id", entry.get("link", "")),
|
||||
"title": entry.get("title", ""),
|
||||
"url": entry.get("link", ""),
|
||||
"content": content,
|
||||
"published_at": pub,
|
||||
}
|
||||
|
||||
|
||||
async def _parse_feed(url: str) -> feedparser.FeedParserDict:
|
||||
"""Run feedparser in a thread executor so we don't block the event loop."""
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(None, feedparser.parse, url)
|
||||
|
||||
|
||||
async def fetch_and_cache_feed(feed_id: int, url: str) -> int:
|
||||
"""
|
||||
Fetch a feed URL, parse it, and upsert new items into rss_items.
|
||||
Returns the number of new items stored.
|
||||
"""
|
||||
scheme = url.split("://")[0].lower() if "://" in url else ""
|
||||
if scheme not in ("http", "https"):
|
||||
logger.warning("Blocked RSS fetch with non-http(s) scheme: %s", url[:80])
|
||||
return 0
|
||||
try:
|
||||
parsed = await _parse_feed(url)
|
||||
except Exception:
|
||||
logger.warning("Failed to fetch RSS feed %s", url, exc_info=True)
|
||||
return 0
|
||||
|
||||
if parsed.bozo and not parsed.entries:
|
||||
logger.warning("Malformed RSS feed %s: %s", url, parsed.bozo_exception)
|
||||
return 0
|
||||
|
||||
new_count = 0
|
||||
feed_user_id: int | None = None
|
||||
|
||||
async with async_session() as session:
|
||||
for entry in parsed.entries:
|
||||
item_data = extract_item(entry)
|
||||
if not item_data["guid"]:
|
||||
continue
|
||||
# Check if already stored
|
||||
existing = await session.execute(
|
||||
select(RssItem).where(
|
||||
RssItem.feed_id == feed_id,
|
||||
RssItem.guid == item_data["guid"],
|
||||
)
|
||||
)
|
||||
if existing.scalars().first() is not None:
|
||||
continue
|
||||
item = RssItem(
|
||||
feed_id=feed_id,
|
||||
**item_data,
|
||||
)
|
||||
session.add(item)
|
||||
new_count += 1
|
||||
|
||||
# Update last_fetched_at on the feed
|
||||
feed_row = await session.get(RssFeed, feed_id)
|
||||
if feed_row:
|
||||
feed_row.last_fetched_at = datetime.now(timezone.utc)
|
||||
feed_user_id = feed_row.user_id
|
||||
# Auto-populate title from feed metadata if blank
|
||||
if not feed_row.title and parsed.feed.get("title"):
|
||||
feed_row.title = parsed.feed.title[:200]
|
||||
|
||||
await session.commit()
|
||||
|
||||
# Collect IDs of unclassified items after commit.
|
||||
# We query classified_at IS NULL (not just the items inserted above) because
|
||||
# classification is best-effort and may have failed on previous fetches.
|
||||
# Re-queuing all unclassified items for this feed on each fetch is intentional:
|
||||
# it provides automatic retry without a separate retry loop. The classifier
|
||||
# only writes to items it successfully classifies, so already-classified items
|
||||
# are not re-processed (they have classified_at set).
|
||||
unclassified_ids: list[int] = []
|
||||
new_item_data: list[tuple[int, str, str]] = [] # (id, title, content) for embedding
|
||||
if new_count > 0:
|
||||
result = await session.execute(
|
||||
select(RssItem.id, RssItem.title, RssItem.content, RssItem.classified_at).where(
|
||||
RssItem.feed_id == feed_id,
|
||||
)
|
||||
)
|
||||
for row in result.fetchall():
|
||||
item_id, title, content, classified_at = row
|
||||
if classified_at is None:
|
||||
unclassified_ids.append(item_id)
|
||||
new_item_data.append((item_id, title or "", content or ""))
|
||||
|
||||
# Prune old items to keep DB tidy
|
||||
await _prune_old_items(feed_id)
|
||||
|
||||
# Fire-and-forget classification for unclassified items
|
||||
if unclassified_ids and feed_user_id is not None:
|
||||
from fabledassistant.services.rss_classifier import classify_and_store
|
||||
asyncio.create_task(classify_and_store(unclassified_ids, feed_user_id))
|
||||
|
||||
# Collect (id, url) for newly inserted items to enrich with full article text
|
||||
new_items_for_enrichment: list[tuple[int, str]] = []
|
||||
if new_count > 0:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(RssItem.id, RssItem.url).where(
|
||||
RssItem.feed_id == feed_id,
|
||||
RssItem.url != "",
|
||||
).order_by(RssItem.fetched_at.desc()).limit(new_count)
|
||||
)
|
||||
new_items_for_enrichment = list(result.fetchall())
|
||||
|
||||
# Fire-and-forget: fetch full article text, then re-embed with richer content
|
||||
if new_items_for_enrichment and feed_user_id is not None:
|
||||
from fabledassistant.services.embeddings import upsert_rss_item_embedding
|
||||
|
||||
async def _enrich_and_embed() -> None:
|
||||
for item_id, article_url in new_items_for_enrichment:
|
||||
full_text = await _fetch_full_article(article_url)
|
||||
if full_text and len(full_text) > 200:
|
||||
async with async_session() as session:
|
||||
item = await session.get(RssItem, item_id)
|
||||
if item:
|
||||
item.content = full_text
|
||||
# Populate the discuss-click cache too so the
|
||||
# first click skips straight to the map-reduce
|
||||
# step without re-fetching.
|
||||
item.content_full = full_text
|
||||
item.content_fetched_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
await upsert_rss_item_embedding(
|
||||
item_id, feed_user_id, item.title or "", item.content
|
||||
)
|
||||
else:
|
||||
# Enrich failed — still embed with RSS-provided content
|
||||
async with async_session() as session:
|
||||
item = await session.get(RssItem, item_id)
|
||||
if item:
|
||||
await upsert_rss_item_embedding(
|
||||
item_id, feed_user_id, item.title or "", item.content or ""
|
||||
)
|
||||
await asyncio.sleep(0.5) # Polite pacing between article fetches
|
||||
|
||||
asyncio.create_task(_enrich_and_embed())
|
||||
elif new_item_data and feed_user_id is not None:
|
||||
# No URLs to enrich — embed with RSS content only
|
||||
from fabledassistant.services.embeddings import upsert_rss_item_embedding
|
||||
|
||||
async def _embed_only() -> None:
|
||||
for item_id, title, content in new_item_data:
|
||||
await upsert_rss_item_embedding(item_id, feed_user_id, title, content)
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
asyncio.create_task(_embed_only())
|
||||
|
||||
return new_count
|
||||
|
||||
|
||||
async def _prune_old_items(feed_id: int) -> None:
|
||||
"""Delete items older than ITEM_MAX_AGE_DAYS from a feed."""
|
||||
async with async_session() as session:
|
||||
await session.execute(
|
||||
text("""
|
||||
DELETE FROM rss_items
|
||||
WHERE feed_id = :feed_id
|
||||
AND published_at < NOW() - INTERVAL '90 days'
|
||||
""").bindparams(feed_id=feed_id)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def get_recent_items(user_id: int, limit: int = 20) -> list[dict]:
|
||||
"""Return the most recent RSS items across all of a user's feeds."""
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(RssItem, RssFeed.title.label("feed_title"))
|
||||
.join(RssFeed, RssItem.feed_id == RssFeed.id)
|
||||
.where(RssFeed.user_id == user_id)
|
||||
.order_by(RssItem.published_at.desc().nullslast())
|
||||
.limit(limit)
|
||||
)
|
||||
rows = result.all()
|
||||
|
||||
return [
|
||||
{**item.to_dict(), "feed_title": feed_title}
|
||||
for item, feed_title in rows
|
||||
]
|
||||
|
||||
|
||||
async def refresh_all_feeds(user_id: int) -> dict[int, int]:
|
||||
"""Fetch all feeds for a user. Returns {feed_id: new_items_count}."""
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(RssFeed).where(RssFeed.user_id == user_id)
|
||||
)
|
||||
feeds = list(result.scalars().all())
|
||||
|
||||
results = {}
|
||||
for feed in feeds:
|
||||
count = await fetch_and_cache_feed(feed.id, feed.url)
|
||||
results[feed.id] = count
|
||||
return results
|
||||
@@ -1,152 +0,0 @@
|
||||
"""
|
||||
RSS item topic classifier.
|
||||
|
||||
Classifies RSS items into topic tags using a fast non-streaming LLM call.
|
||||
Called from rss.py after new items are stored — fire-and-forget.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import httpx
|
||||
|
||||
from fabledassistant.config import Config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
STANDARD_TOPICS = [
|
||||
"technology", "science", "politics", "business",
|
||||
"health", "environment", "local", "entertainment", "sports", "other",
|
||||
]
|
||||
|
||||
_CLASSIFY_PROMPT = """\
|
||||
Classify each news item into 1-3 topics. Use only topics from this list: {vocab}.
|
||||
Return ONLY a JSON object mapping item_id (as string) to a list of topics.
|
||||
Example: {{"1": ["technology", "ai"], "2": ["politics"]}}
|
||||
|
||||
Items:
|
||||
{items_block}"""
|
||||
|
||||
|
||||
async def _llm_classify(prompt: str, model: str) -> str:
|
||||
"""Make a fast non-streaming LLM call and return the raw text response."""
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"stream": False,
|
||||
"options": {"num_ctx": 2048, "temperature": 0.0},
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
resp = await client.post(f"{Config.OLLAMA_URL}/api/chat", json=payload)
|
||||
resp.raise_for_status()
|
||||
return resp.json().get("message", {}).get("content", "")
|
||||
|
||||
|
||||
async def classify_items_batch(
|
||||
items: list[dict],
|
||||
user_include_topics: list[str],
|
||||
model: str | None = None,
|
||||
) -> dict[int, list[str]]:
|
||||
"""
|
||||
Classify a batch of RSS items into topic tags.
|
||||
|
||||
Args:
|
||||
items: list of dicts with 'id', 'title', 'content'
|
||||
user_include_topics: extra topics from user preferences to add to vocabulary
|
||||
model: Ollama model name; defaults to Config.OLLAMA_MODEL
|
||||
|
||||
Returns:
|
||||
dict mapping item_id (int) -> list of topic strings.
|
||||
Items not returned had classification fail; callers should leave classified_at=NULL.
|
||||
"""
|
||||
if not items:
|
||||
return {}
|
||||
|
||||
if model is None:
|
||||
model = Config.OLLAMA_BACKGROUND_MODEL
|
||||
|
||||
vocab = STANDARD_TOPICS + [t for t in user_include_topics if t not in STANDARD_TOPICS]
|
||||
items_block = "\n".join(
|
||||
f"[{item['id']}] {item['title']} — {item.get('content', '')[:300]}"
|
||||
for item in items
|
||||
)
|
||||
prompt = _CLASSIFY_PROMPT.format(vocab=", ".join(vocab), items_block=items_block)
|
||||
|
||||
try:
|
||||
raw = await _llm_classify(prompt, model)
|
||||
# Strip <think>...</think> blocks emitted by reasoning models (e.g. qwen3)
|
||||
raw = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL)
|
||||
# Extract JSON from response (LLM may wrap it in markdown)
|
||||
raw = raw.strip()
|
||||
if raw.startswith("```"):
|
||||
raw = raw.split("```")[1]
|
||||
if raw.startswith("json"):
|
||||
raw = raw[4:]
|
||||
raw = raw.strip()
|
||||
# Allow control characters that may appear in LLM-generated JSON strings
|
||||
parsed = json.loads(raw, strict=False)
|
||||
return {int(k): v for k, v in parsed.items() if isinstance(v, list)}
|
||||
except Exception:
|
||||
logger.warning("RSS classification failed", exc_info=True)
|
||||
return {}
|
||||
|
||||
|
||||
async def classify_and_store(
|
||||
item_ids: list[int],
|
||||
user_id: int,
|
||||
) -> None:
|
||||
"""
|
||||
Classify unclassified RSS items and write results to DB.
|
||||
Called as a fire-and-forget task from rss.py.
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
|
||||
from fabledassistant.models import async_session
|
||||
from fabledassistant.models.rss_feed import RssItem
|
||||
from fabledassistant.services.settings import get_setting
|
||||
|
||||
if not item_ids:
|
||||
return
|
||||
|
||||
# Load the items
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(RssItem).where(RssItem.id.in_(item_ids))
|
||||
)
|
||||
items = list(result.scalars().all())
|
||||
|
||||
if not items:
|
||||
return
|
||||
|
||||
# Get user's include topics to extend vocabulary
|
||||
raw_include = await get_setting(user_id, "briefing_include_topics", "[]")
|
||||
try:
|
||||
include_topics = json.loads(raw_include) if isinstance(raw_include, str) else []
|
||||
except Exception:
|
||||
include_topics = []
|
||||
|
||||
model = await get_setting(user_id, "background_model", Config.OLLAMA_BACKGROUND_MODEL)
|
||||
|
||||
# Classify in batches of 10
|
||||
batch_size = 10
|
||||
all_results: dict[int, list[str]] = {}
|
||||
for i in range(0, len(items), batch_size):
|
||||
batch = items[i: i + batch_size]
|
||||
batch_dicts = [{"id": it.id, "title": it.title, "content": it.content} for it in batch]
|
||||
results = await classify_items_batch(batch_dicts, include_topics, model=model)
|
||||
all_results.update(results)
|
||||
|
||||
# Write back to DB
|
||||
now = datetime.now(timezone.utc)
|
||||
async with async_session() as session:
|
||||
for item in items:
|
||||
item_db = await session.get(RssItem, item.id)
|
||||
if item_db is None:
|
||||
continue
|
||||
topics = all_results.get(item.id)
|
||||
if topics is not None:
|
||||
item_db.topics = topics
|
||||
item_db.classified_at = now
|
||||
await session.commit()
|
||||
@@ -1,110 +0,0 @@
|
||||
"""
|
||||
Briefing preferences: load topic settings, aggregate reaction scores,
|
||||
filter and rank RSS items for briefing inclusion.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fabledassistant.models import async_session
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def load_topic_preferences(user_id: int) -> tuple[list[str], list[str]]:
|
||||
"""
|
||||
Return (include_topics, exclude_topics) from user settings.
|
||||
"""
|
||||
from fabledassistant.services.settings import get_setting
|
||||
|
||||
raw_include = await get_setting(user_id, "briefing_include_topics", "[]")
|
||||
raw_exclude = await get_setting(user_id, "briefing_exclude_topics", "[]")
|
||||
|
||||
def _parse(raw) -> list[str]:
|
||||
try:
|
||||
val = json.loads(raw) if isinstance(raw, str) else raw
|
||||
return [str(t) for t in val] if isinstance(val, list) else []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
return _parse(raw_include), _parse(raw_exclude)
|
||||
|
||||
|
||||
async def load_topic_reaction_scores(user_id: int) -> dict[str, float]:
|
||||
"""
|
||||
Aggregate per-topic reaction scores from the last 30 days.
|
||||
Returns a dict of topic -> net_score (positive = liked, negative = disliked).
|
||||
Uses rss_item_reactions joined to rss_items.topics.
|
||||
"""
|
||||
try:
|
||||
from sqlalchemy import text as _text
|
||||
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
_text("""
|
||||
SELECT unnest(i.topics) AS topic,
|
||||
SUM(CASE r.reaction WHEN 'up' THEN 1 ELSE -1 END) AS score
|
||||
FROM rss_item_reactions r
|
||||
JOIN rss_items i ON i.id = r.rss_item_id
|
||||
WHERE r.user_id = :uid
|
||||
AND r.created_at > NOW() - INTERVAL '30 days'
|
||||
GROUP BY topic
|
||||
""").bindparams(uid=user_id)
|
||||
)
|
||||
return {row.topic: float(row.score) for row in result}
|
||||
except Exception:
|
||||
logger.warning("Failed to load topic reaction scores", exc_info=True)
|
||||
return {}
|
||||
|
||||
|
||||
def score_and_filter_items(
|
||||
items: list[dict],
|
||||
include_topics: list[str],
|
||||
exclude_topics: list[str],
|
||||
topic_scores: dict[str, float],
|
||||
max_items: int = 10,
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Score, filter, and rank RSS items for briefing inclusion.
|
||||
|
||||
Scoring:
|
||||
- Hard-exclude: any item tagged with an excluded topic is removed.
|
||||
- Base score: 0.0
|
||||
- +2.0 per topic that appears in include_topics
|
||||
- +1.0 / -1.0 per topic based on reaction score (clamped per topic)
|
||||
- Tiebreak: newer published_at wins
|
||||
|
||||
Returns up to max_items items, highest score first.
|
||||
Items with classified_at=None (unclassified) pass through with score=0.
|
||||
"""
|
||||
include_set = set(include_topics)
|
||||
exclude_set = set(exclude_topics)
|
||||
scored = []
|
||||
|
||||
for item in items:
|
||||
item_topics = item.get("topics") or []
|
||||
|
||||
# Hard exclude
|
||||
if exclude_set and any(t in exclude_set for t in item_topics):
|
||||
continue
|
||||
|
||||
score = 0.0
|
||||
for topic in item_topics:
|
||||
if topic in include_set:
|
||||
score += 2.0
|
||||
if topic in topic_scores:
|
||||
score += max(-1.0, min(1.0, topic_scores[topic]))
|
||||
|
||||
# Parse published_at for tiebreak
|
||||
pub_str = item.get("published_at") or ""
|
||||
try:
|
||||
pub_ts = datetime.fromisoformat(pub_str).timestamp() if pub_str else 0.0
|
||||
except ValueError:
|
||||
pub_ts = 0.0
|
||||
|
||||
scored.append((score, pub_ts, item))
|
||||
|
||||
# Sort: highest score first, then newest first
|
||||
scored.sort(key=lambda x: (x[0], x[1]), reverse=True)
|
||||
return [item for _, _, item in scored[:max_items]]
|
||||
@@ -8,6 +8,7 @@ of the app depends on.
|
||||
# Import every tool module so their @tool decorators run at import time.
|
||||
# Order does not matter — registration is additive.
|
||||
from fabledassistant.services.tools import ( # noqa: F401
|
||||
article,
|
||||
calendar,
|
||||
entities,
|
||||
journal,
|
||||
@@ -15,7 +16,6 @@ from fabledassistant.services.tools import ( # noqa: F401
|
||||
profile,
|
||||
projects,
|
||||
rag,
|
||||
rss,
|
||||
tasks,
|
||||
utility,
|
||||
weather,
|
||||
|
||||
@@ -87,9 +87,6 @@ async def _check_requires(user_id: int, requires: str) -> bool:
|
||||
return await is_caldav_configured(user_id)
|
||||
if requires == "searxng":
|
||||
return Config.searxng_enabled()
|
||||
if requires == "rss":
|
||||
from fabledassistant.services.settings import get_setting
|
||||
return (await get_setting(user_id, "rss_enabled", "false")).lower() == "true"
|
||||
return True
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Generic article-reading LLM tool.
|
||||
|
||||
The ``read_article`` tool fetches any URL and returns its main body text via
|
||||
trafilatura. URL-generic — not coupled to any feed system.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fabledassistant.services.article_fetcher import fetch_article_text
|
||||
from fabledassistant.services.tools._registry import tool
|
||||
|
||||
|
||||
@tool(
|
||||
name="read_article",
|
||||
description=(
|
||||
"Fetch the main body text of an article at a URL. Use when the user asks "
|
||||
"to read, summarize, or discuss a specific article they've linked. "
|
||||
"Returns the cleaned article text or an empty result if extraction fails."
|
||||
),
|
||||
parameters={
|
||||
"url": {
|
||||
"type": "string",
|
||||
"description": "The article URL to fetch.",
|
||||
},
|
||||
},
|
||||
required=["url"],
|
||||
read_only=True,
|
||||
)
|
||||
async def read_article_tool(*, user_id, arguments, **_ctx):
|
||||
url = (arguments.get("url") or "").strip()
|
||||
if not url:
|
||||
return {"success": False, "error": "url is required"}
|
||||
content = await fetch_article_text(url)
|
||||
if not content:
|
||||
return {"success": True, "type": "article", "data": {"url": url, "content": None, "note": "no content extracted"}}
|
||||
return {"success": True, "type": "article", "data": {"url": url, "content": content[:6000]}}
|
||||
@@ -1,101 +0,0 @@
|
||||
"""RSS and article tools."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fabledassistant.services.tools._registry import tool
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@tool(
|
||||
name="get_rss_items",
|
||||
description="Get recent items from the user's RSS feeds (news, blogs, Reddit, podcasts). Returns titles, URLs, and summaries of recent posts.",
|
||||
parameters={
|
||||
"limit": {"type": "integer", "description": "Number of items to return (default 15, max 50)"},
|
||||
"category": {"type": "string", "description": "Filter by feed category (e.g. 'news', 'tech'). Omit for all."},
|
||||
},
|
||||
read_only=True,
|
||||
briefing=True,
|
||||
requires="rss",
|
||||
)
|
||||
async def get_rss_items_tool(*, user_id, arguments, **_ctx):
|
||||
from fabledassistant.services.rss import get_recent_items
|
||||
|
||||
limit = min(int(arguments.get("limit", 15)), 50)
|
||||
items = await get_recent_items(user_id, limit=limit)
|
||||
return {"data": {"items": items, "count": len(items)}}
|
||||
|
||||
|
||||
@tool(
|
||||
name="add_rss_feed",
|
||||
description="Add an RSS/Atom feed. Use when user asks to subscribe to or track a feed, blog, subreddit, or podcast.",
|
||||
parameters={
|
||||
"url": {"type": "string", "description": "The RSS/Atom feed URL to add."},
|
||||
"category": {"type": "string", "description": "Optional category label (e.g. 'news', 'tech', 'reddit'). Omit if unsure."},
|
||||
},
|
||||
required=["url"],
|
||||
requires="rss",
|
||||
)
|
||||
async def add_rss_feed_tool(*, user_id, arguments, **_ctx):
|
||||
import asyncio as _asyncio
|
||||
|
||||
from sqlalchemy import select as _select
|
||||
|
||||
from fabledassistant.models import async_session as _async_session
|
||||
from fabledassistant.models.rss_feed import RssFeed
|
||||
from fabledassistant.services.rss import fetch_and_cache_feed
|
||||
|
||||
url = str(arguments.get("url", "")).strip()
|
||||
if not url:
|
||||
return {"error": "url is required"}
|
||||
category = arguments.get("category") or None
|
||||
async with _async_session() as session:
|
||||
existing = await session.execute(
|
||||
_select(RssFeed).where(RssFeed.user_id == user_id, RssFeed.url == url)
|
||||
)
|
||||
if existing.scalars().first():
|
||||
return {"error": "Feed already added", "url": url}
|
||||
feed = RssFeed(user_id=user_id, url=url, title="", category=category)
|
||||
session.add(feed)
|
||||
await session.commit()
|
||||
await session.refresh(feed)
|
||||
feed_id = feed.id
|
||||
_asyncio.create_task(fetch_and_cache_feed(feed_id, url))
|
||||
return {"data": {"id": feed_id, "url": url, "message": "Feed added and fetching started."}}
|
||||
|
||||
|
||||
@tool(
|
||||
name="read_article",
|
||||
description=(
|
||||
"Fetch and read the full text of a web page or article from a URL. "
|
||||
"Use when the user shares a URL and wants you to read it, "
|
||||
"or to get the full content of a linked page. "
|
||||
"Do NOT use lookup for URLs — use this tool instead."
|
||||
),
|
||||
parameters={
|
||||
"url": {"type": "string", "description": "The URL to fetch and read"},
|
||||
},
|
||||
required=["url"],
|
||||
read_only=True,
|
||||
briefing=True,
|
||||
)
|
||||
async def read_article_tool(*, user_id, arguments, **_ctx):
|
||||
from fabledassistant.services.rss import _fetch_full_article
|
||||
|
||||
url = arguments.get("url", "").strip()
|
||||
if not url:
|
||||
return {"success": False, "error": "No URL provided"}
|
||||
content = await _fetch_full_article(url)
|
||||
if not content:
|
||||
return {"success": False, "error": f"Could not fetch article content from {url}"}
|
||||
_TOOL_CONTENT_CAP = 40_000
|
||||
truncated = len(content) > _TOOL_CONTENT_CAP
|
||||
return {
|
||||
"success": True,
|
||||
"type": "article_content",
|
||||
"url": url,
|
||||
"content": content[:_TOOL_CONTENT_CAP],
|
||||
"truncated": truncated,
|
||||
}
|
||||
@@ -70,14 +70,14 @@ async def lookup_tool(*, user_id, arguments, **_ctx):
|
||||
if search_results:
|
||||
# Sequential fetches: trafilatura/lxml is not safe to run concurrently
|
||||
# via run_in_executor — parallel calls can trip a libxml2 double-free.
|
||||
from fabledassistant.services.rss import _fetch_full_article
|
||||
from fabledassistant.services.article_fetcher import fetch_article_text
|
||||
|
||||
for r in search_results[:2]:
|
||||
url = r.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
try:
|
||||
content = await _fetch_full_article(url)
|
||||
content = await fetch_article_text(url)
|
||||
except Exception:
|
||||
content = None
|
||||
web_payload.append({
|
||||
|
||||
Reference in New Issue
Block a user