Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 267a975455 | |||
| 7bd1548f71 | |||
| 9f3b3450fa | |||
| ba90ad8132 | |||
| 9157740069 | |||
| fd885c1bc8 | |||
| 8205590f8d |
@@ -1,8 +1,12 @@
|
|||||||
# CI runs first; build only proceeds if all checks pass.
|
# CI runs first; build only proceeds if all checks pass.
|
||||||
#
|
#
|
||||||
# Push to any branch: typecheck + lint + test
|
# Push to dev: typecheck + lint + test + build :dev + :<sha>
|
||||||
# Push to dev: gates + build :dev + :<sha>
|
# Tag v* (release): typecheck + lint + test + build :latest + :<sha> + :<version>
|
||||||
# Tag v* (release): gates + build :latest + :<sha> + :<version>
|
#
|
||||||
|
# main pushes are NOT gated here: a merge to main only happens after
|
||||||
|
# dev has already passed CI, and the release tag is the sole trigger
|
||||||
|
# for a production image. Re-running CI on the merge commit just burns
|
||||||
|
# runner time without changing the outcome.
|
||||||
#
|
#
|
||||||
# To cut a release:
|
# To cut a release:
|
||||||
# Create a release via the Forgejo UI on main with a v* tag name.
|
# Create a release via the Forgejo UI on main with a v* tag name.
|
||||||
@@ -11,6 +15,13 @@
|
|||||||
# PRs aren't triggered on purpose — this is a solo dev→main flow, so
|
# PRs aren't triggered on purpose — this is a solo dev→main flow, so
|
||||||
# gating on branch push is already enough.
|
# gating on branch push is already enough.
|
||||||
#
|
#
|
||||||
|
# NOTE on the `if:` guards below: Forgejo Actions does not consistently
|
||||||
|
# honor `on.push.branches` as a filter — merge commits landing on main
|
||||||
|
# still trigger the workflow, producing redundant runs on the same SHA
|
||||||
|
# that was already gated on dev. Every job therefore repeats the ref
|
||||||
|
# check so main pushes trigger the workflow but every job skips
|
||||||
|
# immediately (no runner time, no duplicate work).
|
||||||
|
#
|
||||||
# Required secrets (repo → Settings → Secrets → Actions):
|
# Required secrets (repo → Settings → Secrets → Actions):
|
||||||
# REGISTRY_USER — your Forgejo username
|
# REGISTRY_USER — your Forgejo username
|
||||||
# REGISTRY_TOKEN — Forgejo PAT with write:packages scope
|
# REGISTRY_TOKEN — Forgejo PAT with write:packages scope
|
||||||
@@ -18,7 +29,7 @@ name: CI & Build
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [dev, main]
|
branches: [dev]
|
||||||
tags: ["v*"]
|
tags: ["v*"]
|
||||||
paths:
|
paths:
|
||||||
- "src/**"
|
- "src/**"
|
||||||
@@ -51,6 +62,8 @@ env:
|
|||||||
jobs:
|
jobs:
|
||||||
typecheck:
|
typecheck:
|
||||||
name: TypeScript typecheck
|
name: TypeScript typecheck
|
||||||
|
# Skip on main merge-commit pushes — see workflow header comment.
|
||||||
|
if: github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/v')
|
||||||
runs-on: ci-runner
|
runs-on: ci-runner
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v6
|
||||||
@@ -77,6 +90,7 @@ jobs:
|
|||||||
|
|
||||||
lint:
|
lint:
|
||||||
name: Python lint
|
name: Python lint
|
||||||
|
if: github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/v')
|
||||||
runs-on: ci-runner
|
runs-on: ci-runner
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v6
|
||||||
@@ -88,6 +102,7 @@ jobs:
|
|||||||
|
|
||||||
test:
|
test:
|
||||||
name: Python tests
|
name: Python tests
|
||||||
|
if: github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/v')
|
||||||
runs-on: ci-runner
|
runs-on: ci-runner
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v6
|
||||||
@@ -118,11 +133,10 @@ jobs:
|
|||||||
name: Build & push image
|
name: Build & push image
|
||||||
needs: [typecheck, lint, test]
|
needs: [typecheck, lint, test]
|
||||||
# Build on dev branch pushes and version tag pushes only.
|
# Build on dev branch pushes and version tag pushes only.
|
||||||
# main branch pushes run the gates for safety but do not build —
|
# Mirrors the ref guard on the gate jobs above — main merge-commit
|
||||||
# the release tag (v*) is the sole trigger for a production image.
|
# pushes skip here too, so no production image is ever built from a
|
||||||
if: |
|
# raw main push (only from the v* tag the release creates).
|
||||||
github.event_name == 'push' &&
|
if: github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/v')
|
||||||
(github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/'))
|
|
||||||
runs-on: ci-runner
|
runs-on: ci-runner
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
"""Add content_full and context_prepared caches to rss_items.
|
||||||
|
|
||||||
|
Revision ID: 0038
|
||||||
|
Revises: 0037
|
||||||
|
Create Date: 2026-04-13
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0038"
|
||||||
|
down_revision = "0037"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("rss_items", sa.Column("content_full", sa.Text(), nullable=True))
|
||||||
|
op.add_column(
|
||||||
|
"rss_items",
|
||||||
|
sa.Column("context_prepared", sa.Text(), nullable=True),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"rss_items",
|
||||||
|
sa.Column("content_fetched_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("rss_items", "content_fetched_at")
|
||||||
|
op.drop_column("rss_items", "context_prepared")
|
||||||
|
op.drop_column("rss_items", "content_full")
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
"""Link rss_items to their discussion-summary note.
|
||||||
|
|
||||||
|
Revision ID: 0039
|
||||||
|
Revises: 0038
|
||||||
|
Create Date: 2026-04-14
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0039"
|
||||||
|
down_revision = "0038"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"rss_items",
|
||||||
|
sa.Column("discussion_note_id", sa.BigInteger(), nullable=True),
|
||||||
|
)
|
||||||
|
op.create_foreign_key(
|
||||||
|
"fk_rss_items_discussion_note_id",
|
||||||
|
"rss_items",
|
||||||
|
"notes",
|
||||||
|
["discussion_note_id"],
|
||||||
|
["id"],
|
||||||
|
ondelete="SET NULL",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_constraint(
|
||||||
|
"fk_rss_items_discussion_note_id", "rss_items", type_="foreignkey"
|
||||||
|
)
|
||||||
|
op.drop_column("rss_items", "discussion_note_id")
|
||||||
@@ -426,7 +426,9 @@ export async function deleteRssReaction(rssItemId: number): Promise<void> {
|
|||||||
return apiDelete(`/api/briefing/rss-reactions/${rssItemId}`);
|
return apiDelete(`/api/briefing/rss-reactions/${rssItemId}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function openArticleInChat(itemId: number): Promise<{ conversation_id: number }> {
|
export async function openArticleInChat(
|
||||||
|
itemId: number,
|
||||||
|
): Promise<{ conversation_id: number; assistant_message_id?: number; status?: string }> {
|
||||||
return apiPost(`/api/chat/from-article/${itemId}`, {});
|
return apiPost(`/api/chat/from-article/${itemId}`, {});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from sqlalchemy import ARRAY, DateTime, ForeignKey, Index, Integer, Text, UniqueConstraint
|
from sqlalchemy import ARRAY, BigInteger, DateTime, ForeignKey, Index, Integer, Text, UniqueConstraint
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
from fabledassistant.models import Base
|
from fabledassistant.models import Base
|
||||||
@@ -46,6 +46,17 @@ class RssItem(Base):
|
|||||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
# Truncated to 2000 chars to keep DB size reasonable
|
# Truncated to 2000 chars to keep DB size reasonable
|
||||||
content: Mapped[str] = mapped_column(Text, default="")
|
content: Mapped[str] = mapped_column(Text, default="")
|
||||||
|
# Full trafilatura-extracted article body, populated lazily on first
|
||||||
|
# discuss-click / enrichment pass. Nullable — most items never get this
|
||||||
|
# cached. Expires naturally with the item (90-day retention).
|
||||||
|
content_full: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
# Map-reduced conversation-ready context derived from content_full. See
|
||||||
|
# services/article_context.py — populated on first discuss click so
|
||||||
|
# repeat clicks skip both the fetch and the LLM map step.
|
||||||
|
context_prepared: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
content_fetched_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
fetched_at: Mapped[datetime] = mapped_column(
|
fetched_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
|
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
|
||||||
)
|
)
|
||||||
@@ -55,6 +66,13 @@ class RssItem(Base):
|
|||||||
classified_at: Mapped[datetime | None] = mapped_column(
|
classified_at: Mapped[datetime | None] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=True
|
DateTime(timezone=True), nullable=True
|
||||||
)
|
)
|
||||||
|
# Note persisting the first-click discussion summary. Set by the article
|
||||||
|
# discussion pipeline once the seeded chat completes its first assistant
|
||||||
|
# reply; links back into RAG so re-discussing the same article lands the
|
||||||
|
# prior summary in context.
|
||||||
|
discussion_note_id: Mapped[int | None] = mapped_column(
|
||||||
|
BigInteger, ForeignKey("notes.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
|
||||||
feed: Mapped["RssFeed"] = relationship(back_populates="items")
|
feed: Mapped["RssFeed"] = relationship(back_populates="items")
|
||||||
|
|
||||||
|
|||||||
@@ -532,25 +532,21 @@ async def discuss_article(item_id: int):
|
|||||||
if get_buffer(conv_id) is not None:
|
if get_buffer(conv_id) is not None:
|
||||||
return jsonify({"error": "Generation already in progress"}), 409
|
return jsonify({"error": "Generation already in progress"}), 409
|
||||||
|
|
||||||
from fabledassistant.services.rss import _fetch_full_article
|
# Shared helper handles the three-layer cache (context_prepared →
|
||||||
article_content = await _fetch_full_article(item.url) or item.content or ""
|
# content_full → fresh fetch), writes the synthetic read_article tool
|
||||||
|
# exchange and the conversational seed user prompt into the conversation.
|
||||||
|
# The /news from-article route calls the same helper so behavior stays
|
||||||
|
# byte-identical across entry points.
|
||||||
|
from fabledassistant.services.article_context import (
|
||||||
|
EmptyArticleError,
|
||||||
|
seed_article_discussion,
|
||||||
|
)
|
||||||
|
|
||||||
# Store synthetic assistant message with read_article tool result
|
model = await get_setting(uid, "default_model", "") or ""
|
||||||
synthetic_tool_calls = [{
|
try:
|
||||||
"function": "read_article",
|
discuss_prompt = await seed_article_discussion(conv_id, item, model)
|
||||||
"arguments": {"url": item.url},
|
except EmptyArticleError as e:
|
||||||
"result": {
|
return jsonify({"error": str(e)}), 422
|
||||||
"success": True,
|
|
||||||
"type": "article_content",
|
|
||||||
"url": item.url,
|
|
||||||
"content": article_content,
|
|
||||||
"truncated": False,
|
|
||||||
},
|
|
||||||
}]
|
|
||||||
await add_message(conv_id, "assistant", "", status="complete", tool_calls=synthetic_tool_calls)
|
|
||||||
|
|
||||||
# Store user message
|
|
||||||
await add_message(conv_id, "user", "Please summarize and discuss this article.")
|
|
||||||
|
|
||||||
# Reload conversation with fresh messages to build history
|
# Reload conversation with fresh messages to build history
|
||||||
conv = await get_conversation(uid, conv_id)
|
conv = await get_conversation(uid, conv_id)
|
||||||
@@ -572,15 +568,13 @@ async def discuss_article(item_id: int):
|
|||||||
else:
|
else:
|
||||||
history.append(msg_dict)
|
history.append(msg_dict)
|
||||||
|
|
||||||
model = await get_setting(uid, "default_model", "") or ""
|
|
||||||
|
|
||||||
assistant_msg = await add_message(conv_id, "assistant", "", status="generating")
|
assistant_msg = await add_message(conv_id, "assistant", "", status="generating")
|
||||||
buf = create_buffer(conv_id, assistant_msg.id)
|
buf = create_buffer(conv_id, assistant_msg.id)
|
||||||
|
|
||||||
asyncio.create_task(run_generation(
|
asyncio.create_task(run_generation(
|
||||||
buf, history, model,
|
buf, history, model,
|
||||||
uid, conv_id, conv.title or "",
|
uid, conv_id, conv.title or "",
|
||||||
"Please summarize and discuss this article.",
|
discuss_prompt,
|
||||||
))
|
))
|
||||||
|
|
||||||
return jsonify({"assistant_message_id": assistant_msg.id, "status": "generating"}), 202
|
return jsonify({"assistant_message_id": assistant_msg.id, "status": "generating"}), 202
|
||||||
|
|||||||
@@ -510,47 +510,78 @@ async def delete_model_route():
|
|||||||
@chat_bp.route("/from-article/<int:item_id>", methods=["POST"])
|
@chat_bp.route("/from-article/<int:item_id>", methods=["POST"])
|
||||||
@login_required
|
@login_required
|
||||||
async def create_conversation_from_article(item_id: int):
|
async def create_conversation_from_article(item_id: int):
|
||||||
"""Create a chat conversation seeded with an RSS article's content."""
|
"""Create a chat conversation seeded for article discussion and auto-run.
|
||||||
|
|
||||||
|
Mirrors the briefing ``discuss_article`` route: creates a fresh
|
||||||
|
conversation, stages the shared synthetic read_article exchange + seed
|
||||||
|
prompt, then kicks off generation so the client lands on an in-flight
|
||||||
|
stream. The Flutter and web chat screens reconnect to the running buffer
|
||||||
|
on mount.
|
||||||
|
"""
|
||||||
from sqlalchemy import select as _select
|
from sqlalchemy import select as _select
|
||||||
from fabledassistant.models import async_session as _async_session
|
from fabledassistant.models import async_session as _async_session
|
||||||
from fabledassistant.models.conversation import Message
|
|
||||||
from fabledassistant.models.rss_feed import RssItem, RssFeed
|
from fabledassistant.models.rss_feed import RssItem, RssFeed
|
||||||
|
from fabledassistant.services.article_context import (
|
||||||
|
EmptyArticleError,
|
||||||
|
seed_article_discussion,
|
||||||
|
)
|
||||||
|
|
||||||
uid = get_current_user_id()
|
uid = get_current_user_id()
|
||||||
|
|
||||||
async with _async_session() as session:
|
async with _async_session() as session:
|
||||||
result = await session.execute(
|
result = await session.execute(
|
||||||
_select(RssItem, RssFeed.title.label("feed_title"))
|
_select(RssItem)
|
||||||
.join(RssFeed, RssItem.feed_id == RssFeed.id)
|
.join(RssFeed, RssItem.feed_id == RssFeed.id)
|
||||||
.where(RssItem.id == item_id, RssFeed.user_id == uid)
|
.where(RssItem.id == item_id, RssFeed.user_id == uid)
|
||||||
)
|
)
|
||||||
row = result.first()
|
item = result.scalars().first()
|
||||||
|
|
||||||
if row is None:
|
if item is None:
|
||||||
return jsonify({"error": "Article not found"}), 404
|
return jsonify({"error": "Article not found"}), 404
|
||||||
|
|
||||||
item, feed_title = row
|
|
||||||
|
|
||||||
conv_title = (item.title or "Article discussion")[:80]
|
conv_title = (item.title or "Article discussion")[:80]
|
||||||
conv = await create_conversation(uid, title=conv_title, conversation_type="chat")
|
conv = await create_conversation(uid, title=conv_title, conversation_type="chat")
|
||||||
|
|
||||||
from fabledassistant.services.rss import _fetch_full_article
|
model = await get_setting(uid, "default_model", "") or Config.OLLAMA_MODEL
|
||||||
source = feed_title or "News"
|
try:
|
||||||
content_body = (await _fetch_full_article(item.url) if item.url else None) or (item.content or "").strip()
|
discuss_prompt = await seed_article_discussion(conv.id, item, model)
|
||||||
seeded_text = f"**{source}**\n\n**{item.title}**"
|
except EmptyArticleError as e:
|
||||||
if content_body:
|
# Roll back the empty conversation so the user doesn't end up with a
|
||||||
seeded_text += f"\n\n{content_body}"
|
# phantom entry in their chat list.
|
||||||
if item.url:
|
try:
|
||||||
seeded_text += f"\n\nSource: {item.url}"
|
await delete_conversation(uid, conv.id)
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Failed to clean up empty article conversation %s", conv.id)
|
||||||
|
return jsonify({"error": str(e)}), 422
|
||||||
|
|
||||||
async with _async_session() as session:
|
# Reload conversation so we see the two messages the helper just added.
|
||||||
msg = Message(
|
conv = await get_conversation(uid, conv.id)
|
||||||
conversation_id=conv.id,
|
assert conv is not None
|
||||||
role="assistant",
|
|
||||||
content=seeded_text,
|
|
||||||
msg_metadata={"rss_item_ids": [item_id]},
|
|
||||||
)
|
|
||||||
session.add(msg)
|
|
||||||
await session.commit()
|
|
||||||
|
|
||||||
return jsonify({"conversation_id": conv.id}), 201
|
history: list[dict] = []
|
||||||
|
for msg in conv.messages:
|
||||||
|
if msg.role == "system":
|
||||||
|
continue
|
||||||
|
msg_dict: dict = {"role": msg.role, "content": msg.content or ""}
|
||||||
|
if msg.tool_calls:
|
||||||
|
msg_dict["tool_calls"] = [
|
||||||
|
{"function": {"name": tc["function"], "arguments": tc["arguments"]}}
|
||||||
|
for tc in msg.tool_calls
|
||||||
|
]
|
||||||
|
history.append(msg_dict)
|
||||||
|
for tc in msg.tool_calls:
|
||||||
|
history.append({"role": "tool", "content": json.dumps(tc.get("result", {}))})
|
||||||
|
else:
|
||||||
|
history.append(msg_dict)
|
||||||
|
|
||||||
|
assistant_msg = await add_message(conv.id, "assistant", "", status="generating")
|
||||||
|
buf = create_buffer(conv.id, assistant_msg.id)
|
||||||
|
asyncio.create_task(run_generation(
|
||||||
|
buf, history, model, uid, conv.id, conv.title or "", discuss_prompt,
|
||||||
|
))
|
||||||
|
|
||||||
|
return jsonify({
|
||||||
|
"conversation_id": conv.id,
|
||||||
|
"assistant_message_id": assistant_msg.id,
|
||||||
|
"status": "generating",
|
||||||
|
}), 202
|
||||||
|
|||||||
@@ -0,0 +1,270 @@
|
|||||||
|
"""Prepare article bodies as conversation-ready context.
|
||||||
|
|
||||||
|
Used by the briefing ``discuss-article`` flow and the ``/news`` discuss button.
|
||||||
|
A raw trafilatura extraction is often too large to drop whole into a chat
|
||||||
|
history without eating the context window, so this module runs a map-reduce
|
||||||
|
step over oversized articles and returns a compact, structured context that
|
||||||
|
still preserves the article's meaning across sections.
|
||||||
|
|
||||||
|
Small articles pass through unchanged — map-reduce only fires when the raw
|
||||||
|
body exceeds CHAR_BUDGET. The output is cached on ``rss_items.context_prepared``
|
||||||
|
by the caller, so repeat discuss-clicks on the same article skip this work
|
||||||
|
entirely.
|
||||||
|
|
||||||
|
The module also owns ``seed_article_discussion``, the shared routine that
|
||||||
|
stages a synthetic ``read_article`` tool exchange plus a conversational seed
|
||||||
|
prompt into a conversation. Both the briefing and ``/news`` entry points call
|
||||||
|
it so the two flows stay byte-identical — the only thing that differs between
|
||||||
|
them is whether the conversation already existed or was freshly created.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
|
from fabledassistant.models import async_session
|
||||||
|
from fabledassistant.models.rss_feed import RssItem
|
||||||
|
from fabledassistant.services.chat import add_message
|
||||||
|
from fabledassistant.services.llm import generate_completion
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# ~12k tokens at 4 chars/token. Comfortably under OLLAMA_NUM_CTX=16384
|
||||||
|
# with room left for system prompt, chat history, and the assistant reply.
|
||||||
|
CHAR_BUDGET = 48_000
|
||||||
|
|
||||||
|
# Chunk size for the map step on oversized articles. Overlap preserves
|
||||||
|
# context across paragraph boundaries that happen to land mid-sentence.
|
||||||
|
CHUNK_CHARS = 8_000
|
||||||
|
CHUNK_OVERLAP = 400
|
||||||
|
|
||||||
|
_PARA_SPLIT = re.compile(r"\n\s*\n")
|
||||||
|
|
||||||
|
|
||||||
|
def _chunk_by_paragraph(body: str) -> list[str]:
|
||||||
|
"""Split ``body`` into chunks of up to CHUNK_CHARS, respecting paragraphs.
|
||||||
|
|
||||||
|
Paragraphs longer than CHUNK_CHARS are split mid-paragraph as a last
|
||||||
|
resort. Adjacent chunks share CHUNK_OVERLAP chars of trailing text so
|
||||||
|
a sentence straddling the boundary stays readable on both sides.
|
||||||
|
"""
|
||||||
|
paragraphs = [p.strip() for p in _PARA_SPLIT.split(body) if p.strip()]
|
||||||
|
chunks: list[str] = []
|
||||||
|
current: list[str] = []
|
||||||
|
current_len = 0
|
||||||
|
for para in paragraphs:
|
||||||
|
para_len = len(para)
|
||||||
|
if para_len > CHUNK_CHARS:
|
||||||
|
if current:
|
||||||
|
chunks.append("\n\n".join(current))
|
||||||
|
current, current_len = [], 0
|
||||||
|
for i in range(0, para_len, CHUNK_CHARS - CHUNK_OVERLAP):
|
||||||
|
chunks.append(para[i : i + CHUNK_CHARS])
|
||||||
|
continue
|
||||||
|
if current_len + para_len + 2 > CHUNK_CHARS and current:
|
||||||
|
chunks.append("\n\n".join(current))
|
||||||
|
tail = current[-1][-CHUNK_OVERLAP:] if current else ""
|
||||||
|
current = [tail, para] if tail else [para]
|
||||||
|
current_len = len(tail) + para_len + (2 if tail else 0)
|
||||||
|
else:
|
||||||
|
current.append(para)
|
||||||
|
current_len += para_len + 2
|
||||||
|
if current:
|
||||||
|
chunks.append("\n\n".join(current))
|
||||||
|
return chunks
|
||||||
|
|
||||||
|
|
||||||
|
async def _summarize_chunk(title: str, chunk: str, index: int, total: int, model: str) -> str:
|
||||||
|
"""Map-step summary of one article chunk.
|
||||||
|
|
||||||
|
Aims for ~300 words of dense, factual prose — not bullet points — so the
|
||||||
|
downstream chat model can quote from it naturally.
|
||||||
|
"""
|
||||||
|
messages = [
|
||||||
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": (
|
||||||
|
"You are summarizing one section of a larger article so a downstream "
|
||||||
|
"conversation model can discuss the full article without having to read "
|
||||||
|
"every word.\n\n"
|
||||||
|
"Requirements:\n"
|
||||||
|
"- 250–350 words of dense factual prose\n"
|
||||||
|
"- Preserve specific claims, numbers, names, and quotes\n"
|
||||||
|
"- Do NOT editorialize or add analysis\n"
|
||||||
|
"- Do NOT use bullet points or headings\n"
|
||||||
|
"- Do NOT say 'this section' or 'this article' — write content, not meta"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": (
|
||||||
|
f"Article: {title}\n"
|
||||||
|
f"Section {index + 1} of {total}:\n\n{chunk}"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
# Pin num_ctx — same rationale as services/research.py:66. A large
|
||||||
|
# chunk plus system prompt can push well past the default window;
|
||||||
|
# silent truncation here would drop the tail of the chunk without
|
||||||
|
# any error, producing a misleading summary.
|
||||||
|
raw = await generate_completion(
|
||||||
|
messages, model, max_tokens=600, num_ctx=16384
|
||||||
|
)
|
||||||
|
return raw.strip()
|
||||||
|
except Exception:
|
||||||
|
logger.warning(
|
||||||
|
"Article chunk summary failed for section %d/%d of '%s'",
|
||||||
|
index + 1, total, title, exc_info=True,
|
||||||
|
)
|
||||||
|
# Fall back to the raw chunk truncated to ~1500 chars so the overall
|
||||||
|
# pipeline still delivers something rather than dropping the section.
|
||||||
|
return chunk[:1500]
|
||||||
|
|
||||||
|
|
||||||
|
async def prepare_article_context(
|
||||||
|
title: str,
|
||||||
|
url: str,
|
||||||
|
body: str,
|
||||||
|
model: str,
|
||||||
|
) -> str:
|
||||||
|
"""Return a conversation-ready context block for ``body``.
|
||||||
|
|
||||||
|
- Small article (≤ CHAR_BUDGET): returns ``body`` unchanged.
|
||||||
|
- Oversized article: runs a parallel map step over paragraph-aware
|
||||||
|
chunks and concatenates the summaries under section headers.
|
||||||
|
|
||||||
|
The returned string is what should go into the ``read_article`` synthetic
|
||||||
|
tool-result in chat history. Callers are responsible for caching it to
|
||||||
|
``rss_items.context_prepared``.
|
||||||
|
"""
|
||||||
|
body = body or ""
|
||||||
|
if len(body) <= CHAR_BUDGET:
|
||||||
|
return body
|
||||||
|
|
||||||
|
chunks = _chunk_by_paragraph(body)
|
||||||
|
logger.info(
|
||||||
|
"Article '%s' is %d chars, map-reducing into %d chunks",
|
||||||
|
title, len(body), len(chunks),
|
||||||
|
)
|
||||||
|
|
||||||
|
summaries = await asyncio.gather(
|
||||||
|
*[
|
||||||
|
_summarize_chunk(title, chunk, i, len(chunks), model)
|
||||||
|
for i, chunk in enumerate(chunks)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
header = (
|
||||||
|
f"(This article was longer than the chat window could hold verbatim, "
|
||||||
|
f"so the full text was split into {len(chunks)} sections and each was "
|
||||||
|
"summarized below. Each section preserves specific claims, numbers, "
|
||||||
|
"and quotes from the original.)\n\n"
|
||||||
|
)
|
||||||
|
parts = [
|
||||||
|
f"## Section {i + 1}\n\n{summary}"
|
||||||
|
for i, summary in enumerate(summaries)
|
||||||
|
]
|
||||||
|
return header + "\n\n".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
# Conversational seed prompt for article discussions. Kept here so both the
|
||||||
|
# briefing and /news entry points use the exact same wording. See
|
||||||
|
# feedback_discuss_prompt_style memory: numbered checklists produce
|
||||||
|
# assignment-completion responses; this conversational seed opens a dialogue.
|
||||||
|
ARTICLE_DISCUSS_SEED = (
|
||||||
|
"I want to talk about this article. Start with a substantive summary "
|
||||||
|
"of what it's arguing and the key evidence it uses, then tell me what "
|
||||||
|
"stood out to you or seems worth pushing back on. I'll ask follow-ups "
|
||||||
|
"from there."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class EmptyArticleError(Exception):
|
||||||
|
"""Raised when an article has no extractable body text.
|
||||||
|
|
||||||
|
Callers (the briefing and /news discuss routes) map this to a 422 so the
|
||||||
|
user sees a clear error instead of a hallucinated summary built from an
|
||||||
|
empty synthetic tool result.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
async def seed_article_discussion(
|
||||||
|
conv_id: int,
|
||||||
|
item: RssItem,
|
||||||
|
model: str,
|
||||||
|
) -> str:
|
||||||
|
"""Stage the synthetic read_article tool exchange + conversational seed.
|
||||||
|
|
||||||
|
Used by both the briefing ``discuss_article`` route and the ``/news``
|
||||||
|
``from-article`` conversation creator. Handles the three-layer cache
|
||||||
|
(``context_prepared`` → ``content_full`` → fresh fetch) and inserts two
|
||||||
|
messages into ``conv_id``:
|
||||||
|
|
||||||
|
1. An assistant message with a synthetic ``read_article`` tool_call whose
|
||||||
|
``result.content`` carries the prepared article context. The message
|
||||||
|
also carries ``msg_metadata={"rss_item_id": ...}`` so the post-generation
|
||||||
|
hook in ``generation_task.py`` can locate it and persist the first
|
||||||
|
reply as a discussion-summary Note.
|
||||||
|
2. A user message with the shared conversational seed prompt.
|
||||||
|
|
||||||
|
Returns the seed prompt string so callers can pass it to ``run_generation``
|
||||||
|
as ``user_content``.
|
||||||
|
"""
|
||||||
|
# Avoid circulars: rss helper imports article_context indirectly nowhere,
|
||||||
|
# but keep this local for symmetry with the route-level imports it
|
||||||
|
# replaces.
|
||||||
|
from fabledassistant.services.rss import get_or_fetch_full_article
|
||||||
|
|
||||||
|
if item.context_prepared:
|
||||||
|
article_content = item.context_prepared
|
||||||
|
else:
|
||||||
|
raw_body = await get_or_fetch_full_article(item) or item.content or ""
|
||||||
|
if not raw_body.strip():
|
||||||
|
# Hard-fail rather than stage an empty synthetic tool result.
|
||||||
|
# An empty `content` field silently tells the model "the article
|
||||||
|
# has nothing in it" and it confabulates from RAG/history. Better
|
||||||
|
# to surface a clean error to the user.
|
||||||
|
logger.warning(
|
||||||
|
"Article discussion aborted: empty body for rss_item %s (%s)",
|
||||||
|
item.id, item.url,
|
||||||
|
)
|
||||||
|
raise EmptyArticleError(
|
||||||
|
"Couldn't extract any readable text from this article."
|
||||||
|
)
|
||||||
|
article_content = await prepare_article_context(
|
||||||
|
item.title or "", item.url, raw_body, model,
|
||||||
|
)
|
||||||
|
if not article_content.strip():
|
||||||
|
raise EmptyArticleError(
|
||||||
|
"Couldn't extract any readable text from this article."
|
||||||
|
)
|
||||||
|
async with async_session() as session:
|
||||||
|
fresh = await session.get(RssItem, item.id)
|
||||||
|
if fresh is not None:
|
||||||
|
fresh.context_prepared = article_content
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
synthetic_tool_calls = [{
|
||||||
|
"function": "read_article",
|
||||||
|
"arguments": {"url": item.url},
|
||||||
|
"result": {
|
||||||
|
"success": True,
|
||||||
|
"type": "article_content",
|
||||||
|
"url": item.url,
|
||||||
|
"content": article_content,
|
||||||
|
"truncated": False,
|
||||||
|
},
|
||||||
|
}]
|
||||||
|
await add_message(
|
||||||
|
conv_id,
|
||||||
|
"assistant",
|
||||||
|
"",
|
||||||
|
status="complete",
|
||||||
|
tool_calls=synthetic_tool_calls,
|
||||||
|
msg_metadata={"rss_item_id": item.id, "article_seed": True},
|
||||||
|
)
|
||||||
|
await add_message(conv_id, "user", ARTICLE_DISCUSS_SEED)
|
||||||
|
return ARTICLE_DISCUSS_SEED
|
||||||
@@ -187,6 +187,7 @@ async def add_message(
|
|||||||
context_note_id: int | None = None,
|
context_note_id: int | None = None,
|
||||||
status: str | None = None,
|
status: str | None = None,
|
||||||
tool_calls: list | None = None,
|
tool_calls: list | None = None,
|
||||||
|
msg_metadata: dict | None = None,
|
||||||
) -> Message:
|
) -> Message:
|
||||||
async with async_session() as session:
|
async with async_session() as session:
|
||||||
kwargs: dict = dict(
|
kwargs: dict = dict(
|
||||||
@@ -199,6 +200,8 @@ async def add_message(
|
|||||||
kwargs["status"] = status
|
kwargs["status"] = status
|
||||||
if tool_calls is not None:
|
if tool_calls is not None:
|
||||||
kwargs["tool_calls"] = tool_calls
|
kwargs["tool_calls"] = tool_calls
|
||||||
|
if msg_metadata is not None:
|
||||||
|
kwargs["msg_metadata"] = msg_metadata
|
||||||
msg = Message(**kwargs)
|
msg = Message(**kwargs)
|
||||||
session.add(msg)
|
session.add(msg)
|
||||||
# Touch conversation updated_at
|
# Touch conversation updated_at
|
||||||
|
|||||||
@@ -36,6 +36,85 @@ _TOOL_CALL_MARKER = re.compile(r"^\s*\[TOOL_CALLS\]\s*", re.IGNORECASE)
|
|||||||
|
|
||||||
DB_FLUSH_INTERVAL = 5.0 # seconds between partial DB flushes
|
DB_FLUSH_INTERVAL = 5.0 # seconds between partial DB flushes
|
||||||
|
|
||||||
|
|
||||||
|
async def _maybe_save_article_discussion_note(
|
||||||
|
user_id: int, conv_id: int, reply_content: str,
|
||||||
|
) -> None:
|
||||||
|
"""Persist a seeded article-discussion's first reply as a Note.
|
||||||
|
|
||||||
|
Fires after ``run_generation`` completes. Looks for a synthetic
|
||||||
|
read_article seed message on the conversation; if found AND the linked
|
||||||
|
``rss_items`` row has no ``discussion_note_id`` yet, saves ``reply_content``
|
||||||
|
as a Note, tags it, and writes the backlink. Subsequent discuss clicks on
|
||||||
|
the same article are a no-op (already linked).
|
||||||
|
|
||||||
|
Failures are logged and swallowed — the chat UI should never break because
|
||||||
|
Note persistence hit a snag.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
if not reply_content or not reply_content.strip():
|
||||||
|
return
|
||||||
|
from sqlalchemy import select as _select
|
||||||
|
from fabledassistant.models.conversation import Message as _Message
|
||||||
|
from fabledassistant.models.rss_feed import RssItem as _RssItem
|
||||||
|
from fabledassistant.services.notes import create_note
|
||||||
|
|
||||||
|
async with async_session() as session:
|
||||||
|
result = await session.execute(
|
||||||
|
_select(_Message)
|
||||||
|
.where(_Message.conversation_id == conv_id)
|
||||||
|
.order_by(_Message.id.asc())
|
||||||
|
)
|
||||||
|
messages = result.scalars().all()
|
||||||
|
seed_meta = None
|
||||||
|
for m in messages:
|
||||||
|
meta = m.msg_metadata or {}
|
||||||
|
if meta.get("article_seed") and meta.get("rss_item_id"):
|
||||||
|
seed_meta = meta
|
||||||
|
break
|
||||||
|
if seed_meta is None:
|
||||||
|
return
|
||||||
|
item_id = int(seed_meta["rss_item_id"])
|
||||||
|
item = await session.get(_RssItem, item_id)
|
||||||
|
if item is None or item.discussion_note_id is not None:
|
||||||
|
return
|
||||||
|
article_title = (item.title or "Untitled article").strip()
|
||||||
|
article_url = item.url
|
||||||
|
article_topics = list(item.topics or [])
|
||||||
|
|
||||||
|
note_title = f"Article: {article_title}"[:200]
|
||||||
|
body_parts = [f"**Source:** {article_url}"] if article_url else []
|
||||||
|
body_parts.append(reply_content.strip())
|
||||||
|
note_body = "\n\n".join(body_parts)
|
||||||
|
tags = ["article-summary"] + [t for t in article_topics if t]
|
||||||
|
note = await create_note(
|
||||||
|
user_id=user_id,
|
||||||
|
title=note_title,
|
||||||
|
body=note_body,
|
||||||
|
tags=tags,
|
||||||
|
entity_meta={
|
||||||
|
"source": "article_discussion",
|
||||||
|
"rss_item_id": item_id,
|
||||||
|
"url": article_url,
|
||||||
|
"conversation_id": conv_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
async with async_session() as session:
|
||||||
|
fresh = await session.get(_RssItem, item_id)
|
||||||
|
if fresh is not None and fresh.discussion_note_id is None:
|
||||||
|
fresh.discussion_note_id = note.id
|
||||||
|
await session.commit()
|
||||||
|
logger.info(
|
||||||
|
"Saved article-discussion summary as note %d for rss_item %d (conv %d)",
|
||||||
|
note.id, item_id, conv_id,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.warning(
|
||||||
|
"Failed to persist article-discussion note for conv %d",
|
||||||
|
conv_id, exc_info=True,
|
||||||
|
)
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Thinking decision
|
# Thinking decision
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -526,6 +605,18 @@ async def run_generation(
|
|||||||
msg_count = len(non_system)
|
msg_count = len(non_system)
|
||||||
should_gen_title = not conv_title or (msg_count > 0 and msg_count % 10 == 0)
|
should_gen_title = not conv_title or (msg_count > 0 and msg_count % 10 == 0)
|
||||||
|
|
||||||
|
# Persist article-discussion seed conversations as a Note on their
|
||||||
|
# first assistant reply. This makes "Discuss" summaries part of RAG
|
||||||
|
# so the knowledge base stops being amnesiac about articles the user
|
||||||
|
# has already engaged with. The hook detects a seeded conversation by
|
||||||
|
# finding a synthetic read_article assistant message whose
|
||||||
|
# msg_metadata carries ``article_seed: True`` and whose rss_items row
|
||||||
|
# has no discussion_note_id yet. Fire-and-forget so the done event
|
||||||
|
# lands immediately.
|
||||||
|
asyncio.create_task(_maybe_save_article_discussion_note(
|
||||||
|
user_id, conv_id, buf.content_so_far,
|
||||||
|
))
|
||||||
|
|
||||||
if should_gen_title:
|
if should_gen_title:
|
||||||
# Feed the title model the *raw* conversation turns only — never
|
# Feed the title model the *raw* conversation turns only — never
|
||||||
# the post-build_context ``messages`` list. ``build_context``
|
# the post-build_context ``messages`` list. ``build_context``
|
||||||
|
|||||||
@@ -724,18 +724,27 @@ async def build_context(
|
|||||||
orphan_only = rag_project_id is None
|
orphan_only = rag_project_id is None
|
||||||
effective_project_id = rag_project_id if (rag_project_id is not None and rag_project_id != -1) else None
|
effective_project_id = rag_project_id if (rag_project_id is not None and rag_project_id != -1) else None
|
||||||
|
|
||||||
try:
|
# Skip RAG auto-injection on the first turn of a seeded article discussion.
|
||||||
from fabledassistant.services.embeddings import semantic_search_notes
|
# The article body is already the sole context the user wants — pulling in
|
||||||
for score, note in await semantic_search_notes(
|
# unrelated orphan notes tricks the model into summarizing those instead.
|
||||||
user_id, user_message, exclude_ids=search_exclude or None, limit=8,
|
# Follow-up turns keep RAG on because by then the user's own messages drive
|
||||||
project_id=effective_project_id,
|
# the query rather than the generic seed prompt.
|
||||||
orphan_only=orphan_only,
|
from fabledassistant.services.article_context import ARTICLE_DISCUSS_SEED
|
||||||
):
|
_skip_rag_for_article_seed = user_message.strip() == ARTICLE_DISCUSS_SEED
|
||||||
found_scored.append((score, note))
|
|
||||||
except Exception:
|
|
||||||
logger.warning("Semantic note search failed, falling back to keyword search", exc_info=True)
|
|
||||||
|
|
||||||
if not found_scored:
|
if not _skip_rag_for_article_seed:
|
||||||
|
try:
|
||||||
|
from fabledassistant.services.embeddings import semantic_search_notes
|
||||||
|
for score, note in await semantic_search_notes(
|
||||||
|
user_id, user_message, exclude_ids=search_exclude or None, limit=8,
|
||||||
|
project_id=effective_project_id,
|
||||||
|
orphan_only=orphan_only,
|
||||||
|
):
|
||||||
|
found_scored.append((score, note))
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Semantic note search failed, falling back to keyword search", exc_info=True)
|
||||||
|
|
||||||
|
if not found_scored and not _skip_rag_for_article_seed:
|
||||||
keywords = _extract_keywords(user_message)
|
keywords = _extract_keywords(user_message)
|
||||||
if keywords:
|
if keywords:
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -34,6 +34,34 @@ def _html_to_text(html: str) -> str:
|
|||||||
return html
|
return html
|
||||||
|
|
||||||
|
|
||||||
|
async def get_or_fetch_full_article(item: RssItem) -> str | None:
|
||||||
|
"""Return the full article body, fetching+caching on miss.
|
||||||
|
|
||||||
|
Checks ``item.content_full`` first — populated either by the enrichment
|
||||||
|
pass at feed-ingest time or by a previous discuss-click. On miss, fetches
|
||||||
|
via ``_fetch_full_article`` and writes through. Returns ``None`` only if
|
||||||
|
the fetch itself fails; ``item.content_full == ""`` is still a cache hit.
|
||||||
|
|
||||||
|
Callers must pass an RssItem attached to an open session if they want
|
||||||
|
the write-through to persist — otherwise the fetched text is returned
|
||||||
|
but the cache stays empty and the next click will re-fetch.
|
||||||
|
"""
|
||||||
|
if item.content_full is not None:
|
||||||
|
return item.content_full
|
||||||
|
if not item.url:
|
||||||
|
return None
|
||||||
|
text = await _fetch_full_article(item.url)
|
||||||
|
if text is None:
|
||||||
|
return None
|
||||||
|
async with async_session() as session:
|
||||||
|
fresh = await session.get(RssItem, item.id)
|
||||||
|
if fresh is not None:
|
||||||
|
fresh.content_full = text
|
||||||
|
fresh.content_fetched_at = datetime.now(timezone.utc)
|
||||||
|
await session.commit()
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
async def _fetch_full_article(url: str) -> str | None:
|
async def _fetch_full_article(url: str) -> str | None:
|
||||||
"""Fetch a URL and extract its main article text via trafilatura.
|
"""Fetch a URL and extract its main article text via trafilatura.
|
||||||
|
|
||||||
@@ -209,6 +237,11 @@ async def fetch_and_cache_feed(feed_id: int, url: str) -> int:
|
|||||||
item = await session.get(RssItem, item_id)
|
item = await session.get(RssItem, item_id)
|
||||||
if item:
|
if item:
|
||||||
item.content = full_text
|
item.content = full_text
|
||||||
|
# Populate the discuss-click cache too so the
|
||||||
|
# first click skips straight to the map-reduce
|
||||||
|
# step without re-fetching.
|
||||||
|
item.content_full = full_text
|
||||||
|
item.content_fetched_at = datetime.now(timezone.utc)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
await upsert_rss_item_embedding(
|
await upsert_rss_item_embedding(
|
||||||
item_id, feed_user_id, item.title or "", item.content
|
item_id, feed_user_id, item.title or "", item.content
|
||||||
|
|||||||
@@ -134,3 +134,83 @@ def test_history_builder_no_tool_calls_unchanged():
|
|||||||
assert len(history) == 2
|
assert len(history) == 2
|
||||||
assert history[0] == {"role": "user", "content": "Hello"}
|
assert history[0] == {"role": "user", "content": "Hello"}
|
||||||
assert history[1] == {"role": "assistant", "content": "Hi there!"}
|
assert history[1] == {"role": "assistant", "content": "Hi there!"}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# prepare_article_context tests
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_prepare_article_context_small_passthrough():
|
||||||
|
"""Articles under CHAR_BUDGET pass through unchanged with zero LLM calls."""
|
||||||
|
from fabledassistant.services import article_context
|
||||||
|
|
||||||
|
body = "A short article.\n\nWith two paragraphs."
|
||||||
|
with patch(
|
||||||
|
"fabledassistant.services.article_context.generate_completion",
|
||||||
|
new_callable=AsyncMock,
|
||||||
|
) as mock_gen:
|
||||||
|
out = await article_context.prepare_article_context(
|
||||||
|
"Title", "https://example.com", body, "test-model",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert out == body
|
||||||
|
mock_gen.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_prepare_article_context_large_runs_map_reduce():
|
||||||
|
"""Articles over CHAR_BUDGET are chunked and map-reduced via the background model."""
|
||||||
|
from fabledassistant.services import article_context
|
||||||
|
|
||||||
|
# CHAR_BUDGET is 48_000 — build a body well over that with paragraph breaks
|
||||||
|
# so the chunker has natural splits to work with.
|
||||||
|
paragraph = ("Lorem ipsum dolor sit amet, consectetur adipiscing elit. " * 40).strip()
|
||||||
|
body = "\n\n".join([paragraph] * 30) # ~70k+ chars across 30 paragraphs
|
||||||
|
assert len(body) > article_context.CHAR_BUDGET
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"fabledassistant.services.article_context.generate_completion",
|
||||||
|
new_callable=AsyncMock,
|
||||||
|
return_value="Summary of this section with specific claims preserved.",
|
||||||
|
) as mock_gen:
|
||||||
|
out = await article_context.prepare_article_context(
|
||||||
|
"Long Article", "https://example.com/long", body, "test-model",
|
||||||
|
)
|
||||||
|
|
||||||
|
# At least one LLM call fired (the map step ran)
|
||||||
|
assert mock_gen.await_count >= 1
|
||||||
|
# Output carries the oversized-article header and section markers
|
||||||
|
assert "longer than the chat window" in out
|
||||||
|
assert "## Section 1" in out
|
||||||
|
# Map output is much smaller than the raw body
|
||||||
|
assert len(out) < len(body)
|
||||||
|
|
||||||
|
|
||||||
|
def test_chunk_by_paragraph_respects_boundaries():
|
||||||
|
"""Chunker splits on paragraph breaks, not mid-sentence."""
|
||||||
|
from fabledassistant.services.article_context import _chunk_by_paragraph, CHUNK_CHARS
|
||||||
|
|
||||||
|
paragraphs = [f"Paragraph {i}. " + ("x" * 1000) for i in range(20)]
|
||||||
|
body = "\n\n".join(paragraphs)
|
||||||
|
|
||||||
|
chunks = _chunk_by_paragraph(body)
|
||||||
|
|
||||||
|
# Each chunk stays under the budget
|
||||||
|
for chunk in chunks:
|
||||||
|
assert len(chunk) <= CHUNK_CHARS
|
||||||
|
# Total content is preserved (modulo overlap duplication, so ≥ original)
|
||||||
|
assert sum(len(c) for c in chunks) >= len(body) * 0.9
|
||||||
|
|
||||||
|
|
||||||
|
def test_chunk_by_paragraph_handles_oversized_paragraph():
|
||||||
|
"""A single paragraph larger than CHUNK_CHARS gets split mid-paragraph."""
|
||||||
|
from fabledassistant.services.article_context import _chunk_by_paragraph, CHUNK_CHARS
|
||||||
|
|
||||||
|
body = "x" * (CHUNK_CHARS * 3) # one huge paragraph, no breaks
|
||||||
|
chunks = _chunk_by_paragraph(body)
|
||||||
|
|
||||||
|
assert len(chunks) >= 3
|
||||||
|
for chunk in chunks:
|
||||||
|
assert len(chunk) <= CHUNK_CHARS
|
||||||
|
|||||||
Reference in New Issue
Block a user