From b4be1f0799e1c556375a4762025a6b15d74097e2 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 3 Apr 2026 00:44:17 -0400 Subject: [PATCH] perf(llm): move retrieval context to user turn for stable system prompt RAG notes, RSS news, current note, URL content, and briefing articles are now prepended to the user message rather than appended to the system message. The system message now contains only stable content (persona, tool guidance, date, profile, workspace, history summary), making its token sequence identical across consecutive requests and allowing Ollama's KV prefix cache to fire reliably every time. Co-Authored-By: Claude Sonnet 4.6 --- src/fabledassistant/services/llm.py | 118 +++++++++++++++------------- 1 file changed, 64 insertions(+), 54 deletions(-) diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index d7f2e19..4af17eb 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -543,7 +543,13 @@ async def build_context( dynamic_tail = f"\n\nToday's date is {today}.{tz_line}{profile_section}{entities_section}" - system_parts = [static_block + dynamic_tail] + # --- System message: stable content only --- + # Workspace context and history summary stay here because they carry + # behavioural instructions / conversational state, not retrieved content. + # Everything retrieval-based (RAG notes, RSS, URL content, current note, + # briefing articles) goes into the user turn below so the system message + # prefix stays byte-for-byte identical across requests, enabling Ollama's + # KV prefix cache to fire reliably. if voice_mode: _style_hints = { @@ -552,11 +558,36 @@ async def build_context( "detailed": "Give thorough, informative responses as if narrating an explanation aloud.", } style_hint = _style_hints.get(voice_speech_style, _style_hints["conversational"]) - system_parts.insert(0, + voice_preamble = ( "VOICE MODE: Respond naturally as if speaking aloud. " "No markdown, bullet points, headers, or code blocks. Complete sentences only. " f"{style_hint}\n\n" ) + system_content = voice_preamble + static_block + dynamic_tail + else: + system_content = static_block + dynamic_tail + + # Inject workspace context (behavioural — must stay in system) + if workspace_project_id is not None: + from fabledassistant.services.projects import get_project + try: + wp = await get_project(user_id, workspace_project_id) + if wp: + system_content += ( + f"\n\n--- Active Workspace ---\n" + f"You are in the \"{wp.title}\" project workspace.\n" + f"All notes and tasks you create or update MUST belong to this project.\n" + f"Always pass project=\"{wp.title}\" when calling create_note or create_task.\n" + f"--- End Active Workspace ---" + ) + except Exception: + logger.warning("Failed to fetch workspace project %d", workspace_project_id) + + # Inject compressed history summary (conversational state — stays in system) + if history_summary: + system_content += ( + f"\n\n--- Earlier Conversation ---\n{history_summary}\n--- End Earlier Conversation ---" + ) context_meta: dict = { "context_note_id": None, @@ -565,35 +596,34 @@ async def build_context( "auto_injected_notes": [], } - # Include current note context if provided — full body, no truncation + # --- User turn context prefix: retrieval-based content --- + # Collected here and prepended to the user message so the system message + # stays stable and the KV prefix cache can fire on every request. + user_context_parts: list[str] = [] + + # Current note being viewed (full body, no truncation) if current_note_id: note = await get_note(user_id, current_note_id) if note: context_meta["context_note_id"] = note.id context_meta["context_note_title"] = note.title - system_parts.append( - f"\n\n--- Current Note ---\n" + user_context_parts.append( + f"--- Current Note ---\n" f"Title: {note.title}\n" f"Content:\n{note.body}\n" f"--- End Note ---" ) - # Search for related notes. High-confidence results (>=0.60) are auto-injected - # into the system prompt; lower-confidence results populate the sidebar only. - # Users can also explicitly include notes via the sidebar (include_note_ids). + # Semantic / keyword note search search_exclude = set(exclude_set) if current_note_id: search_exclude.add(current_note_id) - # (score, note) pairs — score is float for semantic results, None for keyword fallback. found_scored: list[tuple[float | None, object]] = [] - # Derive scope flags from rag_project_id three-value semantics: - # None → orphan notes only; -1 → all notes; positive int → that project orphan_only = rag_project_id is None effective_project_id = rag_project_id if (rag_project_id is not None and rag_project_id != -1) else None - # Try semantic search first; fall back to keyword search on failure / no results. try: from fabledassistant.services.embeddings import semantic_search_notes for score, note in await semantic_search_notes( @@ -618,7 +648,6 @@ async def build_context( except Exception: logger.warning("Failed to search notes for context", exc_info=True) - # Separate high-confidence results for auto-injection vs sidebar display excluded_inject_set = set(excluded_note_ids or []) auto_inject: list[tuple[float, object]] = [] sidebar_only: list[tuple[float | None, object]] = [] @@ -634,7 +663,6 @@ async def build_context( else: sidebar_only.append((score, n)) - # Inject high-scoring notes into system prompt if auto_inject: snippets = [] for score, n in auto_inject: @@ -645,14 +673,12 @@ async def build_context( "title": n.title, "score": round(score, 2), }) - system_parts.append( - "\n\n--- Relevant Notes ---\n" + user_context_parts.append( + "--- Relevant Notes ---\n" + "\n\n".join(snippets) + "\n--- End Relevant Notes ---" ) - # Populate sidebar candidates (auto-injected notes also appear here for reference, - # but sidebar_only are the ones not yet in the prompt) for score, n in auto_inject: context_meta["auto_notes"].append({ "id": n.id, @@ -669,7 +695,7 @@ async def build_context( }) context_meta["auto_note_ids"] = [n.id for _, n in found_scored] - # Inject explicitly included notes (user opted in via sidebar click). + # Explicitly included notes (user opted in via sidebar) if include_note_ids: from fabledassistant.services.notes import get_note as _get_note included_snippets: list[str] = [] @@ -682,13 +708,13 @@ async def build_context( except Exception: logger.warning("Failed to load included note %d for context", nid, exc_info=True) if included_snippets: - system_parts.append( - "\n\n--- Included Notes ---\n" + user_context_parts.append( + "--- Included Notes ---\n" + "\n".join(included_snippets) + "\n--- End Included Notes ---" ) - # Search for semantically relevant recent news items + # Semantically relevant RSS news items try: from fabledassistant.services.embeddings import get_embedding, semantic_search_rss_items news_query_vec = await get_embedding(user_message) @@ -703,8 +729,8 @@ async def build_context( + (f"{excerpt}\n" if excerpt else "") + f"URL: {rss_item.url}" ) - system_parts.append( - "\n\n--- Recent News You've Seen ---\n" + user_context_parts.append( + "--- Recent News You've Seen ---\n" + "\n\n".join(news_snippets) + "\n--- End Recent News ---" ) @@ -715,38 +741,16 @@ async def build_context( except Exception: logger.debug("RSS semantic search skipped", exc_info=True) - # Fetch URL content from user message + # URL content fetched from links in the user message urls = _find_urls(user_message) - for url in urls[:2]: # Limit to 2 URLs + for url in urls[:2]: content = await fetch_url_content(url) if content and not content.startswith("[Failed"): - system_parts.append( - f"\n\n--- Content from {url} ---\n{content}\n--- End URL Content ---" + user_context_parts.append( + f"--- Content from {url} ---\n{content}\n--- End URL Content ---" ) - # Inject workspace context when user is in a project workspace - if workspace_project_id is not None: - from fabledassistant.services.projects import get_project - try: - wp = await get_project(user_id, workspace_project_id) - if wp: - system_parts.append( - f"\n\n--- Active Workspace ---\n" - f"You are in the \"{wp.title}\" project workspace.\n" - f"All notes and tasks you create or update MUST belong to this project.\n" - f"Always pass project=\"{wp.title}\" when calling create_note or create_task.\n" - f"--- End Active Workspace ---" - ) - except Exception: - logger.warning("Failed to fetch workspace project %d", workspace_project_id) - - # Inject compressed summary of older exchanges when history has been trimmed - if history_summary: - system_parts.append( - f"\n\n--- Earlier Conversation ---\n{history_summary}\n--- End Earlier Conversation ---" - ) - - # Inject briefing article content for follow-up Q&A + # Briefing article context for follow-up Q&A if conv_id is not None: from fabledassistant.models import async_session as _async_session from fabledassistant.models.conversation import Conversation @@ -756,11 +760,17 @@ async def build_context( if _conv and getattr(_conv, "conversation_type", None) == "briefing": article_context = await _build_briefing_article_context(conv_id) if article_context: - system_parts.append(article_context) + user_context_parts.append(article_context.strip()) - messages = [{"role": "system", "content": "".join(system_parts)}] + # Build final user message — context prefix (if any) followed by the actual message + if user_context_parts: + user_turn = "\n\n".join(user_context_parts) + "\n\n" + user_message + else: + user_turn = user_message + + messages = [{"role": "system", "content": system_content}] messages.extend(history) - messages.append({"role": "user", "content": user_message}) + messages.append({"role": "user", "content": user_turn}) return messages, context_meta