bvandeusen/FabledScribe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 50 Wiki Activity

perf(llm): move retrieval context to user turn for stable system prompt

Browse Source 
RAG notes, RSS news, current note, URL content, and briefing articles
are now prepended to the user message rather than appended to the system
message. The system message now contains only stable content (persona,
tool guidance, date, profile, workspace, history summary), making its
token sequence identical across consecutive requests and allowing
Ollama's KV prefix cache to fire reliably every time.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Bryan Van Deusen
2026-04-03 00:44:17 -04:00
parent 36634919cc
commit b4be1f0799
1 changed files with 64 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fabledassistant/services/llm.py
+64 -54
View File
@@ -543,7 +543,13 @@ async def build_context(
dynamic_tail = f"\n\nToday's date is {today}.{tz_line}{profile_section}{entities_section}"
system_parts = [static_block + dynamic_tail]
# --- System message: stable content only ---
# Workspace context and history summary stay here because they carry
# behavioural instructions / conversational state, not retrieved content.
# Everything retrieval-based (RAG notes, RSS, URL content, current note,
# briefing articles) goes into the user turn below so the system message
# prefix stays byte-for-byte identical across requests, enabling Ollama's
# KV prefix cache to fire reliably.
if voice_mode:
_style_hints = {
@@ -552,11 +558,36 @@ async def build_context(
"detailed": "Give thorough, informative responses as if narrating an explanation aloud.",
}
style_hint = _style_hints.get(voice_speech_style, _style_hints["conversational"])
system_parts.insert(0,
voice_preamble = (
"VOICE MODE: Respond naturally as if speaking aloud. "
"No markdown, bullet points, headers, or code blocks. Complete sentences only. "
f"{style_hint}\n\n"
)
system_content = voice_preamble + static_block + dynamic_tail
else:
system_content = static_block + dynamic_tail
# Inject workspace context (behavioural — must stay in system)
if workspace_project_id is not None:
from fabledassistant.services.projects import get_project
try:
wp = await get_project(user_id, workspace_project_id)
if wp:
system_content += (
f"\n\n--- Active Workspace ---\n"
f"You are in the \"{wp.title}\" project workspace.\n"
f"All notes and tasks you create or update MUST belong to this project.\n"
f"Always pass project=\"{wp.title}\" when calling create_note or create_task.\n"
f"--- End Active Workspace ---"
)
except Exception:
logger.warning("Failed to fetch workspace project %d", workspace_project_id)
# Inject compressed history summary (conversational state — stays in system)
if history_summary:
system_content += (
f"\n\n--- Earlier Conversation ---\n{history_summary}\n--- End Earlier Conversation ---"
)
context_meta: dict = {
"context_note_id": None,
@@ -565,35 +596,34 @@ async def build_context(
"auto_injected_notes": [],
}
# Include current note context if provided — full body, no truncation
# --- User turn context prefix: retrieval-based content ---
# Collected here and prepended to the user message so the system message
# stays stable and the KV prefix cache can fire on every request.
user_context_parts: list[str] = []
# Current note being viewed (full body, no truncation)
if current_note_id:
note = await get_note(user_id, current_note_id)
if note:
context_meta["context_note_id"] = note.id
context_meta["context_note_title"] = note.title
system_parts.append(
f"\n\n--- Current Note ---\n"
user_context_parts.append(
f"--- Current Note ---\n"
f"Title: {note.title}\n"
f"Content:\n{note.body}\n"
f"--- End Note ---"
)
# Search for related notes. High-confidence results (>=0.60) are auto-injected
# into the system prompt; lower-confidence results populate the sidebar only.
# Users can also explicitly include notes via the sidebar (include_note_ids).
# Semantic / keyword note search
search_exclude = set(exclude_set)
if current_note_id:
search_exclude.add(current_note_id)
# (score, note) pairs — score is float for semantic results, None for keyword fallback.
found_scored: list[tuple[float | None, object]] = []
# Derive scope flags from rag_project_id three-value semantics:
# None → orphan notes only; -1 → all notes; positive int → that project
orphan_only = rag_project_id is None
effective_project_id = rag_project_id if (rag_project_id is not None and rag_project_id != -1) else None
# Try semantic search first; fall back to keyword search on failure / no results.
try:
from fabledassistant.services.embeddings import semantic_search_notes
for score, note in await semantic_search_notes(
@@ -618,7 +648,6 @@ async def build_context(
except Exception:
logger.warning("Failed to search notes for context", exc_info=True)
# Separate high-confidence results for auto-injection vs sidebar display
excluded_inject_set = set(excluded_note_ids or [])
auto_inject: list[tuple[float, object]] = []
sidebar_only: list[tuple[float | None, object]] = []
@@ -634,7 +663,6 @@ async def build_context(
else:
sidebar_only.append((score, n))
# Inject high-scoring notes into system prompt
if auto_inject:
snippets = []
for score, n in auto_inject:
@@ -645,14 +673,12 @@ async def build_context(
"title": n.title,
"score": round(score, 2),
})
system_parts.append(
"\n\n--- Relevant Notes ---\n"
user_context_parts.append(
"--- Relevant Notes ---\n"
+ "\n\n".join(snippets)
+ "\n--- End Relevant Notes ---"
)
# Populate sidebar candidates (auto-injected notes also appear here for reference,
# but sidebar_only are the ones not yet in the prompt)
for score, n in auto_inject:
context_meta["auto_notes"].append({
"id": n.id,
@@ -669,7 +695,7 @@ async def build_context(
})
context_meta["auto_note_ids"] = [n.id for _, n in found_scored]
# Inject explicitly included notes (user opted in via sidebar click).
# Explicitly included notes (user opted in via sidebar)
if include_note_ids:
from fabledassistant.services.notes import get_note as _get_note
included_snippets: list[str] = []
@@ -682,13 +708,13 @@ async def build_context(
except Exception:
logger.warning("Failed to load included note %d for context", nid, exc_info=True)
if included_snippets:
system_parts.append(
"\n\n--- Included Notes ---\n"
user_context_parts.append(
"--- Included Notes ---\n"
+ "\n".join(included_snippets)
+ "\n--- End Included Notes ---"
)
# Search for semantically relevant recent news items
# Semantically relevant RSS news items
try:
from fabledassistant.services.embeddings import get_embedding, semantic_search_rss_items
news_query_vec = await get_embedding(user_message)
@@ -703,8 +729,8 @@ async def build_context(
+ (f"{excerpt}\n" if excerpt else "")
+ f"URL: {rss_item.url}"
)
system_parts.append(
"\n\n--- Recent News You've Seen ---\n"
user_context_parts.append(
"--- Recent News You've Seen ---\n"
+ "\n\n".join(news_snippets)
+ "\n--- End Recent News ---"
)
@@ -715,38 +741,16 @@ async def build_context(
except Exception:
logger.debug("RSS semantic search skipped", exc_info=True)
# Fetch URL content from user message
# URL content fetched from links in the user message
urls = _find_urls(user_message)
for url in urls[:2]: # Limit to 2 URLs
for url in urls[:2]:
content = await fetch_url_content(url)
if content and not content.startswith("[Failed"):
system_parts.append(
f"\n\n--- Content from {url} ---\n{content}\n--- End URL Content ---"
user_context_parts.append(
f"--- Content from {url} ---\n{content}\n--- End URL Content ---"
)
# Inject workspace context when user is in a project workspace
if workspace_project_id is not None:
from fabledassistant.services.projects import get_project
try:
wp = await get_project(user_id, workspace_project_id)
if wp:
system_parts.append(
f"\n\n--- Active Workspace ---\n"
f"You are in the \"{wp.title}\" project workspace.\n"
f"All notes and tasks you create or update MUST belong to this project.\n"
f"Always pass project=\"{wp.title}\" when calling create_note or create_task.\n"
f"--- End Active Workspace ---"
)
except Exception:
logger.warning("Failed to fetch workspace project %d", workspace_project_id)
# Inject compressed summary of older exchanges when history has been trimmed
if history_summary:
system_parts.append(
f"\n\n--- Earlier Conversation ---\n{history_summary}\n--- End Earlier Conversation ---"
)
# Inject briefing article content for follow-up Q&A
# Briefing article context for follow-up Q&A
if conv_id is not None:
from fabledassistant.models import async_session as _async_session
from fabledassistant.models.conversation import Conversation
@@ -756,11 +760,17 @@ async def build_context(
if _conv and getattr(_conv, "conversation_type", None) == "briefing":
article_context = await _build_briefing_article_context(conv_id)
if article_context:
system_parts.append(article_context)
user_context_parts.append(article_context.strip())
messages = [{"role": "system", "content": "".join(system_parts)}]
# Build final user message — context prefix (if any) followed by the actual message
if user_context_parts:
user_turn = "\n\n".join(user_context_parts) + "\n\n" + user_message
else:
user_turn = user_message
messages = [{"role": "system", "content": system_content}]
messages.extend(history)
messages.append({"role": "user", "content": user_message})
messages.append({"role": "user", "content": user_turn})
return messages, context_meta