perf: restructure system prompt for Ollama KV-cache prefix reuse

Move static content (persona + tool guidance) to a fixed prefix and
append all dynamic content (date, timezone, profile, entities) as a tail.

Ollama prefix caching requires byte-for-byte token match from the start
of the prompt. Previously, Today's date + user profile were embedded
mid-prompt, invalidating the cache on every request/day and causing
~20s TTFT regardless of model warmth.

With this change the static prefix (~5500 tokens) should be cached
after the first request each session, reducing TTFT to ~2-5s for the
~200-token dynamic tail only.

Also removed inline user_timezone from tool_lines (timezone is now
stated once in the dynamic tail, which the model reads).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-02 17:29:26 -04:00
parent a7160772bf
commit c3665ddda5
+20 -11
View File
@@ -478,6 +478,10 @@ async def build_context(
has_caldav = await is_caldav_configured(user_id) has_caldav = await is_caldav_configured(user_id)
# Build tool usage guidance based on available integrations # Build tool usage guidance based on available integrations
# --- Static block (Ollama KV-cache prefix) ---
# Everything here must be byte-for-byte identical across requests for the same
# user so Ollama can reuse the cached KV state. No dates, timezones, RAG notes,
# or user-profile data here — those go in the dynamic tail below.
tool_lines = [ tool_lines = [
"You have access to tool functions. You MUST use them when the user asks you to create, add, find, schedule, or search for anything.", "You have access to tool functions. You MUST use them when the user asks you to create, add, find, schedule, or search for anything.",
"CRITICAL: Call the tool functions directly. NEVER write out function calls as text or code. NEVER describe what you would do — just do it.", "CRITICAL: Call the tool functions directly. NEVER write out function calls as text or code. NEVER describe what you would do — just do it.",
@@ -489,14 +493,13 @@ async def build_context(
"create_event, list_events, search_events, update_event, delete_event, list_calendars." "create_event, list_events, search_events, update_event, delete_event, list_calendars."
) )
tool_lines.append( tool_lines.append(
"For calendar events, use ISO 8601 datetime format with the user's timezone offset" "For calendar events, use ISO 8601 datetime format with the user's timezone offset (stated in context below). "
+ (f" ({user_timezone})" if user_timezone else "") "Always include the UTC offset in datetime strings (e.g. 2026-09-30T14:00:00+01:00)."
+ ". Always include the UTC offset in datetime strings (e.g. 2026-09-30T14:00:00+01:00)."
) )
tool_lines.append("When the user says 'remind me' with a time before an event, use the reminder_minutes parameter.") tool_lines.append("When the user says 'remind me' with a time before an event, use the reminder_minutes parameter.")
tool_lines.append( tool_lines.append(
"For relative dates like 'Friday' or 'next week', resolve them to YYYY-MM-DD format. " "For relative dates like 'Friday' or 'next week', resolve them to YYYY-MM-DD format. "
+ (f"Always include the UTC offset when creating events (user's timezone: {user_timezone})." if user_timezone else "For event datetimes, include the UTC offset (e.g. 2026-09-30T14:00:00+01:00).") "Always include the UTC offset when creating events (user's timezone is stated in context below)."
) )
tool_lines.append("When creating notes, use the `tags` parameter — do not embed #tag text in the note body.") tool_lines.append("When creating notes, use the `tags` parameter — do not embed #tag text in the note body.")
tool_lines.append( tool_lines.append(
@@ -519,6 +522,16 @@ async def build_context(
) )
tool_guidance = "\n".join(tool_lines) tool_guidance = "\n".join(tool_lines)
static_block = (
f"You are a helpful assistant named {assistant_name}, integrated into a note-taking and task-tracking app called Fabled Assistant. "
"Help users with their notes, tasks, and general questions. "
"When note context is provided, use it to give relevant answers.\n\n"
f"{tool_guidance}"
)
# --- Dynamic tail (appended after static block, evaluated every request) ---
# Date, timezone, user profile, and entities change per-day or per-user.
# Keeping these at the end preserves the static prefix for KV-cache reuse.
tz_line = f" The user's timezone is {user_timezone}." if user_timezone else "" tz_line = f" The user's timezone is {user_timezone}." if user_timezone else ""
from fabledassistant.services.user_profile import build_profile_context from fabledassistant.services.user_profile import build_profile_context
@@ -528,13 +541,9 @@ async def build_context(
entities_context = await get_people_and_places_context(user_id) entities_context = await get_people_and_places_context(user_id)
entities_section = f"\n\n{entities_context}" if entities_context else "" entities_section = f"\n\n{entities_context}" if entities_context else ""
system_parts = [ dynamic_tail = f"\n\nToday's date is {today}.{tz_line}{profile_section}{entities_section}"
f"You are a helpful assistant named {assistant_name}, integrated into a note-taking and task-tracking app called Fabled Assistant. "
"Help users with their notes, tasks, and general questions. " system_parts = [static_block + dynamic_tail]
"When note context is provided, use it to give relevant answers. "
f"Today's date is {today}.{tz_line}{profile_section}{entities_section}\n\n"
f"{tool_guidance}"
]
if voice_mode: if voice_mode:
_style_hints = { _style_hints = {