From c3665ddda5049cad0925ff22d147ecad11e40de5 Mon Sep 17 00:00:00 2001
From: Bryan Van Deusen <bryan.vandeusen@fabledsword.com>
Date: Thu, 2 Apr 2026 17:29:26 -0400
Subject: [PATCH] perf: restructure system prompt for Ollama KV-cache prefix
 reuse

Move static content (persona + tool guidance) to a fixed prefix and
append all dynamic content (date, timezone, profile, entities) as a tail.

Ollama prefix caching requires byte-for-byte token match from the start
of the prompt. Previously, Today's date + user profile were embedded
mid-prompt, invalidating the cache on every request/day and causing
~20s TTFT regardless of model warmth.

With this change the static prefix (~5500 tokens) should be cached
after the first request each session, reducing TTFT to ~2-5s for the
~200-token dynamic tail only.

Also removed inline user_timezone from tool_lines (timezone is now
stated once in the dynamic tail, which the model reads).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 src/fabledassistant/services/llm.py | 31 +++++++++++++++++++----------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py
index ce3f02c..d7f2e19 100644
--- a/src/fabledassistant/services/llm.py
+++ b/src/fabledassistant/services/llm.py
@@ -478,6 +478,10 @@ async def build_context(
     has_caldav = await is_caldav_configured(user_id)
 
     # Build tool usage guidance based on available integrations
+    # --- Static block (Ollama KV-cache prefix) ---
+    # Everything here must be byte-for-byte identical across requests for the same
+    # user so Ollama can reuse the cached KV state. No dates, timezones, RAG notes,
+    # or user-profile data here — those go in the dynamic tail below.
     tool_lines = [
         "You have access to tool functions. You MUST use them when the user asks you to create, add, find, schedule, or search for anything.",
         "CRITICAL: Call the tool functions directly. NEVER write out function calls as text or code. NEVER describe what you would do — just do it.",
@@ -489,14 +493,13 @@ async def build_context(
             "create_event, list_events, search_events, update_event, delete_event, list_calendars."
         )
         tool_lines.append(
-            "For calendar events, use ISO 8601 datetime format with the user's timezone offset"
-            + (f" ({user_timezone})" if user_timezone else "")
-            + ". Always include the UTC offset in datetime strings (e.g. 2026-09-30T14:00:00+01:00)."
+            "For calendar events, use ISO 8601 datetime format with the user's timezone offset (stated in context below). "
+            "Always include the UTC offset in datetime strings (e.g. 2026-09-30T14:00:00+01:00)."
         )
         tool_lines.append("When the user says 'remind me' with a time before an event, use the reminder_minutes parameter.")
     tool_lines.append(
         "For relative dates like 'Friday' or 'next week', resolve them to YYYY-MM-DD format. "
-        + (f"Always include the UTC offset when creating events (user's timezone: {user_timezone})." if user_timezone else "For event datetimes, include the UTC offset (e.g. 2026-09-30T14:00:00+01:00).")
+        "Always include the UTC offset when creating events (user's timezone is stated in context below)."
     )
     tool_lines.append("When creating notes, use the `tags` parameter — do not embed #tag text in the note body.")
     tool_lines.append(
@@ -519,6 +522,16 @@ async def build_context(
     )
     tool_guidance = "\n".join(tool_lines)
 
+    static_block = (
+        f"You are a helpful assistant named {assistant_name}, integrated into a note-taking and task-tracking app called Fabled Assistant. "
+        "Help users with their notes, tasks, and general questions. "
+        "When note context is provided, use it to give relevant answers.\n\n"
+        f"{tool_guidance}"
+    )
+
+    # --- Dynamic tail (appended after static block, evaluated every request) ---
+    # Date, timezone, user profile, and entities change per-day or per-user.
+    # Keeping these at the end preserves the static prefix for KV-cache reuse.
     tz_line = f" The user's timezone is {user_timezone}." if user_timezone else ""
 
     from fabledassistant.services.user_profile import build_profile_context
@@ -528,13 +541,9 @@ async def build_context(
     entities_context = await get_people_and_places_context(user_id)
     entities_section = f"\n\n{entities_context}" if entities_context else ""
 
-    system_parts = [
-        f"You are a helpful assistant named {assistant_name}, integrated into a note-taking and task-tracking app called Fabled Assistant. "
-        "Help users with their notes, tasks, and general questions. "
-        "When note context is provided, use it to give relevant answers. "
-        f"Today's date is {today}.{tz_line}{profile_section}{entities_section}\n\n"
-        f"{tool_guidance}"
-    ]
+    dynamic_tail = f"\n\nToday's date is {today}.{tz_line}{profile_section}{entities_section}"
+
+    system_parts = [static_block + dynamic_tail]
 
     if voice_mode:
         _style_hints = {