From 38697f261413eb4fb79ed8cb5a8e2745e631e19f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 18 Feb 2026 19:58:52 -0500 Subject: [PATCH] Reduce context note preview sizes to cut prefill latency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Auto-notes (keyword-matched): 2000 → 800 chars each (×3 max = 6000 → 2400 chars). Pinned note (explicit context): was unbounded → capped at 4000 chars with [truncated] marker. The main post-GPU bottleneck is TTFT caused by the prefill phase — the model processing the full input before generating any tokens. Shorter context = faster prefill. Users can ask follow-up questions for more detail. Co-Authored-By: Claude Sonnet 4.6 --- src/fabledassistant/services/llm.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index 4310faa..55716a2 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -289,16 +289,20 @@ async def build_context( "auto_notes": [], } - # Include current note context if provided + # Include current note context if provided (cap at 4000 chars — explicitly selected + # so more generous, but very long notes would blow up the prefill phase) if current_note_id: note = await get_note(user_id, current_note_id) if note: context_meta["context_note_id"] = note.id context_meta["context_note_title"] = note.title + body = note.body or "" + truncated = body[:4000] + suffix = "\n[...truncated]" if len(body) > 4000 else "" system_parts.append( f"\n\n--- Current Note ---\n" f"Title: {note.title}\n" - f"Content:\n{note.body}\n" + f"Content:\n{truncated}{suffix}\n" f"--- End Note ---" ) @@ -314,7 +318,9 @@ async def build_context( ) snippets: list[str] = [] for n in notes: - body_preview = n.body[:2000] if n.body else "" + # 800 chars keeps context meaningful without bloating the prefill phase. + # The user can ask follow-up questions for more detail. + body_preview = n.body[:800] if n.body else "" snippets.append(f"- {n.title}: {body_preview}") context_meta["auto_notes"].append({"id": n.id, "title": n.title}) if snippets: