Reduce context note preview sizes to cut prefill latency
Auto-notes (keyword-matched): 2000 → 800 chars each (×3 max = 6000 → 2400 chars). Pinned note (explicit context): was unbounded → capped at 4000 chars with [truncated] marker. The main post-GPU bottleneck is TTFT caused by the prefill phase — the model processing the full input before generating any tokens. Shorter context = faster prefill. Users can ask follow-up questions for more detail. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -289,16 +289,20 @@ async def build_context(
|
||||
"auto_notes": [],
|
||||
}
|
||||
|
||||
# Include current note context if provided
|
||||
# Include current note context if provided (cap at 4000 chars — explicitly selected
|
||||
# so more generous, but very long notes would blow up the prefill phase)
|
||||
if current_note_id:
|
||||
note = await get_note(user_id, current_note_id)
|
||||
if note:
|
||||
context_meta["context_note_id"] = note.id
|
||||
context_meta["context_note_title"] = note.title
|
||||
body = note.body or ""
|
||||
truncated = body[:4000]
|
||||
suffix = "\n[...truncated]" if len(body) > 4000 else ""
|
||||
system_parts.append(
|
||||
f"\n\n--- Current Note ---\n"
|
||||
f"Title: {note.title}\n"
|
||||
f"Content:\n{note.body}\n"
|
||||
f"Content:\n{truncated}{suffix}\n"
|
||||
f"--- End Note ---"
|
||||
)
|
||||
|
||||
@@ -314,7 +318,9 @@ async def build_context(
|
||||
)
|
||||
snippets: list[str] = []
|
||||
for n in notes:
|
||||
body_preview = n.body[:2000] if n.body else ""
|
||||
# 800 chars keeps context meaningful without bloating the prefill phase.
|
||||
# The user can ask follow-up questions for more detail.
|
||||
body_preview = n.body[:800] if n.body else ""
|
||||
snippets.append(f"- {n.title}: {body_preview}")
|
||||
context_meta["auto_notes"].append({"id": n.id, "title": n.title})
|
||||
if snippets:
|
||||
|
||||
Reference in New Issue
Block a user