Phase 21: Intent-first pipeline, visible ack, KV-stable system prompt

Pipeline changes (generation_task.py, intent.py):
- Remove optimistic streaming queue/race (_drain_queue deleted)
- Remove _generate_acknowledgment — ack now embedded in intent JSON
- Round 0: await intent (~400ms), stream ack immediately as TTFT,
  then execute tool sequentially; chat-only streams directly
- IntentResult.ack: one-sentence acknowledgment, intent max_tokens 200→350
- _parse_intent extracts and trims ack field

KV cache stability (llm.py, generation_buffer.py, generation_task.py):
- build_context: replace cached_note_ids with include_note_ids
- Auto-found notes populate context_meta["auto_notes"] for sidebar but
  are NOT injected into system prompt (--- Related Notes --- removed)
- Explicitly included notes injected as --- Included Notes ---
- _conv_note_cache dict + get/set/clear functions removed from generation_buffer.py
- All clear_conv_note_cache() calls removed

Cold model retry (llm.py):
- generate_completion (used by classify_intent) retries on HTTP 500:
  3 attempts with 3s/6s delays — prevents intent failure during cold load

API + frontend (routes/chat.py, stores/chat.ts, views/ChatView.vue, components/ChatPanel.vue):
- exclude_note_ids → include_note_ids throughout
- ChatView sidebar: Suggested (auto-found, + to include) + In Context (× to remove)
- ChatPanel: remove exclude button from context pills; no IDs passed to sendMessage

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-26 22:34:54 -05:00
parent 316a85e13b
commit e119331645
9 changed files with 237 additions and 353 deletions
+79 -64
View File
@@ -1,3 +1,4 @@
import asyncio
import json
import logging
import re
@@ -178,21 +179,41 @@ async def stream_chat_with_tools(
async def generate_completion(messages: list[dict], model: str, max_tokens: int = 4096) -> str:
"""Non-streaming chat completion, returns full response text."""
async with httpx.AsyncClient(timeout=httpx.Timeout(1800.0, connect=30.0, read=300.0)) as client:
resp = await client.post(
f"{Config.OLLAMA_URL}/api/chat",
json={
"model": model,
"messages": messages,
"stream": False,
"think": False,
"options": {"num_predict": max_tokens},
},
)
resp.raise_for_status()
data = resp.json()
return data.get("message", {}).get("content", "")
"""Non-streaming chat completion, returns full response text.
Retries up to 2 times on Ollama 500 errors (cold model loading race).
"""
last_exc: Exception | None = None
for attempt in range(3):
if attempt > 0:
delay = 3.0 * attempt
logger.warning(
"generate_completion 500 (attempt %d/3), retrying in %.0fs", attempt, delay
)
await asyncio.sleep(delay)
try:
async with httpx.AsyncClient(timeout=httpx.Timeout(1800.0, connect=30.0, read=300.0)) as client:
resp = await client.post(
f"{Config.OLLAMA_URL}/api/chat",
json={
"model": model,
"messages": messages,
"stream": False,
"think": False,
"options": {"num_predict": max_tokens},
},
)
resp.raise_for_status()
data = resp.json()
return data.get("message", {}).get("content", "")
except httpx.HTTPStatusError as exc:
last_exc = exc
if exc.response.status_code != 500:
break
except Exception as exc:
last_exc = exc
break
raise last_exc
async def fetch_url_content(url: str) -> str:
@@ -307,7 +328,7 @@ async def build_context(
user_message: str,
exclude_note_ids: list[int] | None = None,
history_summary: str | None = None,
cached_note_ids: list[int] | None = None,
include_note_ids: list[int] | None = None,
) -> tuple[list[dict], dict]:
"""Build messages array for Ollama with system prompt and context.
@@ -376,64 +397,58 @@ async def build_context(
f"--- End Note ---"
)
# Find related notes to inject into context.
# Priority: (1) use cached note IDs from a previous turn in this conversation
# (2) try semantic search via nomic-embed-text
# (3) fall back to keyword search
# The cache stabilises the system prompt prefix so Ollama's KV cache stays warm.
# Search for related notes to populate sidebar candidates.
# Results are NOT injected into the system prompt automatically — this keeps
# the system prompt prefix stable so Ollama's KV cache can reuse prefill state.
# Users can explicitly include notes via the sidebar (include_note_ids).
search_exclude = set(exclude_set)
if current_note_id:
search_exclude.add(current_note_id)
found_notes = []
if cached_note_ids:
# Load the same notes as last turn — keeps system prompt prefix identical.
try:
from fabledassistant.services.notes import get_note as _get_note
for nid in cached_note_ids:
if nid not in search_exclude:
n = await _get_note(user_id, nid)
if n:
found_notes.append(n)
except Exception:
logger.warning("Failed to load cached notes for context", exc_info=True)
found_notes = []
# Try semantic search first; fall back to keyword search on failure / no results.
try:
from fabledassistant.services.embeddings import semantic_search_notes
found_notes = await semantic_search_notes(
user_id, user_message, exclude_ids=search_exclude or None, limit=3
)
except Exception:
logger.warning("Semantic note search failed, falling back to keyword search", exc_info=True)
if not found_notes:
# Try semantic search first; fall back to keyword search on failure / no results.
try:
from fabledassistant.services.embeddings import semantic_search_notes
found_notes = await semantic_search_notes(
user_id, user_message, exclude_ids=search_exclude or None, limit=3
)
except Exception:
logger.warning("Semantic note search failed, falling back to keyword search", exc_info=True)
keywords = _extract_keywords(user_message)
if keywords:
try:
found_notes = await search_notes_for_context(
user_id, keywords, exclude_ids=search_exclude or None, limit=3
)
except Exception:
logger.warning("Failed to search notes for context", exc_info=True)
if not found_notes:
keywords = _extract_keywords(user_message)
if keywords:
try:
found_notes = await search_notes_for_context(
user_id, keywords, exclude_ids=search_exclude or None, limit=3
)
except Exception:
logger.warning("Failed to search notes for context", exc_info=True)
if found_notes:
snippets: list[str] = []
for n in found_notes:
body_preview = n.body[:2000] if n.body else ""
snippets.append(f"- {n.title}: {body_preview}")
context_meta["auto_notes"].append({"id": n.id, "title": n.title})
system_parts.append(
"\n\n--- Related Notes ---\n"
+ "\n".join(snippets)
+ "\n--- End Related Notes ---"
)
# Expose note IDs so the caller can update the per-conversation cache.
# Populate sidebar candidates (never auto-injected).
for n in found_notes:
context_meta["auto_notes"].append({"id": n.id, "title": n.title})
context_meta["auto_note_ids"] = [n.id for n in found_notes]
# Inject explicitly included notes (user opted in via sidebar click).
if include_note_ids:
from fabledassistant.services.notes import get_note as _get_note
included_snippets: list[str] = []
for nid in include_note_ids:
try:
n = await _get_note(user_id, nid)
if n:
body_preview = n.body[:2000] if n.body else ""
included_snippets.append(f"- {n.title}: {body_preview}")
except Exception:
logger.warning("Failed to load included note %d for context", nid, exc_info=True)
if included_snippets:
system_parts.append(
"\n\n--- Included Notes ---\n"
+ "\n".join(included_snippets)
+ "\n--- End Included Notes ---"
)
# Fetch URL content from user message
urls = _find_urls(user_message)
for url in urls[:2]: # Limit to 2 URLs