Phase 21: Intent-first pipeline, visible ack, KV-stable system prompt
Pipeline changes (generation_task.py, intent.py): - Remove optimistic streaming queue/race (_drain_queue deleted) - Remove _generate_acknowledgment — ack now embedded in intent JSON - Round 0: await intent (~400ms), stream ack immediately as TTFT, then execute tool sequentially; chat-only streams directly - IntentResult.ack: one-sentence acknowledgment, intent max_tokens 200→350 - _parse_intent extracts and trims ack field KV cache stability (llm.py, generation_buffer.py, generation_task.py): - build_context: replace cached_note_ids with include_note_ids - Auto-found notes populate context_meta["auto_notes"] for sidebar but are NOT injected into system prompt (--- Related Notes --- removed) - Explicitly included notes injected as --- Included Notes --- - _conv_note_cache dict + get/set/clear functions removed from generation_buffer.py - All clear_conv_note_cache() calls removed Cold model retry (llm.py): - generate_completion (used by classify_intent) retries on HTTP 500: 3 attempts with 3s/6s delays — prevents intent failure during cold load API + frontend (routes/chat.py, stores/chat.ts, views/ChatView.vue, components/ChatPanel.vue): - exclude_note_ids → include_note_ids throughout - ChatView sidebar: Suggested (auto-found, + to include) + In Context (× to remove) - ChatPanel: remove exclude button from context pills; no IDs passed to sendMessage Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
@@ -178,21 +179,41 @@ async def stream_chat_with_tools(
|
||||
|
||||
|
||||
async def generate_completion(messages: list[dict], model: str, max_tokens: int = 4096) -> str:
|
||||
"""Non-streaming chat completion, returns full response text."""
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(1800.0, connect=30.0, read=300.0)) as client:
|
||||
resp = await client.post(
|
||||
f"{Config.OLLAMA_URL}/api/chat",
|
||||
json={
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"think": False,
|
||||
"options": {"num_predict": max_tokens},
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
return data.get("message", {}).get("content", "")
|
||||
"""Non-streaming chat completion, returns full response text.
|
||||
|
||||
Retries up to 2 times on Ollama 500 errors (cold model loading race).
|
||||
"""
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(3):
|
||||
if attempt > 0:
|
||||
delay = 3.0 * attempt
|
||||
logger.warning(
|
||||
"generate_completion 500 (attempt %d/3), retrying in %.0fs", attempt, delay
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(1800.0, connect=30.0, read=300.0)) as client:
|
||||
resp = await client.post(
|
||||
f"{Config.OLLAMA_URL}/api/chat",
|
||||
json={
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"think": False,
|
||||
"options": {"num_predict": max_tokens},
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
return data.get("message", {}).get("content", "")
|
||||
except httpx.HTTPStatusError as exc:
|
||||
last_exc = exc
|
||||
if exc.response.status_code != 500:
|
||||
break
|
||||
except Exception as exc:
|
||||
last_exc = exc
|
||||
break
|
||||
raise last_exc
|
||||
|
||||
|
||||
async def fetch_url_content(url: str) -> str:
|
||||
@@ -307,7 +328,7 @@ async def build_context(
|
||||
user_message: str,
|
||||
exclude_note_ids: list[int] | None = None,
|
||||
history_summary: str | None = None,
|
||||
cached_note_ids: list[int] | None = None,
|
||||
include_note_ids: list[int] | None = None,
|
||||
) -> tuple[list[dict], dict]:
|
||||
"""Build messages array for Ollama with system prompt and context.
|
||||
|
||||
@@ -376,64 +397,58 @@ async def build_context(
|
||||
f"--- End Note ---"
|
||||
)
|
||||
|
||||
# Find related notes to inject into context.
|
||||
# Priority: (1) use cached note IDs from a previous turn in this conversation
|
||||
# (2) try semantic search via nomic-embed-text
|
||||
# (3) fall back to keyword search
|
||||
# The cache stabilises the system prompt prefix so Ollama's KV cache stays warm.
|
||||
# Search for related notes to populate sidebar candidates.
|
||||
# Results are NOT injected into the system prompt automatically — this keeps
|
||||
# the system prompt prefix stable so Ollama's KV cache can reuse prefill state.
|
||||
# Users can explicitly include notes via the sidebar (include_note_ids).
|
||||
search_exclude = set(exclude_set)
|
||||
if current_note_id:
|
||||
search_exclude.add(current_note_id)
|
||||
|
||||
found_notes = []
|
||||
if cached_note_ids:
|
||||
# Load the same notes as last turn — keeps system prompt prefix identical.
|
||||
try:
|
||||
from fabledassistant.services.notes import get_note as _get_note
|
||||
for nid in cached_note_ids:
|
||||
if nid not in search_exclude:
|
||||
n = await _get_note(user_id, nid)
|
||||
if n:
|
||||
found_notes.append(n)
|
||||
except Exception:
|
||||
logger.warning("Failed to load cached notes for context", exc_info=True)
|
||||
found_notes = []
|
||||
# Try semantic search first; fall back to keyword search on failure / no results.
|
||||
try:
|
||||
from fabledassistant.services.embeddings import semantic_search_notes
|
||||
found_notes = await semantic_search_notes(
|
||||
user_id, user_message, exclude_ids=search_exclude or None, limit=3
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Semantic note search failed, falling back to keyword search", exc_info=True)
|
||||
|
||||
if not found_notes:
|
||||
# Try semantic search first; fall back to keyword search on failure / no results.
|
||||
try:
|
||||
from fabledassistant.services.embeddings import semantic_search_notes
|
||||
found_notes = await semantic_search_notes(
|
||||
user_id, user_message, exclude_ids=search_exclude or None, limit=3
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Semantic note search failed, falling back to keyword search", exc_info=True)
|
||||
keywords = _extract_keywords(user_message)
|
||||
if keywords:
|
||||
try:
|
||||
found_notes = await search_notes_for_context(
|
||||
user_id, keywords, exclude_ids=search_exclude or None, limit=3
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Failed to search notes for context", exc_info=True)
|
||||
|
||||
if not found_notes:
|
||||
keywords = _extract_keywords(user_message)
|
||||
if keywords:
|
||||
try:
|
||||
found_notes = await search_notes_for_context(
|
||||
user_id, keywords, exclude_ids=search_exclude or None, limit=3
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Failed to search notes for context", exc_info=True)
|
||||
|
||||
if found_notes:
|
||||
snippets: list[str] = []
|
||||
for n in found_notes:
|
||||
body_preview = n.body[:2000] if n.body else ""
|
||||
snippets.append(f"- {n.title}: {body_preview}")
|
||||
context_meta["auto_notes"].append({"id": n.id, "title": n.title})
|
||||
system_parts.append(
|
||||
"\n\n--- Related Notes ---\n"
|
||||
+ "\n".join(snippets)
|
||||
+ "\n--- End Related Notes ---"
|
||||
)
|
||||
|
||||
# Expose note IDs so the caller can update the per-conversation cache.
|
||||
# Populate sidebar candidates (never auto-injected).
|
||||
for n in found_notes:
|
||||
context_meta["auto_notes"].append({"id": n.id, "title": n.title})
|
||||
context_meta["auto_note_ids"] = [n.id for n in found_notes]
|
||||
|
||||
# Inject explicitly included notes (user opted in via sidebar click).
|
||||
if include_note_ids:
|
||||
from fabledassistant.services.notes import get_note as _get_note
|
||||
included_snippets: list[str] = []
|
||||
for nid in include_note_ids:
|
||||
try:
|
||||
n = await _get_note(user_id, nid)
|
||||
if n:
|
||||
body_preview = n.body[:2000] if n.body else ""
|
||||
included_snippets.append(f"- {n.title}: {body_preview}")
|
||||
except Exception:
|
||||
logger.warning("Failed to load included note %d for context", nid, exc_info=True)
|
||||
if included_snippets:
|
||||
system_parts.append(
|
||||
"\n\n--- Included Notes ---\n"
|
||||
+ "\n".join(included_snippets)
|
||||
+ "\n--- End Included Notes ---"
|
||||
)
|
||||
|
||||
# Fetch URL content from user message
|
||||
urls = _find_urls(user_message)
|
||||
for url in urls[:2]: # Limit to 2 URLs
|
||||
|
||||
Reference in New Issue
Block a user