diff --git a/src/fabledassistant/app.py b/src/fabledassistant/app.py index 9a76231..d703170 100644 --- a/src/fabledassistant/app.py +++ b/src/fabledassistant/app.py @@ -189,13 +189,16 @@ def create_app() -> Quart: """ try: from fabledassistant.services.llm import build_context, keep_alive_for, pick_num_ctx + from fabledassistant.services.tools import get_tools_for_user messages, _ = await build_context( user_id=user_id, history=[], current_note_id=None, user_message=" ", ) - num_ctx = pick_num_ctx(messages) + # Include tool schemas so num_ctx matches real chat requests. + tools = await get_tools_for_user(user_id) + num_ctx = pick_num_ctx(messages, tools=tools) async with httpx.AsyncClient(timeout=120.0) as client: await client.post( f"{Config.OLLAMA_URL}/api/chat", diff --git a/src/fabledassistant/routes/quick_capture.py b/src/fabledassistant/routes/quick_capture.py index a0da365..de029e1 100644 --- a/src/fabledassistant/routes/quick_capture.py +++ b/src/fabledassistant/routes/quick_capture.py @@ -11,7 +11,6 @@ from quart import Blueprint, jsonify, request from fabledassistant.auth import get_current_user_id, login_required from fabledassistant.config import Config -from fabledassistant.services.generation_task import _should_think from fabledassistant.services.llm import stream_chat_with_tools from fabledassistant.services.tools import execute_tool, get_tools_for_user @@ -53,11 +52,10 @@ async def quick_capture_route(): {"role": "user", "content": text}, ] - think = _should_think(text, think_requested=True) - + # Quick capture is a fast classification path — never think. tool_calls: list[dict] = [] try: - async for chunk in stream_chat_with_tools(messages, model, tools=capture_tools, think=think, num_ctx=4096): + async for chunk in stream_chat_with_tools(messages, model, tools=capture_tools, think=False, num_ctx=4096): if chunk.type == "tool_calls" and chunk.tool_calls: tool_calls = chunk.tool_calls except Exception: diff --git a/src/fabledassistant/routes/settings.py b/src/fabledassistant/routes/settings.py index 422d6e0..33d25cd 100644 --- a/src/fabledassistant/routes/settings.py +++ b/src/fabledassistant/routes/settings.py @@ -16,6 +16,7 @@ async def _prime_kv_cache_bg(user_id: int, model: str) -> None: """Fire-and-forget: prime Ollama's KV cache with the user's system prompt.""" import httpx from fabledassistant.services.llm import build_context, pick_num_ctx + from fabledassistant.services.tools import get_tools_for_user try: messages, _ = await build_context( user_id=user_id, @@ -23,7 +24,11 @@ async def _prime_kv_cache_bg(user_id: int, model: str) -> None: current_note_id=None, user_message=" ", ) - num_ctx = pick_num_ctx(messages) + # Size the prime to match what real chat requests will use, including + # tool schemas — otherwise Ollama reloads the model on the first real + # request and throws away the cache we just built. + tools = await get_tools_for_user(user_id) + num_ctx = pick_num_ctx(messages, tools=tools) from fabledassistant.services.llm import keep_alive_for async with httpx.AsyncClient(timeout=120.0) as client: await client.post( diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index cf2f2cd..48cedc3 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -37,62 +37,24 @@ _TOOL_CALL_MARKER = re.compile(r"^\s*\[TOOL_CALLS\]\s*", re.IGNORECASE) DB_FLUSH_INTERVAL = 5.0 # seconds between partial DB flushes # --------------------------------------------------------------------------- -# Conditional thinking classifier +# Thinking decision # --------------------------------------------------------------------------- - -# Patterns that force think=True even on short messages -_THINK_FORCE = re.compile( - r"\b(" - r"analyz|compar|explain\s+why|help\s+me\s+(think|plan|understand|figure\s+out)|" - r"step[- ]by[- ]step|debug|troubleshoot|diagnos|" - r"pros\s+and\s+cons|trade[- ]?off|" - r"architect|design\s+(a|the|my|this)|" - r"write\s+a\s+(detailed|long|comprehensive|full)|" - r"brainstorm|outline\s+(a|the|my)|" - r"what\s+(are|is)\s+the\s+(best|difference|relationship|impact|implication)|" - r"how\s+(do|does|should|would|can)\s+.{0,40}\s+work|" - r"why\s+(is|are|does|do|did|would|should)\b" - r")", - re.IGNORECASE, -) - -# Patterns that force think=False regardless of message length -_THINK_SKIP = re.compile( - r"^(hi|hey|hello|thanks|thank\s+you|ok|okay|got\s+it|sounds\s+good|" - r"great|perfect|sure|yes|no|yep|nope|nice|cool|awesome|" - r"what('s| is) \d|what time|how many|remind me|add (a |an )?(task|note|reminder)|" - r"create (a |an )?(task|note)|delete|update|mark .{0,30} (done|complete))\b", - re.IGNORECASE, -) - -_WORD_COUNT_THRESHOLD = 60 # messages over this word count always use think=True -_SHORT_MESSAGE_THRESHOLD = 12 # messages under this always use think=False +# +# The `think` flag from the frontend / MCP is taken at face value: if the +# caller asked for thinking, they get thinking. No heuristic gating. +# +# Models that don't support extended reasoning (e.g. llama3, mistral) simply +# ignore the `think` parameter in the Ollama chat request, so this is safe to +# pass unconditionally across the full model zoo. def _should_think(user_content: str, think_requested: bool) -> bool: """Return whether extended thinking should be used for this request. - If the caller didn't request thinking, we never enable it. If they did, - we check whether the message is complex enough to warrant the overhead. + Honors the caller's request directly — no message-complexity classifier. + The frontend toggle / MCP `think` parameter is the source of truth. """ - if not think_requested: - return False - - text = user_content.strip() - word_count = len(text.split()) - - if word_count <= _SHORT_MESSAGE_THRESHOLD: - return False - if _THINK_SKIP.match(text): - return False - if word_count >= _WORD_COUNT_THRESHOLD: - return True - if "```" in text: - return True - if _THINK_FORCE.search(text): - return True - - return False + return bool(think_requested) # Human-readable labels for each tool, shown in the status indicator @@ -273,9 +235,10 @@ async def run_generation( voice_speech_style=voice_speech_style, ) - # Pick the smallest context tier that fits the current messages. + # Pick the smallest context tier that fits the current messages AND the + # tool schemas (which can be 6-10K tokens on their own with ~40 tools). # Using the minimum needed tier reduces KV cache size and speeds up prefill. - num_ctx = pick_num_ctx(messages) + num_ctx = pick_num_ctx(messages, tools=tools) logger.debug("Adaptive num_ctx=%d for conv %d", num_ctx, conv_id) # Emit context event diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index ee1d7b6..acd79da 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -38,12 +38,20 @@ def keep_alive_for(model: str) -> str: return Config.OLLAMA_KEEP_ALIVE_MAIN -def pick_num_ctx(messages: list[dict]) -> int: - """Return the smallest context tier that fits *messages* with 25% headroom. +def pick_num_ctx(messages: list[dict], tools: list[dict] | None = None) -> int: + """Return the smallest context tier that fits *messages* + *tools* with 25% headroom. + + The ``tools`` JSON schemas are a large, often-overlooked chunk of the prompt. + With ~40 tools in the registry the schemas alone can be 6-10K tokens — enough + that omitting them from the estimate causes silent prompt truncation. Stays at or below Config.OLLAMA_NUM_CTX (the configured ceiling). """ total_chars = sum(len(m.get("content") or "") for m in messages) + if tools: + # Serialize the same way Ollama will see them. json.dumps gives us a + # faithful char count for the schema payload without any guesswork. + total_chars += len(json.dumps(tools)) estimated_tokens = int(total_chars / 3.5) needed = int(estimated_tokens * 1.25) + 256 # 25% headroom + output buffer cap = Config.OLLAMA_NUM_CTX diff --git a/src/fabledassistant/services/tools/notes.py b/src/fabledassistant/services/tools/notes.py index 6fa7554..2c2da03 100644 --- a/src/fabledassistant/services/tools/notes.py +++ b/src/fabledassistant/services/tools/notes.py @@ -391,9 +391,9 @@ async def list_notes_tool(*, user_id, arguments, **_ctx): @tool( name="delete_note", - description="Delete a note or task permanently. Use ONLY when the user explicitly asks to delete or remove an item. Always confirm with the user first — this cannot be undone.", + description="Delete any item from the user's knowledge base permanently — notes, tasks, persons (created via save_person), places (created via save_place), and lists (created via create_list) are all stored as notes and use this single delete tool. Use ONLY when the user explicitly asks to delete or remove an item. Always confirm with the user first — this cannot be undone.", parameters={ - "query": {"type": "string", "description": "Title or keyword to find the note or task to delete"}, + "query": {"type": "string", "description": "Title or keyword to find the item to delete (works for notes, tasks, persons, places, and lists)"}, "confirmed": {"type": "boolean", "description": "Must be true — only set after the user has explicitly confirmed they want this item deleted."}, }, required=["query"],