fix(llm): correct context sizing, honor think requests, broaden delete
Three related fixes uncovered while benchmarking qwen3:14b against 8b: - pick_num_ctx was only counting message content, missing the ~15K tokens of tool schemas. num_ctx=8192 was being selected while actual prompt_tokens hit 14K+, causing silent prompt truncation on every tool-using request. Now includes json.dumps(tools) in the estimate. KV cache priming in app.py and routes/settings.py also fetches tools so the primed num_ctx matches what real chat requests will use. - _should_think's heuristic classifier was overriding explicit think=true requests from the frontend toggle and MCP, gating on message length and regex patterns. Now a pass-through — the caller is the source of truth. quick_capture hardcodes think=False since it's a fast classification path that was relying on the old gating. - delete_note description only mentioned "note or task", so the model refused to call it for entries created by save_person / save_place / create_list. Description now explicitly lists all five note_types it handles. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -37,62 +37,24 @@ _TOOL_CALL_MARKER = re.compile(r"^\s*\[TOOL_CALLS\]\s*", re.IGNORECASE)
|
||||
DB_FLUSH_INTERVAL = 5.0 # seconds between partial DB flushes
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Conditional thinking classifier
|
||||
# Thinking decision
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Patterns that force think=True even on short messages
|
||||
_THINK_FORCE = re.compile(
|
||||
r"\b("
|
||||
r"analyz|compar|explain\s+why|help\s+me\s+(think|plan|understand|figure\s+out)|"
|
||||
r"step[- ]by[- ]step|debug|troubleshoot|diagnos|"
|
||||
r"pros\s+and\s+cons|trade[- ]?off|"
|
||||
r"architect|design\s+(a|the|my|this)|"
|
||||
r"write\s+a\s+(detailed|long|comprehensive|full)|"
|
||||
r"brainstorm|outline\s+(a|the|my)|"
|
||||
r"what\s+(are|is)\s+the\s+(best|difference|relationship|impact|implication)|"
|
||||
r"how\s+(do|does|should|would|can)\s+.{0,40}\s+work|"
|
||||
r"why\s+(is|are|does|do|did|would|should)\b"
|
||||
r")",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Patterns that force think=False regardless of message length
|
||||
_THINK_SKIP = re.compile(
|
||||
r"^(hi|hey|hello|thanks|thank\s+you|ok|okay|got\s+it|sounds\s+good|"
|
||||
r"great|perfect|sure|yes|no|yep|nope|nice|cool|awesome|"
|
||||
r"what('s| is) \d|what time|how many|remind me|add (a |an )?(task|note|reminder)|"
|
||||
r"create (a |an )?(task|note)|delete|update|mark .{0,30} (done|complete))\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_WORD_COUNT_THRESHOLD = 60 # messages over this word count always use think=True
|
||||
_SHORT_MESSAGE_THRESHOLD = 12 # messages under this always use think=False
|
||||
#
|
||||
# The `think` flag from the frontend / MCP is taken at face value: if the
|
||||
# caller asked for thinking, they get thinking. No heuristic gating.
|
||||
#
|
||||
# Models that don't support extended reasoning (e.g. llama3, mistral) simply
|
||||
# ignore the `think` parameter in the Ollama chat request, so this is safe to
|
||||
# pass unconditionally across the full model zoo.
|
||||
|
||||
|
||||
def _should_think(user_content: str, think_requested: bool) -> bool:
|
||||
"""Return whether extended thinking should be used for this request.
|
||||
|
||||
If the caller didn't request thinking, we never enable it. If they did,
|
||||
we check whether the message is complex enough to warrant the overhead.
|
||||
Honors the caller's request directly — no message-complexity classifier.
|
||||
The frontend toggle / MCP `think` parameter is the source of truth.
|
||||
"""
|
||||
if not think_requested:
|
||||
return False
|
||||
|
||||
text = user_content.strip()
|
||||
word_count = len(text.split())
|
||||
|
||||
if word_count <= _SHORT_MESSAGE_THRESHOLD:
|
||||
return False
|
||||
if _THINK_SKIP.match(text):
|
||||
return False
|
||||
if word_count >= _WORD_COUNT_THRESHOLD:
|
||||
return True
|
||||
if "```" in text:
|
||||
return True
|
||||
if _THINK_FORCE.search(text):
|
||||
return True
|
||||
|
||||
return False
|
||||
return bool(think_requested)
|
||||
|
||||
|
||||
# Human-readable labels for each tool, shown in the status indicator
|
||||
@@ -273,9 +235,10 @@ async def run_generation(
|
||||
voice_speech_style=voice_speech_style,
|
||||
)
|
||||
|
||||
# Pick the smallest context tier that fits the current messages.
|
||||
# Pick the smallest context tier that fits the current messages AND the
|
||||
# tool schemas (which can be 6-10K tokens on their own with ~40 tools).
|
||||
# Using the minimum needed tier reduces KV cache size and speeds up prefill.
|
||||
num_ctx = pick_num_ctx(messages)
|
||||
num_ctx = pick_num_ctx(messages, tools=tools)
|
||||
logger.debug("Adaptive num_ctx=%d for conv %d", num_ctx, conv_id)
|
||||
|
||||
# Emit context event
|
||||
|
||||
Reference in New Issue
Block a user