diff --git a/frontend/src/components/ChatPanel.vue b/frontend/src/components/ChatPanel.vue index 1ecce61..bb4ad18 100644 --- a/frontend/src/components/ChatPanel.vue +++ b/frontend/src/components/ChatPanel.vue @@ -208,7 +208,7 @@ async function onSubmit(payload: { content: string; contextNoteId?: number }) { payload.content, payload.contextNoteId, includedNoteIds.value.size ? [...includedNoteIds.value] : undefined, - true, + false, undefined, excludedNoteIds.value.length ? excludedNoteIds.value : undefined, props.projectId ?? store.ragProjectId, @@ -249,7 +249,7 @@ async function widgetSend(payload: { content: string; contextNoteId?: number }) widgetConvId.value = conv.id emit('conversation-started', conv.id) - await store.sendMessage(payload.content, payload.contextNoteId, undefined, true) + await store.sendMessage(payload.content, payload.contextNoteId, undefined, false) const msgs = store.currentConversation?.messages ?? [] const lastAssistant = [...msgs].reverse().find((m: Message) => m.role === 'assistant') diff --git a/frontend/src/views/KnowledgeView.vue b/frontend/src/views/KnowledgeView.vue index a92e243..64a7995 100644 --- a/frontend/src/views/KnowledgeView.vue +++ b/frontend/src/views/KnowledgeView.vue @@ -255,7 +255,7 @@ async function onMinichatSubmit(payload: { content: string; contextNoteId?: numb } chatOpen.value = true; chatCollapsed.value = false; - await chatStore.sendMessage(payload.content, payload.contextNoteId, undefined, true); + await chatStore.sendMessage(payload.content, payload.contextNoteId, undefined, false); } // ─── Auto-refresh cards when chat creates/edits notes or tasks ─────────────── diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index 48cedc3..6b7f100 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -40,21 +40,65 @@ DB_FLUSH_INTERVAL = 5.0 # seconds between partial DB flushes # Thinking decision # --------------------------------------------------------------------------- # -# The `think` flag from the frontend / MCP is taken at face value: if the -# caller asked for thinking, they get thinking. No heuristic gating. +# `_should_think` is the single source of truth for whether a qwen3-class +# model should engage chain-of-thought for a given request. Frontend callers +# should NOT hardcode think=True — leave it False and let the classifier +# decide from message content. An explicit think_requested=True still acts +# as an override for callers (e.g. a future UI toggle or MCP client) that +# want to force extended reasoning regardless of content. +# +# Why gate it: on qwen3:14b, thinking adds 5–20s of latency before the first +# visible content token, and most conversational messages do not benefit. +# Gating by content keeps quick chats fast while preserving reasoning depth +# for prompts that actually need it. # # Models that don't support extended reasoning (e.g. llama3, mistral) simply -# ignore the `think` parameter in the Ollama chat request, so this is safe to -# pass unconditionally across the full model zoo. +# ignore the `think` parameter in the Ollama chat request, so the decision +# here is harmless on non-thinking models. + + +# Keywords that strongly suggest the user wants reasoning / analysis. Matched +# case-insensitively as whole-ish phrases. +_THINK_KEYWORDS: tuple[str, ...] = ( + "why", "how does", "how do i", "how would", "how should", + "explain", "analyze", "analyse", "compare", "contrast", + "design", "architect", "architecture", "plan out", "strategize", + "debug", "diagnose", "troubleshoot", "root cause", + "review", "critique", "evaluate", "trade-off", "tradeoff", "trade off", + "pros and cons", "step by step", "walk me through", + "prove", "derive", "figure out", "work through", +) + +# Messages shorter than this and without any think-keyword are treated as +# simple/conversational and skip the thinking phase. +_SHORT_MESSAGE_CHARS = 80 + +# Messages longer than this are treated as substantive regardless of keywords. +_LONG_MESSAGE_CHARS = 400 def _should_think(user_content: str, think_requested: bool) -> bool: """Return whether extended thinking should be used for this request. - Honors the caller's request directly — no message-complexity classifier. - The frontend toggle / MCP `think` parameter is the source of truth. + ``think_requested`` acts as an explicit override: if True, thinking is + forced on regardless of content. If False (the default), the decision is + made by inspecting the message: long or keyword-bearing messages get + thinking; short conversational messages skip it. """ - return bool(think_requested) + if think_requested: + return True + text = (user_content or "").strip() + if not text: + return False + if len(text) >= _LONG_MESSAGE_CHARS: + return True + lowered = text.lower() + if any(kw in lowered for kw in _THINK_KEYWORDS): + return True + if len(text) < _SHORT_MESSAGE_CHARS: + return False + # Medium-length message with no obvious reasoning cue: default off. + return False # Human-readable labels for each tool, shown in the status indicator @@ -244,17 +288,23 @@ async def run_generation( # Emit context event buf.append_event("context", {"context": context_meta}) - # Apply thinking classifier — downgrade think=True for simple/conversational messages - think = _should_think(user_content, think) + # `_should_think` is authoritative — frontend callers pass think=False by + # default and let this classifier decide based on message content. An + # explicit think=True still forces on as an override. + think_requested = think + think = _should_think(user_content, think_requested) t_start = time.monotonic() timing: dict = { + "think_requested": think_requested, "think": think, "num_ctx": num_ctx, "tools": [], "rounds": 0, "prompt_tokens": None, "output_tokens": None, + "first_token_ms": None, + "thinking_ms": None, "ttft_ms": None, "generation_ms": None, "total_ms": None, @@ -286,11 +336,21 @@ async def run_generation( break if chunk.type == "thinking": + if timing["first_token_ms"] is None: + timing["first_token_ms"] = int((time.monotonic() - t_start) * 1000) buf.append_event("thinking_chunk", {"chunk": chunk.content}) elif chunk.type == "content": if timing["ttft_ms"] is None: - timing["ttft_ms"] = int((time.monotonic() - t_start) * 1000) + now_ms = int((time.monotonic() - t_start) * 1000) + timing["ttft_ms"] = now_ms + if timing["first_token_ms"] is None: + # No thinking phase occurred — first token IS content. + timing["first_token_ms"] = now_ms + else: + # Thinking phase duration = gap between first thinking + # token and first content token. + timing["thinking_ms"] = now_ms - timing["first_token_ms"] buf.content_so_far += chunk.content clean = _TOOL_CALL_MARKER.sub("", chunk.content) if clean: @@ -414,9 +474,13 @@ async def run_generation( timing["total_ms"] = int((time.monotonic() - t_start) * 1000) logger.info( - "Generation timing for conv %d: total=%dms ttft=%s tools=%s generation=%s", - conv_id, timing["total_ms"], timing["ttft_ms"], - [(t["name"], t["ms"]) for t in timing["tools"]], timing["generation_ms"], + "Generation timing for conv %d: total=%dms think=%s(req=%s) first_token=%s " + "thinking=%s ttft=%s generation=%s tools=%s", + conv_id, timing["total_ms"], + timing["think"], timing["think_requested"], + timing["first_token_ms"], timing["thinking_ms"], timing["ttft_ms"], + timing["generation_ms"], + [(t["name"], t["ms"]) for t in timing["tools"]], ) try: await log_generation(user_id, conv_id, model, timing)