fix(chat): surface silent generations instead of empty bubbles
Qwen3:14b sometimes burns output tokens on tool-calling attempts whose emission doesn't parse into any field we read — eval_count > 0 but no thinking/content/tool_calls ever stream to the caller. Generation completes "successfully," the user sees an empty assistant bubble, and no error is logged. Seen in conv 220 today. Two safety rails: - stream_chat_with_tools now tracks whether it yielded anything; when Ollama's done frame reports eval_count > 0 with zero yields, log a warning including the last ~5 raw frames so the next occurrence leaves breadcrumbs for diagnosis. - run_generation checks the same post-condition after the tool loop exits and, if content is empty with no tool calls but output_tokens > 0, substitutes a visible fallback message and streams it as a chunk so the user gets something readable instead of a blank bubble. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -542,6 +542,29 @@ async def run_generation(
|
|||||||
# Strip model artifacts from final content
|
# Strip model artifacts from final content
|
||||||
buf.content_so_far = _TOOL_CALL_MARKER.sub("", buf.content_so_far)
|
buf.content_so_far = _TOOL_CALL_MARKER.sub("", buf.content_so_far)
|
||||||
|
|
||||||
|
# Silent-generation safety net: the model burned output tokens but
|
||||||
|
# nothing landed in content or tool_calls (seen with qwen3:14b when
|
||||||
|
# its tool-call emission doesn't parse). Show a visible fallback so
|
||||||
|
# the user isn't staring at an empty bubble.
|
||||||
|
if (
|
||||||
|
not cancelled
|
||||||
|
and not buf.content_so_far.strip()
|
||||||
|
and not all_tool_calls
|
||||||
|
and (timing.get("output_tokens") or 0) > 0
|
||||||
|
):
|
||||||
|
logger.warning(
|
||||||
|
"Silent generation for conv %d: output_tokens=%s but empty content "
|
||||||
|
"and no tool calls (model=%s)",
|
||||||
|
conv_id, timing.get("output_tokens"), model,
|
||||||
|
)
|
||||||
|
fallback = (
|
||||||
|
"I wasn't able to produce a usable response — the model generated "
|
||||||
|
"tokens that couldn't be parsed as content or a tool call. "
|
||||||
|
"Please try rephrasing, or try again."
|
||||||
|
)
|
||||||
|
buf.content_so_far = fallback
|
||||||
|
buf.append_event("chunk", {"chunk": fallback})
|
||||||
|
|
||||||
# Final save
|
# Final save
|
||||||
logger.info("Generation complete for conv %d: content_length=%d, tool_calls=%d",
|
logger.info("Generation complete for conv %d: content_length=%d, tool_calls=%d",
|
||||||
conv_id, len(buf.content_so_far), len(all_tool_calls))
|
conv_id, len(buf.content_so_far), len(all_tool_calls))
|
||||||
|
|||||||
@@ -265,20 +265,31 @@ async def stream_chat_with_tools(
|
|||||||
) as resp:
|
) as resp:
|
||||||
await _raise_ollama_error(resp, model)
|
await _raise_ollama_error(resp, model)
|
||||||
accumulated_tool_calls: list[dict] = []
|
accumulated_tool_calls: list[dict] = []
|
||||||
|
# Silent-generation diagnostic: if Ollama reports non-zero eval_count
|
||||||
|
# but we never yielded any thinking/content/tool_calls, something
|
||||||
|
# in the frames isn't landing in a field we read. Capture the last
|
||||||
|
# few frames so we can see what Ollama actually sent.
|
||||||
|
yielded_anything = False
|
||||||
|
recent_frames: list[str] = []
|
||||||
async for line in resp.aiter_lines():
|
async for line in resp.aiter_lines():
|
||||||
if not line.strip():
|
if not line.strip():
|
||||||
continue
|
continue
|
||||||
|
if len(recent_frames) >= 5:
|
||||||
|
recent_frames.pop(0)
|
||||||
|
recent_frames.append(line[:500])
|
||||||
data = json.loads(line)
|
data = json.loads(line)
|
||||||
msg = data.get("message", {})
|
msg = data.get("message", {})
|
||||||
|
|
||||||
# Thinking chunks (qwen3 chain-of-thought, only when think=True)
|
# Thinking chunks (qwen3 chain-of-thought, only when think=True)
|
||||||
thinking = msg.get("thinking", "")
|
thinking = msg.get("thinking", "")
|
||||||
if thinking:
|
if thinking:
|
||||||
|
yielded_anything = True
|
||||||
yield ChatChunk(type="thinking", content=thinking)
|
yield ChatChunk(type="thinking", content=thinking)
|
||||||
|
|
||||||
# Content chunks
|
# Content chunks
|
||||||
chunk = msg.get("content", "")
|
chunk = msg.get("content", "")
|
||||||
if chunk:
|
if chunk:
|
||||||
|
yielded_anything = True
|
||||||
yield ChatChunk(type="content", content=chunk)
|
yield ChatChunk(type="content", content=chunk)
|
||||||
|
|
||||||
# Collect tool calls from any message (some models
|
# Collect tool calls from any message (some models
|
||||||
@@ -294,13 +305,21 @@ async def stream_chat_with_tools(
|
|||||||
len(accumulated_tool_calls),
|
len(accumulated_tool_calls),
|
||||||
json.dumps(accumulated_tool_calls)[:500],
|
json.dumps(accumulated_tool_calls)[:500],
|
||||||
)
|
)
|
||||||
|
yielded_anything = True
|
||||||
yield ChatChunk(type="tool_calls", tool_calls=accumulated_tool_calls)
|
yield ChatChunk(type="tool_calls", tool_calls=accumulated_tool_calls)
|
||||||
else:
|
else:
|
||||||
logger.debug("Ollama done with no tool calls")
|
logger.debug("Ollama done with no tool calls")
|
||||||
|
eval_count = data.get("eval_count") or 0
|
||||||
|
if not yielded_anything and eval_count > 0:
|
||||||
|
logger.warning(
|
||||||
|
"Ollama silent generation: model=%s eval_count=%d but no "
|
||||||
|
"thinking/content/tool_calls were yielded. Last frames: %s",
|
||||||
|
model, eval_count, recent_frames,
|
||||||
|
)
|
||||||
yield ChatChunk(
|
yield ChatChunk(
|
||||||
type="done",
|
type="done",
|
||||||
prompt_tokens=data.get("prompt_eval_count"),
|
prompt_tokens=data.get("prompt_eval_count"),
|
||||||
output_tokens=data.get("eval_count"),
|
output_tokens=eval_count,
|
||||||
)
|
)
|
||||||
break
|
break
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user