fix(chat): retry silent rounds with think=True before falling back
Qwen3's tool-call tokens sometimes fail to parse into either content or tool_calls, burning output tokens and producing empty bubbles. Detect the signature within a round (empty content, no tool calls, eval_count > 0) and re-run the same round once with reasoning mode enabled, which emits more reliable output. The post-loop fallback remains as the final catch. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -410,7 +410,13 @@ async def run_generation(
|
|||||||
buf.append_event("status", {"status": "Generating response..." if _round == 0 else "Composing response..."})
|
buf.append_event("status", {"status": "Generating response..." if _round == 0 else "Composing response..."})
|
||||||
t_stream = time.monotonic()
|
t_stream = time.monotonic()
|
||||||
|
|
||||||
async for chunk in _stream_with_retry(messages, model, tools, think, num_ctx=num_ctx):
|
round_content_start = len(buf.content_so_far)
|
||||||
|
round_tokens_start = timing.get("output_tokens") or 0
|
||||||
|
effective_think = think
|
||||||
|
retried_with_think = False
|
||||||
|
|
||||||
|
while True:
|
||||||
|
async for chunk in _stream_with_retry(messages, model, tools, effective_think, num_ctx=num_ctx):
|
||||||
if buf.cancel_event.is_set():
|
if buf.cancel_event.is_set():
|
||||||
cancelled = True
|
cancelled = True
|
||||||
break
|
break
|
||||||
@@ -508,6 +514,31 @@ async def run_generation(
|
|||||||
all_tool_calls.append(tool_record)
|
all_tool_calls.append(tool_record)
|
||||||
buf.append_event("tool_call", {"tool_call": tool_record})
|
buf.append_event("tool_call", {"tool_call": tool_record})
|
||||||
|
|
||||||
|
if cancelled:
|
||||||
|
break
|
||||||
|
|
||||||
|
round_content_added = len(buf.content_so_far) - round_content_start
|
||||||
|
round_tokens_added = (timing.get("output_tokens") or 0) - round_tokens_start
|
||||||
|
if (
|
||||||
|
not round_tool_calls
|
||||||
|
and round_content_added == 0
|
||||||
|
and round_tokens_added > 0
|
||||||
|
and not effective_think
|
||||||
|
and not retried_with_think
|
||||||
|
):
|
||||||
|
# Silent round: qwen3's tool-call tokens sometimes aren't
|
||||||
|
# parsed into content or tool_calls. Retry once with
|
||||||
|
# think=True — reasoning mode produces more reliable output.
|
||||||
|
logger.warning(
|
||||||
|
"Silent round %d for conv %d (tokens=%d); retrying with think=True",
|
||||||
|
_round, conv_id, round_tokens_added,
|
||||||
|
)
|
||||||
|
buf.append_event("status", {"status": "Retrying with reasoning enabled..."})
|
||||||
|
effective_think = True
|
||||||
|
retried_with_think = True
|
||||||
|
continue
|
||||||
|
break
|
||||||
|
|
||||||
timing["generation_ms"] = int((time.monotonic() - t_stream) * 1000)
|
timing["generation_ms"] = int((time.monotonic() - t_stream) * 1000)
|
||||||
|
|
||||||
if cancelled:
|
if cancelled:
|
||||||
|
|||||||
Reference in New Issue
Block a user