From 1b63371bb3840b9641b66f98e97b4850dd5c70df Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 18 Feb 2026 20:29:58 -0500 Subject: [PATCH] Stream conversational acknowledgment in parallel with tool execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the intent router detects a tool call, the acknowledgment sentence and the tool now execute concurrently via asyncio.gather. The acknowledgment uses the small intent model (already in VRAM) with max_tokens=40, so it completes in ~200-400ms — the user sees text almost immediately instead of staring at a status label for the full main-model TTFT (~22s). The acknowledgment text is: - Streamed to the client as a chunk event (clears the status spinner) - Included in the assistant message for round 1 so the main LLM continues coherently from where the acknowledgment left off - Recorded in TTFT timing (acknowledgment counts as first token) Varied phrasing is enforced in the system prompt so responses feel natural rather than formulaic. Co-Authored-By: Claude Sonnet 4.6 --- .../services/generation_task.py | 75 ++++++++++++++++++- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index ec6ee25..cf83ac1 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -50,6 +50,59 @@ _TOOL_LABELS: dict[str, str] = { "delete_todo": "Removing todo", } +# Action phrases used in the acknowledgment prompt — "You are about to: {action}" +_TOOL_ACTIONS: dict[str, str] = { + "create_task": "create a task", + "create_note": "create a new note", + "update_note": "update an existing note", + "list_tasks": "look up tasks", + "search_notes": "search through notes", + "create_event": "schedule a calendar event", + "list_events": "check the calendar", + "search_events": "search calendar events", + "update_event": "update a calendar event", + "delete_event": "remove a calendar event", + "list_calendars": "list available calendars", + "create_todo": "create a calendar todo", + "list_todos": "check calendar todos", + "update_todo": "update a calendar todo", + "complete_todo": "mark a todo complete", + "delete_todo": "remove a calendar todo", +} + + +async def _generate_acknowledgment(user_content: str, tool_name: str, model: str) -> str: + """Generate a brief conversational acknowledgment that runs in parallel with tool execution. + + Uses the intent model (small, fast, already in VRAM) so the sentence is ready + within ~200-400ms. Returned string includes a trailing double-newline so the + main LLM response starts on a new paragraph. + """ + action = _TOOL_ACTIONS.get(tool_name, "work on that") + messages = [ + { + "role": "system", + "content": ( + "You are a helpful assistant. Write ONE short, natural sentence acknowledging " + "what you are about to do. Vary your phrasing — do not always start with " + "'Let me'. Be warm and conversational. Do not answer the question yet. " + "Output only the sentence, nothing else." + ), + }, + { + "role": "user", + "content": f"User said: {user_content}\nYou are about to: {action}", + }, + ] + try: + ack = await generate_completion(messages, model, max_tokens=40) + ack = ack.strip() + if ack: + return ack + "\n\n" + except Exception: + logger.warning("Failed to generate acknowledgment", exc_info=True) + return "" + async def _generate_title(messages: list[dict], model: str) -> str: """Ask the LLM for a concise conversation title.""" @@ -186,11 +239,26 @@ async def run_generation( ) if intent.should_execute: buf.append_event("status", {"status": f"{_TOOL_LABELS.get(intent.tool_name, 'Working')}..."}) + + # Run tool execution and acknowledgment generation in parallel. + # The acknowledgment uses the fast intent model (already in VRAM), + # so the user sees text within ~200-400ms instead of waiting for + # the full main-model TTFT (~22s). t_tool = time.monotonic() - result = await execute_tool(user_id, intent.tool_name, intent.arguments) + result, ack_text = await asyncio.gather( + execute_tool(user_id, intent.tool_name, intent.arguments), + _generate_acknowledgment(user_content, intent.tool_name, intent_model), + ) timing["tools"].append({"name": intent.tool_name, "ms": int((time.monotonic() - t_tool) * 1000)}) logger.info("Intent-routed tool %s result: success=%s", intent.tool_name, result.get("success")) + # Stream acknowledgment immediately — user sees text before main LLM starts + if ack_text: + buf.append_event("chunk", {"chunk": ack_text}) + buf.content_so_far += ack_text + if timing["ttft_ms"] is None: + timing["ttft_ms"] = int((time.monotonic() - t_start) * 1000) + tool_record = { "function": intent.tool_name, "arguments": intent.arguments, @@ -200,10 +268,11 @@ async def run_generation( all_tool_calls.append(tool_record) buf.append_event("tool_call", {"tool_call": tool_record}) - # Inject into messages so LLM can write a natural response + # Include ack as the assistant's partial response so round 1 + # continues coherently from where the acknowledgment left off messages.append({ "role": "assistant", - "content": "", + "content": ack_text, "tool_calls": [ {"function": {"name": intent.tool_name, "arguments": intent.arguments}} ],