diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index 29d31fc..7ff4ff8 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -16,7 +16,7 @@ from fabledassistant.config import Config from fabledassistant.models import async_session from fabledassistant.models.conversation import Message from fabledassistant.services.generation_buffer import GenerationBuffer, GenerationState -from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools +from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools, summarize_history_for_context from fabledassistant.services.chat import update_conversation_title from fabledassistant.services.intent import IntentResult, classify_intent from fabledassistant.services.logging import log_generation @@ -172,43 +172,55 @@ async def run_generation( MAX_TOOL_ROUNDS = 5 msg_id = buf.assistant_message_id - # Phase 1: launch all independent work in parallel so nothing waits on anything - # unnecessarily. build_context (note search + system prompt) and the intent LLM - # call are the two slow legs — run them concurrently. buf.append_event("status", {"status": "Building context..."}) - context_task = asyncio.create_task(build_context( - user_id, history, context_note_id, user_content, exclude_note_ids=exclude_note_ids - )) - tools_task = asyncio.create_task(get_tools_for_user(user_id)) - intent_model_task = asyncio.create_task(get_setting(user_id, "intent_model", "")) - - # Tools + intent-model setting are fast DB calls — get them first so intent - # can start immediately while build_context is still running. - tools, intent_model_setting = await asyncio.gather(tools_task, intent_model_task) + # Phase 1: Quick DB calls — resolve tools list and intent model in parallel. + tools, intent_model_setting = await asyncio.gather( + get_tools_for_user(user_id), + get_setting(user_id, "intent_model", ""), + ) intent_model = intent_model_setting or Config.OLLAMA_INTENT_MODEL or model logger.info( "Starting generation for conv %d: model=%s, intent_model=%s, tools=%d", conv_id, model, intent_model, len(tools), ) - # Start intent classification in parallel with remaining build_context work. + # Phase 2: Summarize long conversation history if needed. + # For short conversations (<= threshold) this returns immediately with no LLM call. + # For long conversations it spends ~500ms to compress old exchanges, which + # more than pays for itself by reducing the prefill token count every turn. + history_to_use = history + history_summary: str | None = None + if len(history) > 20: # matches _HISTORY_SUMMARY_THRESHOLD in llm.py + buf.append_event("status", {"status": "Summarizing conversation history..."}) + history_to_use, history_summary = await summarize_history_for_context(history, intent_model) + + # Phase 3: Build context and classify intent in parallel — the two slow legs. pre_intent: IntentResult = IntentResult() intent_timing_ms: int | None = None if tools: intent_history = [ - m for m in history + m for m in history_to_use if m.get("role") in ("user", "assistant") and m.get("content") ][-6:] buf.append_event("status", {"status": "Analyzing your request..."}) t_intent = time.monotonic() + context_task = asyncio.create_task(build_context( + user_id, history_to_use, context_note_id, user_content, + exclude_note_ids=exclude_note_ids, + history_summary=history_summary, + )) intent_task = asyncio.create_task( classify_intent(user_content, tools, intent_model, history=intent_history) ) (messages, context_meta), pre_intent = await asyncio.gather(context_task, intent_task) intent_timing_ms = int((time.monotonic() - t_intent) * 1000) else: - messages, context_meta = await context_task + messages, context_meta = await build_context( + user_id, history_to_use, context_note_id, user_content, + exclude_note_ids=exclude_note_ids, + history_summary=history_summary, + ) # Emit context event buf.append_event("context", {"context": context_meta}) diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index cc81e68..52035ab 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -238,12 +238,75 @@ def _find_urls(text: str) -> list[str]: return re.findall(r"https?://[^\s<>\"')\]]+", text) +# History summarization thresholds +_HISTORY_SUMMARY_THRESHOLD = 20 # total messages before summarizing +_HISTORY_KEEP_RECENT = 6 # verbatim tail to preserve (3 exchanges) + + +async def summarize_history_for_context( + history: list[dict], + model: str, +) -> tuple[list[dict], str | None]: + """Summarize old conversation history when it exceeds the threshold. + + Returns (recent_history, summary_text | None). + recent_history is the verbatim tail passed to the model. + summary_text (when not None) should be injected into the system prompt + so the model retains the gist of earlier exchanges without the full tokens. + For short conversations, returns (history, None) immediately with no LLM call. + """ + if len(history) <= _HISTORY_SUMMARY_THRESHOLD: + return history, None + + to_summarize = history[:-_HISTORY_KEEP_RECENT] + recent = history[-_HISTORY_KEEP_RECENT:] + + lines: list[str] = [] + for m in to_summarize: + role = m.get("role", "") + content = (m.get("content") or "").strip() + if role in ("user", "assistant") and content: + label = "User" if role == "user" else "Assistant" + lines.append(f"{label}: {content[:400]}") + + if not lines: + return history, None + + prompt_messages = [ + { + "role": "system", + "content": ( + "Summarize this conversation history in 3-5 concise sentences. " + "Capture: topics discussed, any notes/tasks/events created or modified, " + "decisions made, and context needed to continue the conversation naturally. " + "Be specific and factual. Output only the summary, nothing else." + ), + }, + {"role": "user", "content": "\n".join(lines)}, + ] + + try: + summary = await generate_completion(prompt_messages, model, max_tokens=200) + summary = summary.strip() + if summary: + logger.info( + "Summarized %d history messages (%d chars) for context", + len(to_summarize), len(summary), + ) + return recent, summary + except Exception: + logger.warning("Failed to summarize conversation history", exc_info=True) + + return history, None + + async def build_context( user_id: int, history: list[dict], current_note_id: int | None, user_message: str, exclude_note_ids: list[int] | None = None, + history_summary: str | None = None, ) -> tuple[list[dict], dict]: """Build messages array for Ollama with system prompt and context. @@ -340,6 +403,12 @@ async def build_context( f"\n\n--- Content from {url} ---\n{content}\n--- End URL Content ---" ) + # Inject compressed summary of older exchanges when history has been trimmed + if history_summary: + system_parts.append( + f"\n\n--- Earlier Conversation ---\n{history_summary}\n--- End Earlier Conversation ---" + ) + messages = [{"role": "system", "content": "".join(system_parts)}] messages.extend(history) messages.append({"role": "user", "content": user_message})