From de5921904d25124441761a79fce49e8c8a6cbc7b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 18 Feb 2026 21:33:00 -0500 Subject: [PATCH] Add conversation history summarization for long chats When a conversation exceeds 20 messages (10 exchanges), the oldest messages are summarized into a compact 3-5 sentence paragraph using the intent model, and only the most recent 6 messages are passed verbatim. The summary is injected into the system prompt so the model retains context without the full token cost. For short conversations the check is O(1) and returns immediately. The status indicator shows "Summarizing conversation history..." when the LLM call is needed. Co-Authored-By: Claude Sonnet 4.6 --- .../services/generation_task.py | 44 +++++++----- src/fabledassistant/services/llm.py | 69 +++++++++++++++++++ 2 files changed, 97 insertions(+), 16 deletions(-) diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index 29d31fc..7ff4ff8 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -16,7 +16,7 @@ from fabledassistant.config import Config from fabledassistant.models import async_session from fabledassistant.models.conversation import Message from fabledassistant.services.generation_buffer import GenerationBuffer, GenerationState -from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools +from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools, summarize_history_for_context from fabledassistant.services.chat import update_conversation_title from fabledassistant.services.intent import IntentResult, classify_intent from fabledassistant.services.logging import log_generation @@ -172,43 +172,55 @@ async def run_generation( MAX_TOOL_ROUNDS = 5 msg_id = buf.assistant_message_id - # Phase 1: launch all independent work in parallel so nothing waits on anything - # unnecessarily. build_context (note search + system prompt) and the intent LLM - # call are the two slow legs — run them concurrently. buf.append_event("status", {"status": "Building context..."}) - context_task = asyncio.create_task(build_context( - user_id, history, context_note_id, user_content, exclude_note_ids=exclude_note_ids - )) - tools_task = asyncio.create_task(get_tools_for_user(user_id)) - intent_model_task = asyncio.create_task(get_setting(user_id, "intent_model", "")) - - # Tools + intent-model setting are fast DB calls — get them first so intent - # can start immediately while build_context is still running. - tools, intent_model_setting = await asyncio.gather(tools_task, intent_model_task) + # Phase 1: Quick DB calls — resolve tools list and intent model in parallel. + tools, intent_model_setting = await asyncio.gather( + get_tools_for_user(user_id), + get_setting(user_id, "intent_model", ""), + ) intent_model = intent_model_setting or Config.OLLAMA_INTENT_MODEL or model logger.info( "Starting generation for conv %d: model=%s, intent_model=%s, tools=%d", conv_id, model, intent_model, len(tools), ) - # Start intent classification in parallel with remaining build_context work. + # Phase 2: Summarize long conversation history if needed. + # For short conversations (<= threshold) this returns immediately with no LLM call. + # For long conversations it spends ~500ms to compress old exchanges, which + # more than pays for itself by reducing the prefill token count every turn. + history_to_use = history + history_summary: str | None = None + if len(history) > 20: # matches _HISTORY_SUMMARY_THRESHOLD in llm.py + buf.append_event("status", {"status": "Summarizing conversation history..."}) + history_to_use, history_summary = await summarize_history_for_context(history, intent_model) + + # Phase 3: Build context and classify intent in parallel — the two slow legs. pre_intent: IntentResult = IntentResult() intent_timing_ms: int | None = None if tools: intent_history = [ - m for m in history + m for m in history_to_use if m.get("role") in ("user", "assistant") and m.get("content") ][-6:] buf.append_event("status", {"status": "Analyzing your request..."}) t_intent = time.monotonic() + context_task = asyncio.create_task(build_context( + user_id, history_to_use, context_note_id, user_content, + exclude_note_ids=exclude_note_ids, + history_summary=history_summary, + )) intent_task = asyncio.create_task( classify_intent(user_content, tools, intent_model, history=intent_history) ) (messages, context_meta), pre_intent = await asyncio.gather(context_task, intent_task) intent_timing_ms = int((time.monotonic() - t_intent) * 1000) else: - messages, context_meta = await context_task + messages, context_meta = await build_context( + user_id, history_to_use, context_note_id, user_content, + exclude_note_ids=exclude_note_ids, + history_summary=history_summary, + ) # Emit context event buf.append_event("context", {"context": context_meta}) diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index cc81e68..52035ab 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -238,12 +238,75 @@ def _find_urls(text: str) -> list[str]: return re.findall(r"https?://[^\s<>\"')\]]+", text) +# History summarization thresholds +_HISTORY_SUMMARY_THRESHOLD = 20 # total messages before summarizing +_HISTORY_KEEP_RECENT = 6 # verbatim tail to preserve (3 exchanges) + + +async def summarize_history_for_context( + history: list[dict], + model: str, +) -> tuple[list[dict], str | None]: + """Summarize old conversation history when it exceeds the threshold. + + Returns (recent_history, summary_text | None). + recent_history is the verbatim tail passed to the model. + summary_text (when not None) should be injected into the system prompt + so the model retains the gist of earlier exchanges without the full tokens. + For short conversations, returns (history, None) immediately with no LLM call. + """ + if len(history) <= _HISTORY_SUMMARY_THRESHOLD: + return history, None + + to_summarize = history[:-_HISTORY_KEEP_RECENT] + recent = history[-_HISTORY_KEEP_RECENT:] + + lines: list[str] = [] + for m in to_summarize: + role = m.get("role", "") + content = (m.get("content") or "").strip() + if role in ("user", "assistant") and content: + label = "User" if role == "user" else "Assistant" + lines.append(f"{label}: {content[:400]}") + + if not lines: + return history, None + + prompt_messages = [ + { + "role": "system", + "content": ( + "Summarize this conversation history in 3-5 concise sentences. " + "Capture: topics discussed, any notes/tasks/events created or modified, " + "decisions made, and context needed to continue the conversation naturally. " + "Be specific and factual. Output only the summary, nothing else." + ), + }, + {"role": "user", "content": "\n".join(lines)}, + ] + + try: + summary = await generate_completion(prompt_messages, model, max_tokens=200) + summary = summary.strip() + if summary: + logger.info( + "Summarized %d history messages (%d chars) for context", + len(to_summarize), len(summary), + ) + return recent, summary + except Exception: + logger.warning("Failed to summarize conversation history", exc_info=True) + + return history, None + + async def build_context( user_id: int, history: list[dict], current_note_id: int | None, user_message: str, exclude_note_ids: list[int] | None = None, + history_summary: str | None = None, ) -> tuple[list[dict], dict]: """Build messages array for Ollama with system prompt and context. @@ -340,6 +403,12 @@ async def build_context( f"\n\n--- Content from {url} ---\n{content}\n--- End URL Content ---" ) + # Inject compressed summary of older exchanges when history has been trimmed + if history_summary: + system_parts.append( + f"\n\n--- Earlier Conversation ---\n{history_summary}\n--- End Earlier Conversation ---" + ) + messages = [{"role": "system", "content": "".join(system_parts)}] messages.extend(history) messages.append({"role": "user", "content": user_message})