bvandeusen/FabledScribe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 50 Wiki Activity

Add conversation history summarization for long chats

Browse Source 
When a conversation exceeds 20 messages (10 exchanges), the oldest
messages are summarized into a compact 3-5 sentence paragraph using the
intent model, and only the most recent 6 messages are passed verbatim.
The summary is injected into the system prompt so the model retains
context without the full token cost. For short conversations the check
is O(1) and returns immediately. The status indicator shows
"Summarizing conversation history..." when the LLM call is needed.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Bryan Van Deusen
2026-02-18 21:33:00 -05:00
parent 24d3c5bc68
commit de5921904d
2 changed files with 97 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fabledassistant/services/generation_task.py
+28 -16
View File
@@ -16,7 +16,7 @@ from fabledassistant.config import Config
from fabledassistant.models import async_session
from fabledassistant.models.conversation import Message
from fabledassistant.services.generation_buffer import GenerationBuffer, GenerationState
from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools
from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools, summarize_history_for_context
from fabledassistant.services.chat import update_conversation_title
from fabledassistant.services.intent import IntentResult, classify_intent
from fabledassistant.services.logging import log_generation
@@ -172,43 +172,55 @@ async def run_generation(
MAX_TOOL_ROUNDS = 5
msg_id = buf.assistant_message_id
# Phase 1: launch all independent work in parallel so nothing waits on anything
# unnecessarily. build_context (note search + system prompt) and the intent LLM
# call are the two slow legs — run them concurrently.
buf.append_event("status", {"status": "Building context..."})
context_task = asyncio.create_task(build_context(
user_id, history, context_note_id, user_content, exclude_note_ids=exclude_note_ids
))
tools_task = asyncio.create_task(get_tools_for_user(user_id))
intent_model_task = asyncio.create_task(get_setting(user_id, "intent_model", ""))
# Tools + intent-model setting are fast DB calls — get them first so intent
# can start immediately while build_context is still running.
tools, intent_model_setting = await asyncio.gather(tools_task, intent_model_task)
# Phase 1: Quick DB calls — resolve tools list and intent model in parallel.
tools, intent_model_setting = await asyncio.gather(
get_tools_for_user(user_id),
get_setting(user_id, "intent_model", ""),
)
intent_model = intent_model_setting or Config.OLLAMA_INTENT_MODEL or model
logger.info(
"Starting generation for conv %d: model=%s, intent_model=%s, tools=%d",
conv_id, model, intent_model, len(tools),
)
# Start intent classification in parallel with remaining build_context work.
# Phase 2: Summarize long conversation history if needed.
# For short conversations (<= threshold) this returns immediately with no LLM call.
# For long conversations it spends ~500ms to compress old exchanges, which
# more than pays for itself by reducing the prefill token count every turn.
history_to_use = history
history_summary: str | None = None
if len(history) > 20: # matches _HISTORY_SUMMARY_THRESHOLD in llm.py
buf.append_event("status", {"status": "Summarizing conversation history..."})
history_to_use, history_summary = await summarize_history_for_context(history, intent_model)
# Phase 3: Build context and classify intent in parallel — the two slow legs.
pre_intent: IntentResult = IntentResult()
intent_timing_ms: int | None = None
if tools:
intent_history = [
m for m in history
m for m in history_to_use
if m.get("role") in ("user", "assistant") and m.get("content")
][-6:]
buf.append_event("status", {"status": "Analyzing your request..."})
t_intent = time.monotonic()
context_task = asyncio.create_task(build_context(
user_id, history_to_use, context_note_id, user_content,
exclude_note_ids=exclude_note_ids,
history_summary=history_summary,
))
intent_task = asyncio.create_task(
classify_intent(user_content, tools, intent_model, history=intent_history)
)
(messages, context_meta), pre_intent = await asyncio.gather(context_task, intent_task)
intent_timing_ms = int((time.monotonic() - t_intent) * 1000)
else:
messages, context_meta = await context_task
messages, context_meta = await build_context(
user_id, history_to_use, context_note_id, user_content,
exclude_note_ids=exclude_note_ids,
history_summary=history_summary,
)
# Emit context event
buf.append_event("context", {"context": context_meta})
src/fabledassistant/services/llm.py
+69
View File
@@ -238,12 +238,75 @@ def _find_urls(text: str) -> list[str]:
return re.findall(r"https?://[^\s<>\"')\]]+", text)
# History summarization thresholds
_HISTORY_SUMMARY_THRESHOLD = 20 # total messages before summarizing
_HISTORY_KEEP_RECENT = 6 # verbatim tail to preserve (3 exchanges)
async def summarize_history_for_context(
history: list[dict],
model: str,
) -> tuple[list[dict], str | None]:
"""Summarize old conversation history when it exceeds the threshold.
Returns (recent_history, summary_text | None).
recent_history is the verbatim tail passed to the model.
summary_text (when not None) should be injected into the system prompt
so the model retains the gist of earlier exchanges without the full tokens.
For short conversations, returns (history, None) immediately with no LLM call.
"""
if len(history) <= _HISTORY_SUMMARY_THRESHOLD:
return history, None
to_summarize = history[:-_HISTORY_KEEP_RECENT]
recent = history[-_HISTORY_KEEP_RECENT:]
lines: list[str] = []
for m in to_summarize:
role = m.get("role", "")
content = (m.get("content") or "").strip()
if role in ("user", "assistant") and content:
label = "User" if role == "user" else "Assistant"
lines.append(f"{label}: {content[:400]}")
if not lines:
return history, None
prompt_messages = [
{
"role": "system",
"content": (
"Summarize this conversation history in 3-5 concise sentences. "
"Capture: topics discussed, any notes/tasks/events created or modified, "
"decisions made, and context needed to continue the conversation naturally. "
"Be specific and factual. Output only the summary, nothing else."
),
},
{"role": "user", "content": "\n".join(lines)},
]
try:
summary = await generate_completion(prompt_messages, model, max_tokens=200)
summary = summary.strip()
if summary:
logger.info(
"Summarized %d history messages (%d chars) for context",
len(to_summarize), len(summary),
)
return recent, summary
except Exception:
logger.warning("Failed to summarize conversation history", exc_info=True)
return history, None
async def build_context(
user_id: int,
history: list[dict],
current_note_id: int | None,
user_message: str,
exclude_note_ids: list[int] | None = None,
history_summary: str | None = None,
) -> tuple[list[dict], dict]:
"""Build messages array for Ollama with system prompt and context.
@@ -340,6 +403,12 @@ async def build_context(
f"\n\n--- Content from {url} ---\n{content}\n--- End URL Content ---"
)
# Inject compressed summary of older exchanges when history has been trimmed
if history_summary:
system_parts.append(
f"\n\n--- Earlier Conversation ---\n{history_summary}\n--- End Earlier Conversation ---"
)
messages = [{"role": "system", "content": "".join(system_parts)}]
messages.extend(history)
messages.append({"role": "user", "content": user_message})