diff --git a/src/fabledassistant/config.py b/src/fabledassistant/config.py index 2675f3c..9a69db9 100644 --- a/src/fabledassistant/config.py +++ b/src/fabledassistant/config.py @@ -27,7 +27,7 @@ class Config: # Lightweight model for background tasks (title generation, tag suggestions, # project summaries, RSS classification). Using a separate model keeps the # main model's KV cache intact between user messages, enabling prefix cache hits. - OLLAMA_BACKGROUND_MODEL: str = os.environ.get("OLLAMA_BACKGROUND_MODEL", "qwen2.5:0.5b") + OLLAMA_BACKGROUND_MODEL: str = os.environ.get("OLLAMA_BACKGROUND_MODEL", "qwen2.5:3b") # KV cache context window for generation. Keep this as small as practical — # a larger context forces more KV cache into CPU RAM, drastically slowing prefill. # 16384 covers ~30+ message conversations with our system prompt comfortably. diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index 5a63b9a..329628c 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -118,34 +118,45 @@ _TOOL_LABELS: dict[str, str] = { async def _generate_title(messages: list[dict], user_id: int) -> str: - """Ask the LLM for a concise conversation title.""" - # Build conversation text like summarize_conversation_as_note - conv_lines = [] + """Ask the LLM for a concise conversation title. + + Only uses user messages to avoid feeding tool-call JSON, system prompt + fragments, or other noise into the title generator. Caps input length + to keep the task fast and focused. + """ + user_texts = [] for m in messages: - if m["role"] in ("system", "tool"): - continue - label = "User" if m["role"] == "user" else "Assistant" - content = m.get("content", "") - if not content: - continue - conv_lines.append(f"{label}: {content}") - # Keep only last 6 pairs worth of text - conv_lines = conv_lines[-12:] + if m["role"] == "user": + content = (m.get("content") or "").strip() + if content: + user_texts.append(content[:300]) + if not user_texts: + return "" + # First + last user messages capture intent best + if len(user_texts) > 2: + user_texts = [user_texts[0], user_texts[-1]] prompt_messages = [ { - "role": "system", + "role": "user", "content": ( - "Generate a concise 3-8 word title for this conversation. " - "Reply with ONLY the title, no quotes or punctuation." + "Generate a concise 3-8 word title for a conversation that started with:\n\n" + + "\n\n".join(user_texts) + + "\n\nReply with ONLY the title. No quotes, no punctuation, no explanation." ), }, - {"role": "user", "content": "\n\n".join(conv_lines)}, ] bg_model = await get_setting(user_id, "background_model", Config.OLLAMA_BACKGROUND_MODEL) - title = await generate_completion(prompt_messages, bg_model, max_tokens=30) + title = await generate_completion(prompt_messages, bg_model, max_tokens=30, num_ctx=1024) + # Strip common LLM noise: quotes, thinking tags, role labels title = title.strip().strip('"\'').strip() - return title[:100] if title else "" + for prefix in ("Title:", "title:", "Assistant:", "User:"): + if title.startswith(prefix): + title = title[len(prefix):].strip() + # Drop anything after a newline (model sometimes adds explanation) + if "\n" in title: + title = title.split("\n")[0].strip() + return title[:80] if title else "" async def _update_message( diff --git a/src/fabledassistant/services/projects.py b/src/fabledassistant/services/projects.py index a2ce5af..d3e1e38 100644 --- a/src/fabledassistant/services/projects.py +++ b/src/fabledassistant/services/projects.py @@ -124,7 +124,7 @@ async def generate_project_summary(user_id: int, project_id: int) -> None: from fabledassistant.services.settings import get_setting messages = [{"role": "user", "content": prompt}] bg_model = await get_setting(user_id, "background_model", Config.OLLAMA_BACKGROUND_MODEL) - summary = await generate_completion(messages, model=bg_model, max_tokens=400) + summary = await generate_completion(messages, model=bg_model, max_tokens=400, num_ctx=2048) if not summary: return