diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 244308c..25a7c71 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -48,6 +48,10 @@ services: - ollama_models:/root/.ollama networks: - fabledassistant_backend + environment: + OLLAMA_MAX_LOADED_MODELS: "2" + OLLAMA_KEEP_ALIVE: "30m" + OLLAMA_FLASH_ATTENTION: "1" healthcheck: test: ["CMD-SHELL", "ollama list || exit 1"] interval: 30s @@ -59,20 +63,14 @@ services: constraints: - node.role == worker resources: - limits: - memory: 8G + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] restart_policy: condition: on-failure max_attempts: 5 - # To enable GPU support, uncomment the section below - # (requires nvidia-container-toolkit) - # deploy: - # resources: - # reservations: - # devices: - # - driver: nvidia - # count: all - # capabilities: [gpu] volumes: pgdata: diff --git a/docker-compose.yml b/docker-compose.yml index 3308f82..67a3722 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,15 +35,14 @@ services: environment: OLLAMA_MAX_LOADED_MODELS: "2" OLLAMA_KEEP_ALIVE: "30m" - # To enable GPU support, uncomment the deploy section below - # (requires nvidia-container-toolkit) - # deploy: - # resources: - # reservations: - # devices: - # - driver: nvidia - # count: all - # capabilities: [gpu] + OLLAMA_FLASH_ATTENTION: "1" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] volumes: pgdata: diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index c3d80fb..ec6ee25 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -18,7 +18,7 @@ from fabledassistant.models.conversation import Message from fabledassistant.services.generation_buffer import GenerationBuffer, GenerationState from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools from fabledassistant.services.chat import update_conversation_title -from fabledassistant.services.intent import classify_intent +from fabledassistant.services.intent import IntentResult, classify_intent from fabledassistant.services.logging import log_generation from fabledassistant.services.settings import get_setting from fabledassistant.services.tools import get_tools_for_user, execute_tool @@ -111,18 +111,50 @@ async def run_generation( MAX_TOOL_ROUNDS = 5 msg_id = buf.assistant_message_id - # Build context inside the background task so the 202 response returns immediately + # Phase 1: launch all independent work in parallel so nothing waits on anything + # unnecessarily. build_context (note search + system prompt) and the intent LLM + # call are the two slow legs — run them concurrently. buf.append_event("status", {"status": "Building context..."}) - messages, context_meta = await build_context( + + context_task = asyncio.create_task(build_context( user_id, history, context_note_id, user_content, exclude_note_ids=exclude_note_ids + )) + tools_task = asyncio.create_task(get_tools_for_user(user_id)) + intent_model_task = asyncio.create_task(get_setting(user_id, "intent_model", "")) + + # Tools + intent-model setting are fast DB calls — get them first so intent + # can start immediately while build_context is still running. + tools, intent_model_setting = await asyncio.gather(tools_task, intent_model_task) + intent_model = intent_model_setting or Config.OLLAMA_INTENT_MODEL or model + logger.info( + "Starting generation for conv %d: model=%s, intent_model=%s, tools=%d", + conv_id, model, intent_model, len(tools), ) + # Start intent classification in parallel with remaining build_context work. + pre_intent: IntentResult = IntentResult() + intent_timing_ms: int | None = None + if tools: + intent_history = [ + m for m in history + if m.get("role") in ("user", "assistant") and m.get("content") + ][-6:] + buf.append_event("status", {"status": "Analyzing your request..."}) + t_intent = time.monotonic() + intent_task = asyncio.create_task( + classify_intent(user_content, tools, intent_model, history=intent_history) + ) + (messages, context_meta), pre_intent = await asyncio.gather(context_task, intent_task) + intent_timing_ms = int((time.monotonic() - t_intent) * 1000) + else: + messages, context_meta = await context_task + # Emit context event buf.append_event("context", {"context": context_meta}) t_start = time.monotonic() timing: dict = { - "intent_ms": None, + "intent_ms": intent_timing_ms, "tools": [], "ttft_ms": None, "generation_ms": None, @@ -132,17 +164,6 @@ async def run_generation( last_flush = time.monotonic() all_tool_calls: list[dict] = [] - # Resolve tools and intent model in parallel - tools, intent_model_setting = await asyncio.gather( - get_tools_for_user(user_id), - get_setting(user_id, "intent_model", ""), - ) - intent_model = intent_model_setting or Config.OLLAMA_INTENT_MODEL or model - logger.info( - "Starting generation for conv %d: model=%s, intent_model=%s, tools=%d", - conv_id, model, intent_model, len(tools), - ) - try: cancelled = False @@ -150,18 +171,9 @@ async def run_generation( round_tool_calls: list[dict] = [] logger.info("Generation round %d started for conv %d (model=%s)", _round, conv_id, model) - # Intent routing — first round only + # Intent routing — first round only (result pre-computed in parallel with build_context) if _round == 0 and tools: - # Pass last 3 user/assistant pairs (6 messages) for anaphora resolution. - # messages = [system, *history, current_user] — exclude system and current user. - intent_history = [ - m for m in messages[1:-1] - if m.get("role") in ("user", "assistant") and m.get("content") - ][-6:] - buf.append_event("status", {"status": "Analyzing your request..."}) - t_intent = time.monotonic() - intent = await classify_intent(user_content, tools, intent_model, history=intent_history) - timing["intent_ms"] = int((time.monotonic() - t_intent) * 1000) + intent = pre_intent if intent.should_execute: logger.info( "Intent router detected tool (confidence=%s): %s(%s)", diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index 5a9f0fe..4310faa 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -80,7 +80,7 @@ async def stream_chat( options: dict | None = None, ) -> AsyncGenerator[str, None]: """Stream chat completion from Ollama, yielding content chunks.""" - merged_options = {"num_ctx": 16384} + merged_options = {"num_ctx": 32768} if options: merged_options.update(options) payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options} @@ -121,10 +121,9 @@ async def stream_chat_with_tools( ChatChunk(type="tool_calls") is yielded. Always ends with ChatChunk(type="done"). """ - options: dict = {"num_ctx": 16384} - # Disable thinking mode for models like qwen3 — it interferes with tool calling + options: dict = {"num_ctx": 32768} if tools: - options["num_predict"] = 4096 + options["num_predict"] = 8192 payload: dict = { "model": model, "messages": messages,