GPU support, parallel intent+context, and increased context window

Docker Compose:
- Enable Ollama GPU passthrough (nvidia, count: all) in both dev and prod files
- Add OLLAMA_FLASH_ATTENTION=1 (faster attention on GPU in both files)
- Add OLLAMA_MAX_LOADED_MODELS=2 and OLLAMA_KEEP_ALIVE=30m to prod (was already in dev)
- Remove 8G memory limit from prod Ollama service (CPU-bound constraint, no longer valid)

llm.py:
- Increase num_ctx 16384 → 32768 in stream_chat and stream_chat_with_tools (GPU VRAM allows it)
- Increase num_predict cap 4096 → 8192 for tool-augmented responses

generation_task.py:
- Parallelize build_context, get_tools_for_user, and get_setting all from the start
- As soon as tools list is ready (fast DB call), launch classify_intent as an asyncio.Task
- Await build_context and classify_intent together via asyncio.gather
- Intent result is pre-computed before the generation loop; loop just reads pre_intent on round 0
- intent_ms timing now reflects wall-clock time from intent start to completion

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

docker-compose.prod.yml

@@ -48,6 +48,10 @@ services:
       - ollama_models:/root/.ollama
     networks:
       - fabledassistant_backend
+    environment:
+      OLLAMA_MAX_LOADED_MODELS: "2"
+      OLLAMA_KEEP_ALIVE: "30m"
+      OLLAMA_FLASH_ATTENTION: "1"
     healthcheck:
       test: ["CMD-SHELL", "ollama list || exit 1"]
       interval: 30s
@@ -59,20 +63,14 @@ services:
         constraints:
           - node.role == worker
       resources:
-        limits:
+        reservations:
-          memory: 8G
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
       restart_policy:
         condition: on-failure
         max_attempts: 5
-    # To enable GPU support, uncomment the section below
-    # (requires nvidia-container-toolkit)
-    # deploy:
-    #   resources:
-    #     reservations:
-    #       devices:
-    #         - driver: nvidia
-    #           count: all
-    #           capabilities: [gpu]
 
 volumes:
   pgdata:

docker-compose.yml

@@ -35,15 +35,14 @@ services:
     environment:
       OLLAMA_MAX_LOADED_MODELS: "2"
       OLLAMA_KEEP_ALIVE: "30m"
-    # To enable GPU support, uncomment the deploy section below
+      OLLAMA_FLASH_ATTENTION: "1"
-    # (requires nvidia-container-toolkit)
+    deploy:
-    # deploy:
+      resources:
-    #   resources:
+        reservations:
-    #     reservations:
+          devices:
-    #       devices:
+            - driver: nvidia
-    #         - driver: nvidia
+              count: all
-    #           count: all
+              capabilities: [gpu]
-    #           capabilities: [gpu]
 
 volumes:
   pgdata:

src/fabledassistant/services/generation_task.py

@@ -18,7 +18,7 @@ from fabledassistant.models.conversation import Message
 from fabledassistant.services.generation_buffer import GenerationBuffer, GenerationState
 from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools
 from fabledassistant.services.chat import update_conversation_title
-from fabledassistant.services.intent import classify_intent
+from fabledassistant.services.intent import IntentResult, classify_intent
 from fabledassistant.services.logging import log_generation
 from fabledassistant.services.settings import get_setting
 from fabledassistant.services.tools import get_tools_for_user, execute_tool
@@ -111,18 +111,50 @@ async def run_generation(
     MAX_TOOL_ROUNDS = 5
     msg_id = buf.assistant_message_id
 
-    # Build context inside the background task so the 202 response returns immediately
+    # Phase 1: launch all independent work in parallel so nothing waits on anything
+    # unnecessarily. build_context (note search + system prompt) and the intent LLM
+    # call are the two slow legs — run them concurrently.
     buf.append_event("status", {"status": "Building context..."})
-    messages, context_meta = await build_context(
+
+    context_task = asyncio.create_task(build_context(
         user_id, history, context_note_id, user_content, exclude_note_ids=exclude_note_ids
+    ))
+    tools_task = asyncio.create_task(get_tools_for_user(user_id))
+    intent_model_task = asyncio.create_task(get_setting(user_id, "intent_model", ""))
+
+    # Tools + intent-model setting are fast DB calls — get them first so intent
+    # can start immediately while build_context is still running.
+    tools, intent_model_setting = await asyncio.gather(tools_task, intent_model_task)
+    intent_model = intent_model_setting or Config.OLLAMA_INTENT_MODEL or model
+    logger.info(
+        "Starting generation for conv %d: model=%s, intent_model=%s, tools=%d",
+        conv_id, model, intent_model, len(tools),
     )
 
+    # Start intent classification in parallel with remaining build_context work.
+    pre_intent: IntentResult = IntentResult()
+    intent_timing_ms: int | None = None
+    if tools:
+        intent_history = [
+            m for m in history
+            if m.get("role") in ("user", "assistant") and m.get("content")
+        ][-6:]
+        buf.append_event("status", {"status": "Analyzing your request..."})
+        t_intent = time.monotonic()
+        intent_task = asyncio.create_task(
+            classify_intent(user_content, tools, intent_model, history=intent_history)
+        )
+        (messages, context_meta), pre_intent = await asyncio.gather(context_task, intent_task)
+        intent_timing_ms = int((time.monotonic() - t_intent) * 1000)
+    else:
+        messages, context_meta = await context_task
+
     # Emit context event
     buf.append_event("context", {"context": context_meta})
 
     t_start = time.monotonic()
     timing: dict = {
-        "intent_ms": None,
+        "intent_ms": intent_timing_ms,
         "tools": [],
         "ttft_ms": None,
         "generation_ms": None,
@@ -132,17 +164,6 @@ async def run_generation(
     last_flush = time.monotonic()
     all_tool_calls: list[dict] = []
 
-    # Resolve tools and intent model in parallel
-    tools, intent_model_setting = await asyncio.gather(
-        get_tools_for_user(user_id),
-        get_setting(user_id, "intent_model", ""),
-    )
-    intent_model = intent_model_setting or Config.OLLAMA_INTENT_MODEL or model
-    logger.info(
-        "Starting generation for conv %d: model=%s, intent_model=%s, tools=%d",
-        conv_id, model, intent_model, len(tools),
-    )
-
     try:
         cancelled = False
 
@@ -150,18 +171,9 @@ async def run_generation(
             round_tool_calls: list[dict] = []
             logger.info("Generation round %d started for conv %d (model=%s)", _round, conv_id, model)
 
-            # Intent routing — first round only
+            # Intent routing — first round only (result pre-computed in parallel with build_context)
             if _round == 0 and tools:
-                # Pass last 3 user/assistant pairs (6 messages) for anaphora resolution.
+                intent = pre_intent
-                # messages = [system, *history, current_user] — exclude system and current user.
-                intent_history = [
-                    m for m in messages[1:-1]
-                    if m.get("role") in ("user", "assistant") and m.get("content")
-                ][-6:]
-                buf.append_event("status", {"status": "Analyzing your request..."})
-                t_intent = time.monotonic()
-                intent = await classify_intent(user_content, tools, intent_model, history=intent_history)
-                timing["intent_ms"] = int((time.monotonic() - t_intent) * 1000)
                 if intent.should_execute:
                     logger.info(
                         "Intent router detected tool (confidence=%s): %s(%s)",

src/fabledassistant/services/llm.py

@@ -80,7 +80,7 @@ async def stream_chat(
     options: dict | None = None,
 ) -> AsyncGenerator[str, None]:
     """Stream chat completion from Ollama, yielding content chunks."""
-    merged_options = {"num_ctx": 16384}
+    merged_options = {"num_ctx": 32768}
     if options:
         merged_options.update(options)
     payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options}
@@ -121,10 +121,9 @@ async def stream_chat_with_tools(
     ChatChunk(type="tool_calls") is yielded. Always ends with
     ChatChunk(type="done").
     """
-    options: dict = {"num_ctx": 16384}
+    options: dict = {"num_ctx": 32768}
-    # Disable thinking mode for models like qwen3 — it interferes with tool calling
     if tools:
-        options["num_predict"] = 4096
+        options["num_predict"] = 8192
     payload: dict = {
         "model": model,
         "messages": messages,