bvandeusen/FabledScribe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 50 Wiki Activity

GPU support, parallel intent+context, and increased context window

Browse Source 
Docker Compose:
- Enable Ollama GPU passthrough (nvidia, count: all) in both dev and prod files
- Add OLLAMA_FLASH_ATTENTION=1 (faster attention on GPU in both files)
- Add OLLAMA_MAX_LOADED_MODELS=2 and OLLAMA_KEEP_ALIVE=30m to prod (was already in dev)
- Remove 8G memory limit from prod Ollama service (CPU-bound constraint, no longer valid)

llm.py:
- Increase num_ctx 16384 → 32768 in stream_chat and stream_chat_with_tools (GPU VRAM allows it)
- Increase num_predict cap 4096 → 8192 for tool-augmented responses

generation_task.py:
- Parallelize build_context, get_tools_for_user, and get_setting all from the start
- As soon as tools list is ready (fast DB call), launch classify_intent as an asyncio.Task
- Await build_context and classify_intent together via asyncio.gather
- Intent result is pre-computed before the generation loop; loop just reads pre_intent on round 0
- intent_ms timing now reflects wall-clock time from intent start to completion

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Bryan Van Deusen
2026-02-18 19:29:31 -05:00
parent 7ff60eb8a6
commit 931a059e9f
4 changed files with 58 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker-compose.prod.yml
+9 -11
View File
@@ -48,6 +48,10 @@ services:
- ollama_models:/root/.ollama
networks:
- fabledassistant_backend
environment:
OLLAMA_MAX_LOADED_MODELS: "2"
OLLAMA_KEEP_ALIVE: "30m"
OLLAMA_FLASH_ATTENTION: "1"
healthcheck:
test: ["CMD-SHELL", "ollama list || exit 1"]
interval: 30s
@@ -59,20 +63,14 @@ services:
constraints:
- node.role == worker
resources:
limits:
memory: 8G
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
restart_policy:
condition: on-failure
max_attempts: 5
# To enable GPU support, uncomment the section below
# (requires nvidia-container-toolkit)
# deploy:
# resources:
# reservations:
# devices:
# - driver: nvidia
# count: all
# capabilities: [gpu]
volumes:
pgdata:
docker-compose.yml
+8 -9
View File
@@ -35,15 +35,14 @@ services:
environment:
OLLAMA_MAX_LOADED_MODELS: "2"
OLLAMA_KEEP_ALIVE: "30m"
# To enable GPU support, uncomment the deploy section below
# (requires nvidia-container-toolkit)
# deploy:
# resources:
# reservations:
# devices:
# - driver: nvidia
# count: all
# capabilities: [gpu]
OLLAMA_FLASH_ATTENTION: "1"
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
volumes:
pgdata:
src/fabledassistant/services/generation_task.py
+38 -26
View File
@@ -18,7 +18,7 @@ from fabledassistant.models.conversation import Message
from fabledassistant.services.generation_buffer import GenerationBuffer, GenerationState
from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools
from fabledassistant.services.chat import update_conversation_title
from fabledassistant.services.intent import classify_intent
from fabledassistant.services.intent import IntentResult, classify_intent
from fabledassistant.services.logging import log_generation
from fabledassistant.services.settings import get_setting
from fabledassistant.services.tools import get_tools_for_user, execute_tool
@@ -111,18 +111,50 @@ async def run_generation(
MAX_TOOL_ROUNDS = 5
msg_id = buf.assistant_message_id
# Build context inside the background task so the 202 response returns immediately
# Phase 1: launch all independent work in parallel so nothing waits on anything
# unnecessarily. build_context (note search + system prompt) and the intent LLM
# call are the two slow legs — run them concurrently.
buf.append_event("status", {"status": "Building context..."})
messages, context_meta = await build_context(
context_task = asyncio.create_task(build_context(
user_id, history, context_note_id, user_content, exclude_note_ids=exclude_note_ids
))
tools_task = asyncio.create_task(get_tools_for_user(user_id))
intent_model_task = asyncio.create_task(get_setting(user_id, "intent_model", ""))
# Tools + intent-model setting are fast DB calls — get them first so intent
# can start immediately while build_context is still running.
tools, intent_model_setting = await asyncio.gather(tools_task, intent_model_task)
intent_model = intent_model_setting or Config.OLLAMA_INTENT_MODEL or model
logger.info(
"Starting generation for conv %d: model=%s, intent_model=%s, tools=%d",
conv_id, model, intent_model, len(tools),
)
# Start intent classification in parallel with remaining build_context work.
pre_intent: IntentResult = IntentResult()
intent_timing_ms: int | None = None
if tools:
intent_history = [
m for m in history
if m.get("role") in ("user", "assistant") and m.get("content")
][-6:]
buf.append_event("status", {"status": "Analyzing your request..."})
t_intent = time.monotonic()
intent_task = asyncio.create_task(
classify_intent(user_content, tools, intent_model, history=intent_history)
)
(messages, context_meta), pre_intent = await asyncio.gather(context_task, intent_task)
intent_timing_ms = int((time.monotonic() - t_intent) * 1000)
else:
messages, context_meta = await context_task
# Emit context event
buf.append_event("context", {"context": context_meta})
t_start = time.monotonic()
timing: dict = {
"intent_ms": None,
"intent_ms": intent_timing_ms,
"tools": [],
"ttft_ms": None,
"generation_ms": None,
@@ -132,17 +164,6 @@ async def run_generation(
last_flush = time.monotonic()
all_tool_calls: list[dict] = []
# Resolve tools and intent model in parallel
tools, intent_model_setting = await asyncio.gather(
get_tools_for_user(user_id),
get_setting(user_id, "intent_model", ""),
)
intent_model = intent_model_setting or Config.OLLAMA_INTENT_MODEL or model
logger.info(
"Starting generation for conv %d: model=%s, intent_model=%s, tools=%d",
conv_id, model, intent_model, len(tools),
)
try:
cancelled = False
@@ -150,18 +171,9 @@ async def run_generation(
round_tool_calls: list[dict] = []
logger.info("Generation round %d started for conv %d (model=%s)", _round, conv_id, model)
# Intent routing — first round only
# Intent routing — first round only (result pre-computed in parallel with build_context)
if _round == 0 and tools:
# Pass last 3 user/assistant pairs (6 messages) for anaphora resolution.
# messages = [system, *history, current_user] — exclude system and current user.
intent_history = [
m for m in messages[1:-1]
if m.get("role") in ("user", "assistant") and m.get("content")
][-6:]
buf.append_event("status", {"status": "Analyzing your request..."})
t_intent = time.monotonic()
intent = await classify_intent(user_content, tools, intent_model, history=intent_history)
timing["intent_ms"] = int((time.monotonic() - t_intent) * 1000)
intent = pre_intent
if intent.should_execute:
logger.info(
"Intent router detected tool (confidence=%s): %s(%s)",
src/fabledassistant/services/llm.py
+3 -4
View File
@@ -80,7 +80,7 @@ async def stream_chat(
options: dict | None = None,
) -> AsyncGenerator[str, None]:
"""Stream chat completion from Ollama, yielding content chunks."""
merged_options = {"num_ctx": 16384}
merged_options = {"num_ctx": 32768}
if options:
merged_options.update(options)
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options}
@@ -121,10 +121,9 @@ async def stream_chat_with_tools(
ChatChunk(type="tool_calls") is yielded. Always ends with
ChatChunk(type="done").
"""
options: dict = {"num_ctx": 16384}
# Disable thinking mode for models like qwen3 — it interferes with tool calling
options: dict = {"num_ctx": 32768}
if tools:
options["num_predict"] = 4096
options["num_predict"] = 8192
payload: dict = {
"model": model,
"messages": messages,