bvandeusen
931a059e9f
Docker Compose: - Enable Ollama GPU passthrough (nvidia, count: all) in both dev and prod files - Add OLLAMA_FLASH_ATTENTION=1 (faster attention on GPU in both files) - Add OLLAMA_MAX_LOADED_MODELS=2 and OLLAMA_KEEP_ALIVE=30m to prod (was already in dev) - Remove 8G memory limit from prod Ollama service (CPU-bound constraint, no longer valid) llm.py: - Increase num_ctx 16384 → 32768 in stream_chat and stream_chat_with_tools (GPU VRAM allows it) - Increase num_predict cap 4096 → 8192 for tool-augmented responses generation_task.py: - Parallelize build_context, get_tools_for_user, and get_setting all from the start - As soon as tools list is ready (fast DB call), launch classify_intent as an asyncio.Task - Await build_context and classify_intent together via asyncio.gather - Intent result is pre-computed before the generation loop; loop just reads pre_intent on round 0 - intent_ms timing now reflects wall-clock time from intent start to completion Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
82 lines
1.9 KiB
YAML
82 lines
1.9 KiB
YAML
services:
|
|
app:
|
|
image: git.fabledsword.com/bvandeusen/fabledassistant:latest
|
|
ports:
|
|
- "5000:5000"
|
|
environment:
|
|
DATABASE_URL: "postgresql+asyncpg://fabled:${DB_PASSWORD}@db:5432/fabledassistant"
|
|
SECRET_KEY: "${SECRET_KEY}"
|
|
OLLAMA_URL: "http://ollama:11434"
|
|
OLLAMA_MODEL: "${OLLAMA_MODEL:-llama3.1}"
|
|
LOG_LEVEL: "${LOG_LEVEL:-INFO}"
|
|
networks:
|
|
- fabledassistant_backend
|
|
healthcheck:
|
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:5000/api/health')"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 30s
|
|
deploy:
|
|
restart_policy:
|
|
condition: on-failure
|
|
max_attempts: 5
|
|
|
|
db:
|
|
image: postgres:16-alpine
|
|
volumes:
|
|
- pgdata:/var/lib/postgresql/data
|
|
environment:
|
|
POSTGRES_USER: fabled
|
|
POSTGRES_PASSWORD: "${DB_PASSWORD}"
|
|
POSTGRES_DB: fabledassistant
|
|
networks:
|
|
- fabledassistant_backend
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U fabled"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 5
|
|
deploy:
|
|
restart_policy:
|
|
condition: on-failure
|
|
max_attempts: 5
|
|
|
|
ollama:
|
|
image: ollama/ollama
|
|
volumes:
|
|
- ollama_models:/root/.ollama
|
|
networks:
|
|
- fabledassistant_backend
|
|
environment:
|
|
OLLAMA_MAX_LOADED_MODELS: "2"
|
|
OLLAMA_KEEP_ALIVE: "30m"
|
|
OLLAMA_FLASH_ATTENTION: "1"
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "ollama list || exit 1"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 5
|
|
start_period: 30s
|
|
deploy:
|
|
placement:
|
|
constraints:
|
|
- node.role == worker
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities: [gpu]
|
|
restart_policy:
|
|
condition: on-failure
|
|
max_attempts: 5
|
|
|
|
volumes:
|
|
pgdata:
|
|
ollama_models:
|
|
|
|
networks:
|
|
fabledassistant_backend:
|
|
driver: overlay
|