bvandeusen
6cf70e22db
Mitigation for the nightly fabledscribe Postgres outage on the vdnt-docker02 Swarm node (incidents 2026-05-15/16/17 around 03:50 UTC). Confirmed kill chain (not the trigger): a brief host-level setns/exec stall makes the Docker healthcheck exec fail with exit 1 → unhealthy → SIGKILL → fast-shutdown can't finish on NFS in 10s → exit 137 → swarm restart_policy.max_attempts: 5 burns out → DB stays dead. Hardens the `db` service so a transient host blip can't escalate to killing the database: - stop_grace_period: 120s (gives PG room to fsync on shutdown) - healthcheck: interval 30s / timeout 10s / retries 10 / start_period 180s (only gates app startup order — not authoritative liveness) - prod: restart_policy condition=on-failure, max_attempts=0, window=120s - quickstart/dev: restart: unless-stopped Host-side trigger (what stalls runc/exec at ~03:50 UTC) is still under investigation — see project_pg_nightly_outage.md. Note: the Portainer prod stack differs from docker-compose.prod.yml here (NFS bind, traefik labels, no ollama). The same `db:` block needs to be pasted into Portainer for the prod mitigation to apply. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
87 lines
2.5 KiB
YAML
87 lines
2.5 KiB
YAML
# Fabled Scribe — Quick Start
|
|
#
|
|
# No build required. Pulls the latest pre-built image from the registry.
|
|
#
|
|
# Usage:
|
|
# 1. Download this file
|
|
# 2. docker compose -f docker-compose.quickstart.yml up -d
|
|
# 3. Open http://localhost:5000 — the first account registered becomes admin
|
|
# 4. Go to Settings → General to pull an LLM model (qwen3:8b or llama3.1:8b are good starting points)
|
|
#
|
|
# Set SECRET_KEY via environment variable or a .env file alongside this file:
|
|
# SECRET_KEY=your-random-secret-here
|
|
|
|
services:
|
|
app:
|
|
image: git.fabledsword.com/bvandeusen/fabledscribe:latest
|
|
ports:
|
|
- "5000:5000"
|
|
environment:
|
|
DATABASE_URL: "postgresql+asyncpg://fabled:fabled@db:5432/fabledassistant"
|
|
SECRET_KEY: "${SECRET_KEY:-change-me-in-production}"
|
|
OLLAMA_URL: "http://ollama:11434"
|
|
OLLAMA_MODEL: "${OLLAMA_MODEL:-llama3.1:8b}"
|
|
LOG_LEVEL: "${LOG_LEVEL:-INFO}"
|
|
volumes:
|
|
- app_data:/data
|
|
depends_on:
|
|
db:
|
|
condition: service_healthy
|
|
ollama:
|
|
condition: service_healthy
|
|
restart: unless-stopped
|
|
healthcheck:
|
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:5000/api/health')"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 30s
|
|
|
|
db:
|
|
image: postgres:16-alpine
|
|
stop_grace_period: 120s
|
|
volumes:
|
|
- pgdata:/var/lib/postgresql/data
|
|
environment:
|
|
POSTGRES_USER: fabled
|
|
POSTGRES_PASSWORD: fabled
|
|
POSTGRES_DB: fabledassistant
|
|
# Lenient by design: a transient host exec/healthcheck stall must never
|
|
# escalate to killing the DB. Health here only gates app startup order.
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U fabled"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 10
|
|
start_period: 180s
|
|
restart: unless-stopped
|
|
|
|
ollama:
|
|
image: ollama/ollama
|
|
volumes:
|
|
- ollama_models:/root/.ollama
|
|
environment:
|
|
OLLAMA_MAX_LOADED_MODELS: "2"
|
|
OLLAMA_KEEP_ALIVE: "30m"
|
|
OLLAMA_FLASH_ATTENTION: "1"
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "ollama list > /dev/null 2>&1"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 5
|
|
start_period: 15s
|
|
restart: unless-stopped
|
|
# Uncomment to enable NVIDIA GPU passthrough (requires nvidia-container-toolkit):
|
|
# deploy:
|
|
# resources:
|
|
# reservations:
|
|
# devices:
|
|
# - driver: nvidia
|
|
# count: all
|
|
# capabilities: [gpu]
|
|
|
|
volumes:
|
|
app_data:
|
|
pgdata:
|
|
ollama_models:
|