diff --git a/docker-compose.yml b/docker-compose.yml index 60eeda1..1fb24d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,35 @@ # run `docker compose up` from this directory and switches images to # local builds + DEBUG logging. +# Rolling-deploy safety (Swarm / `docker stack deploy`): update one task at a +# time, START the new task before stopping the old (zero-downtime via the ingress +# mesh), and if the new task doesn't reach a healthy state within `monitor`, roll +# back to the previous image automatically. `monitor` is sized above web's +# healthcheck start_period so a broken image that never goes healthy is caught. +# Plain `docker compose up` ignores `deploy:` (it warns + skips), so the dev +# override is unaffected. Referenced by each long-lived service below. +x-deploy-policy: &deploy_policy + update_config: + order: start-first + failure_action: rollback + monitor: 90s + rollback_config: + order: start-first + restart_policy: + condition: any + delay: 10s + +# Worker liveness: ping THIS container's celery node over the broker. Lenient +# (60s interval, 3 retries, 60s start_period) so a transient broker blip never +# false-flags a worker into a rollback. `$$HOSTNAME` → `$HOSTNAME` for the shell; +# celery's default node name is celery@ (the container id). +x-celery-healthcheck: &celery_healthcheck + test: ["CMD-SHELL", "celery -A backend.app.celery_app:celery inspect ping -d celery@$$HOSTNAME --timeout 10 >/dev/null 2>&1"] + interval: 60s + timeout: 15s + retries: 3 + start_period: 60s + services: redis: image: redis:7-alpine @@ -55,6 +84,16 @@ services: # by the 5-min recovery sweeps, so a kill never corrupts. web = short HTTP # requests + the occasional file download. stop_grace_period: 30s + # Liveness for rolling deploys: /api/health is a no-DB 200 (just proves the + # app booted + serves HTTP after `alembic upgrade head`). start_period covers + # the migration + boot so a slow start isn't mis-flagged. + healthcheck: + test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:8080/api/health', timeout=5).status==200 else 1)\""] + interval: 15s + timeout: 6s + retries: 3 + start_period: 40s + deploy: *deploy_policy ports: - "${PORT:-8080}:8080" environment: &app_env @@ -87,6 +126,8 @@ services: command: ["worker"] # Drain in-flight import/thumbnail/download tasks before SIGKILL on deploy. stop_grace_period: 90s + healthcheck: *celery_healthcheck + deploy: *deploy_policy environment: <<: *app_env CELERY_QUEUES: default,import,thumbnail,download @@ -105,6 +146,8 @@ services: command: ["scheduler"] # Quick maintenance/scan lane + beat — short tasks, modest drain window. stop_grace_period: 60s + healthcheck: *celery_healthcheck + deploy: *deploy_policy environment: <<: *app_env CELERY_QUEUES: maintenance,scan @@ -126,6 +169,8 @@ services: # the most room to finish a chunk gracefully. Chunked + idempotent, so a job # that still outruns this resumes cleanly next run rather than corrupting. stop_grace_period: 180s + healthcheck: *celery_healthcheck + deploy: *deploy_policy environment: <<: *app_env CELERY_QUEUES: maintenance_long @@ -143,6 +188,8 @@ services: command: ["ml-worker"] # A single GPU inference pass can run tens of seconds — let it finish. stop_grace_period: 120s + healthcheck: *celery_healthcheck + deploy: *deploy_policy environment: <<: *app_env volumes: