From 40cc11be5b2a1cd21b66b36c225b5b92aed59964 Mon Sep 17 00:00:00 2001
From: Bryan Van Deusen <bryan.vandeusen@fabledsword.com>
Date: Tue, 7 Jul 2026 22:15:33 -0400
Subject: [PATCH] feat(deploy): container healthchecks + Swarm rolling-update
 auto-rollback

web gets a /api/health liveness check; workers a lenient celery-ping check. A
shared deploy policy (update_config order=start-first, failure_action=rollback,
monitor 90s; rollback_config; restart_policy) means a bad image that never goes
healthy is rolled back automatically instead of taking the service down. Ignored
by plain `docker compose up` (deploy: is swarm-only), so the dev override is
unaffected. Assumes prod deploys from this file via docker stack deploy.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 docker-compose.yml | 47 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/docker-compose.yml b/docker-compose.yml
index 60eeda1..1fb24d6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -8,6 +8,35 @@
 # run `docker compose up` from this directory and switches images to
 # local builds + DEBUG logging.
 
+# Rolling-deploy safety (Swarm / `docker stack deploy`): update one task at a
+# time, START the new task before stopping the old (zero-downtime via the ingress
+# mesh), and if the new task doesn't reach a healthy state within `monitor`, roll
+# back to the previous image automatically. `monitor` is sized above web's
+# healthcheck start_period so a broken image that never goes healthy is caught.
+# Plain `docker compose up` ignores `deploy:` (it warns + skips), so the dev
+# override is unaffected. Referenced by each long-lived service below.
+x-deploy-policy: &deploy_policy
+  update_config:
+    order: start-first
+    failure_action: rollback
+    monitor: 90s
+  rollback_config:
+    order: start-first
+  restart_policy:
+    condition: any
+    delay: 10s
+
+# Worker liveness: ping THIS container's celery node over the broker. Lenient
+# (60s interval, 3 retries, 60s start_period) so a transient broker blip never
+# false-flags a worker into a rollback. `$$HOSTNAME` → `$HOSTNAME` for the shell;
+# celery's default node name is celery@<hostname> (the container id).
+x-celery-healthcheck: &celery_healthcheck
+  test: ["CMD-SHELL", "celery -A backend.app.celery_app:celery inspect ping -d celery@$$HOSTNAME --timeout 10 >/dev/null 2>&1"]
+  interval: 60s
+  timeout: 15s
+  retries: 3
+  start_period: 60s
+
 services:
   redis:
     image: redis:7-alpine
@@ -55,6 +84,16 @@ services:
     # by the 5-min recovery sweeps, so a kill never corrupts. web = short HTTP
     # requests + the occasional file download.
     stop_grace_period: 30s
+    # Liveness for rolling deploys: /api/health is a no-DB 200 (just proves the
+    # app booted + serves HTTP after `alembic upgrade head`). start_period covers
+    # the migration + boot so a slow start isn't mis-flagged.
+    healthcheck:
+      test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:8080/api/health', timeout=5).status==200 else 1)\""]
+      interval: 15s
+      timeout: 6s
+      retries: 3
+      start_period: 40s
+    deploy: *deploy_policy
     ports:
       - "${PORT:-8080}:8080"
     environment: &app_env
@@ -87,6 +126,8 @@ services:
     command: ["worker"]
     # Drain in-flight import/thumbnail/download tasks before SIGKILL on deploy.
     stop_grace_period: 90s
+    healthcheck: *celery_healthcheck
+    deploy: *deploy_policy
     environment:
       <<: *app_env
       CELERY_QUEUES: default,import,thumbnail,download
@@ -105,6 +146,8 @@ services:
     command: ["scheduler"]
     # Quick maintenance/scan lane + beat — short tasks, modest drain window.
     stop_grace_period: 60s
+    healthcheck: *celery_healthcheck
+    deploy: *deploy_policy
     environment:
       <<: *app_env
       CELERY_QUEUES: maintenance,scan
@@ -126,6 +169,8 @@ services:
     # the most room to finish a chunk gracefully. Chunked + idempotent, so a job
     # that still outruns this resumes cleanly next run rather than corrupting.
     stop_grace_period: 180s
+    healthcheck: *celery_healthcheck
+    deploy: *deploy_policy
     environment:
       <<: *app_env
       CELERY_QUEUES: maintenance_long
@@ -143,6 +188,8 @@ services:
     command: ["ml-worker"]
     # A single GPU inference pass can run tens of seconds — let it finish.
     stop_grace_period: 120s
+    healthcheck: *celery_healthcheck
+    deploy: *deploy_policy
     environment:
       <<: *app_env
     volumes: