diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index 0576d85..eae20e6 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -61,7 +61,33 @@ def make_celery() -> Celery: # Heavy ML tasks need fair dispatch — see ImageRepo's precedent. task_acks_late=True, worker_prefetch_multiplier=1, + # Broker resilience (2026-06-24): a swarm overlay-network blip after a + # redeploy left Redis healthy but transiently unreachable, and a worker + # starting in that window crash-looped on the initial broker connect + # (kombu OperationalError) instead of waiting it out — needing a manual + # Redis reset to recover. Retry the broker FOREVER (None) on startup and + # at runtime so a transient outage self-heals when routing returns, + # rather than the worker exiting. broker_connection_retry_on_startup=True, + broker_connection_retry=True, + broker_connection_max_retries=None, + # Redis-transport socket options (apply to the BROKER connection): a + # short connect timeout + TCP keepalive so a dead/blocked socket is + # noticed and retried, and a periodic health check that proactively + # reconnects a live worker through a network hiccup. + broker_transport_options={ + "socket_connect_timeout": 5, + "socket_timeout": 30, + "socket_keepalive": True, + "retry_on_timeout": True, + "health_check_interval": 30, + }, + # Same hardening for the Redis RESULT backend (separate connection pool). + redis_socket_connect_timeout=5, + redis_socket_timeout=30, + redis_socket_keepalive=True, + redis_retry_on_timeout=True, + redis_backend_health_check_interval=30, beat_schedule={ "recover-interrupted-tasks": { "task": "backend.app.tasks.maintenance.recover_interrupted_tasks", diff --git a/tests/test_celery_smoke.py b/tests/test_celery_smoke.py index 40b0054..68dcb80 100644 --- a/tests/test_celery_smoke.py +++ b/tests/test_celery_smoke.py @@ -22,3 +22,17 @@ def test_ping_returns_pong(): def test_ping_with_value(): result = ping.delay("hello") assert result.get(timeout=1) == "hello" + + +def test_broker_resilience_configured(): + # Workers must ride out a transient broker outage (swarm overlay blip after + # a redeploy left Redis briefly unreachable) instead of crash-looping on the + # initial connect and needing a manual Redis reset (2026-06-24). + conf = celery.conf + assert conf.broker_connection_retry_on_startup is True + assert conf.broker_connection_max_retries is None # retry forever + opts = conf.broker_transport_options + assert opts["socket_keepalive"] is True + assert opts["socket_connect_timeout"] == 5 + assert opts["health_check_interval"] == 30 + assert conf.redis_retry_on_timeout is True