fix(celery): harden broker connection so workers ride out a Redis blip

A swarm overlay-network blip after the :latest redeploy left Redis healthy but
transiently unreachable; a worker starting in that window crash-looped on the
initial broker connect (kombu OperationalError) and needed a manual Redis reset
to recover.

Retry the broker forever on startup + at runtime (broker_connection_max_retries
=None), add redis-transport socket options to the broker (short connect timeout,
TCP keepalive, retry_on_timeout, periodic health check), and mirror the same on
the Redis result backend. Now a transient outage self-heals when overlay routing
returns instead of the worker exiting.

Test pins the key resilience settings.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XCUHUGQLrBrkgyk1t49kpX

backend/app/celery_app.py

@@ -61,7 +61,33 @@ def make_celery() -> Celery:
         # Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
         task_acks_late=True,
         worker_prefetch_multiplier=1,
+        # Broker resilience (2026-06-24): a swarm overlay-network blip after a
+        # redeploy left Redis healthy but transiently unreachable, and a worker
+        # starting in that window crash-looped on the initial broker connect
+        # (kombu OperationalError) instead of waiting it out — needing a manual
+        # Redis reset to recover. Retry the broker FOREVER (None) on startup and
+        # at runtime so a transient outage self-heals when routing returns,
+        # rather than the worker exiting.
         broker_connection_retry_on_startup=True,
+        broker_connection_retry=True,
+        broker_connection_max_retries=None,
+        # Redis-transport socket options (apply to the BROKER connection): a
+        # short connect timeout + TCP keepalive so a dead/blocked socket is
+        # noticed and retried, and a periodic health check that proactively
+        # reconnects a live worker through a network hiccup.
+        broker_transport_options={
+            "socket_connect_timeout": 5,
+            "socket_timeout": 30,
+            "socket_keepalive": True,
+            "retry_on_timeout": True,
+            "health_check_interval": 30,
+        },
+        # Same hardening for the Redis RESULT backend (separate connection pool).
+        redis_socket_connect_timeout=5,
+        redis_socket_timeout=30,
+        redis_socket_keepalive=True,
+        redis_retry_on_timeout=True,
+        redis_backend_health_check_interval=30,
         beat_schedule={
             "recover-interrupted-tasks": {
                 "task": "backend.app.tasks.maintenance.recover_interrupted_tasks",

tests/test_celery_smoke.py

@@ -22,3 +22,17 @@ def test_ping_returns_pong():
 def test_ping_with_value():
     result = ping.delay("hello")
     assert result.get(timeout=1) == "hello"
+
+
+def test_broker_resilience_configured():
+    # Workers must ride out a transient broker outage (swarm overlay blip after
+    # a redeploy left Redis briefly unreachable) instead of crash-looping on the
+    # initial connect and needing a manual Redis reset (2026-06-24).
+    conf = celery.conf
+    assert conf.broker_connection_retry_on_startup is True
+    assert conf.broker_connection_max_retries is None  # retry forever
+    opts = conf.broker_transport_options
+    assert opts["socket_keepalive"] is True
+    assert opts["socket_connect_timeout"] == 5
+    assert opts["health_check_interval"] == 30
+    assert conf.redis_retry_on_timeout is True