fix(ml): load SigLIP image-only processor to avoid SentencePiece dep — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -34,10 +34,21 @@ class Embedder:
|
|||||||
if self._model is not None:
|
if self._model is not None:
|
||||||
return
|
return
|
||||||
import torch
|
import torch
|
||||||
from transformers import AutoModel, AutoProcessor
|
from transformers import AutoModel, SiglipImageProcessor
|
||||||
|
|
||||||
self._torch = torch
|
self._torch = torch
|
||||||
self._processor = AutoProcessor.from_pretrained(str(self._model_dir))
|
# FC's embedder only does IMAGE inference — never text. AutoProcessor
|
||||||
|
# loads the full processor including SiglipTokenizer, which requires
|
||||||
|
# the sentencepiece library at import time even if we never call it.
|
||||||
|
# SiglipImageProcessor loads ONLY preprocessor_config.json (image
|
||||||
|
# side) and skips the tokenizer config entirely. Operator hit the
|
||||||
|
# ImportError 2026-05-25 once the ml-worker started actually running
|
||||||
|
# tag_and_embed; switching to the image-only loader avoids the
|
||||||
|
# tokenizer dep without adding ~30 MB of unused C++ build to the
|
||||||
|
# lean ml-worker image.
|
||||||
|
self._processor = SiglipImageProcessor.from_pretrained(
|
||||||
|
str(self._model_dir)
|
||||||
|
)
|
||||||
self._model = AutoModel.from_pretrained(str(self._model_dir))
|
self._model = AutoModel.from_pretrained(str(self._model_dir))
|
||||||
self._model.eval()
|
self._model.eval()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user