fix(ml): load SigLIP image-only processor to avoid SentencePiece dep — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

This commit is contained in:
2026-05-25 17:31:06 -04:00
parent a0470b5f60
commit 111b952535
+13 -2
View File
@@ -34,10 +34,21 @@ class Embedder:
if self._model is not None:
return
import torch
from transformers import AutoModel, AutoProcessor
from transformers import AutoModel, SiglipImageProcessor
self._torch = torch
self._processor = AutoProcessor.from_pretrained(str(self._model_dir))
# FC's embedder only does IMAGE inference — never text. AutoProcessor
# loads the full processor including SiglipTokenizer, which requires
# the sentencepiece library at import time even if we never call it.
# SiglipImageProcessor loads ONLY preprocessor_config.json (image
# side) and skips the tokenizer config entirely. Operator hit the
# ImportError 2026-05-25 once the ml-worker started actually running
# tag_and_embed; switching to the image-only loader avoids the
# tokenizer dep without adding ~30 MB of unused C++ build to the
# lean ml-worker image.
self._processor = SiglipImageProcessor.from_pretrained(
str(self._model_dir)
)
self._model = AutoModel.from_pretrained(str(self._model_dir))
self._model.eval()