feat(tagging): SigLIP concept crops + max-over-bag scoring (#114)
Lift recall on small/local concepts (glasses, cum, stomach-bulge, xray, lactation) that the whole-image SigLIP vector washes out: the GPU agent now embeds figure crops with SigLIP too, stored as kind='concept' regions, and the suggestion rail scores each image as a BAG (whole-image + every concept crop), taking each head's MAX over the bag. The whole-image vector is always in the bag, so this can never score lower than before. Model-agnostic by construction: the server ANNOUNCES the embedding model (HF name + version) in the lease, so the agent loads whatever the heads were trained in and stays in lock-step — a model swap is a server setting + a re-embed migration, never an agent change. - agent: model-agnostic CropEmbedder (torch/transformers get_image_features, fp16 on CUDA, inference-locked); worker branches on job.task — 'ccip' emits figure(CCIP)+concept(SigLIP) in one pass, 'siglip' emits concept-only so the back-catalogue backfill never churns figure/CCIP regions; torch cu124 + transformers in the image. - server: lease announces embed_model_name/embed_version; score_image is max-over-bag (version-filtered region embeddings); enqueue_gpu_backfill 'siglip' gates on a missing concept region (drains the back-catalogue, retries failures, no double-enqueue); daily siglip-backfill beat; UI button; /api/ccip/overview reports images_with_concept_siglip. - v1 scope: suggestion rail only — auto-apply stays whole-image (conservative; heads' thresholds were calibrated on whole-image). Bulk-apply bag = follow-up. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
This commit is contained in:
+66
-12
@@ -22,6 +22,12 @@ from .crops import crop_region
|
||||
# Push it up while watching the GPU util + VRAM in the UI.
|
||||
MAX_CONCURRENCY = 32
|
||||
|
||||
# Fallbacks only — the server ANNOUNCES the embedding model (name + version) in
|
||||
# the lease so the agent stays model-agnostic and in lock-step with the space
|
||||
# the heads were trained in. These cover an older server that doesn't send them.
|
||||
DEFAULT_EMBED_MODEL = "google/siglip-so400m-patch14-384"
|
||||
DEFAULT_EMBED_VERSION = "siglip-so400m-patch14-384"
|
||||
|
||||
|
||||
class _Slot:
|
||||
"""One worker loop. `inflight` = jobs leased but not yet processed, so a
|
||||
@@ -44,6 +50,10 @@ class Worker:
|
||||
self.processed = 0
|
||||
self.errors = 0
|
||||
self._active = 0 # slots currently mid-image
|
||||
# The crop embedder (SigLIP-family) is built lazily on the first job that
|
||||
# needs it, from the model the server announces — one shared instance.
|
||||
self._embedder = None
|
||||
self._embedder_lock = threading.Lock()
|
||||
|
||||
# --- control -----------------------------------------------------------
|
||||
def start(self):
|
||||
@@ -114,6 +124,15 @@ class Worker:
|
||||
self.client.release(slot.inflight)
|
||||
slot.inflight = []
|
||||
|
||||
def _ensure_embedder(self, model_name: str):
|
||||
if self._embedder is not None:
|
||||
return self._embedder
|
||||
with self._embedder_lock:
|
||||
if self._embedder is None:
|
||||
from .embedder import CropEmbedder
|
||||
self._embedder = CropEmbedder(model_name, self.cfg.embed_dtype)
|
||||
return self._embedder
|
||||
|
||||
def _process(self, job: dict):
|
||||
self._bump(active=1)
|
||||
try:
|
||||
@@ -126,8 +145,31 @@ class Worker:
|
||||
else:
|
||||
frames = [(None, media.load_image(data))]
|
||||
|
||||
# task picks what to produce per crop:
|
||||
# 'siglip' (backfill existing images) → concept (SigLIP) regions
|
||||
# ONLY, so it never churns their figure/CCIP regions or the
|
||||
# character-reference cache.
|
||||
# 'ccip' / 'both' (a new image's first pass) → figure (CCIP) AND
|
||||
# concept (SigLIP) in one go, off the same crop.
|
||||
task = job.get("task") or "ccip"
|
||||
want_ccip = task in ("ccip", "both")
|
||||
want_siglip = task in ("ccip", "siglip", "both")
|
||||
replace_kinds = (
|
||||
["concept"] if task == "siglip" else ["figure", "face", "concept"]
|
||||
)
|
||||
|
||||
embed_version = job.get("embed_version") or DEFAULT_EMBED_VERSION
|
||||
embedder = None
|
||||
if want_siglip:
|
||||
model_name = (
|
||||
self.cfg.embed_model_override
|
||||
or job.get("embed_model_name")
|
||||
or DEFAULT_EMBED_MODEL
|
||||
)
|
||||
embedder = self._ensure_embedder(model_name)
|
||||
|
||||
regions = []
|
||||
ev = self.cfg.ccip_model or "ccip-default"
|
||||
ccip_ev = self.cfg.ccip_model or "ccip-default"
|
||||
dv = f"person-{self.cfg.detector_level}"
|
||||
for t, frame in frames:
|
||||
figs = models.detect_figures(frame, self.cfg.detector_level)
|
||||
@@ -137,17 +179,29 @@ class Worker:
|
||||
crop = crop_region(frame, bbox)
|
||||
if crop is None:
|
||||
continue
|
||||
vec = models.ccip_vector(crop, self.cfg.ccip_model or None)
|
||||
regions.append({
|
||||
"kind": "figure",
|
||||
"bbox": list(bbox),
|
||||
"frame_time": t,
|
||||
"score": score,
|
||||
"ccip_embedding": vec,
|
||||
"embedding_version": ev,
|
||||
"detector_version": dv,
|
||||
})
|
||||
self.client.submit(job["job_id"], regions, ["figure", "face"])
|
||||
if want_ccip:
|
||||
regions.append({
|
||||
"kind": "figure",
|
||||
"bbox": list(bbox),
|
||||
"frame_time": t,
|
||||
"score": score,
|
||||
"ccip_embedding": models.ccip_vector(
|
||||
crop, self.cfg.ccip_model or None
|
||||
),
|
||||
"embedding_version": ccip_ev,
|
||||
"detector_version": dv,
|
||||
})
|
||||
if want_siglip:
|
||||
regions.append({
|
||||
"kind": "concept",
|
||||
"bbox": list(bbox),
|
||||
"frame_time": t,
|
||||
"score": score,
|
||||
"siglip_embedding": embedder.embed(crop),
|
||||
"embedding_version": embed_version,
|
||||
"detector_version": dv,
|
||||
})
|
||||
self.client.submit(job["job_id"], regions, replace_kinds)
|
||||
self._bump(processed=1)
|
||||
except Exception as exc: # noqa: BLE001 — report + move on
|
||||
self._bump(errors=1)
|
||||
|
||||
Reference in New Issue
Block a user