feat(tagging): SigLIP concept crops + max-over-bag scoring (#114)

Lift recall on small/local concepts (glasses, cum, stomach-bulge, xray,
lactation) that the whole-image SigLIP vector washes out: the GPU agent now
embeds figure crops with SigLIP too, stored as kind='concept' regions, and the
suggestion rail scores each image as a BAG (whole-image + every concept crop),
taking each head's MAX over the bag. The whole-image vector is always in the
bag, so this can never score lower than before.

Model-agnostic by construction: the server ANNOUNCES the embedding model
(HF name + version) in the lease, so the agent loads whatever the heads were
trained in and stays in lock-step — a model swap is a server setting + a
re-embed migration, never an agent change.

- agent: model-agnostic CropEmbedder (torch/transformers get_image_features,
  fp16 on CUDA, inference-locked); worker branches on job.task — 'ccip' emits
  figure(CCIP)+concept(SigLIP) in one pass, 'siglip' emits concept-only so the
  back-catalogue backfill never churns figure/CCIP regions; torch cu124 +
  transformers in the image.
- server: lease announces embed_model_name/embed_version; score_image is
  max-over-bag (version-filtered region embeddings); enqueue_gpu_backfill
  'siglip' gates on a missing concept region (drains the back-catalogue,
  retries failures, no double-enqueue); daily siglip-backfill beat; UI button;
  /api/ccip/overview reports images_with_concept_siglip.
- v1 scope: suggestion rail only — auto-apply stays whole-image (conservative;
  heads' thresholds were calibrated on whole-image). Bulk-apply bag = follow-up.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
This commit is contained in:
2026-06-30 08:17:47 -04:00
parent b91a230f12
commit c6f38b0dac
13 changed files with 329 additions and 33 deletions
+6 -1
View File
@@ -10,11 +10,16 @@ RUN apt-get update \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
# torch from the CUDA-12.4 wheel index (matches the base image); its wheels
# bundle their own CUDA + cuDNN and coexist with onnxruntime-gpu. Installed
# first + separately so the GPU build of torch is deterministic and layer-cached.
RUN pip3 install --no-cache-dir torch==2.6.0 --index-url https://download.pytorch.org/whl/cu124
COPY requirements.txt .
RUN pip3 install --no-cache-dir -r requirements.txt
COPY fc_agent ./fc_agent
# imgutils caches downloaded ONNX models here; mount a volume to persist them.
# imgutils ONNX models + the transformers SigLIP weights both cache here; mount
# a volume to persist them across restarts (the SigLIP download is ~3.5 GB once).
ENV HF_HOME=/models
EXPOSE 8770
+5
View File
@@ -13,6 +13,9 @@ class Config:
ccip_model: str # imgutils CCIP model name ("" → imgutils default)
detector_level: str # imgutils person-detector level: n|s|m|x
poll_idle_seconds: float # wait between empty leases
embed_dtype: str # torch dtype for the crop embedder: float16|float32
embed_model_override: str # force a SigLIP-family model ("" → use the one
# the server announces in the lease)
@classmethod
def from_env(cls) -> "Config":
@@ -25,4 +28,6 @@ class Config:
ccip_model=os.environ.get("CCIP_MODEL", ""),
detector_level=os.environ.get("DETECTOR_LEVEL", "m"),
poll_idle_seconds=float(os.environ.get("POLL_IDLE_SECONDS", "10")),
embed_dtype=os.environ.get("SIGLIP_DTYPE", "float16"),
embed_model_override=os.environ.get("EMBED_MODEL_NAME", ""),
)
+69
View File
@@ -0,0 +1,69 @@
"""Crop EMBEDDER for the concept bag — model-agnostic (CLIP/SigLIP-family).
The server trains its per-concept heads in the embedding space of whatever model
its `embedder_model_version` names; a crop must be embedded with the SAME model
or its vector lands in a different coordinate system and every head misfires. So
the model identity (HF name + version) is ANNOUNCED BY THE SERVER in the lease —
nothing here is hardcoded to SigLIP. Whatever name the server sends is loaded via
transformers `get_image_features` (the CLIP/SigLIP-family image-tower call); a
non-CLIP backbone (e.g. a DINO encoder) would need its own pooling adapter.
torch on CUDA, fp16 by default to keep VRAM low on a shared desktop GPU — the
tiny fp16-vs-fp32 difference is negligible for the linear heads (cosine ~0.999).
A single inference lock serializes the forward pass: the pipeline is I/O-bound,
so the GPU isn't the bottleneck, and one model shared across worker threads is
safest behind a lock.
"""
import threading
import numpy as np
from PIL import Image
class CropEmbedder:
def __init__(self, model_name: str, dtype: str = "float16"):
self._name = model_name
self._dtype_name = dtype
self._model = None
self._processor = None
self._torch = None
self._device = None
self._dt = None
self._load_lock = threading.Lock()
self._infer_lock = threading.Lock()
@property
def model_name(self) -> str:
return self._name
def load(self) -> None:
if self._model is not None:
return
with self._load_lock:
if self._model is not None:
return
import torch
from transformers import AutoImageProcessor, AutoModel
self._torch = torch
self._device = "cuda" if torch.cuda.is_available() else "cpu"
dt = getattr(torch, self._dtype_name, torch.float16)
if self._device == "cpu":
dt = torch.float32 # fp16 matmul is unsupported/slow on CPU
self._dt = dt
self._processor = AutoImageProcessor.from_pretrained(self._name)
model = AutoModel.from_pretrained(self._name, torch_dtype=dt)
model.eval().to(self._device)
self._model = model
def embed(self, image: Image.Image) -> list[float]:
"""A crop → its embedding as a plain float list, ready to POST."""
self.load()
torch = self._torch
enc = self._processor(images=image, return_tensors="pt")
pixel_values = enc["pixel_values"].to(self._device, self._dt)
with self._infer_lock, torch.no_grad():
out = self._model.get_image_features(pixel_values=pixel_values)
pooled = out.pooler_output if hasattr(out, "pooler_output") else out
vec = pooled[0].float().cpu().numpy().astype(np.float32).reshape(-1)
return vec.tolist()
+66 -12
View File
@@ -22,6 +22,12 @@ from .crops import crop_region
# Push it up while watching the GPU util + VRAM in the UI.
MAX_CONCURRENCY = 32
# Fallbacks only — the server ANNOUNCES the embedding model (name + version) in
# the lease so the agent stays model-agnostic and in lock-step with the space
# the heads were trained in. These cover an older server that doesn't send them.
DEFAULT_EMBED_MODEL = "google/siglip-so400m-patch14-384"
DEFAULT_EMBED_VERSION = "siglip-so400m-patch14-384"
class _Slot:
"""One worker loop. `inflight` = jobs leased but not yet processed, so a
@@ -44,6 +50,10 @@ class Worker:
self.processed = 0
self.errors = 0
self._active = 0 # slots currently mid-image
# The crop embedder (SigLIP-family) is built lazily on the first job that
# needs it, from the model the server announces — one shared instance.
self._embedder = None
self._embedder_lock = threading.Lock()
# --- control -----------------------------------------------------------
def start(self):
@@ -114,6 +124,15 @@ class Worker:
self.client.release(slot.inflight)
slot.inflight = []
def _ensure_embedder(self, model_name: str):
if self._embedder is not None:
return self._embedder
with self._embedder_lock:
if self._embedder is None:
from .embedder import CropEmbedder
self._embedder = CropEmbedder(model_name, self.cfg.embed_dtype)
return self._embedder
def _process(self, job: dict):
self._bump(active=1)
try:
@@ -126,8 +145,31 @@ class Worker:
else:
frames = [(None, media.load_image(data))]
# task picks what to produce per crop:
# 'siglip' (backfill existing images) → concept (SigLIP) regions
# ONLY, so it never churns their figure/CCIP regions or the
# character-reference cache.
# 'ccip' / 'both' (a new image's first pass) → figure (CCIP) AND
# concept (SigLIP) in one go, off the same crop.
task = job.get("task") or "ccip"
want_ccip = task in ("ccip", "both")
want_siglip = task in ("ccip", "siglip", "both")
replace_kinds = (
["concept"] if task == "siglip" else ["figure", "face", "concept"]
)
embed_version = job.get("embed_version") or DEFAULT_EMBED_VERSION
embedder = None
if want_siglip:
model_name = (
self.cfg.embed_model_override
or job.get("embed_model_name")
or DEFAULT_EMBED_MODEL
)
embedder = self._ensure_embedder(model_name)
regions = []
ev = self.cfg.ccip_model or "ccip-default"
ccip_ev = self.cfg.ccip_model or "ccip-default"
dv = f"person-{self.cfg.detector_level}"
for t, frame in frames:
figs = models.detect_figures(frame, self.cfg.detector_level)
@@ -137,17 +179,29 @@ class Worker:
crop = crop_region(frame, bbox)
if crop is None:
continue
vec = models.ccip_vector(crop, self.cfg.ccip_model or None)
regions.append({
"kind": "figure",
"bbox": list(bbox),
"frame_time": t,
"score": score,
"ccip_embedding": vec,
"embedding_version": ev,
"detector_version": dv,
})
self.client.submit(job["job_id"], regions, ["figure", "face"])
if want_ccip:
regions.append({
"kind": "figure",
"bbox": list(bbox),
"frame_time": t,
"score": score,
"ccip_embedding": models.ccip_vector(
crop, self.cfg.ccip_model or None
),
"embedding_version": ccip_ev,
"detector_version": dv,
})
if want_siglip:
regions.append({
"kind": "concept",
"bbox": list(bbox),
"frame_time": t,
"score": score,
"siglip_embedding": embedder.embed(crop),
"embedding_version": embed_version,
"detector_version": dv,
})
self.client.submit(job["job_id"], regions, replace_kinds)
self._bump(processed=1)
except Exception as exc: # noqa: BLE001 — report + move on
self._bump(errors=1)
+4
View File
@@ -3,6 +3,10 @@ dghs-imgutils>=0.4
# GPU inference for the ONNX models. Swap to onnxruntime (CPU) for a slow
# server-side fallback run.
onnxruntime-gpu
# The crop EMBEDDER (concept bag). torch is installed separately in the
# Dockerfile from the CUDA-12.4 wheel index so the GPU build is deterministic;
# transformers loads whatever SigLIP-family model the server announces.
transformers>=4.45
# Control surface + HTTP.
fastapi
uvicorn[standard]
+10
View File
@@ -37,6 +37,15 @@ async def overview():
.where(ImageRegion.ccip_embedding.is_not(None))
)
).scalar_one()
# Concept-crop (SigLIP bag) coverage — how far the back-catalogue embed
# has progressed, so the max-over-bag scorer's reach is checkable.
images_with_concept_siglip = (
await session.execute(
select(func.count(distinct(ImageRegion.image_record_id)))
.where(ImageRegion.kind == "concept")
.where(ImageRegion.siglip_embedding.is_not(None))
)
).scalar_one()
# Per-character reference counts (no vectors loaded) — which characters
# have enough examples to match on.
ref_rows = (
@@ -72,6 +81,7 @@ async def overview():
return jsonify({
"regions_by_kind": by_kind,
"images_with_figure_ccip": images_with_figure_ccip,
"images_with_concept_siglip": images_with_concept_siglip,
"characters_with_references": len(ref_rows),
"character_references": [
{"tag_id": t, "name": n, "n_refs": c} for (t, n, c) in ref_rows
+7
View File
@@ -17,6 +17,7 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert
from ..extensions import get_session
from ..models import AppSetting, GpuJob, ImageRecord, MLSettings
from ..services.gallery_service import image_url
from ..services.ml.embedder import MODEL_NAME as EMBED_MODEL_NAME
from ..services.ml.gpu_jobs import GpuJobService
from ..services.ml.regions import RegionService
@@ -137,6 +138,12 @@ async def lease():
# For video/animated: the agent samples at this cadence.
"frame_interval_seconds": ml.video_frame_interval_seconds,
"max_frames": ml.video_max_frames,
# The embedding model the agent must use for concept crops, so
# its region vectors land in the SAME space the heads trained in.
# Server-announced → the agent stays model-agnostic; a swap is a
# server setting + a re-embed migration, never an agent change.
"embed_model_name": EMBED_MODEL_NAME,
"embed_version": ml.embedder_model_version,
})
return jsonify({"jobs": out})
+5
View File
@@ -126,6 +126,11 @@ def make_celery() -> Celery:
"schedule": 3600.0, # auto-feed new images (+ retry errored) so
"args": ("ccip",), # the queue keeps moving without the button
},
"enqueue-siglip-backfill-daily": {
"task": "backend.app.tasks.ml.enqueue_gpu_backfill",
"schedule": 86400.0, # drain the concept-crop back-catalogue +
"args": ("siglip",), # retry failed embeds, no button needed
},
"ccip-auto-apply-daily": {
"task": "backend.app.tasks.ml.scheduled_ccip_auto_apply",
"schedule": 86400.0, # no-op unless ccip_auto_apply_enabled
+29 -6
View File
@@ -29,6 +29,7 @@ from ...models import (
HeadAutoApplyRun,
HeadTrainingRun,
ImageRecord,
ImageRegion,
MLSettings,
Tag,
TagHead,
@@ -296,7 +297,14 @@ async def score_image(
category, score}], ranked. A concept surfaces when its score clears the
head's own suggest_threshold — or, when threshold_override is given (the
typed-dropdown "show everything" mode), that flat floor instead (0 → every
head). Empty if the image has no embedding or no heads exist yet."""
head). Empty if the image has no embedding or no heads exist yet.
MAX-OVER-BAG: the image is scored as a BAG of embeddings — the whole-image
vector PLUS every concept-region crop the agent embedded (same model
version) — and each head takes its MAX score across the bag. A small/local
concept (glasses, a stomach bulge) that the whole-image vector washes out
can still surface from the crop where it dominates. The whole-image vector is
always in the bag, so this can never score lower than whole-image alone."""
import numpy as np
img = await session.get(ImageRecord, image_id)
@@ -306,11 +314,26 @@ async def score_image(
heads = await _current_heads(session, settings.embedder_model_version)
if heads["W"] is None:
return []
x = np.asarray(img.siglip_embedding, dtype=np.float32)
n = float(np.linalg.norm(x)) or 1.0
xn = x / n
z = heads["W"] @ xn + heads["b"]
probs = 1.0 / (1.0 + np.exp(-z))
bag = [np.asarray(img.siglip_embedding, dtype=np.float32)]
region_vecs = (
await session.execute(
select(ImageRegion.siglip_embedding)
.where(ImageRegion.image_record_id == image_id)
.where(ImageRegion.siglip_embedding.is_not(None))
.where(ImageRegion.embedding_version == settings.embedder_model_version)
)
).all()
for (vec,) in region_vecs:
if vec is not None:
bag.append(np.asarray(vec, dtype=np.float32))
X = np.vstack(bag) # (B, D)
norms = np.linalg.norm(X, axis=1, keepdims=True)
norms[norms == 0] = 1.0
Xn = X / norms
Z = Xn @ heads["W"].T + heads["b"] # (B, H)
probs = (1.0 / (1.0 + np.exp(-Z))).max(axis=0) # (H,) best over the bag
out = []
for i, p in enumerate(probs):
cut = threshold_override if threshold_override is not None else heads["thr"][i]
+31 -12
View File
@@ -742,24 +742,43 @@ def scheduled_apply_head_tags() -> str:
@celery.task(name="backend.app.tasks.ml.enqueue_gpu_backfill")
def enqueue_gpu_backfill(task_name: str) -> int:
"""Enqueue a gpu_job for every image that doesn't already have one for
`task_name` (one INSERT…SELECT, so it scales to a full library). The desktop
agent drains the queue over HTTP. Returns the number enqueued."""
"""Enqueue a gpu_job for every image that still needs `task_name` (one
INSERT…SELECT, so it scales to a full library). The desktop agent drains the
queue over HTTP. Returns the number enqueued.
'siglip' gates on the RESULT (no concept region yet) rather than on a prior
job, so it picks up the back-catalogue of images that were CCIP-embedded
before concept crops existed, and retries images whose concept embed failed —
without re-touching their figure/CCIP regions."""
from sqlalchemy import exists, insert, literal
from sqlalchemy import select as sa_select
from ..models import GpuJob, ImageRecord
from ..models import GpuJob, ImageRecord, ImageRegion
SessionLocal = _sync_session_factory()
with SessionLocal() as session:
already = exists().where(
GpuJob.image_record_id == ImageRecord.id,
GpuJob.task == task_name,
GpuJob.status.in_(["pending", "leased", "done"]),
)
sel = sa_select(
ImageRecord.id, literal(task_name), literal("pending")
).where(~already)
if task_name == "siglip":
has_concept = exists().where(
ImageRegion.image_record_id == ImageRecord.id,
ImageRegion.kind == "concept",
)
queued = exists().where(
GpuJob.image_record_id == ImageRecord.id,
GpuJob.task == "siglip",
GpuJob.status.in_(["pending", "leased"]),
)
sel = sa_select(
ImageRecord.id, literal("siglip"), literal("pending")
).where(~has_concept).where(~queued)
else:
already = exists().where(
GpuJob.image_record_id == ImageRecord.id,
GpuJob.task == task_name,
GpuJob.status.in_(["pending", "leased", "done"]),
)
sel = sa_select(
ImageRecord.id, literal(task_name), literal("pending")
).where(~already)
# RETURNING + count: result.rowcount is unreliable for INSERT…SELECT.
rows = session.execute(
insert(GpuJob)
@@ -61,6 +61,16 @@
processes until the agent is running.
</p>
<v-btn
class="mt-3" color="accent" variant="tonal" rounded="pill" size="small"
prepend-icon="mdi-crop" :loading="backfillingSiglip" @click="onBackfillSiglip"
>Queue concept crops (SigLIP)</v-btn>
<p class="fc-muted text-caption mt-2 mb-0">
Enqueues every image that doesn't have concept-crop embeddings yet the
localized vectors that help small/local tags (glasses, etc.) surface. New
images get these automatically; this catches the back-catalogue.
</p>
<!-- Match strictness -->
<div class="fc-section-h mt-5 mb-1">Character-match strictness</div>
<div v-if="ml.settings" class="d-flex align-center" style="gap:12px">
@@ -115,6 +125,7 @@ const tokenValue = ref(null)
const masked = ref(true)
const rotating = ref(false)
const backfilling = ref(false)
const backfillingSiglip = ref(false)
const threshold = ref(0.85)
const savingThreshold = ref(false)
const autoApply = ref(true)
@@ -215,6 +226,19 @@ async function onBackfill() {
backfilling.value = false
}
}
async function onBackfillSiglip() {
backfillingSiglip.value = true
try {
await store.backfill('siglip')
toast({ text: 'Queued concept crops — run the agent to process them', type: 'success' })
await refreshQueue()
} catch (e) {
toast({ text: `Could not queue backfill: ${e.message}`, type: 'error' })
} finally {
backfillingSiglip.value = false
}
}
</script>
<style scoped>
+39 -2
View File
@@ -2,9 +2,9 @@
from datetime import UTC, datetime, timedelta
import pytest
from sqlalchemy import select
from sqlalchemy import func, select
from backend.app.models import GpuJob, ImageRecord
from backend.app.models import GpuJob, ImageRecord, ImageRegion
from backend.app.services.ml.gpu_jobs import GpuJobService
pytestmark = pytest.mark.integration
@@ -20,6 +20,43 @@ async def _img(db, sha) -> ImageRecord:
return img
@pytest.mark.asyncio
async def test_enqueue_siglip_backfill_gates_on_concept_region(db):
# 'siglip' backfill enqueues images that lack a concept region (the
# back-catalogue) and skips ones that already have one — and never double-
# enqueues an image that already has a pending siglip job.
from backend.app.tasks.ml import enqueue_gpu_backfill
need = await _img(db, "e1" * 32) # no concept region → wants one
have = await _img(db, "e2" * 32) # already embedded → skip
db.add(ImageRegion(
image_record_id=have.id, kind="concept", rx=0.0, ry=0.0, rw=1.0, rh=1.0,
siglip_embedding=[0.0] * 1152, embedding_version="siglip-test",
))
await db.commit()
assert enqueue_gpu_backfill("siglip") >= 1
queued = {
j.image_record_id for j in (
await db.execute(select(GpuJob).where(GpuJob.task == "siglip"))
).scalars()
}
assert need.id in queued
assert have.id not in queued
# Idempotent: the now-pending job means a second run doesn't re-enqueue it.
enqueue_gpu_backfill("siglip")
n_for_need = (
await db.execute(
select(func.count()).select_from(GpuJob).where(
GpuJob.task == "siglip", GpuJob.image_record_id == need.id
)
)
).scalar_one()
assert n_for_need == 1
@pytest.mark.asyncio
async def test_enqueue_dedupes_same_pair(db):
img = await _img(db, "a" * 64)
+34
View File
@@ -111,6 +111,40 @@ async def test_threshold_override_surfaces_below_cut(db):
assert any(s.canonical_tag_id == tag.id for s in flooded.by_category["general"])
@pytest.mark.asyncio
async def test_concept_region_surfaces_via_max_over_bag(db):
# Max-over-bag: the whole-image vector is orthogonal to the head (scores the
# 0.5 midpoint, under a 0.7 cut → nothing), but a concept CROP that aligns
# with the head lifts the max over the bag above the cut. A small/local
# concept surfaces ONLY because of the crop.
tag = await TagService(db).find_or_create("glasses", TagKind.general)
img = await _img(db, "b1" * 32, _emb(5)) # whole-image ⟂ head
await _head(db, tag.id, slot=0, suggest_threshold=0.7)
await db.commit()
# Whole-image alone: sigmoid(0)=0.5 < 0.7 → no suggestion.
assert not (await SuggestionService(db).for_image(img.id)).by_category.get("general")
# A concept crop aligned with the head, but stamped with a STALE model
# version → filtered out of the bag, so still nothing.
db.add(ImageRegion(
image_record_id=img.id, kind="concept",
rx=0.1, ry=0.1, rw=0.3, rh=0.3,
siglip_embedding=_emb(0), embedding_version="stale-embedder-v0",
))
await db.commit()
assert not (await SuggestionService(db).for_image(img.id)).by_category.get("general")
# A matching-version concept crop → max-over-bag lifts it over the cut.
db.add(ImageRegion(
image_record_id=img.id, kind="concept",
rx=0.4, ry=0.4, rw=0.3, rh=0.3,
siglip_embedding=_emb(0), embedding_version=await _embver(db),
))
await db.commit()
general = (await SuggestionService(db).for_image(img.id)).by_category["general"]
assert any(s.canonical_tag_id == tag.id and s.score > 0.7 for s in general)
@pytest.mark.asyncio
async def test_rejected_tag_surfaced_flagged_then_reversible(db):
# A dismissed suggestion is NOT dropped: it stays flagged rejected so the