feat(tagging): SigLIP concept crops + max-over-bag scoring (#114)

Lift recall on small/local concepts (glasses, cum, stomach-bulge, xray,
lactation) that the whole-image SigLIP vector washes out: the GPU agent now
embeds figure crops with SigLIP too, stored as kind='concept' regions, and the
suggestion rail scores each image as a BAG (whole-image + every concept crop),
taking each head's MAX over the bag. The whole-image vector is always in the
bag, so this can never score lower than before.

Model-agnostic by construction: the server ANNOUNCES the embedding model
(HF name + version) in the lease, so the agent loads whatever the heads were
trained in and stays in lock-step — a model swap is a server setting + a
re-embed migration, never an agent change.

- agent: model-agnostic CropEmbedder (torch/transformers get_image_features,
  fp16 on CUDA, inference-locked); worker branches on job.task — 'ccip' emits
  figure(CCIP)+concept(SigLIP) in one pass, 'siglip' emits concept-only so the
  back-catalogue backfill never churns figure/CCIP regions; torch cu124 +
  transformers in the image.
- server: lease announces embed_model_name/embed_version; score_image is
  max-over-bag (version-filtered region embeddings); enqueue_gpu_backfill
  'siglip' gates on a missing concept region (drains the back-catalogue,
  retries failures, no double-enqueue); daily siglip-backfill beat; UI button;
  /api/ccip/overview reports images_with_concept_siglip.
- v1 scope: suggestion rail only — auto-apply stays whole-image (conservative;
  heads' thresholds were calibrated on whole-image). Bulk-apply bag = follow-up.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa

backend/app/api/ccip.py

@@ -37,6 +37,15 @@ async def overview():
                 .where(ImageRegion.ccip_embedding.is_not(None))
             )
         ).scalar_one()
+        # Concept-crop (SigLIP bag) coverage — how far the back-catalogue embed
+        # has progressed, so the max-over-bag scorer's reach is checkable.
+        images_with_concept_siglip = (
+            await session.execute(
+                select(func.count(distinct(ImageRegion.image_record_id)))
+                .where(ImageRegion.kind == "concept")
+                .where(ImageRegion.siglip_embedding.is_not(None))
+            )
+        ).scalar_one()
         # Per-character reference counts (no vectors loaded) — which characters
         # have enough examples to match on.
         ref_rows = (
@@ -72,6 +81,7 @@ async def overview():
     return jsonify({
         "regions_by_kind": by_kind,
         "images_with_figure_ccip": images_with_figure_ccip,
+        "images_with_concept_siglip": images_with_concept_siglip,
         "characters_with_references": len(ref_rows),
         "character_references": [
             {"tag_id": t, "name": n, "n_refs": c} for (t, n, c) in ref_rows

backend/app/api/gpu.py

@@ -17,6 +17,7 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert
 from ..extensions import get_session
 from ..models import AppSetting, GpuJob, ImageRecord, MLSettings
 from ..services.gallery_service import image_url
+from ..services.ml.embedder import MODEL_NAME as EMBED_MODEL_NAME
 from ..services.ml.gpu_jobs import GpuJobService
 from ..services.ml.regions import RegionService
 
@@ -137,6 +138,12 @@ async def lease():
                 # For video/animated: the agent samples at this cadence.
                 "frame_interval_seconds": ml.video_frame_interval_seconds,
                 "max_frames": ml.video_max_frames,
+                # The embedding model the agent must use for concept crops, so
+                # its region vectors land in the SAME space the heads trained in.
+                # Server-announced → the agent stays model-agnostic; a swap is a
+                # server setting + a re-embed migration, never an agent change.
+                "embed_model_name": EMBED_MODEL_NAME,
+                "embed_version": ml.embedder_model_version,
             })
     return jsonify({"jobs": out})
 

backend/app/celery_app.py

@@ -126,6 +126,11 @@ def make_celery() -> Celery:
                 "schedule": 3600.0,  # auto-feed new images (+ retry errored) so
                 "args": ("ccip",),   # the queue keeps moving without the button
             },
+            "enqueue-siglip-backfill-daily": {
+                "task": "backend.app.tasks.ml.enqueue_gpu_backfill",
+                "schedule": 86400.0,  # drain the concept-crop back-catalogue +
+                "args": ("siglip",),  # retry failed embeds, no button needed
+            },
             "ccip-auto-apply-daily": {
                 "task": "backend.app.tasks.ml.scheduled_ccip_auto_apply",
                 "schedule": 86400.0,  # no-op unless ccip_auto_apply_enabled

backend/app/services/ml/heads.py

@@ -29,6 +29,7 @@ from ...models import (
     HeadAutoApplyRun,
     HeadTrainingRun,
     ImageRecord,
+    ImageRegion,
     MLSettings,
     Tag,
     TagHead,
@@ -296,7 +297,14 @@ async def score_image(
     category, score}], ranked. A concept surfaces when its score clears the
     head's own suggest_threshold — or, when threshold_override is given (the
     typed-dropdown "show everything" mode), that flat floor instead (0 → every
-    head). Empty if the image has no embedding or no heads exist yet."""
+    head). Empty if the image has no embedding or no heads exist yet.
+
+    MAX-OVER-BAG: the image is scored as a BAG of embeddings — the whole-image
+    vector PLUS every concept-region crop the agent embedded (same model
+    version) — and each head takes its MAX score across the bag. A small/local
+    concept (glasses, a stomach bulge) that the whole-image vector washes out
+    can still surface from the crop where it dominates. The whole-image vector is
+    always in the bag, so this can never score lower than whole-image alone."""
     import numpy as np
 
     img = await session.get(ImageRecord, image_id)
@@ -306,11 +314,26 @@ async def score_image(
     heads = await _current_heads(session, settings.embedder_model_version)
     if heads["W"] is None:
         return []
-    x = np.asarray(img.siglip_embedding, dtype=np.float32)
-    n = float(np.linalg.norm(x)) or 1.0
-    xn = x / n
-    z = heads["W"] @ xn + heads["b"]
-    probs = 1.0 / (1.0 + np.exp(-z))
+
+    bag = [np.asarray(img.siglip_embedding, dtype=np.float32)]
+    region_vecs = (
+        await session.execute(
+            select(ImageRegion.siglip_embedding)
+            .where(ImageRegion.image_record_id == image_id)
+            .where(ImageRegion.siglip_embedding.is_not(None))
+            .where(ImageRegion.embedding_version == settings.embedder_model_version)
+        )
+    ).all()
+    for (vec,) in region_vecs:
+        if vec is not None:
+            bag.append(np.asarray(vec, dtype=np.float32))
+
+    X = np.vstack(bag)                                  # (B, D)
+    norms = np.linalg.norm(X, axis=1, keepdims=True)
+    norms[norms == 0] = 1.0
+    Xn = X / norms
+    Z = Xn @ heads["W"].T + heads["b"]                  # (B, H)
+    probs = (1.0 / (1.0 + np.exp(-Z))).max(axis=0)      # (H,) best over the bag
     out = []
     for i, p in enumerate(probs):
         cut = threshold_override if threshold_override is not None else heads["thr"][i]

backend/app/tasks/ml.py

@@ -742,24 +742,43 @@ def scheduled_apply_head_tags() -> str:
 
 @celery.task(name="backend.app.tasks.ml.enqueue_gpu_backfill")
 def enqueue_gpu_backfill(task_name: str) -> int:
-    """Enqueue a gpu_job for every image that doesn't already have one for
-    `task_name` (one INSERT…SELECT, so it scales to a full library). The desktop
-    agent drains the queue over HTTP. Returns the number enqueued."""
+    """Enqueue a gpu_job for every image that still needs `task_name` (one
+    INSERT…SELECT, so it scales to a full library). The desktop agent drains the
+    queue over HTTP. Returns the number enqueued.
+
+    'siglip' gates on the RESULT (no concept region yet) rather than on a prior
+    job, so it picks up the back-catalogue of images that were CCIP-embedded
+    before concept crops existed, and retries images whose concept embed failed —
+    without re-touching their figure/CCIP regions."""
     from sqlalchemy import exists, insert, literal
     from sqlalchemy import select as sa_select
 
-    from ..models import GpuJob, ImageRecord
+    from ..models import GpuJob, ImageRecord, ImageRegion
 
     SessionLocal = _sync_session_factory()
     with SessionLocal() as session:
-        already = exists().where(
-            GpuJob.image_record_id == ImageRecord.id,
-            GpuJob.task == task_name,
-            GpuJob.status.in_(["pending", "leased", "done"]),
-        )
-        sel = sa_select(
-            ImageRecord.id, literal(task_name), literal("pending")
-        ).where(~already)
+        if task_name == "siglip":
+            has_concept = exists().where(
+                ImageRegion.image_record_id == ImageRecord.id,
+                ImageRegion.kind == "concept",
+            )
+            queued = exists().where(
+                GpuJob.image_record_id == ImageRecord.id,
+                GpuJob.task == "siglip",
+                GpuJob.status.in_(["pending", "leased"]),
+            )
+            sel = sa_select(
+                ImageRecord.id, literal("siglip"), literal("pending")
+            ).where(~has_concept).where(~queued)
+        else:
+            already = exists().where(
+                GpuJob.image_record_id == ImageRecord.id,
+                GpuJob.task == task_name,
+                GpuJob.status.in_(["pending", "leased", "done"]),
+            )
+            sel = sa_select(
+                ImageRecord.id, literal(task_name), literal("pending")
+            ).where(~already)
         # RETURNING + count: result.rowcount is unreliable for INSERT…SELECT.
         rows = session.execute(
             insert(GpuJob)