feat(agent): dedupe near-duplicate crops before the SigLIP embed

Figure boxes are already NMS-merged (iou 0.6) and each YOLO detector self-NMSes,
but the combined per-frame crop pile (figure→concept ∪ anatomy component→concept
∪ panel) was embedded with no cross-proposer dedup — so genuine near-duplicates
slipped through (a figure box ≈ an anatomy component on a solo bust; overlapping
booru head classes on one head), embedding the same region twice and burning a
slot against max_regions.

Add detectors.dedupe_crops(): a greedy, high-IoU (default 0.85), kind-aware pass
over the pending (crop, template) list right before embed_batch — drop boxes that
overlap ≥ iou within the same kind, keep the highest score. The high threshold is
deliberate: it collapses only true near-identical boxes while preserving
intentional nested crops across scopes (a whole figure vs a small head component
sit well below it) and distinct kinds (concept vs panel). Env-tunable DEDUPE_IOU
(≥1.0 disables). Runs on CPU before the GPU work, so it cuts both embed cost and
region count. Temporal (cross-frame) dedup deferred. Build marker 2026-07-01.2.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa

agent/fc_agent/app.py

@@ -17,7 +17,7 @@ from .worker import Worker
 # Bump on every agent change. The page embeds this and /status reports it; the UI
 # warns to reload when they differ — so a stale browser-cached page can't be
 # mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.)
-VERSION = "2026-07-01.1 · download/GPU pipeline"
+VERSION = "2026-07-01.2 · crop dedup before embed"
 
 logbuf.install()
 cfg = Config.from_env()

agent/fc_agent/config.py

@@ -36,6 +36,8 @@ class Config:
     max_panels: int       # cap panel crops per frame
     max_figures: int      # cap figure boxes per frame (each = a CCIP call + crop)
     max_regions: int      # hard cap on total regions per JOB (submit-size backstop)
+    dedupe_iou: float     # crops overlapping >= this (same kind) are near-dupes,
+    #                       dropped before the embed; >=1.0 disables it
 
     @classmethod
     def from_env(cls) -> Config:
@@ -62,4 +64,5 @@ class Config:
             max_panels=int(os.environ.get("MAX_PANELS", "8")),
             max_figures=int(os.environ.get("MAX_FIGURES", "8")),
             max_regions=int(os.environ.get("MAX_REGIONS", "128")),
+            dedupe_iou=float(os.environ.get("DEDUPE_IOU", "0.85")),
         )

agent/fc_agent/detectors.py

@@ -67,6 +67,37 @@ def nms_merge(boxes, iou_thresh: float = 0.6):
     return kept
 
 
+def dedupe_crops(pending, iou_thresh: float = 0.85):
+    """Greedy high-IoU dedupe over a list of (crop, region_template) pairs, run
+    just before the batched SigLIP embed so we never embed the same region twice.
+
+    Figure boxes are already NMS-merged and each YOLO self-NMSes, but the combined
+    per-frame pile (figure→concept ∪ anatomy component→concept ∪ panel) can still
+    carry genuine near-duplicates across proposers — e.g. a figure box that nearly
+    coincides with an anatomy component on a solo bust, or overlapping booru head
+    classes on one head. Those embed the same region twice, wasting GPU and a slot
+    against max_regions.
+
+    Boxes are compared ONLY within the same output kind and dropped when they
+    overlap at >= iou_thresh, keeping the highest-scoring one. The HIGH default
+    threshold is deliberate: it collapses only true near-identical boxes while
+    preserving intentional nested crops across scopes (a whole figure vs a small
+    head component sit well below it) and distinct kinds (concept vs panel). A
+    value >= 1.0 effectively disables it (nothing but an exact box matches)."""
+    kept = []
+    kept_boxes: dict = {}   # kind -> [bbox, ...] already kept
+    for crop, tmpl in sorted(
+        pending, key=lambda p: p[1].get("score") or 0.0, reverse=True
+    ):
+        bb = tmpl.get("bbox")
+        prior = kept_boxes.setdefault(tmpl.get("kind"), [])
+        if bb is not None and any(_iou(bb, kb) >= iou_thresh for kb in prior):
+            continue
+        prior.append(bb)
+        kept.append((crop, tmpl))
+    return kept
+
+
 class YoloProposer:
     """One lazily-loaded ultralytics YOLO. detect(image) → [(bbox_norm, score,
     label)] with bbox normalized (x, y, w, h) in [0,1]. Self-disables on any

agent/fc_agent/worker.py

@@ -40,6 +40,7 @@ from . import media, models
 from .client import FcClient
 from .config import Config
 from .crops import crop_region
+from .detectors import dedupe_crops
 
 # Cap on the lease-retry backoff: when curator is unreachable (e.g. you redeploy
 # it while away), a downloader retries leasing with exponential backoff up to this
@@ -629,6 +630,12 @@ class Worker:
                             "score": score, "detector_version": "panel",
                         }))
                 if pending:
+                    # Drop near-duplicate crops (a figure box ≈ an anatomy
+                    # component, overlapping booru classes) before the embed so we
+                    # never SigLIP the same region twice — saves GPU and a slot
+                    # against max_regions. High-IoU + kind-aware, so intentional
+                    # nested crops (figure ⊃ head) survive.
+                    pending = dedupe_crops(pending, self.cfg.dedupe_iou)
                     vecs = embedder.embed_batch([c for c, _ in pending])
                     for (_c, tmpl), vec in zip(pending, vecs, strict=True):
                         tmpl["siglip_embedding"] = vec