From e8d3400d2214b3793aaa7a8905d8c9cdef2c4d92 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 29 Jun 2026 10:17:05 -0400 Subject: [PATCH] feat(crops): shared crop primitive for the region/crop pipeline (#114) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The trunk of both crop jobs — CCIP figure-crops and SigLIP concept-crops call the SAME crop_region(): normalized-bbox crop with optional context padding, edge-clamping, and the lower-bound size floor (max of a fraction-of-short-side and an absolute pixel floor) below which a region is too small to embed and returns None. Only the proposer (where) and embedder (what) differ; the crop is shared. Pure Pillow — importable + testable anywhere (the GPU agent imports it for the crop step). Unit-lane tests (no DB): region pixels, floor rejection, edge clamp, pad expansion, out-size resize. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa --- backend/app/services/ml/crops.py | 73 ++++++++++++++++++++++++++++++++ tests/test_crops.py | 44 +++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 backend/app/services/ml/crops.py create mode 100644 tests/test_crops.py diff --git a/backend/app/services/ml/crops.py b/backend/app/services/ml/crops.py new file mode 100644 index 0000000..ffb7b6a --- /dev/null +++ b/backend/app/services/ml/crops.py @@ -0,0 +1,73 @@ +"""Shared crop primitive for the region/crop pipeline (#114). + +One model- and transport-agnostic function sits at the trunk of both crop jobs: +- CCIP characters: a face/figure detector proposes regions → crop → CCIP-embed. +- SigLIP concepts: head-guided / saliency proposes regions → crop → SigLIP-embed. +Only the PROPOSER (where to crop) and the EMBEDDER (what to run) differ; the crop +itself — including the lower-bound size floor below which a region is too small to +embed reliably — is identical, so it lives here and both jobs call it. + +The actual detector + embedders run in the GPU agent; this is pure Pillow so it's +importable + testable anywhere (and the agent imports it for the crop step). +""" + +from __future__ import annotations + +from PIL import Image + +# Size floor: a region must be at least this big on its SHORTER edge to be worth +# embedding — a smaller crop is a blurry upscale carrying little real signal, and +# unbounded tiny crops would explode the bag. Expressed as BOTH a fraction of the +# image's short side and an absolute pixel floor; the larger of the two wins. +MIN_CROP_FRACTION = 0.10 +MIN_CROP_PX = 64 + + +def _to_pixels(bbox: tuple[float, float, float, float], w: int, h: int): + """Normalized (x, y, w, h) in [0,1] → pixel (x, y, w, h).""" + x, y, bw, bh = bbox + return x * w, y * h, bw * w, bh * h + + +def crop_region( + img: Image.Image, + bbox: tuple[float, float, float, float], + *, + pad: float = 0.0, + min_fraction: float = MIN_CROP_FRACTION, + min_px: int = MIN_CROP_PX, + out_size: int | None = None, +) -> Image.Image | None: + """Crop a NORMALIZED bbox (x, y, w, h in [0,1]) from img. + + - pad: grow the box by this fraction on each side (e.g. 0.15 = +15% context), + clamped to the image bounds. + - Returns None when the resulting region is below the size floor (too small to + embed reliably) — the caller skips embedding it. + - out_size: if given, resize the crop to out_size×out_size; otherwise return + the raw crop and let the embedder do its own preprocessing. + """ + iw, ih = img.size + px, py, pw, ph = _to_pixels(bbox, iw, ih) + + if pad: + px -= pw * pad / 2.0 + py -= ph * pad / 2.0 + pw *= (1.0 + pad) + ph *= (1.0 + pad) + + left = max(0, int(round(px))) + top = max(0, int(round(py))) + right = min(iw, int(round(px + pw))) + bottom = min(ih, int(round(py + ph))) + if right <= left or bottom <= top: + return None + + floor = max(min_px, int(min_fraction * min(iw, ih))) + if min(right - left, bottom - top) < floor: + return None + + crop = img.crop((left, top, right, bottom)).convert("RGB") + if out_size: + crop = crop.resize((out_size, out_size)) + return crop diff --git a/tests/test_crops.py b/tests/test_crops.py new file mode 100644 index 0000000..88e2310 --- /dev/null +++ b/tests/test_crops.py @@ -0,0 +1,44 @@ +"""Shared crop primitive (#114) — pure Pillow, no DB, so it runs in the fast +unit lane (no integration marker).""" +from PIL import Image + +from backend.app.services.ml.crops import crop_region + + +def _quadrant_img(): + """400x400 red with a blue bottom-right quadrant, so a crop's content is + checkable by pixel.""" + img = Image.new("RGB", (400, 400), (255, 0, 0)) + img.paste(Image.new("RGB", (200, 200), (0, 0, 255)), (200, 200)) + return img + + +def test_crop_returns_region_pixels(): + crop = crop_region(_quadrant_img(), (0.5, 0.5, 0.5, 0.5)) + assert crop is not None + assert crop.size == (200, 200) + assert crop.getpixel((100, 100)) == (0, 0, 255) # the blue quadrant + + +def test_crop_below_floor_is_rejected(): + # 0.05 * 400 = 20px on a side — below max(64, 0.10*400=40) → None. + assert crop_region(_quadrant_img(), (0.0, 0.0, 0.05, 0.05)) is None + + +def test_crop_clamped_to_image_bounds(): + # Box runs off the right/bottom edge; clamps to the remaining 0.2*400=80px. + crop = crop_region(_quadrant_img(), (0.8, 0.8, 0.5, 0.5)) + assert crop is not None + assert crop.size == (80, 80) + + +def test_pad_expands_the_crop(): + base = crop_region(_quadrant_img(), (0.4, 0.4, 0.2, 0.2)) + padded = crop_region(_quadrant_img(), (0.4, 0.4, 0.2, 0.2), pad=0.5) + assert base.size == (80, 80) + assert padded.size[0] > base.size[0] and padded.size[1] > base.size[1] + + +def test_out_size_resizes_square(): + crop = crop_region(_quadrant_img(), (0.25, 0.25, 0.5, 0.5), out_size=224) + assert crop.size == (224, 224)