feat(ml): argmax grounding in score_image → suggestions carry the winning crop (#133 step 1)
score_image now keeps the ARGMAX beside the max-over-bag: which bag row won each
head. The region query also selects bbox/kind/detector_version, a parallel
bag_meta maps each row → its region (None for the whole-image vector), and every
hit gains grounding {bbox,kind,detector} (null when the global vector won). Threaded
through SuggestionService (new Suggestion.grounding field) → /api/.../suggestions
payload. This is the data the #1206 hover-overlay draws. CCIP-only hits ground null
for now (figure grounding = step 2). Tests: winning crop grounds the tag with its
bbox+kind; whole-image win → grounding None.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
@@ -375,21 +375,33 @@ async def score_image(
|
||||
# skipped rather than scored by heads trained in a different space; a legacy
|
||||
# NULL version is treated as current (those predate per-row stamping).
|
||||
bag = []
|
||||
# Parallel to `bag`: what each row IS, so a surfaced tag can point back at the
|
||||
# crop that produced it (#1206 grounding). None = the whole-image vector (not
|
||||
# localized); a dict = a region's {bbox, kind, detector}.
|
||||
bag_meta: list[dict | None] = []
|
||||
if img.siglip_embedding is not None and img.siglip_model_version in (
|
||||
cur_version, None,
|
||||
):
|
||||
bag.append(np.asarray(img.siglip_embedding, dtype=np.float32))
|
||||
region_vecs = (
|
||||
bag_meta.append(None)
|
||||
region_rows = (
|
||||
await session.execute(
|
||||
select(ImageRegion.siglip_embedding)
|
||||
select(
|
||||
ImageRegion.siglip_embedding,
|
||||
ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh,
|
||||
ImageRegion.kind, ImageRegion.detector_version,
|
||||
)
|
||||
.where(ImageRegion.image_record_id == image_id)
|
||||
.where(ImageRegion.siglip_embedding.is_not(None))
|
||||
.where(ImageRegion.embedding_version == cur_version)
|
||||
)
|
||||
).all()
|
||||
for (vec,) in region_vecs:
|
||||
for vec, rx, ry, rw, rh, kind, detector in region_rows:
|
||||
if vec is not None:
|
||||
bag.append(np.asarray(vec, dtype=np.float32))
|
||||
bag_meta.append(
|
||||
{"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector}
|
||||
)
|
||||
if not bag:
|
||||
return []
|
||||
|
||||
@@ -398,7 +410,11 @@ async def score_image(
|
||||
norms[norms == 0] = 1.0
|
||||
Xn = X / norms
|
||||
Z = Xn @ heads["W"].T + heads["b"] # (B, H)
|
||||
probs = (1.0 / (1.0 + np.exp(-Z))).max(axis=0) # (H,) best over the bag
|
||||
probs_bag = 1.0 / (1.0 + np.exp(-Z)) # (B, H)
|
||||
probs = probs_bag.max(axis=0) # (H,) best over the bag
|
||||
# ARGMAX beside the max: WHICH bag row won each head → the region that grounds
|
||||
# the tag (bag_meta[win]); None when the whole-image vector won (#1206).
|
||||
winners = probs_bag.argmax(axis=0) # (H,)
|
||||
out = []
|
||||
for i, p in enumerate(probs):
|
||||
if threshold_override is not None:
|
||||
@@ -416,6 +432,7 @@ async def score_image(
|
||||
"name": m["name"],
|
||||
"category": m["category"],
|
||||
"score": float(p),
|
||||
"grounding": bag_meta[int(winners[i])],
|
||||
})
|
||||
out.sort(key=lambda d: d["score"], reverse=True)
|
||||
return out
|
||||
|
||||
Reference in New Issue
Block a user