From 409724b98106cf251ce52e3bb8996589a8720b48 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 5 Jul 2026 23:13:29 -0400 Subject: [PATCH 1/5] =?UTF-8?q?feat(ml):=20argmax=20grounding=20in=20score?= =?UTF-8?q?=5Fimage=20=E2=86=92=20suggestions=20carry=20the=20winning=20cr?= =?UTF-8?q?op=20(#133=20step=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit score_image now keeps the ARGMAX beside the max-over-bag: which bag row won each head. The region query also selects bbox/kind/detector_version, a parallel bag_meta maps each row → its region (None for the whole-image vector), and every hit gains grounding {bbox,kind,detector} (null when the global vector won). Threaded through SuggestionService (new Suggestion.grounding field) → /api/.../suggestions payload. This is the data the #1206 hover-overlay draws. CCIP-only hits ground null for now (figure grounding = step 2). Tests: winning crop grounds the tag with its bbox+kind; whole-image win → grounding None. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- backend/app/api/suggestions.py | 4 ++++ backend/app/services/ml/heads.py | 25 +++++++++++++++++++++---- backend/app/services/ml/suggestions.py | 7 +++++++ tests/test_ml_suggestions.py | 26 +++++++++++++++++++++++++- 4 files changed, 57 insertions(+), 5 deletions(-) diff --git a/backend/app/api/suggestions.py b/backend/app/api/suggestions.py index cf27125..28df98e 100644 --- a/backend/app/api/suggestions.py +++ b/backend/app/api/suggestions.py @@ -46,6 +46,10 @@ async def get_suggestions(image_id: int): # (not dropped) so the rail can show it rejected + offer # one-click un-reject. "rejected": s.rejected, + # the crop region that produced this tag (#1206) — + # {bbox,kind,detector} or null (whole-image won). Drives + # the hover→overlay highlight. + "grounding": s.grounding, } for s in items ] diff --git a/backend/app/services/ml/heads.py b/backend/app/services/ml/heads.py index 1f4d01a..aad0226 100644 --- a/backend/app/services/ml/heads.py +++ b/backend/app/services/ml/heads.py @@ -375,21 +375,33 @@ async def score_image( # skipped rather than scored by heads trained in a different space; a legacy # NULL version is treated as current (those predate per-row stamping). bag = [] + # Parallel to `bag`: what each row IS, so a surfaced tag can point back at the + # crop that produced it (#1206 grounding). None = the whole-image vector (not + # localized); a dict = a region's {bbox, kind, detector}. + bag_meta: list[dict | None] = [] if img.siglip_embedding is not None and img.siglip_model_version in ( cur_version, None, ): bag.append(np.asarray(img.siglip_embedding, dtype=np.float32)) - region_vecs = ( + bag_meta.append(None) + region_rows = ( await session.execute( - select(ImageRegion.siglip_embedding) + select( + ImageRegion.siglip_embedding, + ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh, + ImageRegion.kind, ImageRegion.detector_version, + ) .where(ImageRegion.image_record_id == image_id) .where(ImageRegion.siglip_embedding.is_not(None)) .where(ImageRegion.embedding_version == cur_version) ) ).all() - for (vec,) in region_vecs: + for vec, rx, ry, rw, rh, kind, detector in region_rows: if vec is not None: bag.append(np.asarray(vec, dtype=np.float32)) + bag_meta.append( + {"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector} + ) if not bag: return [] @@ -398,7 +410,11 @@ async def score_image( norms[norms == 0] = 1.0 Xn = X / norms Z = Xn @ heads["W"].T + heads["b"] # (B, H) - probs = (1.0 / (1.0 + np.exp(-Z))).max(axis=0) # (H,) best over the bag + probs_bag = 1.0 / (1.0 + np.exp(-Z)) # (B, H) + probs = probs_bag.max(axis=0) # (H,) best over the bag + # ARGMAX beside the max: WHICH bag row won each head → the region that grounds + # the tag (bag_meta[win]); None when the whole-image vector won (#1206). + winners = probs_bag.argmax(axis=0) # (H,) out = [] for i, p in enumerate(probs): if threshold_override is not None: @@ -416,6 +432,7 @@ async def score_image( "name": m["name"], "category": m["category"], "score": float(p), + "grounding": bag_meta[int(winners[i])], }) out.sort(key=lambda d: d["score"], reverse=True) return out diff --git a/backend/app/services/ml/suggestions.py b/backend/app/services/ml/suggestions.py index 71fbd24..772074c 100644 --- a/backend/app/services/ml/suggestions.py +++ b/backend/app/services/ml/suggestions.py @@ -43,6 +43,11 @@ class Suggestion: # the rejection is VISIBLE and REVERSIBLE in the rail (misclick recovery, # operator-asked 2026-06-27) instead of silently vanishing or re-suggesting. rejected: bool = False + # grounding = the crop region that produced this suggestion (#1206): + # {bbox:[x,y,w,h] normalized, kind, detector}. None when the whole-image + # vector won (not localized) or for a CCIP-only hit (figure grounding TBD). + # Lets the rail highlight the exact region on hover. + grounding: dict | None = None @dataclass @@ -103,6 +108,7 @@ class SuggestionService: for h in hits: merged[(h["category"], h["tag_id"])] = { "name": h["name"], "score": h["score"], "source": "head", + "grounding": h.get("grounding"), } for c in ccip_hits: key = ("character", c["tag_id"]) @@ -128,6 +134,7 @@ class SuggestionService: source=m["source"], creates_new_tag=False, rejected=tag_id in rejected, + grounding=m.get("grounding"), ) ) for cat in result.by_category: diff --git a/tests/test_ml_suggestions.py b/tests/test_ml_suggestions.py index f8f4d00..cf2b732 100644 --- a/tests/test_ml_suggestions.py +++ b/tests/test_ml_suggestions.py @@ -187,7 +187,31 @@ async def test_concept_region_surfaces_via_max_over_bag(db): )) await db.commit() general = (await SuggestionService(db).for_image(img.id)).by_category["general"] - assert any(s.canonical_tag_id == tag.id and s.score > 0.7 for s in general) + s = next(x for x in general if x.canonical_tag_id == tag.id) + assert s.score > 0.7 + # #1206: the winning crop grounds the tag — hover highlights THIS region + # (the matching-version crop at 0.4,0.4,0.3,0.3), not the whole image. + assert s.grounding is not None + assert s.grounding["bbox"] == pytest.approx([0.4, 0.4, 0.3, 0.3]) + assert s.grounding["kind"] == "concept" + + +@pytest.mark.asyncio +async def test_grounding_none_when_whole_image_wins(db): + # #1206: when the whole-image vector (not a crop) produces the winning score, + # grounding is None — the tag came from the global vector, not a region. + tag = await TagService(db).find_or_create("sky", TagKind.general) + img = await _img(db, "d1" * 32, _emb(0)) # whole-image ALIGNED w/ head + await _head(db, tag.id, slot=0, suggest_threshold=0.5) + db.add(ImageRegion( # an orthogonal crop (0.5) + image_record_id=img.id, kind="concept", + rx=0.1, ry=0.1, rw=0.2, rh=0.2, + siglip_embedding=_emb(5), embedding_version=await _embver(db), + )) + await db.commit() + general = (await SuggestionService(db).for_image(img.id)).by_category["general"] + s = next(x for x in general if x.canonical_tag_id == tag.id) + assert s.grounding is None @pytest.mark.asyncio -- 2.52.0 From dfe2fda564678e52f78a7bfab6285e558de9ec1f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 5 Jul 2026 23:18:41 -0400 Subject: [PATCH 2/5] feat(ml): CCIP character matches ground to the matched figure region (#133 step 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit match_image now tracks WHICH query figure produced the winning cosine per character (argmax over the per-figure best-reference sim) and attaches its bbox as grounding {bbox,kind:'figure',detector}. SuggestionService carries it: a CCIP-only character hit grounds to its figure; a 'both' hit keeps the head's localized crop if it had one, else falls back to the CCIP figure — so corroborated characters stay grounded. Test: a character match carries the matched figure's bbox+kind. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- backend/app/services/ml/ccip.py | 26 +++++++++++++++++++++----- backend/app/services/ml/suggestions.py | 4 ++++ tests/test_ccip.py | 4 ++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/backend/app/services/ml/ccip.py b/backend/app/services/ml/ccip.py index 8c98edd..0a30def 100644 --- a/backend/app/services/ml/ccip.py +++ b/backend/app/services/ml/ccip.py @@ -161,16 +161,22 @@ async def match_image( if threshold is None: threshold = await _settings_threshold(session) - qvecs = ( + # Keep each figure region's bbox alongside its vector so a match can point at + # the figure that matched (#1206 grounding), not just the score. + fig_rows = ( await session.execute( - select(ImageRegion.ccip_embedding).where( + select( + ImageRegion.ccip_embedding, + ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh, + ImageRegion.kind, ImageRegion.detector_version, + ).where( ImageRegion.image_record_id == image_id, ImageRegion.kind.in_(_FIGURE_KINDS), ImageRegion.ccip_embedding.is_not(None), ) ) - ).scalars().all() - if not qvecs: + ).all() + if not fig_rows: return [] refs = await character_references(session) if not refs: @@ -186,13 +192,21 @@ async def match_image( ) names = await _tag_names(session, [t for t in refs if t not in applied]) + qvecs = [r[0] for r in fig_rows] + fig_meta = [ + {"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector} + for _v, rx, ry, rw, rh, kind, detector in fig_rows + ] Q = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in qvecs]), np) out = [] for tag_id, vecs in refs.items(): if tag_id in applied: continue R = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np) - best = float((Q @ R.T).max()) # best (query figure, reference) cosine + sims = Q @ R.T # (n_query_figures, n_references) + per_figure = sims.max(axis=1) # best reference cosine per figure + best_figure = int(per_figure.argmax()) + best = float(per_figure[best_figure]) if best >= threshold: out.append({ "tag_id": tag_id, @@ -200,6 +214,8 @@ async def match_image( "category": "character", "score": round(best, 4), "source": "ccip", + # the figure region that matched → grounds the character tag. + "grounding": fig_meta[best_figure], }) out.sort(key=lambda d: d["score"], reverse=True) return out diff --git a/backend/app/services/ml/suggestions.py b/backend/app/services/ml/suggestions.py index 772074c..eba40ad 100644 --- a/backend/app/services/ml/suggestions.py +++ b/backend/app/services/ml/suggestions.py @@ -116,9 +116,13 @@ class SuggestionService: if ex is not None: ex["source"] = "both" ex["score"] = max(ex["score"], c["score"]) + # Keep the head's localized crop if it had one; else fall back to + # the CCIP figure so a corroborated character still grounds (#1206). + ex["grounding"] = ex.get("grounding") or c.get("grounding") else: merged[key] = { "name": c["name"], "score": c["score"], "source": "ccip", + "grounding": c.get("grounding"), } result = SuggestionList() diff --git a/tests/test_ccip.py b/tests/test_ccip.py index ad1dacf..8231c4b 100644 --- a/tests/test_ccip.py +++ b/tests/test_ccip.py @@ -55,6 +55,10 @@ async def test_matches_same_character_across_images(db): m = next(x for x in matches if x["tag_id"] == raven.id) assert m["source"] == "ccip" and m["category"] == "character" assert m["score"] > 0.9 + # #1206: the match grounds to the figure region that matched (hover → the + # figure box lights up), so a character suggestion is localized too. + assert m["grounding"]["bbox"] == pytest.approx([0.0, 0.0, 1.0, 1.0]) + assert m["grounding"]["kind"] == "figure" @pytest.mark.asyncio -- 2.52.0 From 524a26c61894c03d77e49d7d370aa22b6152ed71 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 5 Jul 2026 23:24:18 -0400 Subject: [PATCH 3/5] =?UTF-8?q?feat(ui):=20hover=20a=20suggestion=20?= =?UTF-8?q?=E2=86=92=20highlight=20the=20crop=20region=20it=20came=20from?= =?UTF-8?q?=20(#133=20step=203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The payoff: hover a suggestion in the rail and the exact crop that produced it lights up on the image (a booru:head, a panel, a figure); a null-grounding tag shows a subtle dashed whole-image frame ('global vector won, not a crop'). ImageCanvas gains a grounding overlay that tracks the 's live bounding rect (correct under object-fit letterboxing + pan/zoom) and draws the normalized bbox + a detector/kind label. SuggestionItem sets the hovered grounding via provide/inject (no 4-level event relay through TagPanel/SuggestionsPanel/group); ImageViewer AND ExploreView provide it + pass it to their canvas. Overlay is pointer-events:none so it never blocks pan/zoom/click. Videos out of v1 scope. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- frontend/src/components/modal/ImageCanvas.vue | 113 +++++++++++++++++- frontend/src/components/modal/ImageViewer.vue | 10 +- .../src/components/modal/SuggestionItem.vue | 17 ++- frontend/src/views/ExploreView.vue | 8 +- 4 files changed, 142 insertions(+), 6 deletions(-) diff --git a/frontend/src/components/modal/ImageCanvas.vue b/frontend/src/components/modal/ImageCanvas.vue index a4a4163..efe2e52 100644 --- a/frontend/src/components/modal/ImageCanvas.vue +++ b/frontend/src/components/modal/ImageCanvas.vue @@ -16,20 +16,91 @@ transform: `translate(${panZoom.state.x}px, ${panZoom.state.y}px) scale(${panZoom.state.scale})` }" draggable="false" + @load="onImgLoad" > + +
+
+ {{ regionLabel }} +
+