diff --git a/backend/app/services/ml/ccip.py b/backend/app/services/ml/ccip.py index 8c98edd..0a30def 100644 --- a/backend/app/services/ml/ccip.py +++ b/backend/app/services/ml/ccip.py @@ -161,16 +161,22 @@ async def match_image( if threshold is None: threshold = await _settings_threshold(session) - qvecs = ( + # Keep each figure region's bbox alongside its vector so a match can point at + # the figure that matched (#1206 grounding), not just the score. + fig_rows = ( await session.execute( - select(ImageRegion.ccip_embedding).where( + select( + ImageRegion.ccip_embedding, + ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh, + ImageRegion.kind, ImageRegion.detector_version, + ).where( ImageRegion.image_record_id == image_id, ImageRegion.kind.in_(_FIGURE_KINDS), ImageRegion.ccip_embedding.is_not(None), ) ) - ).scalars().all() - if not qvecs: + ).all() + if not fig_rows: return [] refs = await character_references(session) if not refs: @@ -186,13 +192,21 @@ async def match_image( ) names = await _tag_names(session, [t for t in refs if t not in applied]) + qvecs = [r[0] for r in fig_rows] + fig_meta = [ + {"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector} + for _v, rx, ry, rw, rh, kind, detector in fig_rows + ] Q = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in qvecs]), np) out = [] for tag_id, vecs in refs.items(): if tag_id in applied: continue R = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np) - best = float((Q @ R.T).max()) # best (query figure, reference) cosine + sims = Q @ R.T # (n_query_figures, n_references) + per_figure = sims.max(axis=1) # best reference cosine per figure + best_figure = int(per_figure.argmax()) + best = float(per_figure[best_figure]) if best >= threshold: out.append({ "tag_id": tag_id, @@ -200,6 +214,8 @@ async def match_image( "category": "character", "score": round(best, 4), "source": "ccip", + # the figure region that matched → grounds the character tag. + "grounding": fig_meta[best_figure], }) out.sort(key=lambda d: d["score"], reverse=True) return out diff --git a/backend/app/services/ml/suggestions.py b/backend/app/services/ml/suggestions.py index 772074c..eba40ad 100644 --- a/backend/app/services/ml/suggestions.py +++ b/backend/app/services/ml/suggestions.py @@ -116,9 +116,13 @@ class SuggestionService: if ex is not None: ex["source"] = "both" ex["score"] = max(ex["score"], c["score"]) + # Keep the head's localized crop if it had one; else fall back to + # the CCIP figure so a corroborated character still grounds (#1206). + ex["grounding"] = ex.get("grounding") or c.get("grounding") else: merged[key] = { "name": c["name"], "score": c["score"], "source": "ccip", + "grounding": c.get("grounding"), } result = SuggestionList() diff --git a/tests/test_ccip.py b/tests/test_ccip.py index ad1dacf..8231c4b 100644 --- a/tests/test_ccip.py +++ b/tests/test_ccip.py @@ -55,6 +55,10 @@ async def test_matches_same_character_across_images(db): m = next(x for x in matches if x["tag_id"] == raven.id) assert m["source"] == "ccip" and m["category"] == "character" assert m["score"] > 0.9 + # #1206: the match grounds to the figure region that matched (hover → the + # figure box lights up), so a character suggestion is localized too. + assert m["grounding"]["bbox"] == pytest.approx([0.0, 0.0, 1.0, 1.0]) + assert m["grounding"]["kind"] == "figure" @pytest.mark.asyncio