diff --git a/backend/app/api/suggestions.py b/backend/app/api/suggestions.py index cf27125..28df98e 100644 --- a/backend/app/api/suggestions.py +++ b/backend/app/api/suggestions.py @@ -46,6 +46,10 @@ async def get_suggestions(image_id: int): # (not dropped) so the rail can show it rejected + offer # one-click un-reject. "rejected": s.rejected, + # the crop region that produced this tag (#1206) — + # {bbox,kind,detector} or null (whole-image won). Drives + # the hover→overlay highlight. + "grounding": s.grounding, } for s in items ] diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py index ed156c8..50539cd 100644 --- a/backend/app/api/tags.py +++ b/backend/app/api/tags.py @@ -11,6 +11,7 @@ from ..models.tag import image_tag from ..models.tag_suggestion_rejection import TagSuggestionRejection from ..services.bulk_tag_service import BulkTagService from ..services.ml.aliases import AliasService +from ..services.ml.heads import ground_applied_tag from ..services.series_match_service import SeriesMatchService from ..services.series_service import SeriesError, SeriesService from ..services.tag_directory_service import TagDirectoryService @@ -310,6 +311,21 @@ async def confirm_tag_on_image(image_id: int, tag_id: int): return "", 204 +@tags_bp.route( + "/images//tags//grounding", methods=["GET"] +) +async def tag_grounding(image_id: int, tag_id: int): + """Which crop region best explains an ALREADY-APPLIED tag on this image + (#1206 Step 4). Powers the hover→overlay highlight on applied tag chips, + mirroring the suggestion rail's live grounding. Computed on demand (applied + tags aren't scored live). → {grounding: {bbox,kind,detector}|null, + has_head: bool}; has_head False means the tag has no head to localize with, + so the chip shows no overlay.""" + async with get_session() as session: + grounding, has_head = await ground_applied_tag(session, image_id, tag_id) + return jsonify({"grounding": grounding, "has_head": has_head}) + + @tags_bp.route("/tags/", methods=["GET"]) async def get_tag(tag_id: int): """Resolve a single tag (used by the gallery to label its active diff --git a/backend/app/services/ml/ccip.py b/backend/app/services/ml/ccip.py index 8c98edd..0a30def 100644 --- a/backend/app/services/ml/ccip.py +++ b/backend/app/services/ml/ccip.py @@ -161,16 +161,22 @@ async def match_image( if threshold is None: threshold = await _settings_threshold(session) - qvecs = ( + # Keep each figure region's bbox alongside its vector so a match can point at + # the figure that matched (#1206 grounding), not just the score. + fig_rows = ( await session.execute( - select(ImageRegion.ccip_embedding).where( + select( + ImageRegion.ccip_embedding, + ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh, + ImageRegion.kind, ImageRegion.detector_version, + ).where( ImageRegion.image_record_id == image_id, ImageRegion.kind.in_(_FIGURE_KINDS), ImageRegion.ccip_embedding.is_not(None), ) ) - ).scalars().all() - if not qvecs: + ).all() + if not fig_rows: return [] refs = await character_references(session) if not refs: @@ -186,13 +192,21 @@ async def match_image( ) names = await _tag_names(session, [t for t in refs if t not in applied]) + qvecs = [r[0] for r in fig_rows] + fig_meta = [ + {"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector} + for _v, rx, ry, rw, rh, kind, detector in fig_rows + ] Q = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in qvecs]), np) out = [] for tag_id, vecs in refs.items(): if tag_id in applied: continue R = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np) - best = float((Q @ R.T).max()) # best (query figure, reference) cosine + sims = Q @ R.T # (n_query_figures, n_references) + per_figure = sims.max(axis=1) # best reference cosine per figure + best_figure = int(per_figure.argmax()) + best = float(per_figure[best_figure]) if best >= threshold: out.append({ "tag_id": tag_id, @@ -200,6 +214,8 @@ async def match_image( "category": "character", "score": round(best, 4), "source": "ccip", + # the figure region that matched → grounds the character tag. + "grounding": fig_meta[best_figure], }) out.sort(key=lambda d: d["score"], reverse=True) return out diff --git a/backend/app/services/ml/heads.py b/backend/app/services/ml/heads.py index 1f4d01a..bb18c25 100644 --- a/backend/app/services/ml/heads.py +++ b/backend/app/services/ml/heads.py @@ -341,6 +341,53 @@ async def _current_heads(session: AsyncSession, embedding_version: str): return loaded +async def _image_bag( + session: AsyncSession, image_id: int, cur_version: str, +) -> tuple[list, list[dict | None]]: + """The max-over-bag inputs for one image: the whole-image SigLIP vector (when + it's in the current model's space) PLUS every concept-region crop embedded in + that space. Returns (bag, bag_meta) as PARALLEL lists — bag_meta[i] is None for + the whole-image row, else the region's {bbox, kind, detector} so a surfaced tag + can point back at the crop that produced it (#1206 grounding). + + Only current-version embeddings enter the bag: mid model-swap (#1190) an image + still carrying an OLD-version whole-image vector is skipped rather than scored + by heads trained in a different space; a legacy NULL version is treated as + current (those predate per-row stamping). Shared by live scoring (score_image) + and on-demand applied-tag grounding (ground_applied_tag, #1206 Step 4).""" + import numpy as np + + img = await session.get(ImageRecord, image_id) + bag: list = [] + bag_meta: list[dict | None] = [] + if img is None: + return bag, bag_meta + if img.siglip_embedding is not None and img.siglip_model_version in ( + cur_version, None, + ): + bag.append(np.asarray(img.siglip_embedding, dtype=np.float32)) + bag_meta.append(None) + region_rows = ( + await session.execute( + select( + ImageRegion.siglip_embedding, + ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh, + ImageRegion.kind, ImageRegion.detector_version, + ) + .where(ImageRegion.image_record_id == image_id) + .where(ImageRegion.siglip_embedding.is_not(None)) + .where(ImageRegion.embedding_version == cur_version) + ) + ).all() + for vec, rx, ry, rw, rh, kind, detector in region_rows: + if vec is not None: + bag.append(np.asarray(vec, dtype=np.float32)) + bag_meta.append( + {"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector} + ) + return bag, bag_meta + + async def score_image( session: AsyncSession, image_id: int, threshold_override: float | None = None, ) -> list[dict]: @@ -361,35 +408,12 @@ async def score_image( always in the bag, so this can never score lower than whole-image alone.""" import numpy as np - img = await session.get(ImageRecord, image_id) - if img is None: - return [] settings = await _settings_async(session) cur_version = settings.embedder_model_version heads = await _current_heads(session, cur_version) if heads["W"] is None: return [] - - # Only embeddings in the CURRENT model's space enter the bag. Mid model-swap - # (#1190), an image still carrying the OLD-version whole-image vector is - # skipped rather than scored by heads trained in a different space; a legacy - # NULL version is treated as current (those predate per-row stamping). - bag = [] - if img.siglip_embedding is not None and img.siglip_model_version in ( - cur_version, None, - ): - bag.append(np.asarray(img.siglip_embedding, dtype=np.float32)) - region_vecs = ( - await session.execute( - select(ImageRegion.siglip_embedding) - .where(ImageRegion.image_record_id == image_id) - .where(ImageRegion.siglip_embedding.is_not(None)) - .where(ImageRegion.embedding_version == cur_version) - ) - ).all() - for (vec,) in region_vecs: - if vec is not None: - bag.append(np.asarray(vec, dtype=np.float32)) + bag, bag_meta = await _image_bag(session, image_id, cur_version) if not bag: return [] @@ -398,7 +422,11 @@ async def score_image( norms[norms == 0] = 1.0 Xn = X / norms Z = Xn @ heads["W"].T + heads["b"] # (B, H) - probs = (1.0 / (1.0 + np.exp(-Z))).max(axis=0) # (H,) best over the bag + probs_bag = 1.0 / (1.0 + np.exp(-Z)) # (B, H) + probs = probs_bag.max(axis=0) # (H,) best over the bag + # ARGMAX beside the max: WHICH bag row won each head → the region that grounds + # the tag (bag_meta[win]); None when the whole-image vector won (#1206). + winners = probs_bag.argmax(axis=0) # (H,) out = [] for i, p in enumerate(probs): if threshold_override is not None: @@ -416,11 +444,57 @@ async def score_image( "name": m["name"], "category": m["category"], "score": float(p), + "grounding": bag_meta[int(winners[i])], }) out.sort(key=lambda d: d["score"], reverse=True) return out +async def ground_applied_tag( + session: AsyncSession, image_id: int, tag_id: int, +) -> tuple[dict | None, bool]: + """On-demand grounding for an ALREADY-APPLIED tag (#1206 Step 4). Applied tags + aren't scored live, so recompute the max-over-bag argmax for just this tag's + head — which crop region best explains the tag on this image — mirroring what + score_image records for live suggestions. Returns (grounding, has_head): + + - has_head False → the tag has no head in the current embedding space (manual/ + artist/meta tags, or a concept below the head floor). Nothing to localize + with, so the UI shows no overlay (distinct from "the whole image won"). + - grounding None (has_head True) → the whole-image vector best explains it, + not any crop; the UI shows the subtle whole-image frame. + - grounding {bbox, kind, detector} → the winning region. + + Character heads are covered too (character is a head kind); this deliberately + reuses the SigLIP head bag rather than the CCIP figure path so every applied + concept grounds through one consistent mechanism.""" + import numpy as np + + cur_version = (await _settings_async(session)).embedder_model_version + row = ( + await session.execute( + select(TagHead.weights, TagHead.bias).where( + TagHead.tag_id == tag_id, + TagHead.embedding_version == cur_version, + ) + ) + ).one_or_none() + if row is None: + return None, False + bag, bag_meta = await _image_bag(session, image_id, cur_version) + if not bag: + return None, True + + X = np.vstack(bag) + norms = np.linalg.norm(X, axis=1, keepdims=True) + norms[norms == 0] = 1.0 + Xn = X / norms + # The sigmoid is monotonic in the logit, so the highest-probability bag row is + # just argmax of the raw score — no need to exponentiate to pick the winner. + z = Xn @ np.asarray(row.weights, dtype=np.float32) + float(row.bias) # (B,) + return bag_meta[int(z.argmax())], True + + async def _settings_async(session: AsyncSession) -> MLSettings: return ( await session.execute(select(MLSettings).where(MLSettings.id == 1)) diff --git a/backend/app/services/ml/suggestions.py b/backend/app/services/ml/suggestions.py index 71fbd24..eba40ad 100644 --- a/backend/app/services/ml/suggestions.py +++ b/backend/app/services/ml/suggestions.py @@ -43,6 +43,11 @@ class Suggestion: # the rejection is VISIBLE and REVERSIBLE in the rail (misclick recovery, # operator-asked 2026-06-27) instead of silently vanishing or re-suggesting. rejected: bool = False + # grounding = the crop region that produced this suggestion (#1206): + # {bbox:[x,y,w,h] normalized, kind, detector}. None when the whole-image + # vector won (not localized) or for a CCIP-only hit (figure grounding TBD). + # Lets the rail highlight the exact region on hover. + grounding: dict | None = None @dataclass @@ -103,6 +108,7 @@ class SuggestionService: for h in hits: merged[(h["category"], h["tag_id"])] = { "name": h["name"], "score": h["score"], "source": "head", + "grounding": h.get("grounding"), } for c in ccip_hits: key = ("character", c["tag_id"]) @@ -110,9 +116,13 @@ class SuggestionService: if ex is not None: ex["source"] = "both" ex["score"] = max(ex["score"], c["score"]) + # Keep the head's localized crop if it had one; else fall back to + # the CCIP figure so a corroborated character still grounds (#1206). + ex["grounding"] = ex.get("grounding") or c.get("grounding") else: merged[key] = { "name": c["name"], "score": c["score"], "source": "ccip", + "grounding": c.get("grounding"), } result = SuggestionList() @@ -128,6 +138,7 @@ class SuggestionService: source=m["source"], creates_new_tag=False, rejected=tag_id in rejected, + grounding=m.get("grounding"), ) ) for cat in result.by_category: diff --git a/frontend/src/components/modal/ImageCanvas.vue b/frontend/src/components/modal/ImageCanvas.vue index a4a4163..efe2e52 100644 --- a/frontend/src/components/modal/ImageCanvas.vue +++ b/frontend/src/components/modal/ImageCanvas.vue @@ -16,20 +16,91 @@ transform: `translate(${panZoom.state.x}px, ${panZoom.state.y}px) scale(${panZoom.state.scale})` }" draggable="false" + @load="onImgLoad" > + +
+
+ {{ regionLabel }} +
+