feat(ui): hover an applied tag chip → highlight its grounding crop (#133 step 4)

Applied tags aren't scored live, so compute the grounding on demand: run the
tag's head over the image's max-over-bag (whole-image + concept crops), argmax
→ the region that best explains the tag on this image, mirroring what
score_image records for live suggestions.

- heads.py: extract _image_bag (now shared by score_image) + ground_applied_tag.
  Returns (grounding, has_head): has_head False = no head to localize with →
  no overlay; grounding None = the whole-image vector won → whole-image frame.
- tags.py: GET /api/images/<id>/tags/<id>/grounding → {grounding, has_head}.
- TagChip/TagPanel: applied chips inject fcSuggestionHover and fetch grounding
  on hover (cached per image+tag, race-guarded), reusing Step 3's overlay in
  both the modal and Explore. No new frontend overlay code.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM

backend/app/api/tags.py

@@ -11,6 +11,7 @@ from ..models.tag import image_tag
 from ..models.tag_suggestion_rejection import TagSuggestionRejection
 from ..services.bulk_tag_service import BulkTagService
 from ..services.ml.aliases import AliasService
+from ..services.ml.heads import ground_applied_tag
 from ..services.series_match_service import SeriesMatchService
 from ..services.series_service import SeriesError, SeriesService
 from ..services.tag_directory_service import TagDirectoryService
@@ -310,6 +311,21 @@ async def confirm_tag_on_image(image_id: int, tag_id: int):
     return "", 204
 
 
+@tags_bp.route(
+    "/images/<int:image_id>/tags/<int:tag_id>/grounding", methods=["GET"]
+)
+async def tag_grounding(image_id: int, tag_id: int):
+    """Which crop region best explains an ALREADY-APPLIED tag on this image
+    (#1206 Step 4). Powers the hover→overlay highlight on applied tag chips,
+    mirroring the suggestion rail's live grounding. Computed on demand (applied
+    tags aren't scored live). → {grounding: {bbox,kind,detector}|null,
+    has_head: bool}; has_head False means the tag has no head to localize with,
+    so the chip shows no overlay."""
+    async with get_session() as session:
+        grounding, has_head = await ground_applied_tag(session, image_id, tag_id)
+    return jsonify({"grounding": grounding, "has_head": has_head})
+
+
 @tags_bp.route("/tags/<int:tag_id>", methods=["GET"])
 async def get_tag(tag_id: int):
     """Resolve a single tag (used by the gallery to label its active

backend/app/services/ml/heads.py

@@ -341,44 +341,27 @@ async def _current_heads(session: AsyncSession, embedding_version: str):
     return loaded
 
 
-async def score_image(
-    session: AsyncSession, image_id: int, threshold_override: float | None = None,
-) -> list[dict]:
-    """Suggestions for one image from the trained heads: [{tag_id, name,
-    category, score}], ranked. A concept surfaces when its score clears the
-    head's own suggest_threshold — or, when threshold_override is given (the
-    typed-dropdown "show everything" mode), that flat floor instead (0 → every
-    head). System-tag heads (wip/banner/editor) instead use a flat
-    _SYSTEM_TAG_SUGGEST_FLOOR so their false positives surface for rejection
-    (still overridden by threshold_override). Empty if the image has no
-    embedding or no heads exist yet.
+async def _image_bag(
+    session: AsyncSession, image_id: int, cur_version: str,
+) -> tuple[list, list[dict | None]]:
+    """The max-over-bag inputs for one image: the whole-image SigLIP vector (when
+    it's in the current model's space) PLUS every concept-region crop embedded in
+    that space. Returns (bag, bag_meta) as PARALLEL lists — bag_meta[i] is None for
+    the whole-image row, else the region's {bbox, kind, detector} so a surfaced tag
+    can point back at the crop that produced it (#1206 grounding).
 
-    MAX-OVER-BAG: the image is scored as a BAG of embeddings — the whole-image
-    vector PLUS every concept-region crop the agent embedded (same model
-    version) — and each head takes its MAX score across the bag. A small/local
-    concept (glasses, a stomach bulge) that the whole-image vector washes out
-    can still surface from the crop where it dominates. The whole-image vector is
-    always in the bag, so this can never score lower than whole-image alone."""
+    Only current-version embeddings enter the bag: mid model-swap (#1190) an image
+    still carrying an OLD-version whole-image vector is skipped rather than scored
+    by heads trained in a different space; a legacy NULL version is treated as
+    current (those predate per-row stamping). Shared by live scoring (score_image)
+    and on-demand applied-tag grounding (ground_applied_tag, #1206 Step 4)."""
     import numpy as np
 
     img = await session.get(ImageRecord, image_id)
-    if img is None:
-        return []
-    settings = await _settings_async(session)
-    cur_version = settings.embedder_model_version
-    heads = await _current_heads(session, cur_version)
-    if heads["W"] is None:
-        return []
-
-    # Only embeddings in the CURRENT model's space enter the bag. Mid model-swap
-    # (#1190), an image still carrying the OLD-version whole-image vector is
-    # skipped rather than scored by heads trained in a different space; a legacy
-    # NULL version is treated as current (those predate per-row stamping).
-    bag = []
-    # Parallel to `bag`: what each row IS, so a surfaced tag can point back at the
-    # crop that produced it (#1206 grounding). None = the whole-image vector (not
-    # localized); a dict = a region's {bbox, kind, detector}.
+    bag: list = []
     bag_meta: list[dict | None] = []
+    if img is None:
+        return bag, bag_meta
     if img.siglip_embedding is not None and img.siglip_model_version in (
         cur_version, None,
     ):
@@ -402,6 +385,35 @@ async def score_image(
             bag_meta.append(
                 {"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector}
             )
+    return bag, bag_meta
+
+
+async def score_image(
+    session: AsyncSession, image_id: int, threshold_override: float | None = None,
+) -> list[dict]:
+    """Suggestions for one image from the trained heads: [{tag_id, name,
+    category, score}], ranked. A concept surfaces when its score clears the
+    head's own suggest_threshold — or, when threshold_override is given (the
+    typed-dropdown "show everything" mode), that flat floor instead (0 → every
+    head). System-tag heads (wip/banner/editor) instead use a flat
+    _SYSTEM_TAG_SUGGEST_FLOOR so their false positives surface for rejection
+    (still overridden by threshold_override). Empty if the image has no
+    embedding or no heads exist yet.
+
+    MAX-OVER-BAG: the image is scored as a BAG of embeddings — the whole-image
+    vector PLUS every concept-region crop the agent embedded (same model
+    version) — and each head takes its MAX score across the bag. A small/local
+    concept (glasses, a stomach bulge) that the whole-image vector washes out
+    can still surface from the crop where it dominates. The whole-image vector is
+    always in the bag, so this can never score lower than whole-image alone."""
+    import numpy as np
+
+    settings = await _settings_async(session)
+    cur_version = settings.embedder_model_version
+    heads = await _current_heads(session, cur_version)
+    if heads["W"] is None:
+        return []
+    bag, bag_meta = await _image_bag(session, image_id, cur_version)
     if not bag:
         return []
 
@@ -438,6 +450,51 @@ async def score_image(
     return out
 
 
+async def ground_applied_tag(
+    session: AsyncSession, image_id: int, tag_id: int,
+) -> tuple[dict | None, bool]:
+    """On-demand grounding for an ALREADY-APPLIED tag (#1206 Step 4). Applied tags
+    aren't scored live, so recompute the max-over-bag argmax for just this tag's
+    head — which crop region best explains the tag on this image — mirroring what
+    score_image records for live suggestions. Returns (grounding, has_head):
+
+    - has_head False → the tag has no head in the current embedding space (manual/
+      artist/meta tags, or a concept below the head floor). Nothing to localize
+      with, so the UI shows no overlay (distinct from "the whole image won").
+    - grounding None (has_head True) → the whole-image vector best explains it,
+      not any crop; the UI shows the subtle whole-image frame.
+    - grounding {bbox, kind, detector} → the winning region.
+
+    Character heads are covered too (character is a head kind); this deliberately
+    reuses the SigLIP head bag rather than the CCIP figure path so every applied
+    concept grounds through one consistent mechanism."""
+    import numpy as np
+
+    cur_version = (await _settings_async(session)).embedder_model_version
+    row = (
+        await session.execute(
+            select(TagHead.weights, TagHead.bias).where(
+                TagHead.tag_id == tag_id,
+                TagHead.embedding_version == cur_version,
+            )
+        )
+    ).one_or_none()
+    if row is None:
+        return None, False
+    bag, bag_meta = await _image_bag(session, image_id, cur_version)
+    if not bag:
+        return None, True
+
+    X = np.vstack(bag)
+    norms = np.linalg.norm(X, axis=1, keepdims=True)
+    norms[norms == 0] = 1.0
+    Xn = X / norms
+    # The sigmoid is monotonic in the logit, so the highest-probability bag row is
+    # just argmax of the raw score — no need to exponentiate to pick the winner.
+    z = Xn @ np.asarray(row.weights, dtype=np.float32) + float(row.bias)   # (B,)
+    return bag_meta[int(z.argmax())], True
+
+
 async def _settings_async(session: AsyncSession) -> MLSettings:
     return (
         await session.execute(select(MLSettings).where(MLSettings.id == 1))

frontend/src/components/modal/TagChip.vue

@@ -1,5 +1,5 @@
 <template>
-  <span class="fc-tag-chip">
+  <span class="fc-tag-chip" @mouseenter="onEnter" @mouseleave="onLeave">
     <v-chip
       size="small" closable
       :color="store.colorFor(tag.kind)" variant="tonal"
@@ -38,14 +38,57 @@
 </template>
 
 <script setup>
-import { computed } from 'vue'
+import { computed, inject } from 'vue'
 import { useTagStore } from '../../stores/tags.js'
+import { useApi } from '../../composables/useApi.js'
 import KebabMenu from '../common/KebabMenu.vue'
 
-const props = defineProps({ tag: { type: Object, required: true } })
+const props = defineProps({
+  tag: { type: Object, required: true },
+  // When set (the tagging panels), hovering the chip asks the backend which crop
+  // region best explains this applied tag and lights it up on the image — the
+  // same overlay the suggestion rail uses (#1206 Step 4). Omitted elsewhere →
+  // the hover is inert (no injected target, or no image to ground against).
+  imageId: { type: Number, default: null },
+})
 defineEmits(['remove', 'rename', 'set-fandom', 'navigate'])
 
 const store = useTagStore()
+const api = useApi()
+
+// #1206 Step 4: applied-tag grounding. `fcSuggestionHover` is provided by the
+// image viewer / Explore host (a no-op elsewhere). Applied tags aren't scored
+// live, so we fetch the winning region on demand and cache it per (image, tag).
+const hover = inject('fcSuggestionHover', null)
+const _groundingCache = new Map()
+// Bumped on every enter/leave so a slow fetch that resolves after the pointer
+// has moved on can't draw a stale overlay.
+let hoverSeq = 0
+
+async function onEnter () {
+  if (!hover || props.imageId == null) return
+  const seq = ++hoverSeq
+  const key = `${props.imageId}:${props.tag.id}`
+  let res = _groundingCache.get(key)
+  if (res === undefined) {
+    try {
+      res = await api.get(
+        `/api/images/${props.imageId}/tags/${props.tag.id}/grounding`
+      )
+      _groundingCache.set(key, res)
+    } catch {
+      return   // best-effort — leave the overlay untouched on error
+    }
+  }
+  if (seq !== hoverSeq) return   // pointer already left / moved to another chip
+  // No head → nothing to localize; don't draw an overlay at all. With a head,
+  // null grounding still draws the whole-image frame ("the global vector won").
+  if (res.has_head) hover.value = { g: res.grounding ?? null }
+}
+function onLeave () {
+  hoverSeq++
+  if (hover) hover.value = null
+}
 
 // Show a character's fandom inline (truncated). Falls back to a bare arrow when
 // only fandom_id is known but the name wasn't resolved (older payloads).

frontend/src/components/modal/TagPanel.vue

@@ -4,7 +4,7 @@
     <div class="fc-tag-panel__chips">
       <TagChip
         v-for="tag in host.current?.tags || []"
-        :key="tag.id" :tag="tag"
+        :key="tag.id" :tag="tag" :image-id="host.currentImageId"
         @remove="onRemove" @rename="openRename" @set-fandom="openSetFandom"
         @navigate="onNavigate"
       />

tests/test_api_tags.py

@@ -94,3 +94,73 @@ async def test_unknown_prefix_kept_literal(client):
     body = await resp.get_json()
     assert body["name"] == "Http://example.com"
     assert body["kind"] == "general"
+
+
+# --- #1206 Step 4: applied-tag grounding endpoint (hover on applied chips) ----
+
+
+@pytest.mark.asyncio
+async def test_applied_tag_grounding_returns_winning_region(client, db):
+    # Hovering an applied chip fetches the crop that best explains the tag. Here
+    # the whole-image vector is orthogonal to the head but a concept crop aligns,
+    # so the crop wins the max-over-bag → grounding is that region's box.
+    from sqlalchemy import select
+
+    from backend.app.models import (
+        ImageRecord, ImageRegion, MLSettings, TagHead, TagKind,
+    )
+    from backend.app.services.tag_service import TagService
+
+    ver = (await db.execute(
+        select(MLSettings).where(MLSettings.id == 1)
+    )).scalar_one().embedder_model_version
+    img = ImageRecord(
+        path="/images/grchip.jpg", sha256="gc" * 32, size_bytes=1,
+        mime="image/jpeg", width=1, height=1, origin="imported_filesystem",
+        integrity_status="unknown",
+        siglip_embedding=[0.0] * 5 + [3.0] + [0.0] * 1146,   # whole-image ⟂ head
+    )
+    db.add(img)
+    await db.flush()
+    tag = await TagService(db).find_or_create("glasses", TagKind.general)
+    db.add(TagHead(
+        tag_id=tag.id, embedding_version=ver,
+        weights=[1.0] + [0.0] * 1151, bias=0.0, suggest_threshold=0.5,
+        auto_apply_threshold=None, n_pos=10, n_neg=30,
+        ap=0.8, precision_cv=0.9, recall=0.6,
+    ))
+    db.add(ImageRegion(
+        image_record_id=img.id, kind="concept",
+        rx=0.4, ry=0.4, rw=0.3, rh=0.3,
+        siglip_embedding=[3.0] + [0.0] * 1151, embedding_version=ver,
+    ))
+    await db.commit()
+
+    resp = await client.get(f"/api/images/{img.id}/tags/{tag.id}/grounding")
+    assert resp.status_code == 200
+    body = await resp.get_json()
+    assert body["has_head"] is True
+    assert body["grounding"]["bbox"] == pytest.approx([0.4, 0.4, 0.3, 0.3])
+    assert body["grounding"]["kind"] == "concept"
+
+
+@pytest.mark.asyncio
+async def test_applied_tag_grounding_no_head(client, db):
+    # A tag with no head can't be localized → has_head False, grounding null; the
+    # chip shows no overlay. Validates the response contract the frontend reads.
+    from backend.app.models import ImageRecord, TagKind
+    from backend.app.services.tag_service import TagService
+
+    img = ImageRecord(
+        path="/images/grchip2.jpg", sha256="gd" * 32, size_bytes=1,
+        mime="image/jpeg", width=1, height=1, origin="imported_filesystem",
+        integrity_status="unknown",
+    )
+    db.add(img)
+    await db.flush()
+    tag = await TagService(db).find_or_create("NoHeadHere", TagKind.general)
+    await db.commit()
+
+    resp = await client.get(f"/api/images/{img.id}/tags/{tag.id}/grounding")
+    assert resp.status_code == 200
+    assert await resp.get_json() == {"grounding": None, "has_head": False}

tests/test_ml_suggestions.py

@@ -7,6 +7,7 @@ from sqlalchemy import select
 from backend.app.models import ImageRecord, ImageRegion, MLSettings, TagHead, TagKind
 from backend.app.models.tag import image_tag
 from backend.app.services.ml.allowlist import AllowlistService
+from backend.app.services.ml.heads import ground_applied_tag
 from backend.app.services.ml.suggestions import SuggestionService
 from backend.app.services.tag_service import TagService
 
@@ -280,3 +281,63 @@ async def test_ccip_character_surfaces_in_rail(db):
         if c.canonical_tag_id == raven.id
     )
     assert m.source == "ccip"
+
+
+# --- #1206 Step 4: on-demand grounding for ALREADY-APPLIED tag chips ---------
+# Applied tags aren't scored live, so ground_applied_tag recomputes the winning
+# bag row for one tag's head on demand — the data behind hovering an applied chip.
+
+
+@pytest.mark.asyncio
+async def test_ground_applied_tag_returns_winning_region(db):
+    # Whole-image vector is orthogonal to the head; a concept crop aligns with it,
+    # so the crop is the max-over-bag winner → grounding is THAT region's box.
+    tag = await TagService(db).find_or_create("glasses", TagKind.general)
+    img = await _img(db, "g1" * 32, _emb(5))            # whole-image ⟂ head
+    await _head(db, tag.id, slot=0)
+    db.add(ImageRegion(
+        image_record_id=img.id, kind="concept",
+        rx=0.4, ry=0.4, rw=0.3, rh=0.3,
+        siglip_embedding=_emb(0), embedding_version=await _embver(db),
+    ))
+    await db.commit()
+
+    grounding, has_head = await ground_applied_tag(db, img.id, tag.id)
+    assert has_head is True
+    assert grounding is not None
+    assert grounding["bbox"] == pytest.approx([0.4, 0.4, 0.3, 0.3])
+    assert grounding["kind"] == "concept"
+
+
+@pytest.mark.asyncio
+async def test_ground_applied_tag_none_when_whole_image_wins(db):
+    # Whole-image vector aligns with the head and out-scores an orthogonal crop →
+    # grounding None (has_head True): the tag is best explained by the global
+    # vector, so the chip hover shows the subtle whole-image frame, not a box.
+    tag = await TagService(db).find_or_create("sky", TagKind.general)
+    img = await _img(db, "g2" * 32, _emb(0))            # whole-image ALIGNED
+    await _head(db, tag.id, slot=0)
+    db.add(ImageRegion(
+        image_record_id=img.id, kind="concept",
+        rx=0.1, ry=0.1, rw=0.2, rh=0.2,
+        siglip_embedding=_emb(5), embedding_version=await _embver(db),
+    ))
+    await db.commit()
+
+    grounding, has_head = await ground_applied_tag(db, img.id, tag.id)
+    assert has_head is True
+    assert grounding is None
+
+
+@pytest.mark.asyncio
+async def test_ground_applied_tag_no_head(db):
+    # A tag with no head in the current space (manual/artist/meta, or below the
+    # head floor) can't be localized → (None, has_head False); the UI draws no
+    # overlay at all, distinct from "the whole image won".
+    tag = await TagService(db).find_or_create("artist:someone", TagKind.general)
+    img = await _img(db, "g3" * 32, _emb(0))
+    await db.commit()
+
+    grounding, has_head = await ground_applied_tag(db, img.id, tag.id)
+    assert has_head is False
+    assert grounding is None