diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py index ed156c8..50539cd 100644 --- a/backend/app/api/tags.py +++ b/backend/app/api/tags.py @@ -11,6 +11,7 @@ from ..models.tag import image_tag from ..models.tag_suggestion_rejection import TagSuggestionRejection from ..services.bulk_tag_service import BulkTagService from ..services.ml.aliases import AliasService +from ..services.ml.heads import ground_applied_tag from ..services.series_match_service import SeriesMatchService from ..services.series_service import SeriesError, SeriesService from ..services.tag_directory_service import TagDirectoryService @@ -310,6 +311,21 @@ async def confirm_tag_on_image(image_id: int, tag_id: int): return "", 204 +@tags_bp.route( + "/images//tags//grounding", methods=["GET"] +) +async def tag_grounding(image_id: int, tag_id: int): + """Which crop region best explains an ALREADY-APPLIED tag on this image + (#1206 Step 4). Powers the hover→overlay highlight on applied tag chips, + mirroring the suggestion rail's live grounding. Computed on demand (applied + tags aren't scored live). → {grounding: {bbox,kind,detector}|null, + has_head: bool}; has_head False means the tag has no head to localize with, + so the chip shows no overlay.""" + async with get_session() as session: + grounding, has_head = await ground_applied_tag(session, image_id, tag_id) + return jsonify({"grounding": grounding, "has_head": has_head}) + + @tags_bp.route("/tags/", methods=["GET"]) async def get_tag(tag_id: int): """Resolve a single tag (used by the gallery to label its active diff --git a/backend/app/services/ml/heads.py b/backend/app/services/ml/heads.py index aad0226..bb18c25 100644 --- a/backend/app/services/ml/heads.py +++ b/backend/app/services/ml/heads.py @@ -341,44 +341,27 @@ async def _current_heads(session: AsyncSession, embedding_version: str): return loaded -async def score_image( - session: AsyncSession, image_id: int, threshold_override: float | None = None, -) -> list[dict]: - """Suggestions for one image from the trained heads: [{tag_id, name, - category, score}], ranked. A concept surfaces when its score clears the - head's own suggest_threshold — or, when threshold_override is given (the - typed-dropdown "show everything" mode), that flat floor instead (0 → every - head). System-tag heads (wip/banner/editor) instead use a flat - _SYSTEM_TAG_SUGGEST_FLOOR so their false positives surface for rejection - (still overridden by threshold_override). Empty if the image has no - embedding or no heads exist yet. +async def _image_bag( + session: AsyncSession, image_id: int, cur_version: str, +) -> tuple[list, list[dict | None]]: + """The max-over-bag inputs for one image: the whole-image SigLIP vector (when + it's in the current model's space) PLUS every concept-region crop embedded in + that space. Returns (bag, bag_meta) as PARALLEL lists — bag_meta[i] is None for + the whole-image row, else the region's {bbox, kind, detector} so a surfaced tag + can point back at the crop that produced it (#1206 grounding). - MAX-OVER-BAG: the image is scored as a BAG of embeddings — the whole-image - vector PLUS every concept-region crop the agent embedded (same model - version) — and each head takes its MAX score across the bag. A small/local - concept (glasses, a stomach bulge) that the whole-image vector washes out - can still surface from the crop where it dominates. The whole-image vector is - always in the bag, so this can never score lower than whole-image alone.""" + Only current-version embeddings enter the bag: mid model-swap (#1190) an image + still carrying an OLD-version whole-image vector is skipped rather than scored + by heads trained in a different space; a legacy NULL version is treated as + current (those predate per-row stamping). Shared by live scoring (score_image) + and on-demand applied-tag grounding (ground_applied_tag, #1206 Step 4).""" import numpy as np img = await session.get(ImageRecord, image_id) - if img is None: - return [] - settings = await _settings_async(session) - cur_version = settings.embedder_model_version - heads = await _current_heads(session, cur_version) - if heads["W"] is None: - return [] - - # Only embeddings in the CURRENT model's space enter the bag. Mid model-swap - # (#1190), an image still carrying the OLD-version whole-image vector is - # skipped rather than scored by heads trained in a different space; a legacy - # NULL version is treated as current (those predate per-row stamping). - bag = [] - # Parallel to `bag`: what each row IS, so a surfaced tag can point back at the - # crop that produced it (#1206 grounding). None = the whole-image vector (not - # localized); a dict = a region's {bbox, kind, detector}. + bag: list = [] bag_meta: list[dict | None] = [] + if img is None: + return bag, bag_meta if img.siglip_embedding is not None and img.siglip_model_version in ( cur_version, None, ): @@ -402,6 +385,35 @@ async def score_image( bag_meta.append( {"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector} ) + return bag, bag_meta + + +async def score_image( + session: AsyncSession, image_id: int, threshold_override: float | None = None, +) -> list[dict]: + """Suggestions for one image from the trained heads: [{tag_id, name, + category, score}], ranked. A concept surfaces when its score clears the + head's own suggest_threshold — or, when threshold_override is given (the + typed-dropdown "show everything" mode), that flat floor instead (0 → every + head). System-tag heads (wip/banner/editor) instead use a flat + _SYSTEM_TAG_SUGGEST_FLOOR so their false positives surface for rejection + (still overridden by threshold_override). Empty if the image has no + embedding or no heads exist yet. + + MAX-OVER-BAG: the image is scored as a BAG of embeddings — the whole-image + vector PLUS every concept-region crop the agent embedded (same model + version) — and each head takes its MAX score across the bag. A small/local + concept (glasses, a stomach bulge) that the whole-image vector washes out + can still surface from the crop where it dominates. The whole-image vector is + always in the bag, so this can never score lower than whole-image alone.""" + import numpy as np + + settings = await _settings_async(session) + cur_version = settings.embedder_model_version + heads = await _current_heads(session, cur_version) + if heads["W"] is None: + return [] + bag, bag_meta = await _image_bag(session, image_id, cur_version) if not bag: return [] @@ -438,6 +450,51 @@ async def score_image( return out +async def ground_applied_tag( + session: AsyncSession, image_id: int, tag_id: int, +) -> tuple[dict | None, bool]: + """On-demand grounding for an ALREADY-APPLIED tag (#1206 Step 4). Applied tags + aren't scored live, so recompute the max-over-bag argmax for just this tag's + head — which crop region best explains the tag on this image — mirroring what + score_image records for live suggestions. Returns (grounding, has_head): + + - has_head False → the tag has no head in the current embedding space (manual/ + artist/meta tags, or a concept below the head floor). Nothing to localize + with, so the UI shows no overlay (distinct from "the whole image won"). + - grounding None (has_head True) → the whole-image vector best explains it, + not any crop; the UI shows the subtle whole-image frame. + - grounding {bbox, kind, detector} → the winning region. + + Character heads are covered too (character is a head kind); this deliberately + reuses the SigLIP head bag rather than the CCIP figure path so every applied + concept grounds through one consistent mechanism.""" + import numpy as np + + cur_version = (await _settings_async(session)).embedder_model_version + row = ( + await session.execute( + select(TagHead.weights, TagHead.bias).where( + TagHead.tag_id == tag_id, + TagHead.embedding_version == cur_version, + ) + ) + ).one_or_none() + if row is None: + return None, False + bag, bag_meta = await _image_bag(session, image_id, cur_version) + if not bag: + return None, True + + X = np.vstack(bag) + norms = np.linalg.norm(X, axis=1, keepdims=True) + norms[norms == 0] = 1.0 + Xn = X / norms + # The sigmoid is monotonic in the logit, so the highest-probability bag row is + # just argmax of the raw score — no need to exponentiate to pick the winner. + z = Xn @ np.asarray(row.weights, dtype=np.float32) + float(row.bias) # (B,) + return bag_meta[int(z.argmax())], True + + async def _settings_async(session: AsyncSession) -> MLSettings: return ( await session.execute(select(MLSettings).where(MLSettings.id == 1)) diff --git a/frontend/src/components/modal/TagChip.vue b/frontend/src/components/modal/TagChip.vue index dfe0ee6..bcb1cdf 100644 --- a/frontend/src/components/modal/TagChip.vue +++ b/frontend/src/components/modal/TagChip.vue @@ -1,5 +1,5 @@