@@ -46,6 +46,10 @@ async def get_suggestions(image_id: int):
|
||||
# (not dropped) so the rail can show it rejected + offer
|
||||
# one-click un-reject.
|
||||
"rejected": s.rejected,
|
||||
# the crop region that produced this tag (#1206) —
|
||||
# {bbox,kind,detector} or null (whole-image won). Drives
|
||||
# the hover→overlay highlight.
|
||||
"grounding": s.grounding,
|
||||
}
|
||||
for s in items
|
||||
]
|
||||
|
||||
@@ -11,6 +11,7 @@ from ..models.tag import image_tag
|
||||
from ..models.tag_suggestion_rejection import TagSuggestionRejection
|
||||
from ..services.bulk_tag_service import BulkTagService
|
||||
from ..services.ml.aliases import AliasService
|
||||
from ..services.ml.heads import ground_applied_tag
|
||||
from ..services.series_match_service import SeriesMatchService
|
||||
from ..services.series_service import SeriesError, SeriesService
|
||||
from ..services.tag_directory_service import TagDirectoryService
|
||||
@@ -310,6 +311,21 @@ async def confirm_tag_on_image(image_id: int, tag_id: int):
|
||||
return "", 204
|
||||
|
||||
|
||||
@tags_bp.route(
|
||||
"/images/<int:image_id>/tags/<int:tag_id>/grounding", methods=["GET"]
|
||||
)
|
||||
async def tag_grounding(image_id: int, tag_id: int):
|
||||
"""Which crop region best explains an ALREADY-APPLIED tag on this image
|
||||
(#1206 Step 4). Powers the hover→overlay highlight on applied tag chips,
|
||||
mirroring the suggestion rail's live grounding. Computed on demand (applied
|
||||
tags aren't scored live). → {grounding: {bbox,kind,detector}|null,
|
||||
has_head: bool}; has_head False means the tag has no head to localize with,
|
||||
so the chip shows no overlay."""
|
||||
async with get_session() as session:
|
||||
grounding, has_head = await ground_applied_tag(session, image_id, tag_id)
|
||||
return jsonify({"grounding": grounding, "has_head": has_head})
|
||||
|
||||
|
||||
@tags_bp.route("/tags/<int:tag_id>", methods=["GET"])
|
||||
async def get_tag(tag_id: int):
|
||||
"""Resolve a single tag (used by the gallery to label its active
|
||||
|
||||
@@ -161,16 +161,22 @@ async def match_image(
|
||||
if threshold is None:
|
||||
threshold = await _settings_threshold(session)
|
||||
|
||||
qvecs = (
|
||||
# Keep each figure region's bbox alongside its vector so a match can point at
|
||||
# the figure that matched (#1206 grounding), not just the score.
|
||||
fig_rows = (
|
||||
await session.execute(
|
||||
select(ImageRegion.ccip_embedding).where(
|
||||
select(
|
||||
ImageRegion.ccip_embedding,
|
||||
ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh,
|
||||
ImageRegion.kind, ImageRegion.detector_version,
|
||||
).where(
|
||||
ImageRegion.image_record_id == image_id,
|
||||
ImageRegion.kind.in_(_FIGURE_KINDS),
|
||||
ImageRegion.ccip_embedding.is_not(None),
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
if not qvecs:
|
||||
).all()
|
||||
if not fig_rows:
|
||||
return []
|
||||
refs = await character_references(session)
|
||||
if not refs:
|
||||
@@ -186,13 +192,21 @@ async def match_image(
|
||||
)
|
||||
names = await _tag_names(session, [t for t in refs if t not in applied])
|
||||
|
||||
qvecs = [r[0] for r in fig_rows]
|
||||
fig_meta = [
|
||||
{"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector}
|
||||
for _v, rx, ry, rw, rh, kind, detector in fig_rows
|
||||
]
|
||||
Q = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in qvecs]), np)
|
||||
out = []
|
||||
for tag_id, vecs in refs.items():
|
||||
if tag_id in applied:
|
||||
continue
|
||||
R = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np)
|
||||
best = float((Q @ R.T).max()) # best (query figure, reference) cosine
|
||||
sims = Q @ R.T # (n_query_figures, n_references)
|
||||
per_figure = sims.max(axis=1) # best reference cosine per figure
|
||||
best_figure = int(per_figure.argmax())
|
||||
best = float(per_figure[best_figure])
|
||||
if best >= threshold:
|
||||
out.append({
|
||||
"tag_id": tag_id,
|
||||
@@ -200,6 +214,8 @@ async def match_image(
|
||||
"category": "character",
|
||||
"score": round(best, 4),
|
||||
"source": "ccip",
|
||||
# the figure region that matched → grounds the character tag.
|
||||
"grounding": fig_meta[best_figure],
|
||||
})
|
||||
out.sort(key=lambda d: d["score"], reverse=True)
|
||||
return out
|
||||
|
||||
@@ -341,6 +341,53 @@ async def _current_heads(session: AsyncSession, embedding_version: str):
|
||||
return loaded
|
||||
|
||||
|
||||
async def _image_bag(
|
||||
session: AsyncSession, image_id: int, cur_version: str,
|
||||
) -> tuple[list, list[dict | None]]:
|
||||
"""The max-over-bag inputs for one image: the whole-image SigLIP vector (when
|
||||
it's in the current model's space) PLUS every concept-region crop embedded in
|
||||
that space. Returns (bag, bag_meta) as PARALLEL lists — bag_meta[i] is None for
|
||||
the whole-image row, else the region's {bbox, kind, detector} so a surfaced tag
|
||||
can point back at the crop that produced it (#1206 grounding).
|
||||
|
||||
Only current-version embeddings enter the bag: mid model-swap (#1190) an image
|
||||
still carrying an OLD-version whole-image vector is skipped rather than scored
|
||||
by heads trained in a different space; a legacy NULL version is treated as
|
||||
current (those predate per-row stamping). Shared by live scoring (score_image)
|
||||
and on-demand applied-tag grounding (ground_applied_tag, #1206 Step 4)."""
|
||||
import numpy as np
|
||||
|
||||
img = await session.get(ImageRecord, image_id)
|
||||
bag: list = []
|
||||
bag_meta: list[dict | None] = []
|
||||
if img is None:
|
||||
return bag, bag_meta
|
||||
if img.siglip_embedding is not None and img.siglip_model_version in (
|
||||
cur_version, None,
|
||||
):
|
||||
bag.append(np.asarray(img.siglip_embedding, dtype=np.float32))
|
||||
bag_meta.append(None)
|
||||
region_rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
ImageRegion.siglip_embedding,
|
||||
ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh,
|
||||
ImageRegion.kind, ImageRegion.detector_version,
|
||||
)
|
||||
.where(ImageRegion.image_record_id == image_id)
|
||||
.where(ImageRegion.siglip_embedding.is_not(None))
|
||||
.where(ImageRegion.embedding_version == cur_version)
|
||||
)
|
||||
).all()
|
||||
for vec, rx, ry, rw, rh, kind, detector in region_rows:
|
||||
if vec is not None:
|
||||
bag.append(np.asarray(vec, dtype=np.float32))
|
||||
bag_meta.append(
|
||||
{"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector}
|
||||
)
|
||||
return bag, bag_meta
|
||||
|
||||
|
||||
async def score_image(
|
||||
session: AsyncSession, image_id: int, threshold_override: float | None = None,
|
||||
) -> list[dict]:
|
||||
@@ -361,35 +408,12 @@ async def score_image(
|
||||
always in the bag, so this can never score lower than whole-image alone."""
|
||||
import numpy as np
|
||||
|
||||
img = await session.get(ImageRecord, image_id)
|
||||
if img is None:
|
||||
return []
|
||||
settings = await _settings_async(session)
|
||||
cur_version = settings.embedder_model_version
|
||||
heads = await _current_heads(session, cur_version)
|
||||
if heads["W"] is None:
|
||||
return []
|
||||
|
||||
# Only embeddings in the CURRENT model's space enter the bag. Mid model-swap
|
||||
# (#1190), an image still carrying the OLD-version whole-image vector is
|
||||
# skipped rather than scored by heads trained in a different space; a legacy
|
||||
# NULL version is treated as current (those predate per-row stamping).
|
||||
bag = []
|
||||
if img.siglip_embedding is not None and img.siglip_model_version in (
|
||||
cur_version, None,
|
||||
):
|
||||
bag.append(np.asarray(img.siglip_embedding, dtype=np.float32))
|
||||
region_vecs = (
|
||||
await session.execute(
|
||||
select(ImageRegion.siglip_embedding)
|
||||
.where(ImageRegion.image_record_id == image_id)
|
||||
.where(ImageRegion.siglip_embedding.is_not(None))
|
||||
.where(ImageRegion.embedding_version == cur_version)
|
||||
)
|
||||
).all()
|
||||
for (vec,) in region_vecs:
|
||||
if vec is not None:
|
||||
bag.append(np.asarray(vec, dtype=np.float32))
|
||||
bag, bag_meta = await _image_bag(session, image_id, cur_version)
|
||||
if not bag:
|
||||
return []
|
||||
|
||||
@@ -398,7 +422,11 @@ async def score_image(
|
||||
norms[norms == 0] = 1.0
|
||||
Xn = X / norms
|
||||
Z = Xn @ heads["W"].T + heads["b"] # (B, H)
|
||||
probs = (1.0 / (1.0 + np.exp(-Z))).max(axis=0) # (H,) best over the bag
|
||||
probs_bag = 1.0 / (1.0 + np.exp(-Z)) # (B, H)
|
||||
probs = probs_bag.max(axis=0) # (H,) best over the bag
|
||||
# ARGMAX beside the max: WHICH bag row won each head → the region that grounds
|
||||
# the tag (bag_meta[win]); None when the whole-image vector won (#1206).
|
||||
winners = probs_bag.argmax(axis=0) # (H,)
|
||||
out = []
|
||||
for i, p in enumerate(probs):
|
||||
if threshold_override is not None:
|
||||
@@ -416,11 +444,57 @@ async def score_image(
|
||||
"name": m["name"],
|
||||
"category": m["category"],
|
||||
"score": float(p),
|
||||
"grounding": bag_meta[int(winners[i])],
|
||||
})
|
||||
out.sort(key=lambda d: d["score"], reverse=True)
|
||||
return out
|
||||
|
||||
|
||||
async def ground_applied_tag(
|
||||
session: AsyncSession, image_id: int, tag_id: int,
|
||||
) -> tuple[dict | None, bool]:
|
||||
"""On-demand grounding for an ALREADY-APPLIED tag (#1206 Step 4). Applied tags
|
||||
aren't scored live, so recompute the max-over-bag argmax for just this tag's
|
||||
head — which crop region best explains the tag on this image — mirroring what
|
||||
score_image records for live suggestions. Returns (grounding, has_head):
|
||||
|
||||
- has_head False → the tag has no head in the current embedding space (manual/
|
||||
artist/meta tags, or a concept below the head floor). Nothing to localize
|
||||
with, so the UI shows no overlay (distinct from "the whole image won").
|
||||
- grounding None (has_head True) → the whole-image vector best explains it,
|
||||
not any crop; the UI shows the subtle whole-image frame.
|
||||
- grounding {bbox, kind, detector} → the winning region.
|
||||
|
||||
Character heads are covered too (character is a head kind); this deliberately
|
||||
reuses the SigLIP head bag rather than the CCIP figure path so every applied
|
||||
concept grounds through one consistent mechanism."""
|
||||
import numpy as np
|
||||
|
||||
cur_version = (await _settings_async(session)).embedder_model_version
|
||||
row = (
|
||||
await session.execute(
|
||||
select(TagHead.weights, TagHead.bias).where(
|
||||
TagHead.tag_id == tag_id,
|
||||
TagHead.embedding_version == cur_version,
|
||||
)
|
||||
)
|
||||
).one_or_none()
|
||||
if row is None:
|
||||
return None, False
|
||||
bag, bag_meta = await _image_bag(session, image_id, cur_version)
|
||||
if not bag:
|
||||
return None, True
|
||||
|
||||
X = np.vstack(bag)
|
||||
norms = np.linalg.norm(X, axis=1, keepdims=True)
|
||||
norms[norms == 0] = 1.0
|
||||
Xn = X / norms
|
||||
# The sigmoid is monotonic in the logit, so the highest-probability bag row is
|
||||
# just argmax of the raw score — no need to exponentiate to pick the winner.
|
||||
z = Xn @ np.asarray(row.weights, dtype=np.float32) + float(row.bias) # (B,)
|
||||
return bag_meta[int(z.argmax())], True
|
||||
|
||||
|
||||
async def _settings_async(session: AsyncSession) -> MLSettings:
|
||||
return (
|
||||
await session.execute(select(MLSettings).where(MLSettings.id == 1))
|
||||
|
||||
@@ -43,6 +43,11 @@ class Suggestion:
|
||||
# the rejection is VISIBLE and REVERSIBLE in the rail (misclick recovery,
|
||||
# operator-asked 2026-06-27) instead of silently vanishing or re-suggesting.
|
||||
rejected: bool = False
|
||||
# grounding = the crop region that produced this suggestion (#1206):
|
||||
# {bbox:[x,y,w,h] normalized, kind, detector}. None when the whole-image
|
||||
# vector won (not localized) or for a CCIP-only hit (figure grounding TBD).
|
||||
# Lets the rail highlight the exact region on hover.
|
||||
grounding: dict | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -103,6 +108,7 @@ class SuggestionService:
|
||||
for h in hits:
|
||||
merged[(h["category"], h["tag_id"])] = {
|
||||
"name": h["name"], "score": h["score"], "source": "head",
|
||||
"grounding": h.get("grounding"),
|
||||
}
|
||||
for c in ccip_hits:
|
||||
key = ("character", c["tag_id"])
|
||||
@@ -110,9 +116,13 @@ class SuggestionService:
|
||||
if ex is not None:
|
||||
ex["source"] = "both"
|
||||
ex["score"] = max(ex["score"], c["score"])
|
||||
# Keep the head's localized crop if it had one; else fall back to
|
||||
# the CCIP figure so a corroborated character still grounds (#1206).
|
||||
ex["grounding"] = ex.get("grounding") or c.get("grounding")
|
||||
else:
|
||||
merged[key] = {
|
||||
"name": c["name"], "score": c["score"], "source": "ccip",
|
||||
"grounding": c.get("grounding"),
|
||||
}
|
||||
|
||||
result = SuggestionList()
|
||||
@@ -128,6 +138,7 @@ class SuggestionService:
|
||||
source=m["source"],
|
||||
creates_new_tag=False,
|
||||
rejected=tag_id in rejected,
|
||||
grounding=m.get("grounding"),
|
||||
)
|
||||
)
|
||||
for cat in result.by_category:
|
||||
|
||||
@@ -16,20 +16,91 @@
|
||||
transform: `translate(${panZoom.state.x}px, ${panZoom.state.y}px) scale(${panZoom.state.scale})`
|
||||
}"
|
||||
draggable="false"
|
||||
@load="onImgLoad"
|
||||
>
|
||||
<!-- #1206 grounding overlay: tracks the <img>'s live rendered rect (so it's
|
||||
correct under object-fit letterboxing + pan/zoom) and draws the crop
|
||||
region a hovered suggestion came from. Null grounding → a subtle
|
||||
whole-image frame ("the global vector won, not a crop"). -->
|
||||
<div
|
||||
v-if="hover && overlayBox"
|
||||
class="fc-canvas__overlay"
|
||||
:style="{
|
||||
left: overlayBox.left + 'px', top: overlayBox.top + 'px',
|
||||
width: overlayBox.width + 'px', height: overlayBox.height + 'px',
|
||||
}"
|
||||
>
|
||||
<div
|
||||
class="fc-canvas__region"
|
||||
:class="{ 'fc-canvas__region--whole': !regionStyle }"
|
||||
:style="regionStyle || {}"
|
||||
>
|
||||
<span class="fc-canvas__region-label">{{ regionLabel }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref, watch } from 'vue'
|
||||
import { computed, nextTick, onMounted, onUnmounted, ref, watch } from 'vue'
|
||||
import { usePanZoom } from '../../composables/usePanZoom.js'
|
||||
|
||||
const props = defineProps({ src: String, alt: String })
|
||||
const props = defineProps({
|
||||
src: String,
|
||||
alt: String,
|
||||
// Hovered-suggestion grounding, or null when nothing is hovered:
|
||||
// null → no overlay
|
||||
// { g: {bbox,kind,detector} } → highlight that crop region
|
||||
// { g: null } → whole-image frame (not localized)
|
||||
hover: { type: Object, default: null },
|
||||
})
|
||||
const emit = defineEmits(['close-request'])
|
||||
const panZoom = usePanZoom()
|
||||
|
||||
const canvasEl = ref(null)
|
||||
const imgEl = ref(null)
|
||||
const overlayBox = ref(null) // the <img>'s rect relative to the canvas
|
||||
|
||||
// The <img> renders aspect-fit + transformed, so its getBoundingClientRect is
|
||||
// the ground truth for where the picture actually is. Position the overlay to
|
||||
// match it, then place the region as a % within.
|
||||
function measure () {
|
||||
const img = imgEl.value
|
||||
const canvas = canvasEl.value
|
||||
if (!img || !canvas) { overlayBox.value = null; return }
|
||||
const ir = img.getBoundingClientRect()
|
||||
const cr = canvas.getBoundingClientRect()
|
||||
if (!ir.width || !ir.height) { overlayBox.value = null; return }
|
||||
overlayBox.value = {
|
||||
left: ir.left - cr.left, top: ir.top - cr.top,
|
||||
width: ir.width, height: ir.height,
|
||||
}
|
||||
}
|
||||
|
||||
const regionStyle = computed(() => {
|
||||
const g = props.hover?.g
|
||||
if (!g || !Array.isArray(g.bbox)) return null // null grounding → whole frame
|
||||
const [rx, ry, rw, rh] = g.bbox
|
||||
return {
|
||||
left: `${rx * 100}%`, top: `${ry * 100}%`,
|
||||
width: `${rw * 100}%`, height: `${rh * 100}%`,
|
||||
}
|
||||
})
|
||||
const regionLabel = computed(() => {
|
||||
const g = props.hover?.g
|
||||
return g ? (g.detector || g.kind || 'region') : 'whole image'
|
||||
})
|
||||
|
||||
// Re-measure whenever hovering starts or the image could have moved.
|
||||
watch(() => props.hover, (h) => { if (h) nextTick(measure) })
|
||||
watch(
|
||||
() => [panZoom.state.scale, panZoom.state.x, panZoom.state.y],
|
||||
() => { if (props.hover) measure() },
|
||||
)
|
||||
function onResize () { if (props.hover) measure() }
|
||||
function onImgLoad () { if (props.hover) measure() }
|
||||
onMounted(() => window.addEventListener('resize', onResize))
|
||||
onUnmounted(() => window.removeEventListener('resize', onResize))
|
||||
|
||||
// Reset zoom when the src changes (prev/next nav).
|
||||
watch(() => props.src, () => panZoom.reset())
|
||||
@@ -79,4 +150,42 @@ function onCanvasClick(ev) {
|
||||
transition: transform 100ms ease-out;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
/* Grounding overlay — never intercepts pan/zoom/click. */
|
||||
.fc-canvas__overlay {
|
||||
position: absolute;
|
||||
pointer-events: none;
|
||||
z-index: 2;
|
||||
}
|
||||
.fc-canvas__region {
|
||||
position: absolute;
|
||||
border: 2px solid rgb(var(--v-theme-accent));
|
||||
border-radius: 3px;
|
||||
box-shadow: 0 0 0 9999px rgba(0, 0, 0, 0.28); /* dim everything outside it */
|
||||
transition: all 90ms ease-out;
|
||||
}
|
||||
/* Null grounding: a subtle full-image frame, no dimming — it says "this tag
|
||||
came from the whole image, not a crop." */
|
||||
.fc-canvas__region--whole {
|
||||
inset: 0;
|
||||
border-style: dashed;
|
||||
border-color: rgb(var(--v-theme-on-surface-variant));
|
||||
box-shadow: none;
|
||||
}
|
||||
.fc-canvas__region-label {
|
||||
position: absolute;
|
||||
top: 0; left: 0;
|
||||
transform: translateY(-100%);
|
||||
font-size: 11px;
|
||||
font-family: 'JetBrains Mono', monospace;
|
||||
line-height: 1.4;
|
||||
padding: 1px 6px;
|
||||
white-space: nowrap;
|
||||
background: rgb(var(--v-theme-accent));
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
border-radius: 3px 3px 0 0;
|
||||
}
|
||||
.fc-canvas__region--whole .fc-canvas__region-label {
|
||||
background: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -60,6 +60,7 @@
|
||||
<template v-if="modal.current">
|
||||
<ImageCanvas
|
||||
v-if="!isVideo" :src="modal.current.image_url" :alt="`Image ${modal.current.id}`"
|
||||
:hover="hoverGrounding"
|
||||
@close-request="$emit('close')"
|
||||
/>
|
||||
<VideoCanvas
|
||||
@@ -94,7 +95,7 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
|
||||
import { computed, onMounted, onUnmounted, provide, ref, watch } from 'vue'
|
||||
import { useModalStore } from '../../stores/modal.js'
|
||||
import { arrowNavAllowed, isTextEntry } from '../../utils/textEntry.js'
|
||||
import ImageCanvas from './ImageCanvas.vue'
|
||||
@@ -110,6 +111,13 @@ const modal = useModalStore()
|
||||
const rootEl = ref(null)
|
||||
const showHelp = ref(false)
|
||||
|
||||
// #1206: a deep descendant (SuggestionItem) sets this on hover to the crop that
|
||||
// produced a suggestion; ImageCanvas draws it over the image. provide/inject
|
||||
// avoids relaying a hover event through TagPanel → SuggestionsPanel → group.
|
||||
// null = nothing hovered; { g: grounding|null } = hovering (g null → whole-image).
|
||||
const hoverGrounding = ref(null)
|
||||
provide('fcSuggestionHover', hoverGrounding)
|
||||
|
||||
const isVideo = computed(() =>
|
||||
modal.current?.mime && modal.current.mime.startsWith('video/')
|
||||
)
|
||||
|
||||
@@ -3,7 +3,10 @@
|
||||
name, score, and action buttons as one "object" (operator-asked
|
||||
2026-06-01). The row itself is informational; the green ✓ / red ✗
|
||||
verdict pair + 3-dot alias menu are the action affordances. -->
|
||||
<div class="fc-suggestion" :class="{ 'fc-suggestion--rejected': suggestion.rejected }">
|
||||
<div
|
||||
class="fc-suggestion" :class="{ 'fc-suggestion--rejected': suggestion.rejected }"
|
||||
@mouseenter="onEnter" @mouseleave="onLeave"
|
||||
>
|
||||
<span class="fc-suggestion__name">
|
||||
{{ suggestion.display_name }}
|
||||
<span v-if="suggestion.rejected" class="fc-suggestion__rejected-tag"
|
||||
@@ -72,12 +75,22 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
import { computed, inject } from 'vue'
|
||||
import KebabMenu from '../common/KebabMenu.vue'
|
||||
|
||||
const props = defineProps({ suggestion: { type: Object, required: true } })
|
||||
defineEmits(['accept', 'alias', 'remove-alias', 'dismiss', 'undismiss'])
|
||||
|
||||
// #1206: on hover, tell the image viewer which crop produced this suggestion so
|
||||
// it highlights that region. Provided by ImageViewer/Explore; a no-op elsewhere.
|
||||
const hover = inject('fcSuggestionHover', null)
|
||||
function onEnter () {
|
||||
if (hover) hover.value = { g: props.suggestion.grounding ?? null }
|
||||
}
|
||||
function onLeave () {
|
||||
if (hover) hover.value = null
|
||||
}
|
||||
|
||||
const scorePct = computed(() => `${Math.round(props.suggestion.score * 100)}%`)
|
||||
// Kebab now only carries alias actions: show it when this suggestion can be
|
||||
// aliased (raw model key, not yet aliased) or is already aliased (so it can be
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<template>
|
||||
<span class="fc-tag-chip">
|
||||
<span class="fc-tag-chip" @mouseenter="onEnter" @mouseleave="onLeave">
|
||||
<v-chip
|
||||
size="small" closable
|
||||
:color="store.colorFor(tag.kind)" variant="tonal"
|
||||
@@ -38,14 +38,57 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
import { computed, inject } from 'vue'
|
||||
import { useTagStore } from '../../stores/tags.js'
|
||||
import { useApi } from '../../composables/useApi.js'
|
||||
import KebabMenu from '../common/KebabMenu.vue'
|
||||
|
||||
const props = defineProps({ tag: { type: Object, required: true } })
|
||||
const props = defineProps({
|
||||
tag: { type: Object, required: true },
|
||||
// When set (the tagging panels), hovering the chip asks the backend which crop
|
||||
// region best explains this applied tag and lights it up on the image — the
|
||||
// same overlay the suggestion rail uses (#1206 Step 4). Omitted elsewhere →
|
||||
// the hover is inert (no injected target, or no image to ground against).
|
||||
imageId: { type: Number, default: null },
|
||||
})
|
||||
defineEmits(['remove', 'rename', 'set-fandom', 'navigate'])
|
||||
|
||||
const store = useTagStore()
|
||||
const api = useApi()
|
||||
|
||||
// #1206 Step 4: applied-tag grounding. `fcSuggestionHover` is provided by the
|
||||
// image viewer / Explore host (a no-op elsewhere). Applied tags aren't scored
|
||||
// live, so we fetch the winning region on demand and cache it per (image, tag).
|
||||
const hover = inject('fcSuggestionHover', null)
|
||||
const _groundingCache = new Map()
|
||||
// Bumped on every enter/leave so a slow fetch that resolves after the pointer
|
||||
// has moved on can't draw a stale overlay.
|
||||
let hoverSeq = 0
|
||||
|
||||
async function onEnter () {
|
||||
if (!hover || props.imageId == null) return
|
||||
const seq = ++hoverSeq
|
||||
const key = `${props.imageId}:${props.tag.id}`
|
||||
let res = _groundingCache.get(key)
|
||||
if (res === undefined) {
|
||||
try {
|
||||
res = await api.get(
|
||||
`/api/images/${props.imageId}/tags/${props.tag.id}/grounding`
|
||||
)
|
||||
_groundingCache.set(key, res)
|
||||
} catch {
|
||||
return // best-effort — leave the overlay untouched on error
|
||||
}
|
||||
}
|
||||
if (seq !== hoverSeq) return // pointer already left / moved to another chip
|
||||
// No head → nothing to localize; don't draw an overlay at all. With a head,
|
||||
// null grounding still draws the whole-image frame ("the global vector won").
|
||||
if (res.has_head) hover.value = { g: res.grounding ?? null }
|
||||
}
|
||||
function onLeave () {
|
||||
hoverSeq++
|
||||
if (hover) hover.value = null
|
||||
}
|
||||
|
||||
// Show a character's fandom inline (truncated). Falls back to a bare arrow when
|
||||
// only fandom_id is known but the name wasn't resolved (older payloads).
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<div class="fc-tag-panel__chips">
|
||||
<TagChip
|
||||
v-for="tag in host.current?.tags || []"
|
||||
:key="tag.id" :tag="tag"
|
||||
:key="tag.id" :tag="tag" :image-id="host.currentImageId"
|
||||
@remove="onRemove" @rename="openRename" @set-fandom="openSetFandom"
|
||||
@navigate="onNavigate"
|
||||
/>
|
||||
|
||||
@@ -98,6 +98,7 @@
|
||||
:key="store.anchor.id"
|
||||
:src="store.anchor.image_url"
|
||||
:alt="`Image ${store.anchor.id}`"
|
||||
:hover="hoverGrounding"
|
||||
/>
|
||||
<VideoCanvas
|
||||
v-else
|
||||
@@ -131,7 +132,7 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, nextTick, onMounted, onUnmounted, ref, watch } from 'vue'
|
||||
import { computed, nextTick, onMounted, onUnmounted, provide, ref, watch } from 'vue'
|
||||
import { useRoute, useRouter } from 'vue-router'
|
||||
import { useApi } from '../composables/useApi.js'
|
||||
import { useExploreStore } from '../stores/explore.js'
|
||||
@@ -152,6 +153,11 @@ const modal = useModalStore()
|
||||
|
||||
const anchorId = computed(() => route.params.imageId || null)
|
||||
const isVideo = computed(() => !!store.anchor?.mime?.startsWith('video/'))
|
||||
|
||||
// #1206: hovering a suggestion in the rail highlights the crop it came from on
|
||||
// the anchor image (same provide/inject as the modal viewer).
|
||||
const hoverGrounding = ref(null)
|
||||
provide('fcSuggestionHover', hoverGrounding)
|
||||
const seeding = ref(false)
|
||||
const seedError = ref(null)
|
||||
const tagPanelRef = ref(null)
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import ImageRecord, ImageRegion, MLSettings, TagHead, TagKind
|
||||
from backend.app.services.tag_service import TagService
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
@@ -94,3 +98,63 @@ async def test_unknown_prefix_kept_literal(client):
|
||||
body = await resp.get_json()
|
||||
assert body["name"] == "Http://example.com"
|
||||
assert body["kind"] == "general"
|
||||
|
||||
|
||||
# --- #1206 Step 4: applied-tag grounding endpoint (hover on applied chips) ----
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_applied_tag_grounding_returns_winning_region(client, db):
|
||||
# Hovering an applied chip fetches the crop that best explains the tag. Here
|
||||
# the whole-image vector is orthogonal to the head but a concept crop aligns,
|
||||
# so the crop wins the max-over-bag → grounding is that region's box.
|
||||
ver = (await db.execute(
|
||||
select(MLSettings).where(MLSettings.id == 1)
|
||||
)).scalar_one().embedder_model_version
|
||||
img = ImageRecord(
|
||||
path="/images/grchip.jpg", sha256="gc" * 32, size_bytes=1,
|
||||
mime="image/jpeg", width=1, height=1, origin="imported_filesystem",
|
||||
integrity_status="unknown",
|
||||
siglip_embedding=[0.0] * 5 + [3.0] + [0.0] * 1146, # whole-image ⟂ head
|
||||
)
|
||||
db.add(img)
|
||||
await db.flush()
|
||||
tag = await TagService(db).find_or_create("glasses", TagKind.general)
|
||||
db.add(TagHead(
|
||||
tag_id=tag.id, embedding_version=ver,
|
||||
weights=[1.0] + [0.0] * 1151, bias=0.0, suggest_threshold=0.5,
|
||||
auto_apply_threshold=None, n_pos=10, n_neg=30,
|
||||
ap=0.8, precision_cv=0.9, recall=0.6,
|
||||
))
|
||||
db.add(ImageRegion(
|
||||
image_record_id=img.id, kind="concept",
|
||||
rx=0.4, ry=0.4, rw=0.3, rh=0.3,
|
||||
siglip_embedding=[3.0] + [0.0] * 1151, embedding_version=ver,
|
||||
))
|
||||
await db.commit()
|
||||
|
||||
resp = await client.get(f"/api/images/{img.id}/tags/{tag.id}/grounding")
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["has_head"] is True
|
||||
assert body["grounding"]["bbox"] == pytest.approx([0.4, 0.4, 0.3, 0.3])
|
||||
assert body["grounding"]["kind"] == "concept"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_applied_tag_grounding_no_head(client, db):
|
||||
# A tag with no head can't be localized → has_head False, grounding null; the
|
||||
# chip shows no overlay. Validates the response contract the frontend reads.
|
||||
img = ImageRecord(
|
||||
path="/images/grchip2.jpg", sha256="gd" * 32, size_bytes=1,
|
||||
mime="image/jpeg", width=1, height=1, origin="imported_filesystem",
|
||||
integrity_status="unknown",
|
||||
)
|
||||
db.add(img)
|
||||
await db.flush()
|
||||
tag = await TagService(db).find_or_create("NoHeadHere", TagKind.general)
|
||||
await db.commit()
|
||||
|
||||
resp = await client.get(f"/api/images/{img.id}/tags/{tag.id}/grounding")
|
||||
assert resp.status_code == 200
|
||||
assert await resp.get_json() == {"grounding": None, "has_head": False}
|
||||
|
||||
@@ -55,6 +55,10 @@ async def test_matches_same_character_across_images(db):
|
||||
m = next(x for x in matches if x["tag_id"] == raven.id)
|
||||
assert m["source"] == "ccip" and m["category"] == "character"
|
||||
assert m["score"] > 0.9
|
||||
# #1206: the match grounds to the figure region that matched (hover → the
|
||||
# figure box lights up), so a character suggestion is localized too.
|
||||
assert m["grounding"]["bbox"] == pytest.approx([0.0, 0.0, 1.0, 1.0])
|
||||
assert m["grounding"]["kind"] == "figure"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -7,6 +7,7 @@ from sqlalchemy import select
|
||||
from backend.app.models import ImageRecord, ImageRegion, MLSettings, TagHead, TagKind
|
||||
from backend.app.models.tag import image_tag
|
||||
from backend.app.services.ml.allowlist import AllowlistService
|
||||
from backend.app.services.ml.heads import ground_applied_tag
|
||||
from backend.app.services.ml.suggestions import SuggestionService
|
||||
from backend.app.services.tag_service import TagService
|
||||
|
||||
@@ -187,7 +188,31 @@ async def test_concept_region_surfaces_via_max_over_bag(db):
|
||||
))
|
||||
await db.commit()
|
||||
general = (await SuggestionService(db).for_image(img.id)).by_category["general"]
|
||||
assert any(s.canonical_tag_id == tag.id and s.score > 0.7 for s in general)
|
||||
s = next(x for x in general if x.canonical_tag_id == tag.id)
|
||||
assert s.score > 0.7
|
||||
# #1206: the winning crop grounds the tag — hover highlights THIS region
|
||||
# (the matching-version crop at 0.4,0.4,0.3,0.3), not the whole image.
|
||||
assert s.grounding is not None
|
||||
assert s.grounding["bbox"] == pytest.approx([0.4, 0.4, 0.3, 0.3])
|
||||
assert s.grounding["kind"] == "concept"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_grounding_none_when_whole_image_wins(db):
|
||||
# #1206: when the whole-image vector (not a crop) produces the winning score,
|
||||
# grounding is None — the tag came from the global vector, not a region.
|
||||
tag = await TagService(db).find_or_create("sky", TagKind.general)
|
||||
img = await _img(db, "d1" * 32, _emb(0)) # whole-image ALIGNED w/ head
|
||||
await _head(db, tag.id, slot=0, suggest_threshold=0.5)
|
||||
db.add(ImageRegion( # an orthogonal crop (0.5)
|
||||
image_record_id=img.id, kind="concept",
|
||||
rx=0.1, ry=0.1, rw=0.2, rh=0.2,
|
||||
siglip_embedding=_emb(5), embedding_version=await _embver(db),
|
||||
))
|
||||
await db.commit()
|
||||
general = (await SuggestionService(db).for_image(img.id)).by_category["general"]
|
||||
s = next(x for x in general if x.canonical_tag_id == tag.id)
|
||||
assert s.grounding is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -256,3 +281,63 @@ async def test_ccip_character_surfaces_in_rail(db):
|
||||
if c.canonical_tag_id == raven.id
|
||||
)
|
||||
assert m.source == "ccip"
|
||||
|
||||
|
||||
# --- #1206 Step 4: on-demand grounding for ALREADY-APPLIED tag chips ---------
|
||||
# Applied tags aren't scored live, so ground_applied_tag recomputes the winning
|
||||
# bag row for one tag's head on demand — the data behind hovering an applied chip.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ground_applied_tag_returns_winning_region(db):
|
||||
# Whole-image vector is orthogonal to the head; a concept crop aligns with it,
|
||||
# so the crop is the max-over-bag winner → grounding is THAT region's box.
|
||||
tag = await TagService(db).find_or_create("glasses", TagKind.general)
|
||||
img = await _img(db, "g1" * 32, _emb(5)) # whole-image ⟂ head
|
||||
await _head(db, tag.id, slot=0)
|
||||
db.add(ImageRegion(
|
||||
image_record_id=img.id, kind="concept",
|
||||
rx=0.4, ry=0.4, rw=0.3, rh=0.3,
|
||||
siglip_embedding=_emb(0), embedding_version=await _embver(db),
|
||||
))
|
||||
await db.commit()
|
||||
|
||||
grounding, has_head = await ground_applied_tag(db, img.id, tag.id)
|
||||
assert has_head is True
|
||||
assert grounding is not None
|
||||
assert grounding["bbox"] == pytest.approx([0.4, 0.4, 0.3, 0.3])
|
||||
assert grounding["kind"] == "concept"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ground_applied_tag_none_when_whole_image_wins(db):
|
||||
# Whole-image vector aligns with the head and out-scores an orthogonal crop →
|
||||
# grounding None (has_head True): the tag is best explained by the global
|
||||
# vector, so the chip hover shows the subtle whole-image frame, not a box.
|
||||
tag = await TagService(db).find_or_create("sky", TagKind.general)
|
||||
img = await _img(db, "g2" * 32, _emb(0)) # whole-image ALIGNED
|
||||
await _head(db, tag.id, slot=0)
|
||||
db.add(ImageRegion(
|
||||
image_record_id=img.id, kind="concept",
|
||||
rx=0.1, ry=0.1, rw=0.2, rh=0.2,
|
||||
siglip_embedding=_emb(5), embedding_version=await _embver(db),
|
||||
))
|
||||
await db.commit()
|
||||
|
||||
grounding, has_head = await ground_applied_tag(db, img.id, tag.id)
|
||||
assert has_head is True
|
||||
assert grounding is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ground_applied_tag_no_head(db):
|
||||
# A tag with no head in the current space (manual/artist/meta, or below the
|
||||
# head floor) can't be localized → (None, has_head False); the UI draws no
|
||||
# overlay at all, distinct from "the whole image won".
|
||||
tag = await TagService(db).find_or_create("artist:someone", TagKind.general)
|
||||
img = await _img(db, "g3" * 32, _emb(0))
|
||||
await db.commit()
|
||||
|
||||
grounding, has_head = await ground_applied_tag(db, img.id, tag.id)
|
||||
assert has_head is False
|
||||
assert grounding is None
|
||||
|
||||
Reference in New Issue
Block a user