Tag-hover → crop-region grounding overlay (#1206, milestone #133) #195

Merged
bvandeusen merged 5 commits from dev into main 2026-07-06 13:29:31 -04:00
14 changed files with 494 additions and 41 deletions
+4
View File
@@ -46,6 +46,10 @@ async def get_suggestions(image_id: int):
# (not dropped) so the rail can show it rejected + offer
# one-click un-reject.
"rejected": s.rejected,
# the crop region that produced this tag (#1206) —
# {bbox,kind,detector} or null (whole-image won). Drives
# the hover→overlay highlight.
"grounding": s.grounding,
}
for s in items
]
+16
View File
@@ -11,6 +11,7 @@ from ..models.tag import image_tag
from ..models.tag_suggestion_rejection import TagSuggestionRejection
from ..services.bulk_tag_service import BulkTagService
from ..services.ml.aliases import AliasService
from ..services.ml.heads import ground_applied_tag
from ..services.series_match_service import SeriesMatchService
from ..services.series_service import SeriesError, SeriesService
from ..services.tag_directory_service import TagDirectoryService
@@ -310,6 +311,21 @@ async def confirm_tag_on_image(image_id: int, tag_id: int):
return "", 204
@tags_bp.route(
"/images/<int:image_id>/tags/<int:tag_id>/grounding", methods=["GET"]
)
async def tag_grounding(image_id: int, tag_id: int):
"""Which crop region best explains an ALREADY-APPLIED tag on this image
(#1206 Step 4). Powers the hover→overlay highlight on applied tag chips,
mirroring the suggestion rail's live grounding. Computed on demand (applied
tags aren't scored live). → {grounding: {bbox,kind,detector}|null,
has_head: bool}; has_head False means the tag has no head to localize with,
so the chip shows no overlay."""
async with get_session() as session:
grounding, has_head = await ground_applied_tag(session, image_id, tag_id)
return jsonify({"grounding": grounding, "has_head": has_head})
@tags_bp.route("/tags/<int:tag_id>", methods=["GET"])
async def get_tag(tag_id: int):
"""Resolve a single tag (used by the gallery to label its active
+21 -5
View File
@@ -161,16 +161,22 @@ async def match_image(
if threshold is None:
threshold = await _settings_threshold(session)
qvecs = (
# Keep each figure region's bbox alongside its vector so a match can point at
# the figure that matched (#1206 grounding), not just the score.
fig_rows = (
await session.execute(
select(ImageRegion.ccip_embedding).where(
select(
ImageRegion.ccip_embedding,
ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh,
ImageRegion.kind, ImageRegion.detector_version,
).where(
ImageRegion.image_record_id == image_id,
ImageRegion.kind.in_(_FIGURE_KINDS),
ImageRegion.ccip_embedding.is_not(None),
)
)
).scalars().all()
if not qvecs:
).all()
if not fig_rows:
return []
refs = await character_references(session)
if not refs:
@@ -186,13 +192,21 @@ async def match_image(
)
names = await _tag_names(session, [t for t in refs if t not in applied])
qvecs = [r[0] for r in fig_rows]
fig_meta = [
{"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector}
for _v, rx, ry, rw, rh, kind, detector in fig_rows
]
Q = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in qvecs]), np)
out = []
for tag_id, vecs in refs.items():
if tag_id in applied:
continue
R = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np)
best = float((Q @ R.T).max()) # best (query figure, reference) cosine
sims = Q @ R.T # (n_query_figures, n_references)
per_figure = sims.max(axis=1) # best reference cosine per figure
best_figure = int(per_figure.argmax())
best = float(per_figure[best_figure])
if best >= threshold:
out.append({
"tag_id": tag_id,
@@ -200,6 +214,8 @@ async def match_image(
"category": "character",
"score": round(best, 4),
"source": "ccip",
# the figure region that matched → grounds the character tag.
"grounding": fig_meta[best_figure],
})
out.sort(key=lambda d: d["score"], reverse=True)
return out
+99 -25
View File
@@ -341,6 +341,53 @@ async def _current_heads(session: AsyncSession, embedding_version: str):
return loaded
async def _image_bag(
session: AsyncSession, image_id: int, cur_version: str,
) -> tuple[list, list[dict | None]]:
"""The max-over-bag inputs for one image: the whole-image SigLIP vector (when
it's in the current model's space) PLUS every concept-region crop embedded in
that space. Returns (bag, bag_meta) as PARALLEL lists — bag_meta[i] is None for
the whole-image row, else the region's {bbox, kind, detector} so a surfaced tag
can point back at the crop that produced it (#1206 grounding).
Only current-version embeddings enter the bag: mid model-swap (#1190) an image
still carrying an OLD-version whole-image vector is skipped rather than scored
by heads trained in a different space; a legacy NULL version is treated as
current (those predate per-row stamping). Shared by live scoring (score_image)
and on-demand applied-tag grounding (ground_applied_tag, #1206 Step 4)."""
import numpy as np
img = await session.get(ImageRecord, image_id)
bag: list = []
bag_meta: list[dict | None] = []
if img is None:
return bag, bag_meta
if img.siglip_embedding is not None and img.siglip_model_version in (
cur_version, None,
):
bag.append(np.asarray(img.siglip_embedding, dtype=np.float32))
bag_meta.append(None)
region_rows = (
await session.execute(
select(
ImageRegion.siglip_embedding,
ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh,
ImageRegion.kind, ImageRegion.detector_version,
)
.where(ImageRegion.image_record_id == image_id)
.where(ImageRegion.siglip_embedding.is_not(None))
.where(ImageRegion.embedding_version == cur_version)
)
).all()
for vec, rx, ry, rw, rh, kind, detector in region_rows:
if vec is not None:
bag.append(np.asarray(vec, dtype=np.float32))
bag_meta.append(
{"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector}
)
return bag, bag_meta
async def score_image(
session: AsyncSession, image_id: int, threshold_override: float | None = None,
) -> list[dict]:
@@ -361,35 +408,12 @@ async def score_image(
always in the bag, so this can never score lower than whole-image alone."""
import numpy as np
img = await session.get(ImageRecord, image_id)
if img is None:
return []
settings = await _settings_async(session)
cur_version = settings.embedder_model_version
heads = await _current_heads(session, cur_version)
if heads["W"] is None:
return []
# Only embeddings in the CURRENT model's space enter the bag. Mid model-swap
# (#1190), an image still carrying the OLD-version whole-image vector is
# skipped rather than scored by heads trained in a different space; a legacy
# NULL version is treated as current (those predate per-row stamping).
bag = []
if img.siglip_embedding is not None and img.siglip_model_version in (
cur_version, None,
):
bag.append(np.asarray(img.siglip_embedding, dtype=np.float32))
region_vecs = (
await session.execute(
select(ImageRegion.siglip_embedding)
.where(ImageRegion.image_record_id == image_id)
.where(ImageRegion.siglip_embedding.is_not(None))
.where(ImageRegion.embedding_version == cur_version)
)
).all()
for (vec,) in region_vecs:
if vec is not None:
bag.append(np.asarray(vec, dtype=np.float32))
bag, bag_meta = await _image_bag(session, image_id, cur_version)
if not bag:
return []
@@ -398,7 +422,11 @@ async def score_image(
norms[norms == 0] = 1.0
Xn = X / norms
Z = Xn @ heads["W"].T + heads["b"] # (B, H)
probs = (1.0 / (1.0 + np.exp(-Z))).max(axis=0) # (H,) best over the bag
probs_bag = 1.0 / (1.0 + np.exp(-Z)) # (B, H)
probs = probs_bag.max(axis=0) # (H,) best over the bag
# ARGMAX beside the max: WHICH bag row won each head → the region that grounds
# the tag (bag_meta[win]); None when the whole-image vector won (#1206).
winners = probs_bag.argmax(axis=0) # (H,)
out = []
for i, p in enumerate(probs):
if threshold_override is not None:
@@ -416,11 +444,57 @@ async def score_image(
"name": m["name"],
"category": m["category"],
"score": float(p),
"grounding": bag_meta[int(winners[i])],
})
out.sort(key=lambda d: d["score"], reverse=True)
return out
async def ground_applied_tag(
session: AsyncSession, image_id: int, tag_id: int,
) -> tuple[dict | None, bool]:
"""On-demand grounding for an ALREADY-APPLIED tag (#1206 Step 4). Applied tags
aren't scored live, so recompute the max-over-bag argmax for just this tag's
head — which crop region best explains the tag on this image — mirroring what
score_image records for live suggestions. Returns (grounding, has_head):
- has_head False → the tag has no head in the current embedding space (manual/
artist/meta tags, or a concept below the head floor). Nothing to localize
with, so the UI shows no overlay (distinct from "the whole image won").
- grounding None (has_head True) → the whole-image vector best explains it,
not any crop; the UI shows the subtle whole-image frame.
- grounding {bbox, kind, detector} → the winning region.
Character heads are covered too (character is a head kind); this deliberately
reuses the SigLIP head bag rather than the CCIP figure path so every applied
concept grounds through one consistent mechanism."""
import numpy as np
cur_version = (await _settings_async(session)).embedder_model_version
row = (
await session.execute(
select(TagHead.weights, TagHead.bias).where(
TagHead.tag_id == tag_id,
TagHead.embedding_version == cur_version,
)
)
).one_or_none()
if row is None:
return None, False
bag, bag_meta = await _image_bag(session, image_id, cur_version)
if not bag:
return None, True
X = np.vstack(bag)
norms = np.linalg.norm(X, axis=1, keepdims=True)
norms[norms == 0] = 1.0
Xn = X / norms
# The sigmoid is monotonic in the logit, so the highest-probability bag row is
# just argmax of the raw score — no need to exponentiate to pick the winner.
z = Xn @ np.asarray(row.weights, dtype=np.float32) + float(row.bias) # (B,)
return bag_meta[int(z.argmax())], True
async def _settings_async(session: AsyncSession) -> MLSettings:
return (
await session.execute(select(MLSettings).where(MLSettings.id == 1))
+11
View File
@@ -43,6 +43,11 @@ class Suggestion:
# the rejection is VISIBLE and REVERSIBLE in the rail (misclick recovery,
# operator-asked 2026-06-27) instead of silently vanishing or re-suggesting.
rejected: bool = False
# grounding = the crop region that produced this suggestion (#1206):
# {bbox:[x,y,w,h] normalized, kind, detector}. None when the whole-image
# vector won (not localized) or for a CCIP-only hit (figure grounding TBD).
# Lets the rail highlight the exact region on hover.
grounding: dict | None = None
@dataclass
@@ -103,6 +108,7 @@ class SuggestionService:
for h in hits:
merged[(h["category"], h["tag_id"])] = {
"name": h["name"], "score": h["score"], "source": "head",
"grounding": h.get("grounding"),
}
for c in ccip_hits:
key = ("character", c["tag_id"])
@@ -110,9 +116,13 @@ class SuggestionService:
if ex is not None:
ex["source"] = "both"
ex["score"] = max(ex["score"], c["score"])
# Keep the head's localized crop if it had one; else fall back to
# the CCIP figure so a corroborated character still grounds (#1206).
ex["grounding"] = ex.get("grounding") or c.get("grounding")
else:
merged[key] = {
"name": c["name"], "score": c["score"], "source": "ccip",
"grounding": c.get("grounding"),
}
result = SuggestionList()
@@ -128,6 +138,7 @@ class SuggestionService:
source=m["source"],
creates_new_tag=False,
rejected=tag_id in rejected,
grounding=m.get("grounding"),
)
)
for cat in result.by_category:
+111 -2
View File
@@ -16,20 +16,91 @@
transform: `translate(${panZoom.state.x}px, ${panZoom.state.y}px) scale(${panZoom.state.scale})`
}"
draggable="false"
@load="onImgLoad"
>
<!-- #1206 grounding overlay: tracks the <img>'s live rendered rect (so it's
correct under object-fit letterboxing + pan/zoom) and draws the crop
region a hovered suggestion came from. Null grounding a subtle
whole-image frame ("the global vector won, not a crop"). -->
<div
v-if="hover && overlayBox"
class="fc-canvas__overlay"
:style="{
left: overlayBox.left + 'px', top: overlayBox.top + 'px',
width: overlayBox.width + 'px', height: overlayBox.height + 'px',
}"
>
<div
class="fc-canvas__region"
:class="{ 'fc-canvas__region--whole': !regionStyle }"
:style="regionStyle || {}"
>
<span class="fc-canvas__region-label">{{ regionLabel }}</span>
</div>
</div>
</div>
</template>
<script setup>
import { ref, watch } from 'vue'
import { computed, nextTick, onMounted, onUnmounted, ref, watch } from 'vue'
import { usePanZoom } from '../../composables/usePanZoom.js'
const props = defineProps({ src: String, alt: String })
const props = defineProps({
src: String,
alt: String,
// Hovered-suggestion grounding, or null when nothing is hovered:
// null → no overlay
// { g: {bbox,kind,detector} } → highlight that crop region
// { g: null } → whole-image frame (not localized)
hover: { type: Object, default: null },
})
const emit = defineEmits(['close-request'])
const panZoom = usePanZoom()
const canvasEl = ref(null)
const imgEl = ref(null)
const overlayBox = ref(null) // the <img>'s rect relative to the canvas
// The <img> renders aspect-fit + transformed, so its getBoundingClientRect is
// the ground truth for where the picture actually is. Position the overlay to
// match it, then place the region as a % within.
function measure () {
const img = imgEl.value
const canvas = canvasEl.value
if (!img || !canvas) { overlayBox.value = null; return }
const ir = img.getBoundingClientRect()
const cr = canvas.getBoundingClientRect()
if (!ir.width || !ir.height) { overlayBox.value = null; return }
overlayBox.value = {
left: ir.left - cr.left, top: ir.top - cr.top,
width: ir.width, height: ir.height,
}
}
const regionStyle = computed(() => {
const g = props.hover?.g
if (!g || !Array.isArray(g.bbox)) return null // null grounding → whole frame
const [rx, ry, rw, rh] = g.bbox
return {
left: `${rx * 100}%`, top: `${ry * 100}%`,
width: `${rw * 100}%`, height: `${rh * 100}%`,
}
})
const regionLabel = computed(() => {
const g = props.hover?.g
return g ? (g.detector || g.kind || 'region') : 'whole image'
})
// Re-measure whenever hovering starts or the image could have moved.
watch(() => props.hover, (h) => { if (h) nextTick(measure) })
watch(
() => [panZoom.state.scale, panZoom.state.x, panZoom.state.y],
() => { if (props.hover) measure() },
)
function onResize () { if (props.hover) measure() }
function onImgLoad () { if (props.hover) measure() }
onMounted(() => window.addEventListener('resize', onResize))
onUnmounted(() => window.removeEventListener('resize', onResize))
// Reset zoom when the src changes (prev/next nav).
watch(() => props.src, () => panZoom.reset())
@@ -79,4 +150,42 @@ function onCanvasClick(ev) {
transition: transform 100ms ease-out;
pointer-events: none;
}
/* Grounding overlay — never intercepts pan/zoom/click. */
.fc-canvas__overlay {
position: absolute;
pointer-events: none;
z-index: 2;
}
.fc-canvas__region {
position: absolute;
border: 2px solid rgb(var(--v-theme-accent));
border-radius: 3px;
box-shadow: 0 0 0 9999px rgba(0, 0, 0, 0.28); /* dim everything outside it */
transition: all 90ms ease-out;
}
/* Null grounding: a subtle full-image frame, no dimming — it says "this tag
came from the whole image, not a crop." */
.fc-canvas__region--whole {
inset: 0;
border-style: dashed;
border-color: rgb(var(--v-theme-on-surface-variant));
box-shadow: none;
}
.fc-canvas__region-label {
position: absolute;
top: 0; left: 0;
transform: translateY(-100%);
font-size: 11px;
font-family: 'JetBrains Mono', monospace;
line-height: 1.4;
padding: 1px 6px;
white-space: nowrap;
background: rgb(var(--v-theme-accent));
color: rgb(var(--v-theme-on-surface));
border-radius: 3px 3px 0 0;
}
.fc-canvas__region--whole .fc-canvas__region-label {
background: rgb(var(--v-theme-on-surface-variant));
}
</style>
@@ -60,6 +60,7 @@
<template v-if="modal.current">
<ImageCanvas
v-if="!isVideo" :src="modal.current.image_url" :alt="`Image ${modal.current.id}`"
:hover="hoverGrounding"
@close-request="$emit('close')"
/>
<VideoCanvas
@@ -94,7 +95,7 @@
</template>
<script setup>
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
import { computed, onMounted, onUnmounted, provide, ref, watch } from 'vue'
import { useModalStore } from '../../stores/modal.js'
import { arrowNavAllowed, isTextEntry } from '../../utils/textEntry.js'
import ImageCanvas from './ImageCanvas.vue'
@@ -110,6 +111,13 @@ const modal = useModalStore()
const rootEl = ref(null)
const showHelp = ref(false)
// #1206: a deep descendant (SuggestionItem) sets this on hover to the crop that
// produced a suggestion; ImageCanvas draws it over the image. provide/inject
// avoids relaying a hover event through TagPanel → SuggestionsPanel → group.
// null = nothing hovered; { g: grounding|null } = hovering (g null → whole-image).
const hoverGrounding = ref(null)
provide('fcSuggestionHover', hoverGrounding)
const isVideo = computed(() =>
modal.current?.mime && modal.current.mime.startsWith('video/')
)
@@ -3,7 +3,10 @@
name, score, and action buttons as one "object" (operator-asked
2026-06-01). The row itself is informational; the green / red
verdict pair + 3-dot alias menu are the action affordances. -->
<div class="fc-suggestion" :class="{ 'fc-suggestion--rejected': suggestion.rejected }">
<div
class="fc-suggestion" :class="{ 'fc-suggestion--rejected': suggestion.rejected }"
@mouseenter="onEnter" @mouseleave="onLeave"
>
<span class="fc-suggestion__name">
{{ suggestion.display_name }}
<span v-if="suggestion.rejected" class="fc-suggestion__rejected-tag"
@@ -72,12 +75,22 @@
</template>
<script setup>
import { computed } from 'vue'
import { computed, inject } from 'vue'
import KebabMenu from '../common/KebabMenu.vue'
const props = defineProps({ suggestion: { type: Object, required: true } })
defineEmits(['accept', 'alias', 'remove-alias', 'dismiss', 'undismiss'])
// #1206: on hover, tell the image viewer which crop produced this suggestion so
// it highlights that region. Provided by ImageViewer/Explore; a no-op elsewhere.
const hover = inject('fcSuggestionHover', null)
function onEnter () {
if (hover) hover.value = { g: props.suggestion.grounding ?? null }
}
function onLeave () {
if (hover) hover.value = null
}
const scorePct = computed(() => `${Math.round(props.suggestion.score * 100)}%`)
// Kebab now only carries alias actions: show it when this suggestion can be
// aliased (raw model key, not yet aliased) or is already aliased (so it can be
+46 -3
View File
@@ -1,5 +1,5 @@
<template>
<span class="fc-tag-chip">
<span class="fc-tag-chip" @mouseenter="onEnter" @mouseleave="onLeave">
<v-chip
size="small" closable
:color="store.colorFor(tag.kind)" variant="tonal"
@@ -38,14 +38,57 @@
</template>
<script setup>
import { computed } from 'vue'
import { computed, inject } from 'vue'
import { useTagStore } from '../../stores/tags.js'
import { useApi } from '../../composables/useApi.js'
import KebabMenu from '../common/KebabMenu.vue'
const props = defineProps({ tag: { type: Object, required: true } })
const props = defineProps({
tag: { type: Object, required: true },
// When set (the tagging panels), hovering the chip asks the backend which crop
// region best explains this applied tag and lights it up on the image — the
// same overlay the suggestion rail uses (#1206 Step 4). Omitted elsewhere →
// the hover is inert (no injected target, or no image to ground against).
imageId: { type: Number, default: null },
})
defineEmits(['remove', 'rename', 'set-fandom', 'navigate'])
const store = useTagStore()
const api = useApi()
// #1206 Step 4: applied-tag grounding. `fcSuggestionHover` is provided by the
// image viewer / Explore host (a no-op elsewhere). Applied tags aren't scored
// live, so we fetch the winning region on demand and cache it per (image, tag).
const hover = inject('fcSuggestionHover', null)
const _groundingCache = new Map()
// Bumped on every enter/leave so a slow fetch that resolves after the pointer
// has moved on can't draw a stale overlay.
let hoverSeq = 0
async function onEnter () {
if (!hover || props.imageId == null) return
const seq = ++hoverSeq
const key = `${props.imageId}:${props.tag.id}`
let res = _groundingCache.get(key)
if (res === undefined) {
try {
res = await api.get(
`/api/images/${props.imageId}/tags/${props.tag.id}/grounding`
)
_groundingCache.set(key, res)
} catch {
return // best-effort — leave the overlay untouched on error
}
}
if (seq !== hoverSeq) return // pointer already left / moved to another chip
// No head → nothing to localize; don't draw an overlay at all. With a head,
// null grounding still draws the whole-image frame ("the global vector won").
if (res.has_head) hover.value = { g: res.grounding ?? null }
}
function onLeave () {
hoverSeq++
if (hover) hover.value = null
}
// Show a character's fandom inline (truncated). Falls back to a bare arrow when
// only fandom_id is known but the name wasn't resolved (older payloads).
+1 -1
View File
@@ -4,7 +4,7 @@
<div class="fc-tag-panel__chips">
<TagChip
v-for="tag in host.current?.tags || []"
:key="tag.id" :tag="tag"
:key="tag.id" :tag="tag" :image-id="host.currentImageId"
@remove="onRemove" @rename="openRename" @set-fandom="openSetFandom"
@navigate="onNavigate"
/>
+7 -1
View File
@@ -98,6 +98,7 @@
:key="store.anchor.id"
:src="store.anchor.image_url"
:alt="`Image ${store.anchor.id}`"
:hover="hoverGrounding"
/>
<VideoCanvas
v-else
@@ -131,7 +132,7 @@
</template>
<script setup>
import { computed, nextTick, onMounted, onUnmounted, ref, watch } from 'vue'
import { computed, nextTick, onMounted, onUnmounted, provide, ref, watch } from 'vue'
import { useRoute, useRouter } from 'vue-router'
import { useApi } from '../composables/useApi.js'
import { useExploreStore } from '../stores/explore.js'
@@ -152,6 +153,11 @@ const modal = useModalStore()
const anchorId = computed(() => route.params.imageId || null)
const isVideo = computed(() => !!store.anchor?.mime?.startsWith('video/'))
// #1206: hovering a suggestion in the rail highlights the crop it came from on
// the anchor image (same provide/inject as the modal viewer).
const hoverGrounding = ref(null)
provide('fcSuggestionHover', hoverGrounding)
const seeding = ref(false)
const seedError = ref(null)
const tagPanelRef = ref(null)
+64
View File
@@ -1,4 +1,8 @@
import pytest
from sqlalchemy import select
from backend.app.models import ImageRecord, ImageRegion, MLSettings, TagHead, TagKind
from backend.app.services.tag_service import TagService
pytestmark = pytest.mark.integration
@@ -94,3 +98,63 @@ async def test_unknown_prefix_kept_literal(client):
body = await resp.get_json()
assert body["name"] == "Http://example.com"
assert body["kind"] == "general"
# --- #1206 Step 4: applied-tag grounding endpoint (hover on applied chips) ----
@pytest.mark.asyncio
async def test_applied_tag_grounding_returns_winning_region(client, db):
# Hovering an applied chip fetches the crop that best explains the tag. Here
# the whole-image vector is orthogonal to the head but a concept crop aligns,
# so the crop wins the max-over-bag → grounding is that region's box.
ver = (await db.execute(
select(MLSettings).where(MLSettings.id == 1)
)).scalar_one().embedder_model_version
img = ImageRecord(
path="/images/grchip.jpg", sha256="gc" * 32, size_bytes=1,
mime="image/jpeg", width=1, height=1, origin="imported_filesystem",
integrity_status="unknown",
siglip_embedding=[0.0] * 5 + [3.0] + [0.0] * 1146, # whole-image ⟂ head
)
db.add(img)
await db.flush()
tag = await TagService(db).find_or_create("glasses", TagKind.general)
db.add(TagHead(
tag_id=tag.id, embedding_version=ver,
weights=[1.0] + [0.0] * 1151, bias=0.0, suggest_threshold=0.5,
auto_apply_threshold=None, n_pos=10, n_neg=30,
ap=0.8, precision_cv=0.9, recall=0.6,
))
db.add(ImageRegion(
image_record_id=img.id, kind="concept",
rx=0.4, ry=0.4, rw=0.3, rh=0.3,
siglip_embedding=[3.0] + [0.0] * 1151, embedding_version=ver,
))
await db.commit()
resp = await client.get(f"/api/images/{img.id}/tags/{tag.id}/grounding")
assert resp.status_code == 200
body = await resp.get_json()
assert body["has_head"] is True
assert body["grounding"]["bbox"] == pytest.approx([0.4, 0.4, 0.3, 0.3])
assert body["grounding"]["kind"] == "concept"
@pytest.mark.asyncio
async def test_applied_tag_grounding_no_head(client, db):
# A tag with no head can't be localized → has_head False, grounding null; the
# chip shows no overlay. Validates the response contract the frontend reads.
img = ImageRecord(
path="/images/grchip2.jpg", sha256="gd" * 32, size_bytes=1,
mime="image/jpeg", width=1, height=1, origin="imported_filesystem",
integrity_status="unknown",
)
db.add(img)
await db.flush()
tag = await TagService(db).find_or_create("NoHeadHere", TagKind.general)
await db.commit()
resp = await client.get(f"/api/images/{img.id}/tags/{tag.id}/grounding")
assert resp.status_code == 200
assert await resp.get_json() == {"grounding": None, "has_head": False}
+4
View File
@@ -55,6 +55,10 @@ async def test_matches_same_character_across_images(db):
m = next(x for x in matches if x["tag_id"] == raven.id)
assert m["source"] == "ccip" and m["category"] == "character"
assert m["score"] > 0.9
# #1206: the match grounds to the figure region that matched (hover → the
# figure box lights up), so a character suggestion is localized too.
assert m["grounding"]["bbox"] == pytest.approx([0.0, 0.0, 1.0, 1.0])
assert m["grounding"]["kind"] == "figure"
@pytest.mark.asyncio
+86 -1
View File
@@ -7,6 +7,7 @@ from sqlalchemy import select
from backend.app.models import ImageRecord, ImageRegion, MLSettings, TagHead, TagKind
from backend.app.models.tag import image_tag
from backend.app.services.ml.allowlist import AllowlistService
from backend.app.services.ml.heads import ground_applied_tag
from backend.app.services.ml.suggestions import SuggestionService
from backend.app.services.tag_service import TagService
@@ -187,7 +188,31 @@ async def test_concept_region_surfaces_via_max_over_bag(db):
))
await db.commit()
general = (await SuggestionService(db).for_image(img.id)).by_category["general"]
assert any(s.canonical_tag_id == tag.id and s.score > 0.7 for s in general)
s = next(x for x in general if x.canonical_tag_id == tag.id)
assert s.score > 0.7
# #1206: the winning crop grounds the tag — hover highlights THIS region
# (the matching-version crop at 0.4,0.4,0.3,0.3), not the whole image.
assert s.grounding is not None
assert s.grounding["bbox"] == pytest.approx([0.4, 0.4, 0.3, 0.3])
assert s.grounding["kind"] == "concept"
@pytest.mark.asyncio
async def test_grounding_none_when_whole_image_wins(db):
# #1206: when the whole-image vector (not a crop) produces the winning score,
# grounding is None — the tag came from the global vector, not a region.
tag = await TagService(db).find_or_create("sky", TagKind.general)
img = await _img(db, "d1" * 32, _emb(0)) # whole-image ALIGNED w/ head
await _head(db, tag.id, slot=0, suggest_threshold=0.5)
db.add(ImageRegion( # an orthogonal crop (0.5)
image_record_id=img.id, kind="concept",
rx=0.1, ry=0.1, rw=0.2, rh=0.2,
siglip_embedding=_emb(5), embedding_version=await _embver(db),
))
await db.commit()
general = (await SuggestionService(db).for_image(img.id)).by_category["general"]
s = next(x for x in general if x.canonical_tag_id == tag.id)
assert s.grounding is None
@pytest.mark.asyncio
@@ -256,3 +281,63 @@ async def test_ccip_character_surfaces_in_rail(db):
if c.canonical_tag_id == raven.id
)
assert m.source == "ccip"
# --- #1206 Step 4: on-demand grounding for ALREADY-APPLIED tag chips ---------
# Applied tags aren't scored live, so ground_applied_tag recomputes the winning
# bag row for one tag's head on demand — the data behind hovering an applied chip.
@pytest.mark.asyncio
async def test_ground_applied_tag_returns_winning_region(db):
# Whole-image vector is orthogonal to the head; a concept crop aligns with it,
# so the crop is the max-over-bag winner → grounding is THAT region's box.
tag = await TagService(db).find_or_create("glasses", TagKind.general)
img = await _img(db, "g1" * 32, _emb(5)) # whole-image ⟂ head
await _head(db, tag.id, slot=0)
db.add(ImageRegion(
image_record_id=img.id, kind="concept",
rx=0.4, ry=0.4, rw=0.3, rh=0.3,
siglip_embedding=_emb(0), embedding_version=await _embver(db),
))
await db.commit()
grounding, has_head = await ground_applied_tag(db, img.id, tag.id)
assert has_head is True
assert grounding is not None
assert grounding["bbox"] == pytest.approx([0.4, 0.4, 0.3, 0.3])
assert grounding["kind"] == "concept"
@pytest.mark.asyncio
async def test_ground_applied_tag_none_when_whole_image_wins(db):
# Whole-image vector aligns with the head and out-scores an orthogonal crop →
# grounding None (has_head True): the tag is best explained by the global
# vector, so the chip hover shows the subtle whole-image frame, not a box.
tag = await TagService(db).find_or_create("sky", TagKind.general)
img = await _img(db, "g2" * 32, _emb(0)) # whole-image ALIGNED
await _head(db, tag.id, slot=0)
db.add(ImageRegion(
image_record_id=img.id, kind="concept",
rx=0.1, ry=0.1, rw=0.2, rh=0.2,
siglip_embedding=_emb(5), embedding_version=await _embver(db),
))
await db.commit()
grounding, has_head = await ground_applied_tag(db, img.id, tag.id)
assert has_head is True
assert grounding is None
@pytest.mark.asyncio
async def test_ground_applied_tag_no_head(db):
# A tag with no head in the current space (manual/artist/meta, or below the
# head floor) can't be localized → (None, has_head False); the UI draws no
# overlay at all, distinct from "the whole image won".
tag = await TagService(db).find_or_create("artist:someone", TagKind.general)
img = await _img(db, "g3" * 32, _emb(0))
await db.commit()
grounding, has_head = await ground_applied_tag(db, img.id, tag.id)
assert has_head is False
assert grounding is None