CCIP match threshold (tunable, 0.85) + Explore → variance #150
@@ -0,0 +1,33 @@
|
|||||||
|
"""ml_settings.ccip_match_threshold — tunable CCIP character-match cut (#114)
|
||||||
|
|
||||||
|
The v1 matcher used a flat 0.75 cosine; live data showed that over-fires (a
|
||||||
|
high-reference character matched a scatter of images). 0.85 keeps the confident
|
||||||
|
single-character matches and drops the noise. Tunable from the GPU agent card.
|
||||||
|
|
||||||
|
Revision ID: 0063
|
||||||
|
Revises: 0062
|
||||||
|
Create Date: 2026-06-29
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0063"
|
||||||
|
down_revision: Union[str, None] = "0062"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"ml_settings",
|
||||||
|
sa.Column(
|
||||||
|
"ccip_match_threshold", sa.Float(), nullable=False,
|
||||||
|
server_default="0.85",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("ml_settings", "ccip_match_threshold")
|
||||||
@@ -21,6 +21,7 @@ _EDITABLE = (
|
|||||||
"head_auto_apply_precision",
|
"head_auto_apply_precision",
|
||||||
"head_auto_apply_enabled",
|
"head_auto_apply_enabled",
|
||||||
"head_auto_apply_min_positives",
|
"head_auto_apply_min_positives",
|
||||||
|
"ccip_match_threshold",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -48,6 +49,7 @@ async def get_settings():
|
|||||||
"head_auto_apply_precision": s.head_auto_apply_precision,
|
"head_auto_apply_precision": s.head_auto_apply_precision,
|
||||||
"head_auto_apply_enabled": s.head_auto_apply_enabled,
|
"head_auto_apply_enabled": s.head_auto_apply_enabled,
|
||||||
"head_auto_apply_min_positives": s.head_auto_apply_min_positives,
|
"head_auto_apply_min_positives": s.head_auto_apply_min_positives,
|
||||||
|
"ccip_match_threshold": s.ccip_match_threshold,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -115,6 +117,8 @@ def _validate(p: dict) -> str | None:
|
|||||||
return "head_auto_apply_precision must be between 0.5 and 0.999"
|
return "head_auto_apply_precision must be between 0.5 and 0.999"
|
||||||
if int(p["head_auto_apply_min_positives"]) < 1:
|
if int(p["head_auto_apply_min_positives"]) < 1:
|
||||||
return "head_auto_apply_min_positives must be >= 1"
|
return "head_auto_apply_min_positives must be >= 1"
|
||||||
|
if not (0.5 <= float(p["ccip_match_threshold"]) <= 0.999):
|
||||||
|
return "ccip_match_threshold must be between 0.5 and 0.999"
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -86,6 +86,12 @@ class MLSettings(Base):
|
|||||||
head_auto_apply_min_positives: Mapped[int] = mapped_column(
|
head_auto_apply_min_positives: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=30
|
Integer, nullable=False, default=30
|
||||||
)
|
)
|
||||||
|
# CCIP character-match cosine cut (#114). 0.85 default — the v1 flat 0.75
|
||||||
|
# over-fired (high-reference characters matched a scatter of images); 0.85
|
||||||
|
# keeps the confident single-character matches. Tunable from the agent card.
|
||||||
|
ccip_match_threshold: Mapped[float] = mapped_column(
|
||||||
|
Float, nullable=False, default=0.85
|
||||||
|
)
|
||||||
tagger_model_version: Mapped[str] = mapped_column(
|
tagger_model_version: Mapped[str] = mapped_column(
|
||||||
String(128), nullable=False, default="camie-tagger-v2"
|
String(128), nullable=False, default="camie-tagger-v2"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -16,15 +16,25 @@ the hands-on eval. numpy is imported lazily (API worker has it via pgvector).
|
|||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from ...models import ImageRegion, Tag, TagKind
|
from ...models import ImageRegion, MLSettings, Tag, TagKind
|
||||||
from ...models.tag import image_tag
|
from ...models.tag import image_tag
|
||||||
|
|
||||||
# Cosine-similarity floor to call a figure the same character. Conservative
|
# Cosine-similarity floor to call a figure the same character. The live setting
|
||||||
# default; tune from real matches (CCIP same-char clusters tightly).
|
# (ml_settings.ccip_match_threshold) drives it; this is only the fallback when no
|
||||||
DEFAULT_SIM_THRESHOLD = 0.75
|
# threshold is supplied AND no settings row exists.
|
||||||
|
DEFAULT_SIM_THRESHOLD = 0.85
|
||||||
_FIGURE_KINDS = ("face", "figure")
|
_FIGURE_KINDS = ("face", "figure")
|
||||||
|
|
||||||
|
|
||||||
|
async def _settings_threshold(session: AsyncSession) -> float:
|
||||||
|
val = (
|
||||||
|
await session.execute(
|
||||||
|
select(MLSettings.ccip_match_threshold).where(MLSettings.id == 1)
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
return float(val) if val is not None else DEFAULT_SIM_THRESHOLD
|
||||||
|
|
||||||
|
|
||||||
def _l2norm(mat, np):
|
def _l2norm(mat, np):
|
||||||
n = np.linalg.norm(mat, axis=1, keepdims=True)
|
n = np.linalg.norm(mat, axis=1, keepdims=True)
|
||||||
n[n == 0] = 1.0
|
n[n == 0] = 1.0
|
||||||
@@ -68,14 +78,18 @@ async def _tag_names(session: AsyncSession, tag_ids: list[int]) -> dict[int, str
|
|||||||
|
|
||||||
|
|
||||||
async def match_image(
|
async def match_image(
|
||||||
session: AsyncSession, image_id: int, threshold: float = DEFAULT_SIM_THRESHOLD
|
session: AsyncSession, image_id: int, threshold: float | None = None
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
"""Character suggestions for one image from its figure-region CCIP vectors:
|
"""Character suggestions for one image from its figure-region CCIP vectors:
|
||||||
[{tag_id, name, category:'character', score, source:'ccip'}], ranked.
|
[{tag_id, name, category:'character', score, source:'ccip'}], ranked.
|
||||||
Already-applied character tags are excluded. Empty if the image has no figure
|
Already-applied character tags are excluded. Empty if the image has no figure
|
||||||
CCIP vectors or no character references exist yet."""
|
CCIP vectors or no character references exist yet. threshold defaults to the
|
||||||
|
live ml_settings.ccip_match_threshold."""
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
|
if threshold is None:
|
||||||
|
threshold = await _settings_threshold(session)
|
||||||
|
|
||||||
qvecs = (
|
qvecs = (
|
||||||
await session.execute(
|
await session.execute(
|
||||||
select(ImageRegion.ccip_embedding).where(
|
select(ImageRegion.ccip_embedding).where(
|
||||||
|
|||||||
@@ -60,6 +60,22 @@
|
|||||||
Enqueues every image that doesn't have a CCIP embedding yet. Nothing
|
Enqueues every image that doesn't have a CCIP embedding yet. Nothing
|
||||||
processes until the agent is running.
|
processes until the agent is running.
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
|
<!-- Match strictness -->
|
||||||
|
<div class="fc-section-h mt-5 mb-1">Character-match strictness</div>
|
||||||
|
<div v-if="ml.settings" class="d-flex align-center" style="gap:12px">
|
||||||
|
<v-slider
|
||||||
|
v-model="threshold" :min="0.70" :max="0.95" :step="0.01"
|
||||||
|
color="accent" hide-details density="compact" class="flex-grow-1"
|
||||||
|
:loading="savingThreshold" @end="onSaveThreshold"
|
||||||
|
/>
|
||||||
|
<span class="fc-q__n" style="font-size:16px">{{ threshold.toFixed(2) }}</span>
|
||||||
|
</div>
|
||||||
|
<p class="fc-muted text-caption mt-1 mb-0">
|
||||||
|
How close a figure must be (CCIP cosine) to suggest a character. Higher =
|
||||||
|
stricter — fewer but more confident matches. 0.85 recommended; below ~0.80
|
||||||
|
a heavily-tagged character starts matching everything.
|
||||||
|
</p>
|
||||||
</MaintenanceTile>
|
</MaintenanceTile>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
@@ -69,14 +85,18 @@ import { computed, onMounted, onUnmounted, ref } from 'vue'
|
|||||||
|
|
||||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||||
import { useGpuStore } from '../../stores/gpu.js'
|
import { useGpuStore } from '../../stores/gpu.js'
|
||||||
|
import { useMLStore } from '../../stores/ml.js'
|
||||||
import { copyText } from '../../utils/clipboard.js'
|
import { copyText } from '../../utils/clipboard.js'
|
||||||
|
|
||||||
const store = useGpuStore()
|
const store = useGpuStore()
|
||||||
|
const ml = useMLStore()
|
||||||
const loading = ref(true)
|
const loading = ref(true)
|
||||||
const tokenValue = ref(null)
|
const tokenValue = ref(null)
|
||||||
const masked = ref(true)
|
const masked = ref(true)
|
||||||
const rotating = ref(false)
|
const rotating = ref(false)
|
||||||
const backfilling = ref(false)
|
const backfilling = ref(false)
|
||||||
|
const threshold = ref(0.85)
|
||||||
|
const savingThreshold = ref(false)
|
||||||
const queue = ref({ pending: 0, leased: 0, done: 0, error: 0 })
|
const queue = ref({ pending: 0, leased: 0, done: 0, error: 0 })
|
||||||
let pollTimer = null
|
let pollTimer = null
|
||||||
|
|
||||||
@@ -94,9 +114,27 @@ onMounted(async () => {
|
|||||||
}
|
}
|
||||||
await refreshQueue()
|
await refreshQueue()
|
||||||
pollTimer = setInterval(() => { if (!document.hidden) refreshQueue() }, 5000)
|
pollTimer = setInterval(() => { if (!document.hidden) refreshQueue() }, 5000)
|
||||||
|
try {
|
||||||
|
await ml.loadSettings()
|
||||||
|
if (ml.settings?.ccip_match_threshold != null) {
|
||||||
|
threshold.value = ml.settings.ccip_match_threshold
|
||||||
|
}
|
||||||
|
} catch { /* non-fatal */ }
|
||||||
})
|
})
|
||||||
onUnmounted(() => { if (pollTimer) clearInterval(pollTimer) })
|
onUnmounted(() => { if (pollTimer) clearInterval(pollTimer) })
|
||||||
|
|
||||||
|
async function onSaveThreshold() {
|
||||||
|
savingThreshold.value = true
|
||||||
|
try {
|
||||||
|
await ml.patchSettings({ ccip_match_threshold: threshold.value })
|
||||||
|
toast({ text: `Match strictness set to ${threshold.value.toFixed(2)}`, type: 'success' })
|
||||||
|
} catch (e) {
|
||||||
|
toast({ text: `Could not save: ${e.message}`, type: 'error' })
|
||||||
|
} finally {
|
||||||
|
savingThreshold.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function refreshQueue() {
|
async function refreshQueue() {
|
||||||
try { queue.value = await store.status() } catch { /* non-fatal */ }
|
try { queue.value = await store.status() } catch { /* non-fatal */ }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import { toast } from '../utils/toast.js'
|
|||||||
// trail. The store ALSO acts as a TagPanel "host" (current/currentImageId +
|
// trail. The store ALSO acts as a TagPanel "host" (current/currentImageId +
|
||||||
// tag CRUD over the anchor) so the Explore workspace reuses the modal's tag
|
// tag CRUD over the anchor) so the Explore workspace reuses the modal's tag
|
||||||
// rail verbatim for modal-parity tagging while rabbit-holing.
|
// rail verbatim for modal-parity tagging while rabbit-holing.
|
||||||
const NEIGHBOR_LIMIT = 24
|
const NEIGHBOR_LIMIT = 40 // a wider pool → more variety to browse + jump into
|
||||||
|
|
||||||
export const useExploreStore = defineStore('explore', () => {
|
export const useExploreStore = defineStore('explore', () => {
|
||||||
const api = useApi()
|
const api = useApi()
|
||||||
@@ -81,16 +81,26 @@ export const useExploreStore = defineStore('explore', () => {
|
|||||||
return cursor.value > 0 ? breadcrumb.value[cursor.value - 1].id : null
|
return cursor.value > 0 ? breadcrumb.value[cursor.value - 1].id : null
|
||||||
}
|
}
|
||||||
|
|
||||||
// → target: the next already-visited crumb if we'd stepped back, else a
|
// → target: after a ←, walk forward through the already-visited trail
|
||||||
// RANDOM neighbour to keep the rabbit-hole going. Null if neither exists.
|
// (browser-style). Otherwise jump to a varied neighbour to keep the
|
||||||
|
// rabbit-hole going — null if neither exists.
|
||||||
function forwardTarget () {
|
function forwardTarget () {
|
||||||
if (cursor.value >= 0 && cursor.value < breadcrumb.value.length - 1) {
|
if (cursor.value >= 0 && cursor.value < breadcrumb.value.length - 1) {
|
||||||
return breadcrumb.value[cursor.value + 1].id
|
return breadcrumb.value[cursor.value + 1].id
|
||||||
}
|
}
|
||||||
if (neighbors.value.length) {
|
if (!neighbors.value.length) return null
|
||||||
return neighbors.value[Math.floor(Math.random() * neighbors.value.length)].id
|
// Prefer UNVISITED neighbours so → opens something new instead of landing on
|
||||||
}
|
// a crumb (which snaps the cursor back into the trail — the "loops back"
|
||||||
return null
|
// report). Fall back to the full set only if every neighbour's been seen.
|
||||||
|
const seen = new Set(breadcrumb.value.map((c) => c.id))
|
||||||
|
let pool = neighbors.value.filter((n) => !seen.has(n.id))
|
||||||
|
if (!pool.length) pool = neighbors.value
|
||||||
|
// neighbors come similarity-sorted (nearest first). Skip the closest slice —
|
||||||
|
// those near-duplicates are exactly what you get stuck cycling through — and
|
||||||
|
// pick from the more-varied remainder, for real variance in the walk.
|
||||||
|
const skip = pool.length >= 6 ? Math.floor(pool.length / 3) : 0
|
||||||
|
const cands = pool.slice(skip)
|
||||||
|
return cands[Math.floor(Math.random() * cands.length)].id
|
||||||
}
|
}
|
||||||
|
|
||||||
function reset () {
|
function reset () {
|
||||||
|
|||||||
@@ -86,3 +86,20 @@ async def test_no_figure_vectors_means_no_match(db):
|
|||||||
query = await _img(db, "g" * 64)
|
query = await _img(db, "g" * 64)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
assert await match_image(db, query.id) == []
|
assert await match_image(db, query.id) == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_threshold_gates_borderline_match(db):
|
||||||
|
# A figure ~0.9 cosine from the reference: matched at 0.85, dropped at 0.95.
|
||||||
|
raven = await TagService(db).find_or_create("Raven", TagKind.character)
|
||||||
|
ref = await _img(db, "h" * 64)
|
||||||
|
await _figure(db, ref.id, _ccip(0)) # e0
|
||||||
|
await _tag_image(db, ref.id, raven.id)
|
||||||
|
near = [0.0] * 768
|
||||||
|
near[0], near[1] = 0.9, 0.4359 # |·|=1, cos(e0)=0.9
|
||||||
|
query = await _img(db, "i" * 64)
|
||||||
|
await _figure(db, query.id, near)
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
assert any(m["tag_id"] == raven.id for m in await match_image(db, query.id, 0.85))
|
||||||
|
assert await match_image(db, query.id, 0.95) == []
|
||||||
|
|||||||
Reference in New Issue
Block a user