From 625336b6b47be82aac3e2c1b6c674a688d0596d0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 29 Jun 2026 20:41:09 -0400 Subject: [PATCH] feat(ccip): tunable match threshold, default 0.85 (#114) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live data showed the v1 flat 0.75 cosine over-fired — ~64% of matched images got 3-10 character guesses dominated by the most-referenced characters (a 27-ref character clears a low bar on many images). A sweep showed 0.85 collapses the noise (noisy multi-matches 47→3) while keeping the confident single-character matches. - ml_settings.ccip_match_threshold (migration 0063, default 0.85); match_image reads it (override still accepted). DEFAULT_SIM_THRESHOLD fallback 0.75→0.85. - Exposed in GET/PATCH /api/ml/settings (validated 0.5–0.999). - Slider in the GPU agent card ("Character-match strictness") — tune live, no redeploy, same observe-and-tune loop as auto-apply. Test: a ~0.9-cosine figure matches at 0.85, dropped at 0.95. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa --- alembic/versions/0063_ccip_match_threshold.py | 33 ++++++++++++++++ backend/app/api/ml_admin.py | 4 ++ backend/app/models/ml_settings.py | 6 +++ backend/app/services/ml/ccip.py | 26 ++++++++++--- .../src/components/settings/GpuAgentCard.vue | 38 +++++++++++++++++++ tests/test_ccip.py | 17 +++++++++ 6 files changed, 118 insertions(+), 6 deletions(-) create mode 100644 alembic/versions/0063_ccip_match_threshold.py diff --git a/alembic/versions/0063_ccip_match_threshold.py b/alembic/versions/0063_ccip_match_threshold.py new file mode 100644 index 0000000..d841398 --- /dev/null +++ b/alembic/versions/0063_ccip_match_threshold.py @@ -0,0 +1,33 @@ +"""ml_settings.ccip_match_threshold — tunable CCIP character-match cut (#114) + +The v1 matcher used a flat 0.75 cosine; live data showed that over-fires (a +high-reference character matched a scatter of images). 0.85 keeps the confident +single-character matches and drops the noise. Tunable from the GPU agent card. + +Revision ID: 0063 +Revises: 0062 +Create Date: 2026-06-29 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0063" +down_revision: Union[str, None] = "0062" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "ml_settings", + sa.Column( + "ccip_match_threshold", sa.Float(), nullable=False, + server_default="0.85", + ), + ) + + +def downgrade() -> None: + op.drop_column("ml_settings", "ccip_match_threshold") diff --git a/backend/app/api/ml_admin.py b/backend/app/api/ml_admin.py index 1b0e1d1..96e2e23 100644 --- a/backend/app/api/ml_admin.py +++ b/backend/app/api/ml_admin.py @@ -21,6 +21,7 @@ _EDITABLE = ( "head_auto_apply_precision", "head_auto_apply_enabled", "head_auto_apply_min_positives", + "ccip_match_threshold", ) @@ -48,6 +49,7 @@ async def get_settings(): "head_auto_apply_precision": s.head_auto_apply_precision, "head_auto_apply_enabled": s.head_auto_apply_enabled, "head_auto_apply_min_positives": s.head_auto_apply_min_positives, + "ccip_match_threshold": s.ccip_match_threshold, } ) @@ -115,6 +117,8 @@ def _validate(p: dict) -> str | None: return "head_auto_apply_precision must be between 0.5 and 0.999" if int(p["head_auto_apply_min_positives"]) < 1: return "head_auto_apply_min_positives must be >= 1" + if not (0.5 <= float(p["ccip_match_threshold"]) <= 0.999): + return "ccip_match_threshold must be between 0.5 and 0.999" return None diff --git a/backend/app/models/ml_settings.py b/backend/app/models/ml_settings.py index 696787b..9531118 100644 --- a/backend/app/models/ml_settings.py +++ b/backend/app/models/ml_settings.py @@ -86,6 +86,12 @@ class MLSettings(Base): head_auto_apply_min_positives: Mapped[int] = mapped_column( Integer, nullable=False, default=30 ) + # CCIP character-match cosine cut (#114). 0.85 default — the v1 flat 0.75 + # over-fired (high-reference characters matched a scatter of images); 0.85 + # keeps the confident single-character matches. Tunable from the agent card. + ccip_match_threshold: Mapped[float] = mapped_column( + Float, nullable=False, default=0.85 + ) tagger_model_version: Mapped[str] = mapped_column( String(128), nullable=False, default="camie-tagger-v2" ) diff --git a/backend/app/services/ml/ccip.py b/backend/app/services/ml/ccip.py index 406fbed..2e85a88 100644 --- a/backend/app/services/ml/ccip.py +++ b/backend/app/services/ml/ccip.py @@ -16,15 +16,25 @@ the hands-on eval. numpy is imported lazily (API worker has it via pgvector). from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from ...models import ImageRegion, Tag, TagKind +from ...models import ImageRegion, MLSettings, Tag, TagKind from ...models.tag import image_tag -# Cosine-similarity floor to call a figure the same character. Conservative -# default; tune from real matches (CCIP same-char clusters tightly). -DEFAULT_SIM_THRESHOLD = 0.75 +# Cosine-similarity floor to call a figure the same character. The live setting +# (ml_settings.ccip_match_threshold) drives it; this is only the fallback when no +# threshold is supplied AND no settings row exists. +DEFAULT_SIM_THRESHOLD = 0.85 _FIGURE_KINDS = ("face", "figure") +async def _settings_threshold(session: AsyncSession) -> float: + val = ( + await session.execute( + select(MLSettings.ccip_match_threshold).where(MLSettings.id == 1) + ) + ).scalar_one_or_none() + return float(val) if val is not None else DEFAULT_SIM_THRESHOLD + + def _l2norm(mat, np): n = np.linalg.norm(mat, axis=1, keepdims=True) n[n == 0] = 1.0 @@ -68,14 +78,18 @@ async def _tag_names(session: AsyncSession, tag_ids: list[int]) -> dict[int, str async def match_image( - session: AsyncSession, image_id: int, threshold: float = DEFAULT_SIM_THRESHOLD + session: AsyncSession, image_id: int, threshold: float | None = None ) -> list[dict]: """Character suggestions for one image from its figure-region CCIP vectors: [{tag_id, name, category:'character', score, source:'ccip'}], ranked. Already-applied character tags are excluded. Empty if the image has no figure - CCIP vectors or no character references exist yet.""" + CCIP vectors or no character references exist yet. threshold defaults to the + live ml_settings.ccip_match_threshold.""" import numpy as np + if threshold is None: + threshold = await _settings_threshold(session) + qvecs = ( await session.execute( select(ImageRegion.ccip_embedding).where( diff --git a/frontend/src/components/settings/GpuAgentCard.vue b/frontend/src/components/settings/GpuAgentCard.vue index d4a3d96..411f902 100644 --- a/frontend/src/components/settings/GpuAgentCard.vue +++ b/frontend/src/components/settings/GpuAgentCard.vue @@ -60,6 +60,22 @@ Enqueues every image that doesn't have a CCIP embedding yet. Nothing processes until the agent is running.

+ + +
Character-match strictness
+
+ + {{ threshold.toFixed(2) }} +
+

+ How close a figure must be (CCIP cosine) to suggest a character. Higher = + stricter — fewer but more confident matches. 0.85 recommended; below ~0.80 + a heavily-tagged character starts matching everything. +

@@ -69,14 +85,18 @@ import { computed, onMounted, onUnmounted, ref } from 'vue' import MaintenanceTile from '../common/MaintenanceTile.vue' import { useGpuStore } from '../../stores/gpu.js' +import { useMLStore } from '../../stores/ml.js' import { copyText } from '../../utils/clipboard.js' const store = useGpuStore() +const ml = useMLStore() const loading = ref(true) const tokenValue = ref(null) const masked = ref(true) const rotating = ref(false) const backfilling = ref(false) +const threshold = ref(0.85) +const savingThreshold = ref(false) const queue = ref({ pending: 0, leased: 0, done: 0, error: 0 }) let pollTimer = null @@ -94,9 +114,27 @@ onMounted(async () => { } await refreshQueue() pollTimer = setInterval(() => { if (!document.hidden) refreshQueue() }, 5000) + try { + await ml.loadSettings() + if (ml.settings?.ccip_match_threshold != null) { + threshold.value = ml.settings.ccip_match_threshold + } + } catch { /* non-fatal */ } }) onUnmounted(() => { if (pollTimer) clearInterval(pollTimer) }) +async function onSaveThreshold() { + savingThreshold.value = true + try { + await ml.patchSettings({ ccip_match_threshold: threshold.value }) + toast({ text: `Match strictness set to ${threshold.value.toFixed(2)}`, type: 'success' }) + } catch (e) { + toast({ text: `Could not save: ${e.message}`, type: 'error' }) + } finally { + savingThreshold.value = false + } +} + async function refreshQueue() { try { queue.value = await store.status() } catch { /* non-fatal */ } } diff --git a/tests/test_ccip.py b/tests/test_ccip.py index 5d81c72..bdf3f2f 100644 --- a/tests/test_ccip.py +++ b/tests/test_ccip.py @@ -86,3 +86,20 @@ async def test_no_figure_vectors_means_no_match(db): query = await _img(db, "g" * 64) await db.commit() assert await match_image(db, query.id) == [] + + +@pytest.mark.asyncio +async def test_threshold_gates_borderline_match(db): + # A figure ~0.9 cosine from the reference: matched at 0.85, dropped at 0.95. + raven = await TagService(db).find_or_create("Raven", TagKind.character) + ref = await _img(db, "h" * 64) + await _figure(db, ref.id, _ccip(0)) # e0 + await _tag_image(db, ref.id, raven.id) + near = [0.0] * 768 + near[0], near[1] = 0.9, 0.4359 # |·|=1, cos(e0)=0.9 + query = await _img(db, "i" * 64) + await _figure(db, query.id, near) + await db.commit() + + assert any(m["tag_id"] == raven.id for m in await match_image(db, query.id, 0.85)) + assert await match_image(db, query.id, 0.95) == []