bvandeusen
625336b6b4
Live data showed the v1 flat 0.75 cosine over-fired — ~64% of matched images got
3-10 character guesses dominated by the most-referenced characters (a 27-ref
character clears a low bar on many images). A sweep showed 0.85 collapses the
noise (noisy multi-matches 47→3) while keeping the confident single-character
matches.
- ml_settings.ccip_match_threshold (migration 0063, default 0.85); match_image
reads it (override still accepted). DEFAULT_SIM_THRESHOLD fallback 0.75→0.85.
- Exposed in GET/PATCH /api/ml/settings (validated 0.5–0.999).
- Slider in the GPU agent card ("Character-match strictness") — tune live, no
redeploy, same observe-and-tune loop as auto-apply.
Test: a ~0.9-cosine figure matches at 0.85, dropped at 0.95.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
106 lines
3.7 KiB
Python
106 lines
3.7 KiB
Python
"""CCIP few-shot character matcher (#114). numpy cosine on stored vectors — no
|
|
model needed, so it runs in CI with synthetic CCIP vectors."""
|
|
import pytest
|
|
|
|
from backend.app.models import ImageRecord, ImageRegion, TagKind
|
|
from backend.app.models.tag import image_tag
|
|
from backend.app.services.ml.ccip import match_image
|
|
from backend.app.services.tag_service import TagService
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
def _ccip(slot: int) -> list[float]:
|
|
v = [0.0] * 768
|
|
v[slot] = 1.0
|
|
return v
|
|
|
|
|
|
async def _img(db, sha) -> ImageRecord:
|
|
img = ImageRecord(
|
|
path=f"/images/{sha}.jpg", sha256=sha, size_bytes=1, mime="image/jpeg",
|
|
width=1, height=1, origin="imported_filesystem", integrity_status="unknown",
|
|
)
|
|
db.add(img)
|
|
await db.flush()
|
|
return img
|
|
|
|
|
|
async def _figure(db, image_id, ccip):
|
|
db.add(ImageRegion(
|
|
image_record_id=image_id, kind="figure",
|
|
rx=0.0, ry=0.0, rw=1.0, rh=1.0,
|
|
ccip_embedding=ccip, embedding_version="ccip-test",
|
|
))
|
|
|
|
|
|
async def _tag_image(db, image_id, tag_id):
|
|
await db.execute(image_tag.insert().values(
|
|
image_record_id=image_id, tag_id=tag_id, source="manual",
|
|
))
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_matches_same_character_across_images(db):
|
|
raven = await TagService(db).find_or_create("Raven", TagKind.character)
|
|
ref = await _img(db, "a" * 64) # a tagged example = a prototype
|
|
await _figure(db, ref.id, _ccip(0))
|
|
await _tag_image(db, ref.id, raven.id)
|
|
query = await _img(db, "b" * 64) # untagged, near-identical figure
|
|
await _figure(db, query.id, _ccip(0))
|
|
await db.commit()
|
|
|
|
matches = await match_image(db, query.id)
|
|
m = next(x for x in matches if x["tag_id"] == raven.id)
|
|
assert m["source"] == "ccip" and m["category"] == "character"
|
|
assert m["score"] > 0.9
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_match_for_different_character(db):
|
|
raven = await TagService(db).find_or_create("Raven", TagKind.character)
|
|
ref = await _img(db, "c" * 64)
|
|
await _figure(db, ref.id, _ccip(0))
|
|
await _tag_image(db, ref.id, raven.id)
|
|
query = await _img(db, "d" * 64)
|
|
await _figure(db, query.id, _ccip(5)) # orthogonal → not Raven
|
|
await db.commit()
|
|
assert await match_image(db, query.id) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_excludes_already_applied_character(db):
|
|
raven = await TagService(db).find_or_create("Raven", TagKind.character)
|
|
ref = await _img(db, "e" * 64)
|
|
await _figure(db, ref.id, _ccip(0))
|
|
await _tag_image(db, ref.id, raven.id)
|
|
query = await _img(db, "f" * 64)
|
|
await _figure(db, query.id, _ccip(0))
|
|
await _tag_image(db, query.id, raven.id) # already tagged → no re-suggest
|
|
await db.commit()
|
|
assert all(m["tag_id"] != raven.id for m in await match_image(db, query.id))
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_figure_vectors_means_no_match(db):
|
|
query = await _img(db, "g" * 64)
|
|
await db.commit()
|
|
assert await match_image(db, query.id) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_threshold_gates_borderline_match(db):
|
|
# A figure ~0.9 cosine from the reference: matched at 0.85, dropped at 0.95.
|
|
raven = await TagService(db).find_or_create("Raven", TagKind.character)
|
|
ref = await _img(db, "h" * 64)
|
|
await _figure(db, ref.id, _ccip(0)) # e0
|
|
await _tag_image(db, ref.id, raven.id)
|
|
near = [0.0] * 768
|
|
near[0], near[1] = 0.9, 0.4359 # |·|=1, cos(e0)=0.9
|
|
query = await _img(db, "i" * 64)
|
|
await _figure(db, query.id, near)
|
|
await db.commit()
|
|
|
|
assert any(m["tag_id"] == raven.id for m in await match_image(db, query.id, 0.85))
|
|
assert await match_image(db, query.id, 0.95) == []
|