148 lines
4.9 KiB
Python
148 lines
4.9 KiB
Python
"""Tag centroids: the mean SigLIP embedding of a tag's member images.
|
|
|
|
Powers centroid-augmented suggestions (a tag whose centroid is close to an
|
|
image's embedding becomes a suggestion even if Camie didn't predict it).
|
|
"""
|
|
|
|
from dataclasses import dataclass
|
|
|
|
import numpy as np
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.dialects.postgresql import insert
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from ...models import (
|
|
ImageRecord,
|
|
MLSettings,
|
|
Tag,
|
|
TagKind,
|
|
TagReferenceEmbedding,
|
|
)
|
|
from ...models.tag import image_tag
|
|
from .embedder import MODEL_VERSION as SIGLIP_VERSION
|
|
|
|
ELIGIBLE_KINDS = {
|
|
TagKind.character,
|
|
TagKind.fandom,
|
|
TagKind.general,
|
|
TagKind.series,
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CentroidHit:
|
|
tag_id: int
|
|
similarity: float
|
|
|
|
|
|
class CentroidService:
|
|
def __init__(self, session: AsyncSession):
|
|
self.session = session
|
|
|
|
async def _min_reference_images(self) -> int:
|
|
return (
|
|
await self.session.execute(
|
|
select(MLSettings.min_reference_images).where(MLSettings.id == 1)
|
|
)
|
|
).scalar_one()
|
|
|
|
async def recompute_for_tag(self, tag_id: int) -> bool:
|
|
"""Recompute one tag's centroid. Returns True if a centroid was
|
|
written, False if skipped (ineligible kind or too few members)."""
|
|
tag = await self.session.get(Tag, tag_id)
|
|
if tag is None or tag.kind not in ELIGIBLE_KINDS:
|
|
return False
|
|
|
|
min_refs = await self._min_reference_images()
|
|
|
|
stmt = (
|
|
select(ImageRecord.siglip_embedding)
|
|
.join(image_tag, image_tag.c.image_record_id == ImageRecord.id)
|
|
.where(image_tag.c.tag_id == tag_id)
|
|
.where(ImageRecord.siglip_embedding.is_not(None))
|
|
)
|
|
embeddings = [
|
|
np.array(e, dtype=np.float32)
|
|
for e in (await self.session.execute(stmt)).scalars().all()
|
|
]
|
|
if len(embeddings) < min_refs:
|
|
return False
|
|
|
|
centroid = np.mean(np.stack(embeddings), axis=0).astype(np.float32)
|
|
|
|
stmt = insert(TagReferenceEmbedding).values(
|
|
tag_id=tag_id,
|
|
embedding=centroid.tolist(),
|
|
reference_count=len(embeddings),
|
|
model_version=SIGLIP_VERSION,
|
|
)
|
|
stmt = stmt.on_conflict_do_update(
|
|
index_elements=["tag_id"],
|
|
set_={
|
|
"embedding": centroid.tolist(),
|
|
"reference_count": len(embeddings),
|
|
"model_version": SIGLIP_VERSION,
|
|
"updated_at": func.now(),
|
|
},
|
|
)
|
|
await self.session.execute(stmt)
|
|
return True
|
|
|
|
async def list_drifted(self) -> list[int]:
|
|
"""Tag ids whose centroid is stale: member count != reference_count,
|
|
OR no centroid row, OR centroid built on a different SigLIP version.
|
|
Only considers eligible-kind tags with embeddings present."""
|
|
member_counts = (
|
|
select(
|
|
image_tag.c.tag_id.label("tag_id"),
|
|
func.count(image_tag.c.image_record_id).label("members"),
|
|
)
|
|
.join(ImageRecord, ImageRecord.id == image_tag.c.image_record_id)
|
|
.where(ImageRecord.siglip_embedding.is_not(None))
|
|
.group_by(image_tag.c.tag_id)
|
|
.subquery()
|
|
)
|
|
stmt = (
|
|
select(Tag.id)
|
|
.join(member_counts, member_counts.c.tag_id == Tag.id)
|
|
.outerjoin(
|
|
TagReferenceEmbedding,
|
|
TagReferenceEmbedding.tag_id == Tag.id,
|
|
)
|
|
.where(Tag.kind.in_(ELIGIBLE_KINDS))
|
|
.where(
|
|
(TagReferenceEmbedding.tag_id.is_(None))
|
|
| (
|
|
TagReferenceEmbedding.reference_count
|
|
!= member_counts.c.members
|
|
)
|
|
| (TagReferenceEmbedding.model_version != SIGLIP_VERSION)
|
|
)
|
|
)
|
|
return list((await self.session.execute(stmt)).scalars().all())
|
|
|
|
async def find_similar_tags(
|
|
self, image_id: int, limit: int = 20
|
|
) -> list[CentroidHit]:
|
|
"""Cosine similarity between an image's embedding and stored
|
|
centroids. Returns top-`limit` by similarity DESC. pgvector's
|
|
cosine_distance gives 1 - cosine_similarity."""
|
|
img = await self.session.get(ImageRecord, image_id)
|
|
if img is None or img.siglip_embedding is None:
|
|
return []
|
|
emb = img.siglip_embedding
|
|
distance = TagReferenceEmbedding.embedding.cosine_distance(emb)
|
|
stmt = (
|
|
select(
|
|
TagReferenceEmbedding.tag_id,
|
|
(1 - distance).label("similarity"),
|
|
)
|
|
.order_by(distance.asc())
|
|
.limit(limit)
|
|
)
|
|
rows = (await self.session.execute(stmt)).all()
|
|
return [
|
|
CentroidHit(tag_id=r.tag_id, similarity=float(r.similarity))
|
|
for r in rows
|
|
]
|