feat(ccip): automation + reference quality — keep identity flowing hands-free (#114)
Works through the optional CCIP ideas + the "keep moving even if I forget" ask:
AUTOMATION (no button needed):
- Hourly beat auto-enqueues CCIP backfill — new images get embedded (and errored
ones retried) on their own; the queue never goes idle waiting for a click.
- CCIP auto-apply: a daily sweep tags confident matches (source='ccip_auto') so
identity tags keep flowing. ON by default (opt-out, like head auto-apply);
ml_settings.ccip_auto_apply_enabled + _threshold (0.92, above the suggest cut),
migration 0064. Vectorized (one matmul + reduceat per image), reversible, skips
already-applied/rejected. Switch + threshold in the GPU agent card; GET/PATCH
/api/ml/settings; auto_applied count in /api/ccip/overview.
REFERENCE QUALITY (the over-fire root cause):
- character_references now draws ONLY from single-character images — on a
multi-character image the tag is image-level, so every figure would otherwise
pollute each character's prototypes (a 2-char image tagged 'Velma' made
Daphne's figure a Velma reference). This is the contamination behind residual
over-firing.
- Cached on a cheap signature (char-tag count + ccip-region count/max-id) so the
reference load isn't redone on every modal open.
Tests: multi-character image not used as a reference; auto-apply tags a confident
match as ccip_auto.
NEXT (not done, confirmed): comic-panel cropping + SigLIP concept crops ("spot
interesting content").
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
This commit is contained in:
@@ -795,3 +795,124 @@ def recover_orphaned_gpu_jobs() -> int:
|
||||
)
|
||||
session.commit()
|
||||
return res.rowcount or 0
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.scheduled_ccip_auto_apply",
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def scheduled_ccip_auto_apply() -> str:
|
||||
"""Auto-tag confident CCIP character matches (source='ccip_auto') so identity
|
||||
tags keep flowing without a button. No-op unless ccip_auto_apply_enabled.
|
||||
References come only from single-character images (unambiguous); a tag is
|
||||
applied where any figure's best cosine to a character's prototypes clears
|
||||
ccip_auto_apply_threshold and it isn't already applied/rejected. Reversible."""
|
||||
import numpy as np
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy import select as sa_select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from ..models import ImageRegion, MLSettings, Tag, TagKind, TagSuggestionRejection
|
||||
from ..models.tag import image_tag
|
||||
|
||||
fig = ("face", "figure")
|
||||
|
||||
def _l2(m):
|
||||
n = np.linalg.norm(m, axis=1, keepdims=True)
|
||||
n[n == 0] = 1.0
|
||||
return m / n
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
s = session.get(MLSettings, 1)
|
||||
if s is None or not s.ccip_auto_apply_enabled:
|
||||
return "disabled"
|
||||
thr = float(s.ccip_auto_apply_threshold)
|
||||
|
||||
single = (
|
||||
sa_select(image_tag.c.image_record_id)
|
||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||
.where(Tag.kind == TagKind.character)
|
||||
.group_by(image_tag.c.image_record_id)
|
||||
.having(func.count() == 1)
|
||||
)
|
||||
ref_rows = session.execute(
|
||||
sa_select(image_tag.c.tag_id, ImageRegion.ccip_embedding)
|
||||
.select_from(ImageRegion)
|
||||
.join(
|
||||
image_tag,
|
||||
image_tag.c.image_record_id == ImageRegion.image_record_id,
|
||||
)
|
||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||
.where(Tag.kind == TagKind.character)
|
||||
.where(ImageRegion.kind.in_(fig))
|
||||
.where(ImageRegion.ccip_embedding.is_not(None))
|
||||
.where(ImageRegion.image_record_id.in_(single))
|
||||
).all()
|
||||
if not ref_rows:
|
||||
return "no-references"
|
||||
|
||||
by_char: dict[int, list] = {}
|
||||
for tid, vec in ref_rows:
|
||||
by_char.setdefault(tid, []).append(vec)
|
||||
ref_tags = list(by_char)
|
||||
mats = [_l2(np.asarray(by_char[t], dtype=np.float32)) for t in ref_tags]
|
||||
allref = np.vstack(mats) # (total, 768)
|
||||
seg = np.cumsum([0] + [len(m) for m in mats])[:-1] # per-char start
|
||||
|
||||
# Per character: images that already carry OR rejected the tag — skip.
|
||||
skip = {t: set() for t in ref_tags}
|
||||
for t in ref_tags:
|
||||
for (iid,) in session.execute(
|
||||
sa_select(image_tag.c.image_record_id).where(
|
||||
image_tag.c.tag_id == t
|
||||
)
|
||||
):
|
||||
skip[t].add(iid)
|
||||
for (iid,) in session.execute(
|
||||
sa_select(TagSuggestionRejection.image_record_id).where(
|
||||
TagSuggestionRejection.tag_id == t
|
||||
)
|
||||
):
|
||||
skip[t].add(iid)
|
||||
|
||||
img_ids = list(session.execute(
|
||||
sa_select(ImageRegion.image_record_id)
|
||||
.where(ImageRegion.kind.in_(fig), ImageRegion.ccip_embedding.is_not(None))
|
||||
.distinct()
|
||||
).scalars())
|
||||
|
||||
applied = 0
|
||||
chunk_n = 500
|
||||
for start in range(0, len(img_ids), chunk_n):
|
||||
chunk = img_ids[start:start + chunk_n]
|
||||
rows = session.execute(
|
||||
sa_select(ImageRegion.image_record_id, ImageRegion.ccip_embedding)
|
||||
.where(
|
||||
ImageRegion.image_record_id.in_(chunk),
|
||||
ImageRegion.kind.in_(fig),
|
||||
ImageRegion.ccip_embedding.is_not(None),
|
||||
)
|
||||
).all()
|
||||
by_img: dict[int, list] = {}
|
||||
for iid, vec in rows:
|
||||
by_img.setdefault(iid, []).append(vec)
|
||||
for iid, vecs in by_img.items():
|
||||
q = _l2(np.asarray(vecs, dtype=np.float32)) # (nq, 768)
|
||||
colmax = (q @ allref.T).max(axis=0) # (total,)
|
||||
charmax = np.maximum.reduceat(colmax, seg) # (n_chars,)
|
||||
for ci in np.where(charmax >= thr)[0]:
|
||||
t = ref_tags[int(ci)]
|
||||
if iid in skip[t]:
|
||||
continue
|
||||
skip[t].add(iid)
|
||||
session.execute(
|
||||
pg_insert(image_tag)
|
||||
.values(
|
||||
image_record_id=iid, tag_id=t, source="ccip_auto",
|
||||
)
|
||||
.on_conflict_do_nothing()
|
||||
)
|
||||
applied += 1
|
||||
session.commit()
|
||||
return f"applied={applied}"
|
||||
|
||||
Reference in New Issue
Block a user