From b91a230f128564d88894b84c103ea1557c2506e9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 29 Jun 2026 22:25:40 -0400 Subject: [PATCH] =?UTF-8?q?feat(ccip):=20automation=20+=20reference=20qual?= =?UTF-8?q?ity=20=E2=80=94=20keep=20identity=20flowing=20hands-free=20(#11?= =?UTF-8?q?4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Works through the optional CCIP ideas + the "keep moving even if I forget" ask: AUTOMATION (no button needed): - Hourly beat auto-enqueues CCIP backfill — new images get embedded (and errored ones retried) on their own; the queue never goes idle waiting for a click. - CCIP auto-apply: a daily sweep tags confident matches (source='ccip_auto') so identity tags keep flowing. ON by default (opt-out, like head auto-apply); ml_settings.ccip_auto_apply_enabled + _threshold (0.92, above the suggest cut), migration 0064. Vectorized (one matmul + reduceat per image), reversible, skips already-applied/rejected. Switch + threshold in the GPU agent card; GET/PATCH /api/ml/settings; auto_applied count in /api/ccip/overview. REFERENCE QUALITY (the over-fire root cause): - character_references now draws ONLY from single-character images — on a multi-character image the tag is image-level, so every figure would otherwise pollute each character's prototypes (a 2-char image tagged 'Velma' made Daphne's figure a Velma reference). This is the contamination behind residual over-firing. - Cached on a cheap signature (char-tag count + ccip-region count/max-id) so the reference load isn't redone on every modal open. Tests: multi-character image not used as a reference; auto-apply tags a confident match as ccip_auto. NEXT (not done, confirmed): comic-panel cropping + SigLIP concept crops ("spot interesting content"). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa --- alembic/versions/0064_ccip_auto_apply.py | 42 ++++++ backend/app/api/ccip.py | 8 ++ backend/app/api/ml_admin.py | 6 + backend/app/celery_app.py | 9 ++ backend/app/models/ml_settings.py | 9 ++ backend/app/services/ml/ccip.py | 52 +++++++- backend/app/tasks/ml.py | 121 ++++++++++++++++++ .../src/components/settings/GpuAgentCard.vue | 42 ++++++ tests/test_ccip.py | 38 ++++++ 9 files changed, 324 insertions(+), 3 deletions(-) create mode 100644 alembic/versions/0064_ccip_auto_apply.py diff --git a/alembic/versions/0064_ccip_auto_apply.py b/alembic/versions/0064_ccip_auto_apply.py new file mode 100644 index 0000000..e5323cf --- /dev/null +++ b/alembic/versions/0064_ccip_auto_apply.py @@ -0,0 +1,42 @@ +"""ml_settings: CCIP auto-apply switch + threshold (#114) + +Confident CCIP character matches auto-tag (source='ccip_auto') on a daily sweep, +so identity tags keep flowing without pressing a button. ON by default (opt-out, +like head auto-apply); the high threshold (0.92, above the 0.85 suggest cut) + +single-character references keep it safe, and every auto-tag is reversible. + +Revision ID: 0064 +Revises: 0063 +Create Date: 2026-06-30 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0064" +down_revision: Union[str, None] = "0063" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "ml_settings", + sa.Column( + "ccip_auto_apply_enabled", sa.Boolean(), nullable=False, + server_default=sa.true(), + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "ccip_auto_apply_threshold", sa.Float(), nullable=False, + server_default="0.92", + ), + ) + + +def downgrade() -> None: + op.drop_column("ml_settings", "ccip_auto_apply_threshold") + op.drop_column("ml_settings", "ccip_auto_apply_enabled") diff --git a/backend/app/api/ccip.py b/backend/app/api/ccip.py index 31ff756..5791fba 100644 --- a/backend/app/api/ccip.py +++ b/backend/app/api/ccip.py @@ -62,6 +62,13 @@ async def overview(): ) ).all() if v ] + auto_applied = ( + await session.execute( + select(func.count()).select_from(image_tag).where( + image_tag.c.source == "ccip_auto" + ) + ) + ).scalar_one() return jsonify({ "regions_by_kind": by_kind, "images_with_figure_ccip": images_with_figure_ccip, @@ -70,6 +77,7 @@ async def overview(): {"tag_id": t, "name": n, "n_refs": c} for (t, n, c) in ref_rows ], "embedding_versions": versions, + "auto_applied": auto_applied, }) diff --git a/backend/app/api/ml_admin.py b/backend/app/api/ml_admin.py index 96e2e23..1472770 100644 --- a/backend/app/api/ml_admin.py +++ b/backend/app/api/ml_admin.py @@ -22,6 +22,8 @@ _EDITABLE = ( "head_auto_apply_enabled", "head_auto_apply_min_positives", "ccip_match_threshold", + "ccip_auto_apply_enabled", + "ccip_auto_apply_threshold", ) @@ -50,6 +52,8 @@ async def get_settings(): "head_auto_apply_enabled": s.head_auto_apply_enabled, "head_auto_apply_min_positives": s.head_auto_apply_min_positives, "ccip_match_threshold": s.ccip_match_threshold, + "ccip_auto_apply_enabled": s.ccip_auto_apply_enabled, + "ccip_auto_apply_threshold": s.ccip_auto_apply_threshold, } ) @@ -119,6 +123,8 @@ def _validate(p: dict) -> str | None: return "head_auto_apply_min_positives must be >= 1" if not (0.5 <= float(p["ccip_match_threshold"]) <= 0.999): return "ccip_match_threshold must be between 0.5 and 0.999" + if not (0.5 <= float(p["ccip_auto_apply_threshold"]) <= 0.999): + return "ccip_auto_apply_threshold must be between 0.5 and 0.999" return None diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index fc98ae5..84e9f60 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -121,6 +121,15 @@ def make_celery() -> Celery: "task": "backend.app.tasks.ml.recover_orphaned_gpu_jobs", "schedule": 60.0, # quick pickup of work a dead agent orphaned }, + "enqueue-ccip-backfill-hourly": { + "task": "backend.app.tasks.ml.enqueue_gpu_backfill", + "schedule": 3600.0, # auto-feed new images (+ retry errored) so + "args": ("ccip",), # the queue keeps moving without the button + }, + "ccip-auto-apply-daily": { + "task": "backend.app.tasks.ml.scheduled_ccip_auto_apply", + "schedule": 86400.0, # no-op unless ccip_auto_apply_enabled + }, "snapshot-head-metrics-daily": { "task": "backend.app.tasks.maintenance.snapshot_head_metrics", "schedule": 86400.0, diff --git a/backend/app/models/ml_settings.py b/backend/app/models/ml_settings.py index 9531118..9825568 100644 --- a/backend/app/models/ml_settings.py +++ b/backend/app/models/ml_settings.py @@ -92,6 +92,15 @@ class MLSettings(Base): ccip_match_threshold: Mapped[float] = mapped_column( Float, nullable=False, default=0.85 ) + # CCIP auto-apply (#114). Confident matches (>= ccip_auto_apply_threshold, + # above the suggest cut) auto-tag on a daily sweep. ON by default (opt-out); + # single-character references + the high bar keep it safe, every tag reversible. + ccip_auto_apply_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True + ) + ccip_auto_apply_threshold: Mapped[float] = mapped_column( + Float, nullable=False, default=0.92 + ) tagger_model_version: Mapped[str] = mapped_column( String(128), nullable=False, default="camie-tagger-v2" ) diff --git a/backend/app/services/ml/ccip.py b/backend/app/services/ml/ccip.py index 2e85a88..2912ccd 100644 --- a/backend/app/services/ml/ccip.py +++ b/backend/app/services/ml/ccip.py @@ -13,7 +13,7 @@ exact CCIP difference metric/threshold gets validated against the model during the hands-on eval. numpy is imported lazily (API worker has it via pgvector). """ -from sqlalchemy import select +from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession from ...models import ImageRegion, MLSettings, Tag, TagKind @@ -41,10 +41,54 @@ def _l2norm(mat, np): return mat / n +# Single-shot cache of the (expensive) reference load, keyed on a cheap +# signature that changes exactly when references could: a character tag added/ +# removed (n_char_tags) or a figure embedded (max/ n of ccip regions). Shared by +# the live matcher (every modal open) and the auto-apply sweep. +_REF_CACHE: dict = {"sig": None, "refs": None} + + +def _single_character_images(): + """Subquery of image ids carrying EXACTLY ONE character tag. References come + only from these — on a multi-character image the tag is image-level, so every + figure would otherwise pollute each character's prototype set (a 2-character + image tagged 'Velma' would make Daphne's figure a Velma reference).""" + return ( + select(image_tag.c.image_record_id) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where(Tag.kind == TagKind.character) + .group_by(image_tag.c.image_record_id) + .having(func.count() == 1) + ) + + +async def _ref_signature(session: AsyncSession) -> tuple: + n_tags = ( + await session.execute( + select(func.count()) + .select_from(image_tag) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where(Tag.kind == TagKind.character) + ) + ).scalar_one() + n_regs, max_id = ( + await session.execute( + select(func.count(), func.max(ImageRegion.id)).where( + ImageRegion.kind.in_(_FIGURE_KINDS), + ImageRegion.ccip_embedding.is_not(None), + ) + ) + ).one() + return (n_tags, n_regs, max_id) + + async def character_references(session: AsyncSession) -> dict[int, list]: """Per character-tag CCIP reference vectors: figure/face-region CCIP - embeddings on images that carry that character tag (the operator's examples). - Multi-prototype — several vectors per character.""" + embeddings on UNAMBIGUOUS (single-character) images carrying that tag. + Multi-prototype — several vectors per character. Cached on a cheap signature.""" + sig = await _ref_signature(session) + if _REF_CACHE["sig"] == sig and _REF_CACHE["refs"] is not None: + return _REF_CACHE["refs"] rows = ( await session.execute( select(image_tag.c.tag_id, ImageRegion.ccip_embedding) @@ -57,11 +101,13 @@ async def character_references(session: AsyncSession) -> dict[int, list]: .where(Tag.kind == TagKind.character) .where(ImageRegion.kind.in_(_FIGURE_KINDS)) .where(ImageRegion.ccip_embedding.is_not(None)) + .where(ImageRegion.image_record_id.in_(_single_character_images())) ) ).all() refs: dict[int, list] = {} for tag_id, vec in rows: refs.setdefault(tag_id, []).append(vec) + _REF_CACHE.update(sig=sig, refs=refs) return refs diff --git a/backend/app/tasks/ml.py b/backend/app/tasks/ml.py index bcc1903..4dc6ab7 100644 --- a/backend/app/tasks/ml.py +++ b/backend/app/tasks/ml.py @@ -795,3 +795,124 @@ def recover_orphaned_gpu_jobs() -> int: ) session.commit() return res.rowcount or 0 + + +@celery.task( + name="backend.app.tasks.ml.scheduled_ccip_auto_apply", + soft_time_limit=1800, time_limit=2100, +) +def scheduled_ccip_auto_apply() -> str: + """Auto-tag confident CCIP character matches (source='ccip_auto') so identity + tags keep flowing without a button. No-op unless ccip_auto_apply_enabled. + References come only from single-character images (unambiguous); a tag is + applied where any figure's best cosine to a character's prototypes clears + ccip_auto_apply_threshold and it isn't already applied/rejected. Reversible.""" + import numpy as np + from sqlalchemy import func + from sqlalchemy import select as sa_select + from sqlalchemy.dialects.postgresql import insert as pg_insert + + from ..models import ImageRegion, MLSettings, Tag, TagKind, TagSuggestionRejection + from ..models.tag import image_tag + + fig = ("face", "figure") + + def _l2(m): + n = np.linalg.norm(m, axis=1, keepdims=True) + n[n == 0] = 1.0 + return m / n + + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + s = session.get(MLSettings, 1) + if s is None or not s.ccip_auto_apply_enabled: + return "disabled" + thr = float(s.ccip_auto_apply_threshold) + + single = ( + sa_select(image_tag.c.image_record_id) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where(Tag.kind == TagKind.character) + .group_by(image_tag.c.image_record_id) + .having(func.count() == 1) + ) + ref_rows = session.execute( + sa_select(image_tag.c.tag_id, ImageRegion.ccip_embedding) + .select_from(ImageRegion) + .join( + image_tag, + image_tag.c.image_record_id == ImageRegion.image_record_id, + ) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where(Tag.kind == TagKind.character) + .where(ImageRegion.kind.in_(fig)) + .where(ImageRegion.ccip_embedding.is_not(None)) + .where(ImageRegion.image_record_id.in_(single)) + ).all() + if not ref_rows: + return "no-references" + + by_char: dict[int, list] = {} + for tid, vec in ref_rows: + by_char.setdefault(tid, []).append(vec) + ref_tags = list(by_char) + mats = [_l2(np.asarray(by_char[t], dtype=np.float32)) for t in ref_tags] + allref = np.vstack(mats) # (total, 768) + seg = np.cumsum([0] + [len(m) for m in mats])[:-1] # per-char start + + # Per character: images that already carry OR rejected the tag — skip. + skip = {t: set() for t in ref_tags} + for t in ref_tags: + for (iid,) in session.execute( + sa_select(image_tag.c.image_record_id).where( + image_tag.c.tag_id == t + ) + ): + skip[t].add(iid) + for (iid,) in session.execute( + sa_select(TagSuggestionRejection.image_record_id).where( + TagSuggestionRejection.tag_id == t + ) + ): + skip[t].add(iid) + + img_ids = list(session.execute( + sa_select(ImageRegion.image_record_id) + .where(ImageRegion.kind.in_(fig), ImageRegion.ccip_embedding.is_not(None)) + .distinct() + ).scalars()) + + applied = 0 + chunk_n = 500 + for start in range(0, len(img_ids), chunk_n): + chunk = img_ids[start:start + chunk_n] + rows = session.execute( + sa_select(ImageRegion.image_record_id, ImageRegion.ccip_embedding) + .where( + ImageRegion.image_record_id.in_(chunk), + ImageRegion.kind.in_(fig), + ImageRegion.ccip_embedding.is_not(None), + ) + ).all() + by_img: dict[int, list] = {} + for iid, vec in rows: + by_img.setdefault(iid, []).append(vec) + for iid, vecs in by_img.items(): + q = _l2(np.asarray(vecs, dtype=np.float32)) # (nq, 768) + colmax = (q @ allref.T).max(axis=0) # (total,) + charmax = np.maximum.reduceat(colmax, seg) # (n_chars,) + for ci in np.where(charmax >= thr)[0]: + t = ref_tags[int(ci)] + if iid in skip[t]: + continue + skip[t].add(iid) + session.execute( + pg_insert(image_tag) + .values( + image_record_id=iid, tag_id=t, source="ccip_auto", + ) + .on_conflict_do_nothing() + ) + applied += 1 + session.commit() + return f"applied={applied}" diff --git a/frontend/src/components/settings/GpuAgentCard.vue b/frontend/src/components/settings/GpuAgentCard.vue index 411f902..5f85dc2 100644 --- a/frontend/src/components/settings/GpuAgentCard.vue +++ b/frontend/src/components/settings/GpuAgentCard.vue @@ -76,6 +76,26 @@ stricter — fewer but more confident matches. 0.85 recommended; below ~0.80 a heavily-tagged character starts matching everything.

+ + +
+ + +
+

+ When on, a very-confident character match tags the image on its own (daily, + reversible) — so identity tags keep flowing without review. Stricter than + the suggest cut; 0.92 recommended. +

@@ -97,6 +117,9 @@ const rotating = ref(false) const backfilling = ref(false) const threshold = ref(0.85) const savingThreshold = ref(false) +const autoApply = ref(true) +const autoThreshold = ref(0.92) +const savingAuto = ref(false) const queue = ref({ pending: 0, leased: 0, done: 0, error: 0 }) let pollTimer = null @@ -119,8 +142,27 @@ onMounted(async () => { if (ml.settings?.ccip_match_threshold != null) { threshold.value = ml.settings.ccip_match_threshold } + if (ml.settings?.ccip_auto_apply_enabled != null) { + autoApply.value = ml.settings.ccip_auto_apply_enabled + autoThreshold.value = ml.settings.ccip_auto_apply_threshold + } } catch { /* non-fatal */ } }) + +async function onSaveAuto() { + savingAuto.value = true + try { + await ml.patchSettings({ + ccip_auto_apply_enabled: autoApply.value, + ccip_auto_apply_threshold: autoThreshold.value, + }) + toast({ text: 'Auto-apply settings saved', type: 'success' }) + } catch (e) { + toast({ text: `Could not save: ${e.message}`, type: 'error' }) + } finally { + savingAuto.value = false + } +} onUnmounted(() => { if (pollTimer) clearInterval(pollTimer) }) async function onSaveThreshold() { diff --git a/tests/test_ccip.py b/tests/test_ccip.py index bdf3f2f..ad1dacf 100644 --- a/tests/test_ccip.py +++ b/tests/test_ccip.py @@ -1,6 +1,7 @@ """CCIP few-shot character matcher (#114). numpy cosine on stored vectors — no model needed, so it runs in CI with synthetic CCIP vectors.""" import pytest +from sqlalchemy import select from backend.app.models import ImageRecord, ImageRegion, TagKind from backend.app.models.tag import image_tag @@ -103,3 +104,40 @@ async def test_threshold_gates_borderline_match(db): assert any(m["tag_id"] == raven.id for m in await match_image(db, query.id, 0.85)) assert await match_image(db, query.id, 0.95) == [] + + +@pytest.mark.asyncio +async def test_multi_character_image_not_used_as_reference(db): + # A figure on a 2-character image is ambiguous (tag is image-level), so it + # must NOT seed either character's prototypes — else it'd match both. + raven = await TagService(db).find_or_create("Raven", TagKind.character) + daphne = await TagService(db).find_or_create("Daphne", TagKind.character) + multi = await _img(db, "j" * 64) + await _figure(db, multi.id, _ccip(0)) + await _tag_image(db, multi.id, raven.id) + await _tag_image(db, multi.id, daphne.id) + query = await _img(db, "k" * 64) + await _figure(db, query.id, _ccip(0)) # identical to the ambiguous figure + await db.commit() + assert await match_image(db, query.id) == [] # no clean references → nothing + + +@pytest.mark.asyncio +async def test_auto_apply_tags_confident_match(db): + raven = await TagService(db).find_or_create("Raven", TagKind.character) + ref = await _img(db, "l" * 64) + await _figure(db, ref.id, _ccip(0)) + await _tag_image(db, ref.id, raven.id) # single-character reference + query = await _img(db, "m" * 64) + await _figure(db, query.id, _ccip(0)) # identical → cosine 1.0 + await db.commit() + + from backend.app.tasks.ml import scheduled_ccip_auto_apply + assert "applied=" in scheduled_ccip_auto_apply() # sync task, own session + + rows = (await db.execute( + select(image_tag.c.tag_id, image_tag.c.source).where( + image_tag.c.image_record_id == query.id + ) + )).all() + assert (raven.id, "ccip_auto") in [(t, s) for t, s in rows]