bvandeusen
b91a230f12
Works through the optional CCIP ideas + the "keep moving even if I forget" ask:
AUTOMATION (no button needed):
- Hourly beat auto-enqueues CCIP backfill — new images get embedded (and errored
ones retried) on their own; the queue never goes idle waiting for a click.
- CCIP auto-apply: a daily sweep tags confident matches (source='ccip_auto') so
identity tags keep flowing. ON by default (opt-out, like head auto-apply);
ml_settings.ccip_auto_apply_enabled + _threshold (0.92, above the suggest cut),
migration 0064. Vectorized (one matmul + reduceat per image), reversible, skips
already-applied/rejected. Switch + threshold in the GPU agent card; GET/PATCH
/api/ml/settings; auto_applied count in /api/ccip/overview.
REFERENCE QUALITY (the over-fire root cause):
- character_references now draws ONLY from single-character images — on a
multi-character image the tag is image-level, so every figure would otherwise
pollute each character's prototypes (a 2-char image tagged 'Velma' made
Daphne's figure a Velma reference). This is the contamination behind residual
over-firing.
- Cached on a cheap signature (char-tag count + ccip-region count/max-id) so the
reference load isn't redone on every modal open.
Tests: multi-character image not used as a reference; auto-apply tags a confident
match as ccip_auto.
NEXT (not done, confirmed): comic-panel cropping + SigLIP concept crops ("spot
interesting content").
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
115 lines
4.1 KiB
Python
115 lines
4.1 KiB
Python
"""CCIP / region observability API (#114) — read-only, analysis-shaped.
|
|
|
|
So the work can be checked through an API as the agent fills in vectors: overall
|
|
coverage (regions by kind, how many images have figure CCIP vectors, which
|
|
characters have enough reference examples to match on) + a per-image drill-down
|
|
(its regions + the CCIP character matches it would get). Mirrors the heads
|
|
metrics endpoint; no GPU, just reads what's stored.
|
|
"""
|
|
|
|
from quart import Blueprint, jsonify
|
|
from sqlalchemy import distinct, func, select
|
|
|
|
from ..extensions import get_session
|
|
from ..models import ImageRegion, Tag, TagKind
|
|
from ..models.tag import image_tag
|
|
from ..services.ml.ccip import match_image
|
|
|
|
ccip_bp = Blueprint("ccip", __name__, url_prefix="/api/ccip")
|
|
|
|
_FIGURE_KINDS = ("face", "figure")
|
|
|
|
|
|
@ccip_bp.route("/overview", methods=["GET"])
|
|
async def overview():
|
|
async with get_session() as session:
|
|
by_kind = dict(
|
|
(
|
|
await session.execute(
|
|
select(ImageRegion.kind, func.count()).group_by(ImageRegion.kind)
|
|
)
|
|
).all()
|
|
)
|
|
images_with_figure_ccip = (
|
|
await session.execute(
|
|
select(func.count(distinct(ImageRegion.image_record_id)))
|
|
.where(ImageRegion.kind.in_(_FIGURE_KINDS))
|
|
.where(ImageRegion.ccip_embedding.is_not(None))
|
|
)
|
|
).scalar_one()
|
|
# Per-character reference counts (no vectors loaded) — which characters
|
|
# have enough examples to match on.
|
|
ref_rows = (
|
|
await session.execute(
|
|
select(image_tag.c.tag_id, Tag.name, func.count())
|
|
.select_from(ImageRegion)
|
|
.join(
|
|
image_tag,
|
|
image_tag.c.image_record_id == ImageRegion.image_record_id,
|
|
)
|
|
.join(Tag, Tag.id == image_tag.c.tag_id)
|
|
.where(Tag.kind == TagKind.character)
|
|
.where(ImageRegion.kind.in_(_FIGURE_KINDS))
|
|
.where(ImageRegion.ccip_embedding.is_not(None))
|
|
.group_by(image_tag.c.tag_id, Tag.name)
|
|
.order_by(func.count().desc())
|
|
)
|
|
).all()
|
|
versions = [
|
|
v for (v,) in (
|
|
await session.execute(
|
|
select(distinct(ImageRegion.embedding_version))
|
|
)
|
|
).all() if v
|
|
]
|
|
auto_applied = (
|
|
await session.execute(
|
|
select(func.count()).select_from(image_tag).where(
|
|
image_tag.c.source == "ccip_auto"
|
|
)
|
|
)
|
|
).scalar_one()
|
|
return jsonify({
|
|
"regions_by_kind": by_kind,
|
|
"images_with_figure_ccip": images_with_figure_ccip,
|
|
"characters_with_references": len(ref_rows),
|
|
"character_references": [
|
|
{"tag_id": t, "name": n, "n_refs": c} for (t, n, c) in ref_rows
|
|
],
|
|
"embedding_versions": versions,
|
|
"auto_applied": auto_applied,
|
|
})
|
|
|
|
|
|
@ccip_bp.route("/images/<int:image_id>", methods=["GET"])
|
|
async def image_detail(image_id: int):
|
|
"""An image's stored regions + the CCIP character matches it would get —
|
|
for spot-checking the agent's output + the matcher."""
|
|
async with get_session() as session:
|
|
regions = (
|
|
await session.execute(
|
|
select(ImageRegion)
|
|
.where(ImageRegion.image_record_id == image_id)
|
|
.order_by(ImageRegion.id)
|
|
)
|
|
).scalars().all()
|
|
matches = await match_image(session, image_id)
|
|
return jsonify({
|
|
"image_id": image_id,
|
|
"regions": [
|
|
{
|
|
"id": r.id,
|
|
"kind": r.kind,
|
|
"bbox": [r.rx, r.ry, r.rw, r.rh],
|
|
"frame_time": r.frame_time,
|
|
"score": r.score,
|
|
"detector_version": r.detector_version,
|
|
"embedding_version": r.embedding_version,
|
|
"has_ccip": r.ccip_embedding is not None,
|
|
"has_siglip": r.siglip_embedding is not None,
|
|
}
|
|
for r in regions
|
|
],
|
|
"ccip_matches": matches,
|
|
})
|