diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index 4e51bb7..4609e25 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -24,6 +24,7 @@ def all_blueprints() -> list[Blueprint]: from .credentials import credentials_bp from .downloads import downloads_bp from .extension import extension_bp + from .ccip import ccip_bp from .gallery import gallery_bp from .gpu import gpu_bp from .heads import heads_bp @@ -62,6 +63,7 @@ def all_blueprints() -> list[Blueprint]: tag_eval_bp, heads_bp, gpu_bp, + ccip_bp, ml_admin_bp, thumbnails_bp, sources_bp, diff --git a/backend/app/api/ccip.py b/backend/app/api/ccip.py new file mode 100644 index 0000000..31ff756 --- /dev/null +++ b/backend/app/api/ccip.py @@ -0,0 +1,106 @@ +"""CCIP / region observability API (#114) — read-only, analysis-shaped. + +So the work can be checked through an API as the agent fills in vectors: overall +coverage (regions by kind, how many images have figure CCIP vectors, which +characters have enough reference examples to match on) + a per-image drill-down +(its regions + the CCIP character matches it would get). Mirrors the heads +metrics endpoint; no GPU, just reads what's stored. +""" + +from quart import Blueprint, jsonify +from sqlalchemy import distinct, func, select + +from ..extensions import get_session +from ..models import ImageRegion, Tag, TagKind +from ..models.tag import image_tag +from ..services.ml.ccip import match_image + +ccip_bp = Blueprint("ccip", __name__, url_prefix="/api/ccip") + +_FIGURE_KINDS = ("face", "figure") + + +@ccip_bp.route("/overview", methods=["GET"]) +async def overview(): + async with get_session() as session: + by_kind = dict( + ( + await session.execute( + select(ImageRegion.kind, func.count()).group_by(ImageRegion.kind) + ) + ).all() + ) + images_with_figure_ccip = ( + await session.execute( + select(func.count(distinct(ImageRegion.image_record_id))) + .where(ImageRegion.kind.in_(_FIGURE_KINDS)) + .where(ImageRegion.ccip_embedding.is_not(None)) + ) + ).scalar_one() + # Per-character reference counts (no vectors loaded) — which characters + # have enough examples to match on. + ref_rows = ( + await session.execute( + select(image_tag.c.tag_id, Tag.name, func.count()) + .select_from(ImageRegion) + .join( + image_tag, + image_tag.c.image_record_id == ImageRegion.image_record_id, + ) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where(Tag.kind == TagKind.character) + .where(ImageRegion.kind.in_(_FIGURE_KINDS)) + .where(ImageRegion.ccip_embedding.is_not(None)) + .group_by(image_tag.c.tag_id, Tag.name) + .order_by(func.count().desc()) + ) + ).all() + versions = [ + v for (v,) in ( + await session.execute( + select(distinct(ImageRegion.embedding_version)) + ) + ).all() if v + ] + return jsonify({ + "regions_by_kind": by_kind, + "images_with_figure_ccip": images_with_figure_ccip, + "characters_with_references": len(ref_rows), + "character_references": [ + {"tag_id": t, "name": n, "n_refs": c} for (t, n, c) in ref_rows + ], + "embedding_versions": versions, + }) + + +@ccip_bp.route("/images/", methods=["GET"]) +async def image_detail(image_id: int): + """An image's stored regions + the CCIP character matches it would get — + for spot-checking the agent's output + the matcher.""" + async with get_session() as session: + regions = ( + await session.execute( + select(ImageRegion) + .where(ImageRegion.image_record_id == image_id) + .order_by(ImageRegion.id) + ) + ).scalars().all() + matches = await match_image(session, image_id) + return jsonify({ + "image_id": image_id, + "regions": [ + { + "id": r.id, + "kind": r.kind, + "bbox": [r.rx, r.ry, r.rw, r.rh], + "frame_time": r.frame_time, + "score": r.score, + "detector_version": r.detector_version, + "embedding_version": r.embedding_version, + "has_ccip": r.ccip_embedding is not None, + "has_siglip": r.siglip_embedding is not None, + } + for r in regions + ], + "ccip_matches": matches, + }) diff --git a/tests/test_api_ccip.py b/tests/test_api_ccip.py new file mode 100644 index 0000000..d41167a --- /dev/null +++ b/tests/test_api_ccip.py @@ -0,0 +1,72 @@ +"""CCIP/region observability API (#114) — coverage overview + per-image detail.""" +import pytest + +from backend.app.models import ImageRecord, ImageRegion, TagKind +from backend.app.models.tag import image_tag +from backend.app.services.tag_service import TagService + +pytestmark = pytest.mark.integration + + +def _ccip(slot: int) -> list[float]: + v = [0.0] * 768 + v[slot] = 1.0 + return v + + +async def _img(db, sha) -> ImageRecord: + img = ImageRecord( + path=f"/images/{sha}.jpg", sha256=sha, size_bytes=1, mime="image/jpeg", + width=1, height=1, origin="imported_filesystem", integrity_status="unknown", + ) + db.add(img) + await db.flush() + return img + + +async def _figure(db, image_id, ccip): + db.add(ImageRegion( + image_record_id=image_id, kind="figure", rx=0.0, ry=0.0, rw=1.0, rh=1.0, + ccip_embedding=ccip, embedding_version="ccip-test", + )) + + +@pytest.mark.asyncio +async def test_overview_reports_coverage(client, db): + raven = await TagService(db).find_or_create("Raven", TagKind.character) + ref = await _img(db, "a" * 64) + await _figure(db, ref.id, _ccip(0)) + await db.execute(image_tag.insert().values( + image_record_id=ref.id, tag_id=raven.id, source="manual", + )) + q = await _img(db, "b" * 64) + await _figure(db, q.id, _ccip(0)) + await db.commit() + + body = await (await client.get("/api/ccip/overview")).get_json() + assert body["regions_by_kind"].get("figure", 0) >= 2 + assert body["images_with_figure_ccip"] >= 2 + assert any( + c["name"] == "Raven" and c["n_refs"] >= 1 + for c in body["character_references"] + ) + assert "ccip-test" in body["embedding_versions"] + + +@pytest.mark.asyncio +async def test_image_detail_shows_regions_and_matches(client, db): + raven = await TagService(db).find_or_create("Raven", TagKind.character) + ref = await _img(db, "c" * 64) + await _figure(db, ref.id, _ccip(0)) + await db.execute(image_tag.insert().values( + image_record_id=ref.id, tag_id=raven.id, source="manual", + )) + q = await _img(db, "d" * 64) + await _figure(db, q.id, _ccip(0)) + await db.commit() + + body = await (await client.get(f"/api/ccip/images/{q.id}")).get_json() + assert len(body["regions"]) == 1 + r = body["regions"][0] + assert r["kind"] == "figure" and r["has_ccip"] is True and r["has_siglip"] is False + assert any(m["tag_id"] == raven.id for m in body["ccip_matches"])