From 79cd1234e2ebbef286142ed5ba8344b7cb549cbd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 4 Jun 2026 08:47:43 -0400 Subject: [PATCH] feat(gallery): visual 'more like this' search (Phase 3 backend) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GalleryService.similar() ranks images by pgvector cosine distance to a source image's precomputed SigLIP embedding — no query-time ML inference. Composes with the Phase-1/2 scope filters (AND) but replaces the date sort (always nearest-first, bounded top-N, no cursor). Returns None for a missing source (→404), [] for a source with no embedding (video / pending ML); excludes self and NULL-embedding rows. New GET /api/gallery/similar?similar_to=&limit=N. Image-detail payload gains has_embedding so the UI can hide the surface. Alembic 0036 adds an HNSW vector_cosine_ops index on siglip_embedding (1152<2000 dims) so the search is sub-50ms ANN instead of a full scan; one-time ~30-60s build over existing embeddings on deploy. Shared _gallery_images/_image_json helpers de-dup the scroll/similar builders. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../0036_siglip_embedding_hnsw_index.py | 41 ++++++ backend/app/api/gallery.py | 60 ++++++-- backend/app/services/gallery_service.py | 84 ++++++++--- tests/test_gallery_similar.py | 134 ++++++++++++++++++ 4 files changed, 289 insertions(+), 30 deletions(-) create mode 100644 alembic/versions/0036_siglip_embedding_hnsw_index.py create mode 100644 tests/test_gallery_similar.py diff --git a/alembic/versions/0036_siglip_embedding_hnsw_index.py b/alembic/versions/0036_siglip_embedding_hnsw_index.py new file mode 100644 index 0000000..a8c1251 --- /dev/null +++ b/alembic/versions/0036_siglip_embedding_hnsw_index.py @@ -0,0 +1,41 @@ +"""image_record.siglip_embedding: HNSW cosine index for "more like this" + +Revision ID: 0036 +Revises: 0035 +Create Date: 2026-06-04 + +Gallery Phase 3 (visual similarity search) ranks images by +`siglip_embedding.cosine_distance(source_embedding)`. Without an index that's +a sequential scan computing a 1152-dim distance for every row — fine at small +scale, but it grows linearly with the library. Add an HNSW index with +`vector_cosine_ops` so the top-N nearest search is sub-50ms ANN. + +1152 dims is under pgvector's 2000-dim HNSW limit, so HNSW (no training, +better recall than IVFFlat) is the right choice. ONE-TIME COST: building the +index over the existing embeddings (~57k vectors on the operator's library) +locks image_record for ~30-60s during this migration on deploy — acceptable +for a single-operator homelab. NULL embeddings (videos / not-yet-embedded +rows) are simply not indexed. +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0036" +down_revision: Union[str, None] = "0035" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Raw SQL: alembic's create_index doesn't express the `USING hnsw (... + # vector_cosine_ops)` access-method + opclass cleanly. Must match the + # query's cosine_distance operator class to be usable by the planner. + op.execute( + "CREATE INDEX ix_image_record_siglip_hnsw " + "ON image_record USING hnsw (siglip_embedding vector_cosine_ops)" + ) + + +def downgrade() -> None: + op.drop_index("ix_image_record_siglip_hnsw", table_name="image_record") diff --git a/backend/app/api/gallery.py b/backend/app/api/gallery.py index 807dd77..4e35bab 100644 --- a/backend/app/api/gallery.py +++ b/backend/app/api/gallery.py @@ -10,6 +10,21 @@ from ..services.gallery_service import GalleryService gallery_bp = Blueprint("gallery", __name__, url_prefix="/api/gallery") +def _image_json(i): + """Serialize a GalleryImage for the scroll/similar list responses.""" + return { + "id": i.id, + "sha256": i.sha256, + "mime": i.mime, + "width": i.width, + "height": i.height, + "created_at": i.created_at.isoformat(), + "posted_at": i.posted_at.isoformat() if i.posted_at else None, + "thumbnail_url": i.thumbnail_url, + "artist": i.artist, + } + + def _parse_date(raw): """Parse a YYYY-MM-DD query value to a UTC midnight datetime, or None. Raises ValueError (→ 400) on a malformed value.""" @@ -76,20 +91,7 @@ async def scroll(): return jsonify( { - "images": [ - { - "id": i.id, - "sha256": i.sha256, - "mime": i.mime, - "width": i.width, - "height": i.height, - "created_at": i.created_at.isoformat(), - "posted_at": i.posted_at.isoformat() if i.posted_at else None, - "thumbnail_url": i.thumbnail_url, - "artist": i.artist, - } - for i in page.images - ], + "images": [_image_json(i) for i in page.images], "next_cursor": page.next_cursor, "date_groups": [ {"year": y, "month": m, "image_ids": ids} for y, m, ids in page.date_groups @@ -98,6 +100,36 @@ async def scroll(): ) +@gallery_bp.route("/similar", methods=["GET"]) +async def similar(): + """Visual "more like this": images ranked by cosine distance to the + `similar_to` image's embedding. Composes with the scope filters (AND) but + ignores post_id and sort. Bounded top-N, no cursor.""" + try: + similar_to = int(request.args["similar_to"]) + limit = int(request.args.get("limit", "100")) + filters, _sort = _parse_filters() + except (KeyError, ValueError): + return jsonify({"error": "similar_to query param required"}), 400 + # post_id is the exclusive post-detail view — not a similarity scope. + scope = {k: v for k, v in filters.items() if k != "post_id"} + async with get_session() as session: + svc = GalleryService(session) + try: + images = await svc.similar(image_id=similar_to, limit=limit, **scope) + except ValueError as exc: + return jsonify({"error": str(exc)}), 400 + if images is None: + return jsonify({"error": "not found"}), 404 + return jsonify( + { + "images": [_image_json(i) for i in images], + "next_cursor": None, + "date_groups": [], + } + ) + + @gallery_bp.route("/timeline", methods=["GET"]) async def timeline(): try: diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py index b5060f3..9c31476 100644 --- a/backend/app/services/gallery_service.py +++ b/backend/app/services/gallery_service.py @@ -245,6 +245,27 @@ def _provenance_clause(post_id, artist_id): return None +def _gallery_images(rows, artists: dict[int, dict]) -> list[GalleryImage]: + """Build GalleryImage list from (record, posted_at, eff_date) rows + the + artist hydration map. Shared by scroll() and similar().""" + return [ + GalleryImage( + id=record.id, + path=record.path, + sha256=record.sha256, + mime=record.mime, + width=record.width, + height=record.height, + created_at=record.created_at, + effective_date=eff_date, + posted_at=posted_at, + thumbnail_url=thumbnail_url(record.thumbnail_path, record.sha256, record.mime), + artist=artists.get(record.id), + ) + for record, posted_at, eff_date in rows + ] + + async def _artists_for(session, image_ids: list[int]) -> dict[int, dict]: """Map image_id -> {"name","slug"} via the canonical image_record.artist_id (FC-2d-vii-c). Bounded by page size.""" @@ -324,22 +345,7 @@ class GalleryService: artists = await _artists_for( self.session, [r[0].id for r in rows] ) - images = [ - GalleryImage( - id=record.id, - path=record.path, - sha256=record.sha256, - mime=record.mime, - width=record.width, - height=record.height, - created_at=record.created_at, - effective_date=eff_date, - posted_at=posted_at, - thumbnail_url=thumbnail_url(record.thumbnail_path, record.sha256, record.mime), - artist=artists.get(record.id), - ) - for record, posted_at, eff_date in rows - ] + images = _gallery_images(rows, artists) return GalleryPage( images=images, next_cursor=next_cursor, @@ -505,6 +511,49 @@ class GalleryService: date_min=dmin, date_max=dmax, ) + async def similar( + self, image_id: int, limit: int = 100, *, + tag_ids: list[int] | None = None, artist_id: int | None = None, + media_type: str | None = None, platform: str | None = None, + untagged: bool = False, no_artist: bool = False, + date_from: datetime | None = None, date_to: datetime | None = None, + ) -> list[GalleryImage] | None: + """Visual "more like this": images ranked by cosine distance to + `image_id`'s SigLIP embedding (pgvector, HNSW-indexed — alembic 0036). + No ML inference here; the embedding was computed at import. + + Returns None if the source image doesn't exist (→ 404), [] if it has + no embedding (a video / not-yet-embedded). Composes with the Phase-1/2 + scope filters (AND) but REPLACES the date sort — always nearest-first, + bounded to `limit` (no cursor; distance-ranking has no date cursor). + """ + if limit < 1 or limit > 200: + raise ValueError("limit must be between 1 and 200") + src = await self.session.get(ImageRecord, image_id) + if src is None: + return None + if src.siglip_embedding is None: + return [] + + distance = ImageRecord.siglip_embedding.cosine_distance(src.siglip_embedding) + eff = _effective_date_col() + stmt = select(ImageRecord, Post.post_date, eff.label("eff")) + stmt = _outer_join_primary_post(stmt) + stmt = stmt.where( + ImageRecord.siglip_embedding.is_not(None), + ImageRecord.id != image_id, + ) + stmt = _apply_scope( + stmt, tag_ids=tag_ids, post_id=None, + artist_id=artist_id, media_type=media_type, + platform=platform, untagged=untagged, no_artist=no_artist, + date_from=date_from, date_to=date_to, + ) + stmt = stmt.order_by(distance.asc()).limit(limit) + rows = (await self.session.execute(stmt)).all() + artists = await _artists_for(self.session, [r[0].id for r in rows]) + return _gallery_images(rows, artists) + async def get_image_with_tags(self, image_id: int) -> dict | None: record = await self.session.get(ImageRecord, image_id) if record is None: @@ -542,6 +591,9 @@ class GalleryService: "height": record.height, "size_bytes": record.size_bytes, "integrity_status": record.integrity_status, + # Phase 3: lets the modal hide the "Related"/find-similar surface + # for images that have no embedding yet (videos / pending ML). + "has_embedding": record.siglip_embedding is not None, "created_at": record.created_at.isoformat(), "posted_at": posted_at.isoformat() if posted_at else None, "thumbnail_url": thumbnail_url(record.thumbnail_path, record.sha256, record.mime), diff --git a/tests/test_gallery_similar.py b/tests/test_gallery_similar.py new file mode 100644 index 0000000..07ae149 --- /dev/null +++ b/tests/test_gallery_similar.py @@ -0,0 +1,134 @@ +"""Phase-3 visual "more like this" — pgvector cosine ranking over the +precomputed SigLIP image embeddings. No query-time ML inference.""" + +from datetime import UTC, datetime, timedelta + +import pytest + +from backend.app.models import ImageRecord, Tag, TagKind +from backend.app.models.tag import image_tag +from backend.app.services.gallery_service import GalleryService + +pytestmark = pytest.mark.integration + + +def _vec(*head): + """A 1152-dim embedding with the given leading values, rest zero. Cosine + distance to _vec(1,0) grows as the 2nd component grows, so callers can + order fixtures deterministically by direction.""" + v = [0.0] * 1152 + for i, x in enumerate(head): + v[i] = float(x) + return v + + +async def _img(db, n, emb): + rec = ImageRecord( + path=f"/images/sim/{n}.jpg", sha256=f"e{n:063d}", + size_bytes=1, mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status="unknown", + siglip_embedding=emb, + ) + base = datetime(2026, 1, 1, 12, 0, tzinfo=UTC) + rec.created_at = base - timedelta(minutes=n) + rec.effective_date = rec.created_at + db.add(rec) + await db.flush() + return rec + + +@pytest.mark.asyncio +async def test_similar_ranks_nearest_first(db): + src = await _img(db, 1, _vec(1, 0)) + near = await _img(db, 2, _vec(1, 0.05)) # almost same direction + mid = await _img(db, 3, _vec(1, 1)) # 45° + far = await _img(db, 4, _vec(0, 1)) # orthogonal + svc = GalleryService(db) + res = await svc.similar(src.id, limit=10) + assert [i.id for i in res] == [near.id, mid.id, far.id] # self excluded + + +@pytest.mark.asyncio +async def test_similar_excludes_null_embeddings(db): + src = await _img(db, 1, _vec(1, 0)) + have = await _img(db, 2, _vec(1, 0.1)) + await _img(db, 3, None) # un-embedded (e.g. a video) → excluded + svc = GalleryService(db) + res = await svc.similar(src.id, limit=10) + assert [i.id for i in res] == [have.id] + + +@pytest.mark.asyncio +async def test_similar_source_without_embedding_returns_empty(db): + src = await _img(db, 1, None) + await _img(db, 2, _vec(1, 0)) + svc = GalleryService(db) + assert await svc.similar(src.id, limit=10) == [] + + +@pytest.mark.asyncio +async def test_similar_missing_source_returns_none(db): + svc = GalleryService(db) + assert await svc.similar(99999, limit=10) is None + + +@pytest.mark.asyncio +async def test_similar_composes_with_tag_filter(db): + src = await _img(db, 1, _vec(1, 0)) + await _img(db, 2, _vec(1, 0.02)) # nearest, but untagged + tagged = await _img(db, 3, _vec(1, 0.6)) # farther, but carries the tag + tag = Tag(name="t", kind=TagKind.general) + db.add(tag) + await db.flush() + await db.execute(image_tag.insert().values( + image_record_id=tagged.id, tag_id=tag.id, source="manual")) + svc = GalleryService(db) + res = await svc.similar(src.id, limit=10, tag_ids=[tag.id]) + assert [i.id for i in res] == [tagged.id] # scope AND-narrows the ranked set + + +@pytest.mark.asyncio +async def test_similar_respects_limit(db): + src = await _img(db, 1, _vec(1, 0)) + for n in range(2, 7): + await _img(db, n, _vec(1, 0.1 * n)) + svc = GalleryService(db) + res = await svc.similar(src.id, limit=2) + assert len(res) == 2 + + +# --- API --- + +@pytest.mark.asyncio +async def test_api_similar_endpoint(client, db): + src = await _img(db, 1, _vec(1, 0)) + near = await _img(db, 2, _vec(1, 0.05)) + await db.commit() + resp = await client.get(f"/api/gallery/similar?similar_to={src.id}&limit=10") + assert resp.status_code == 200 + body = await resp.get_json() + assert [i["id"] for i in body["images"]] == [near.id] + assert body["next_cursor"] is None + + +@pytest.mark.asyncio +async def test_api_similar_404_when_source_missing(client): + resp = await client.get("/api/gallery/similar?similar_to=99999") + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_api_similar_requires_param(client): + resp = await client.get("/api/gallery/similar") + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_image_detail_reports_has_embedding(client, db): + embedded = await _img(db, 1, _vec(1, 0)) + plain = await _img(db, 2, None) + await db.commit() + e = await (await client.get(f"/api/gallery/image/{embedded.id}")).get_json() + p = await (await client.get(f"/api/gallery/image/{plain.id}")).get_json() + assert e["has_embedding"] is True + assert p["has_embedding"] is False