bvandeusen
513019786e
Move semantic_search_notes off the full-table Python cosine scan onto a native pgvector column: indexed ORDER BY embedding <=> :q LIMIT k (HNSW, cosine). Migration 0067 enables the extension, converts the JSONB embedding column to vector(384) (stale-dim rows dropped and regenerated by the startup backfill), and builds the HNSW cosine index. Postgres image moves postgres:16-alpine -> pgvector/pgvector:pg17 across prod, quickstart, and CI. Scribe: project 2, milestone 93, task 1031. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Xz4j1H7pjYSjKsEpgcNH5E
74 lines
3.1 KiB
Python
74 lines
3.1 KiB
Python
"""pgvector: note_embeddings.embedding JSONB -> vector(384) + HNSW index
|
|
|
|
Revision ID: 0067
|
|
Revises: 0066
|
|
Create Date: 2026-06-22
|
|
|
|
Moves semantic search off the full-table Python cosine scan onto a native
|
|
pgvector column so ranking + top-k run as an indexed `ORDER BY embedding <=> :q
|
|
LIMIT k` in Postgres (see services/embeddings.semantic_search_notes).
|
|
|
|
Requires a Postgres image that bundles the `vector` extension — the stack moved
|
|
from postgres:16-alpine to pgvector/pgvector:pg16 in the same change (compose +
|
|
CI). `CREATE EXTENSION IF NOT EXISTS vector` below is the in-db half.
|
|
|
|
Embeddings are DERIVED data (regenerated from note text by
|
|
backfill_note_embeddings at startup), so this migration is free to drop any row
|
|
it can't cleanly convert: only rows whose stored JSONB array is exactly 384-dim
|
|
are carried over (guarding against stale vectors from an earlier model — the
|
|
same mixed-dim hazard _cosine_similarity defended against). Dropped rows are
|
|
re-embedded on next boot.
|
|
"""
|
|
from alembic import op
|
|
|
|
|
|
revision = "0067"
|
|
down_revision = "0066"
|
|
branch_labels = None
|
|
depends_on = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
|
|
|
# New native-vector column, populated only from cleanly-convertible rows.
|
|
# A JSONB array like [0.1, 0.2, ...] renders to text that is exactly
|
|
# pgvector's input literal, so (embedding::text)::vector is a direct cast.
|
|
op.execute("ALTER TABLE note_embeddings ADD COLUMN embedding_vec vector(384)")
|
|
op.execute(
|
|
"""
|
|
UPDATE note_embeddings
|
|
SET embedding_vec = (embedding::text)::vector
|
|
WHERE jsonb_array_length(embedding) = 384
|
|
"""
|
|
)
|
|
# Stale-dim rows (couldn't convert) are derived data — drop and let the
|
|
# startup backfill regenerate them at the current dimension.
|
|
op.execute("DELETE FROM note_embeddings WHERE embedding_vec IS NULL")
|
|
|
|
op.execute("ALTER TABLE note_embeddings ALTER COLUMN embedding_vec SET NOT NULL")
|
|
op.execute("ALTER TABLE note_embeddings DROP COLUMN embedding")
|
|
op.execute("ALTER TABLE note_embeddings RENAME COLUMN embedding_vec TO embedding")
|
|
|
|
# HNSW index for cosine distance — matches Vector.cosine_distance (`<=>`).
|
|
op.execute(
|
|
"""
|
|
CREATE INDEX ix_note_embeddings_embedding_hnsw
|
|
ON note_embeddings
|
|
USING hnsw (embedding vector_cosine_ops)
|
|
"""
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
# Back to JSONB. pgvector renders a vector to a text literal that is a valid
|
|
# JSON array, so the reverse cast is symmetric. The `vector` extension is
|
|
# intentionally left installed (other objects may depend on it; dropping an
|
|
# extension is the riskier, rarely-wanted direction).
|
|
op.execute("DROP INDEX IF EXISTS ix_note_embeddings_embedding_hnsw")
|
|
op.execute("ALTER TABLE note_embeddings ADD COLUMN embedding_json jsonb")
|
|
op.execute("UPDATE note_embeddings SET embedding_json = (embedding::text)::jsonb")
|
|
op.execute("ALTER TABLE note_embeddings ALTER COLUMN embedding_json SET NOT NULL")
|
|
op.execute("ALTER TABLE note_embeddings DROP COLUMN embedding")
|
|
op.execute("ALTER TABLE note_embeddings RENAME COLUMN embedding_json TO embedding")
|