feat(search): pgvector substrate — vector(384) + HNSW for semantic search

Move semantic_search_notes off the full-table Python cosine scan onto a native
pgvector column: indexed ORDER BY embedding <=> :q LIMIT k (HNSW, cosine).
Migration 0067 enables the extension, converts the JSONB embedding column to
vector(384) (stale-dim rows dropped and regenerated by the startup backfill),
and builds the HNSW cosine index. Postgres image moves postgres:16-alpine ->
pgvector/pgvector:pg17 across prod, quickstart, and CI.

Scribe: project 2, milestone 93, task 1031.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Xz4j1H7pjYSjKsEpgcNH5E

alembic/versions/0067_pgvector_note_embeddings.py

@@ -0,0 +1,73 @@
+"""pgvector: note_embeddings.embedding JSONB -> vector(384) + HNSW index
+
+Revision ID: 0067
+Revises: 0066
+Create Date: 2026-06-22
+
+Moves semantic search off the full-table Python cosine scan onto a native
+pgvector column so ranking + top-k run as an indexed `ORDER BY embedding <=> :q
+LIMIT k` in Postgres (see services/embeddings.semantic_search_notes).
+
+Requires a Postgres image that bundles the `vector` extension — the stack moved
+from postgres:16-alpine to pgvector/pgvector:pg16 in the same change (compose +
+CI). `CREATE EXTENSION IF NOT EXISTS vector` below is the in-db half.
+
+Embeddings are DERIVED data (regenerated from note text by
+backfill_note_embeddings at startup), so this migration is free to drop any row
+it can't cleanly convert: only rows whose stored JSONB array is exactly 384-dim
+are carried over (guarding against stale vectors from an earlier model — the
+same mixed-dim hazard _cosine_similarity defended against). Dropped rows are
+re-embedded on next boot.
+"""
+from alembic import op
+
+
+revision = "0067"
+down_revision = "0066"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute("CREATE EXTENSION IF NOT EXISTS vector")
+
+    # New native-vector column, populated only from cleanly-convertible rows.
+    # A JSONB array like [0.1, 0.2, ...] renders to text that is exactly
+    # pgvector's input literal, so (embedding::text)::vector is a direct cast.
+    op.execute("ALTER TABLE note_embeddings ADD COLUMN embedding_vec vector(384)")
+    op.execute(
+        """
+        UPDATE note_embeddings
+           SET embedding_vec = (embedding::text)::vector
+         WHERE jsonb_array_length(embedding) = 384
+        """
+    )
+    # Stale-dim rows (couldn't convert) are derived data — drop and let the
+    # startup backfill regenerate them at the current dimension.
+    op.execute("DELETE FROM note_embeddings WHERE embedding_vec IS NULL")
+
+    op.execute("ALTER TABLE note_embeddings ALTER COLUMN embedding_vec SET NOT NULL")
+    op.execute("ALTER TABLE note_embeddings DROP COLUMN embedding")
+    op.execute("ALTER TABLE note_embeddings RENAME COLUMN embedding_vec TO embedding")
+
+    # HNSW index for cosine distance — matches Vector.cosine_distance (`<=>`).
+    op.execute(
+        """
+        CREATE INDEX ix_note_embeddings_embedding_hnsw
+            ON note_embeddings
+         USING hnsw (embedding vector_cosine_ops)
+        """
+    )
+
+
+def downgrade() -> None:
+    # Back to JSONB. pgvector renders a vector to a text literal that is a valid
+    # JSON array, so the reverse cast is symmetric. The `vector` extension is
+    # intentionally left installed (other objects may depend on it; dropping an
+    # extension is the riskier, rarely-wanted direction).
+    op.execute("DROP INDEX IF EXISTS ix_note_embeddings_embedding_hnsw")
+    op.execute("ALTER TABLE note_embeddings ADD COLUMN embedding_json jsonb")
+    op.execute("UPDATE note_embeddings SET embedding_json = (embedding::text)::jsonb")
+    op.execute("ALTER TABLE note_embeddings ALTER COLUMN embedding_json SET NOT NULL")
+    op.execute("ALTER TABLE note_embeddings DROP COLUMN embedding")
+    op.execute("ALTER TABLE note_embeddings RENAME COLUMN embedding_json TO embedding")