"""Chunked embeddings: one note_embeddings row per chunk (#280) Revision ID: 0077 Revises: 0076 Create Date: 2026-08-09 The embedding model reads at most 512 tokens and fastembed truncates the rest silently, so the old one-row-per-note shape permanently lost everything past ~400 words of a record. A note now stores one row per chunk of `embeddings.chunk_document`: PK (note_id, chunk_index), plus the chunk's text (inspectability + future "matched section" surfacing) and the chunker version that produced it (so later shape changes re-embed by version comparison instead of repeating this wipe). Embeddings are DERIVED data (0067 precedent): rows are cleared here and the startup backfill regenerates the whole corpus at the new shape on next boot. The HNSW index is untouched — it indexes chunk rows exactly as it indexed note rows. """ from alembic import op revision = "0077" down_revision = "0076" branch_labels = None depends_on = None def upgrade() -> None: # Derived data — the version-aware startup backfill re-embeds everything # at the chunked shape. Old whole-document rows would be indistinguishable # from properly-chunked single-chunk notes, so they cannot be carried over. op.execute("DELETE FROM note_embeddings") # Empty table, so NOT NULL columns need no defaults and the PK swap is # instant. op.execute("ALTER TABLE note_embeddings ADD COLUMN chunk_index integer NOT NULL") op.execute("ALTER TABLE note_embeddings ADD COLUMN chunk_text text NOT NULL") op.execute("ALTER TABLE note_embeddings ADD COLUMN chunker_version integer NOT NULL") op.execute("ALTER TABLE note_embeddings DROP CONSTRAINT note_embeddings_pkey") op.execute( "ALTER TABLE note_embeddings ADD PRIMARY KEY (note_id, chunk_index)" ) def downgrade() -> None: # Same reasoning in reverse: chunk rows make no sense to a whole-document # reader, so clear and let the old backfill regenerate. op.execute("DELETE FROM note_embeddings") op.execute("ALTER TABLE note_embeddings DROP CONSTRAINT note_embeddings_pkey") op.execute("ALTER TABLE note_embeddings DROP COLUMN chunk_index") op.execute("ALTER TABLE note_embeddings DROP COLUMN chunk_text") op.execute("ALTER TABLE note_embeddings DROP COLUMN chunker_version") op.execute("ALTER TABLE note_embeddings ADD PRIMARY KEY (note_id)")