feat(embeddings): per-chunk rows — schema, write path, version-aware backfill (#280 steps 2+3)
note_embeddings becomes one row per chunk: PK (note_id, chunk_index), plus chunk_text (what this vector actually encodes) and chunker_version. Migration 0077 clears the table — embeddings are derived (0067 precedent) and the old whole-document rows are indistinguishable from single-chunk notes, so the startup backfill regenerates the corpus at the new shape. The backfill is now version-aware: a future shape change is a CHUNKER_VERSION bump that re-embeds exactly the stale notes, not another wipe. upsert_note_embedding takes (title, body) and chunks internally — one path for the write path, the recurrence spawn and the backfill. The recurrence spawn's own embed call is deleted outright: create_note already embeds via embed_note (#2056), so the spawn was a second copy of the rule. An emptied record now CLEARS its stale vectors instead of leaving them findable. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UaYUaouG9jjhATyuxCKrQs
This commit is contained in:
@@ -31,6 +31,20 @@ def _vec(*nonzero_first):
|
||||
return v[:EMBEDDING_DIM]
|
||||
|
||||
|
||||
def _emb(note_id, user_id, chunk_index, vec):
|
||||
"""A chunk row at the current chunker version (#280, migration 0077)."""
|
||||
from scribe.services.embeddings import CHUNKER_VERSION
|
||||
|
||||
return NoteEmbedding(
|
||||
note_id=note_id,
|
||||
chunk_index=chunk_index,
|
||||
user_id=user_id,
|
||||
embedding=vec,
|
||||
chunk_text=f"chunk {chunk_index} of note {note_id}",
|
||||
chunker_version=CHUNKER_VERSION,
|
||||
)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(autouse=True)
|
||||
async def _dispose_engine():
|
||||
# Per-loop pool: dispose after each test (see test_integration_db_maintenance).
|
||||
@@ -54,8 +68,8 @@ async def seeded():
|
||||
await s.flush()
|
||||
# query vector will be [1,0,0,...]; near ~ identical (sim≈1.0),
|
||||
# far is orthogonal (sim≈0.0 -> filtered by the default threshold).
|
||||
s.add(NoteEmbedding(note_id=near.id, user_id=user.id, embedding=_vec(1.0)))
|
||||
s.add(NoteEmbedding(note_id=far.id, user_id=user.id, embedding=_vec(0.0, 1.0)))
|
||||
s.add(_emb(near.id, user.id, 0, _vec(1.0)))
|
||||
s.add(_emb(far.id, user.id, 0, _vec(0.0, 1.0)))
|
||||
await s.commit()
|
||||
ids = (user.id, near.id, far.id)
|
||||
yield ids
|
||||
|
||||
Reference in New Issue
Block a user