feat(embeddings): per-chunk rows — schema, write path, version-aware backfill (#280 steps 2+3)

note_embeddings becomes one row per chunk: PK (note_id, chunk_index), plus
chunk_text (what this vector actually encodes) and chunker_version. Migration
0077 clears the table — embeddings are derived (0067 precedent) and the old
whole-document rows are indistinguishable from single-chunk notes, so the
startup backfill regenerates the corpus at the new shape. The backfill is now
version-aware: a future shape change is a CHUNKER_VERSION bump that re-embeds
exactly the stale notes, not another wipe.

upsert_note_embedding takes (title, body) and chunks internally — one path
for the write path, the recurrence spawn and the backfill. The recurrence
spawn's own embed call is deleted outright: create_note already embeds via
embed_note (#2056), so the spawn was a second copy of the rule. An emptied
record now CLEARS its stale vectors instead of leaving them findable.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UaYUaouG9jjhATyuxCKrQs

src/scribe/services/embeddings.py

@@ -75,10 +75,19 @@ async def get_embedding(text: str) -> list[float]:
     Raises if the fastembed model fails to load. Callers should catch and
     degrade to keyword search.
     """
+    return (await get_embeddings([text]))[0]
+
+
+async def get_embeddings(texts: list[str]) -> list[list[float]]:
+    """Embed several texts in one model call (the chunked write path).
+
+    fastembed batches internally, so N chunks cost far less than N single
+    calls. Raises like get_embedding; callers catch and degrade.
+    """
     embedder = await _get_model()
     # embed() is synchronous CPU work; offload so we don't block the event loop.
-    vecs = await asyncio.to_thread(lambda: list(embedder.embed([text])))
-    return vecs[0].tolist()
+    vecs = await asyncio.to_thread(lambda: list(embedder.embed(texts)))
+    return [v.tolist() for v in vecs]
 
 
 def _cosine_similarity(a: list[float], b: list[float]) -> float:
@@ -322,12 +331,36 @@ def chunk_document(title: str | None, body: str | None) -> list[str]:
     return chunks
 
 
-async def upsert_note_embedding(note_id: int, user_id: int, text: str) -> None:
-    """Generate and persist an embedding for a note. Safe to fire-and-forget."""
-    if not text or not text.strip():
-        return
+async def upsert_note_embedding(
+    note_id: int, user_id: int, title: str | None, body: str | None
+) -> None:
+    """Chunk, embed and persist a note's vectors. Safe to fire-and-forget.
+
+    Takes title/body rather than pre-built text so the chunking happens HERE —
+    one path for the write path, the recurrence spawn and the startup backfill,
+    which is the same single-definition discipline embedding_text existed for.
+
+    Replacement is atomic per note: old rows are deleted and the new chunk set
+    inserted in one transaction, so a concurrent read sees the old shape or the
+    new one, never a mixture.
+    """
+    chunks = chunk_document(title, body)
     try:
-        embedding = await get_embedding(text)
+        if not chunks:
+            # A record emptied of content should stop being findable by its
+            # old content — clear stale vectors rather than leaving them.
+            async with async_session() as session:
+                await session.execute(
+                    delete(NoteEmbedding).where(NoteEmbedding.note_id == note_id)
+                )
+                await session.commit()
+            return
+    except Exception:
+        logger.warning("Failed to clear embedding for note %d", note_id, exc_info=True)
+        return
+
+    try:
+        vectors = await get_embeddings(chunks)
     except Exception:
         logger.debug("Skipping embedding for note %d — embedder unavailable", note_id)
         return
@@ -337,9 +370,19 @@ async def upsert_note_embedding(note_id: int, user_id: int, text: str) -> None:
             await session.execute(
                 delete(NoteEmbedding).where(NoteEmbedding.note_id == note_id)
             )
-            session.add(NoteEmbedding(note_id=note_id, user_id=user_id, embedding=embedding))
+            for index, (chunk, vector) in enumerate(zip(chunks, vectors)):
+                session.add(
+                    NoteEmbedding(
+                        note_id=note_id,
+                        chunk_index=index,
+                        user_id=user_id,
+                        embedding=vector,
+                        chunk_text=chunk,
+                        chunker_version=CHUNKER_VERSION,
+                    )
+                )
             await session.commit()
-        logger.debug("Upserted embedding for note %d", note_id)
+        logger.debug("Upserted %d chunk embedding(s) for note %d", len(chunks), note_id)
     except Exception:
         logger.warning("Failed to persist embedding for note %d", note_id, exc_info=True)
 
@@ -500,40 +543,50 @@ async def semantic_search_notes(
 
 
 async def backfill_note_embeddings() -> None:
-    """Generate embeddings for all notes that don't have one yet.
+    """(Re-)embed every note that is missing vectors OR whose stored vectors
+    were produced by an older chunker.
 
-    Runs as a background task at startup. Adds a small sleep between notes
-    so a large backfill doesn't peg CPU.
+    Runs as a background task at startup. Version-awareness is what makes a
+    document-shape change deployable: migration 0077 cleared the table once,
+    and every later CHUNKER_VERSION bump re-embeds the stale notes here — a
+    version comparison instead of another wipe. Adds a small sleep between
+    notes so a large backfill doesn't peg CPU.
     """
     try:
         async with async_session() as session:
-            existing = {
+            current = {
                 row[0]
                 for row in (
-                    await session.execute(select(NoteEmbedding.note_id))
+                    await session.execute(
+                        select(NoteEmbedding.note_id).where(
+                            NoteEmbedding.chunker_version == CHUNKER_VERSION
+                        )
+                    )
                 ).fetchall()
             }
             result = await session.execute(
                 select(Note.id, Note.user_id, Note.title, Note.body)
             )
             notes_to_embed = [
-                row for row in result.fetchall() if row[0] not in existing
+                row for row in result.fetchall() if row[0] not in current
             ]
     except Exception:
         logger.warning("Embedding backfill: failed to query notes", exc_info=True)
         return
 
     if not notes_to_embed:
-        logger.info("Embedding backfill: all notes already have embeddings")
+        logger.info("Embedding backfill: all notes current at chunker v%d", CHUNKER_VERSION)
         return
 
-    logger.info("Embedding backfill: generating embeddings for %d notes", len(notes_to_embed))
+    logger.info(
+        "Embedding backfill: embedding %d notes at chunker v%d",
+        len(notes_to_embed), CHUNKER_VERSION,
+    )
     success = 0
     for note_id, user_id, title, body in notes_to_embed:
-        text = embedding_text(title, body)
-        if not text:
+        if not chunk_document(title, body):
             continue
-        await upsert_note_embedding(note_id, user_id, text)
+        await upsert_note_embedding(note_id, user_id, title, body)
         success += 1
         await asyncio.sleep(0.05)  # gentle pacing