feat(embeddings): every vector records the model whose space it lives in (#4132)

The four embedding tables stamped chunker_version but not the model, and
vector(384) is a width, not an identity: a same-width model swap would
write a second geometry beside the first with no error.

- embedding_model on note/rule/milestone/system embeddings (0109; existing
  rows stamped with the only model any install has ever run).
- Every write stamps EMBEDDING_MODEL; every backfill's "current" test is
  is_current_stamp(), both halves of calibration_stamp().
- migrate_floor refuses while any row its surface searches is off the live
  model, before sampling: re-embed, then migrate.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/scribe/models/embedding.py

@@ -41,6 +41,14 @@ class NoteEmbedding(Base):
     # any note whose rows carry a stale version — shape changes become a
     # version bump instead of a table wipe.
     chunker_version: Mapped[int] = mapped_column(Integer, nullable=False)
+    # embeddings.EMBEDDING_MODEL at write time — the SPACE the vector lives in
+    # (#4132). The column is `vector(384)`, a width and not an identity, so a
+    # swap to another 384-dim model writes a second geometry beside the first
+    # with no error, and cosine across the two is a number that means nothing.
+    # The version above says what text was embedded; this says in whose
+    # geometry. Either one moving makes the row stale, and the backfill
+    # re-embeds on both.
+    embedding_model: Mapped[str] = mapped_column(Text, nullable=False)
     updated_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True),
         default=lambda: datetime.now(timezone.utc),
@@ -91,6 +99,7 @@ class RuleEmbedding(Base):
     # centroid (measured in note 2485).
     chunk_text: Mapped[str] = mapped_column(Text, nullable=False)
     chunker_version: Mapped[int] = mapped_column(Integer, nullable=False)
+    embedding_model: Mapped[str] = mapped_column(Text, nullable=False)  # see NoteEmbedding
     updated_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True),
         default=lambda: datetime.now(timezone.utc),
@@ -124,6 +133,7 @@ class MilestoneEmbedding(Base):
     embedding: Mapped[list] = mapped_column(Vector(EMBEDDING_DIM), nullable=False)
     chunk_text: Mapped[str] = mapped_column(Text, nullable=False)
     chunker_version: Mapped[int] = mapped_column(Integer, nullable=False)
+    embedding_model: Mapped[str] = mapped_column(Text, nullable=False)  # see NoteEmbedding
     updated_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True),
         default=lambda: datetime.now(timezone.utc),
@@ -163,6 +173,7 @@ class SystemEmbedding(Base):
     embedding: Mapped[list] = mapped_column(Vector(EMBEDDING_DIM), nullable=False)
     chunk_text: Mapped[str] = mapped_column(Text, nullable=False)
     chunker_version: Mapped[int] = mapped_column(Integer, nullable=False)
+    embedding_model: Mapped[str] = mapped_column(Text, nullable=False)  # see NoteEmbedding
     updated_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True),
         default=lambda: datetime.now(timezone.utc),

src/scribe/services/embeddings.py

@@ -18,7 +18,7 @@ from collections.abc import Sequence
 
 from typing import TYPE_CHECKING
 
-from sqlalchemy import delete, func, or_, select
+from sqlalchemy import and_, delete, func, or_, select
 
 from scribe.models import async_session
 from scribe.models.embedding import NoteEmbedding, RuleEmbedding
@@ -367,6 +367,34 @@ def calibration_stamp() -> dict:
     """
     return {"embedding_model": EMBEDDING_MODEL, "shape_version": CHUNKER_VERSION}
 
+
+def is_current_stamp(table):
+    """The rows written by THIS chunker in THIS model's space (#4132).
+
+    Every embedding table stores both halves of `calibration_stamp()` per row,
+    and a row is current only when both match. One predicate for all four
+    tables, because a backfill that checked the version alone is exactly how a
+    same-width model swap would have gone unnoticed.
+    """
+    return and_(
+        table.chunker_version == CHUNKER_VERSION,
+        table.embedding_model == EMBEDDING_MODEL,
+    )
+
+
+async def rows_off_the_live_model(table) -> int:
+    """How many rows of an embedding table were NOT written in the live space.
+
+    Nonzero means the corpus is part-way through a model change: a search over
+    it compares vectors from two geometries, and a statistic computed from it
+    (`retrieval_migration.migrate_floor`) is a blend of both.
+    """
+    async with async_session() as session:
+        return int((await session.execute(
+            select(func.count()).select_from(table)
+            .where(table.embedding_model != EMBEDDING_MODEL)
+        )).scalar_one())
+
 # Character budget approximating the model window. Tokens-per-char varies by
 # content — ~4 chars/token for prose, closer to 3 for code and tables — so 1400
 # chars sits at roughly 350-470 tokens, leaving headroom for the title prefixed
@@ -633,6 +661,7 @@ async def upsert_note_embedding(
                         embedding=vector,
                         chunk_text=chunk,
                         chunker_version=CHUNKER_VERSION,
+                        embedding_model=EMBEDDING_MODEL,
                     )
                 )
             await session.commit()
@@ -1092,7 +1121,7 @@ async def backfill_note_embeddings() -> None:
                 for row in (
                     await session.execute(
                         select(NoteEmbedding.note_id).where(
-                            NoteEmbedding.chunker_version == CHUNKER_VERSION
+                            is_current_stamp(NoteEmbedding)
                         )
                     )
                 ).fetchall()
@@ -1294,6 +1323,7 @@ async def upsert_rule_embedding(
                         embedding=vector,
                         chunk_text=chunk,
                         chunker_version=CHUNKER_VERSION,
+                        embedding_model=EMBEDDING_MODEL,
                     )
                 )
             await session.commit()
@@ -1464,7 +1494,7 @@ async def backfill_rule_embeddings() -> None:
     try:
         async with async_session() as session:
             current = select(RuleEmbedding.rule_id).where(
-                RuleEmbedding.chunker_version == CHUNKER_VERSION
+                is_current_stamp(RuleEmbedding)
             )
             # IDS ONLY — the text is re-read per rule below (#4262).
             by_version = {
@@ -1563,6 +1593,7 @@ async def upsert_milestone_embedding(
                 session.add(MilestoneEmbedding(
                     milestone_id=milestone_id, chunk_index=index, embedding=vector,
                     chunk_text=chunk, chunker_version=CHUNKER_VERSION,
+                    embedding_model=EMBEDDING_MODEL,
                 ))
             await session.commit()
     except Exception:
@@ -1725,6 +1756,7 @@ async def upsert_system_embedding(
                 session.add(SystemEmbedding(
                     system_id=system_id, chunk_index=index, embedding=vector,
                     chunk_text=chunk, chunker_version=CHUNKER_VERSION,
+                    embedding_model=EMBEDDING_MODEL,
                 ))
             await session.commit()
     except Exception:
@@ -1841,7 +1873,7 @@ async def backfill_system_embeddings() -> None:
     try:
         async with async_session() as session:
             current = select(SystemEmbedding.system_id).where(
-                SystemEmbedding.chunker_version == CHUNKER_VERSION
+                is_current_stamp(SystemEmbedding)
             )
             # IDS ONLY — the charter is re-read per System below (#4262).
             by_version = {
@@ -1884,7 +1916,7 @@ async def backfill_milestone_embeddings() -> None:
     try:
         async with async_session() as session:
             current = select(MilestoneEmbedding.milestone_id).where(
-                MilestoneEmbedding.chunker_version == CHUNKER_VERSION
+                is_current_stamp(MilestoneEmbedding)
             )
             # IDS ONLY — the plan is re-read per milestone below (#4262).
             by_version = {

src/scribe/services/retrieval_migration.py

@@ -58,9 +58,11 @@ import logging
 from sqlalchemy import select
 
 from scribe.models import async_session
+from scribe.models.embedding import NoteEmbedding, RuleEmbedding
 from scribe.models.retrieval_log import RetrievalLog
 from scribe.services.embeddings import (
     calibration_stamp,
+    rows_off_the_live_model,
     semantic_search_notes,
     semantic_search_rules,
 )
@@ -117,6 +119,19 @@ _RESCORERS = {
     "report_preference": lambda u, q, p: _rescore_rules(u, q, p, "preference"),
 }
 
+# The embedding table each surface's re-scorer reads. A migration from a corpus
+# that is still part-way through a model change re-scores against a blend of
+# two geometries and writes a floor computed from it, with a confident reason
+# attached (#4132) — so the corpus has to be wholly in the live space first.
+_CORPUS = {
+    "auto_inject": NoteEmbedding,
+    "write_path": NoteEmbedding,
+    "write_path_rule": RuleEmbedding,
+    "pre_tool_rule": RuleEmbedding,
+    "prompt_rule": RuleEmbedding,
+    "report_preference": RuleEmbedding,
+}
+
 
 def _floor_admitting(scores: list[float], fraction: float) -> float:
     """The floor that admits `fraction` of `scores`, on this scale.
@@ -161,6 +176,21 @@ async def migrate_floor(
             "arm's own corpus filters."
         )
 
+    off_model = await rows_off_the_live_model(_CORPUS[surface])
+    if off_model:
+        # Refused before sampling anything. The order of operations — re-embed,
+        # THEN migrate — used to be held only in the operator's memory.
+        stamp = calibration_stamp()
+        return {
+            "surface": surface, "migrated": False,
+            "why": f"{off_model} embedding row(s) this surface searches are not "
+                   f"yet in {stamp['embedding_model']}'s space. Re-scoring now "
+                   "would measure a blend of two models; let the startup "
+                   "backfill finish re-embedding, then migrate",
+            "rows_off_model": off_model,
+            "calibration": stamp,
+        }
+
     old_floor = await floor_for(user_id, surface)
     async with async_session() as session:
         rows = (await session.execute(