feat(embeddings): chunk_document — the chunked document shape (#280 step 1)

bge-small reads 512 tokens and fastembed truncates silently, so a single
whole-document vector permanently lost everything past ~400 words. The new
shape: split at markdown headings (fence-aware), merge small sections, split
oversize ones at paragraph boundaries, title-anchor every chunk, repeat the
section heading on continuation pieces. A record that fits the window yields
exactly one chunk identical to the historical title\nbody shape, so the
corpus's sharpest records are byte-for-byte unaffected. CHUNKER_VERSION added
so later shape changes re-embed by version comparison instead of a table wipe.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UaYUaouG9jjhATyuxCKrQs

src/scribe/services/embeddings.py

@@ -159,7 +159,7 @@ async def _apply_supersession_penalty(
 
 
 def embedding_text(title: str | None, body: str | None) -> str:
-    """The document a record is embedded AS.
+    """`title\\n{body}` — the atomic join every embedded document is built from.
 
     One definition, deliberately. This was written out three times — the write
     path (`notes.embed_note`), the recurring-task spawn, and the startup
@@ -168,21 +168,160 @@ def embedding_text(title: str | None, body: str | None) -> str:
     recurring task embedded to a different shape than everything else would be
     ranked against a corpus it doesn't match, and nothing would report it.
 
-    It is also a PRECONDITION for changing the shape at all (#2486). Measured,
-    a dev-log's vector separates from five unrelated dev-logs by 0.023 while a
-    snippet's separates by 0.153 — the difference being that a snippet states
-    its purpose twice in a short document, so the purpose dominates. Testing an
-    alternative shape against three copies would mean testing a shape that is
-    not the one in production.
-
-    Whether `title\\n{body}` is the RIGHT shape is the open question. That it is
-    one shape is what makes the question answerable.
+    Since the chunking build (#280) this is a BUILDING BLOCK, not the whole
+    story: the document shape a record is embedded as is `chunk_document`
+    below, which calls this once per chunk. Callers that want "the text this
+    note is embedded as" want `chunk_document`; this stays public because the
+    two functions are one contract and the guard in test_embedding_text pins
+    both.
     """
     title = title or ""
     body = body or ""
     return f"{title}\n{body}".strip() if body else title
 
 
+# --- chunking (#280): the document shape ------------------------------------
+#
+# bge-small reads at most 512 tokens and fastembed silently truncates the rest,
+# so a single whole-document vector loses everything past ~400 words — for a
+# long dev-log, three quarters of the record was PERMANENTLY invisible to
+# search. The fix is the document shape: one vector per meaningful chunk, and a
+# record is as findable as its best-matching section.
+
+# Bumped whenever chunk_document's output can change for the same input. Stored
+# on every note_embeddings row so the startup backfill can re-embed exactly the
+# notes whose stored shape is stale — a version comparison instead of the table
+# wipe migrations 0067/0077 had to do.
+CHUNKER_VERSION = 1
+
+# Character budget approximating the model window. Tokens-per-char varies by
+# content — ~4 chars/token for prose, closer to 3 for code and tables — so 1400
+# chars sits at roughly 350-470 tokens, leaving headroom for the title prefixed
+# to every chunk. Deliberately conservative: our own measurement (#2485) says
+# shorter, single-topic documents embed SHARPER, so the cost of over-splitting
+# is a few extra cheap vectors while the cost of under-splitting is truncation —
+# the exact data loss this exists to end.
+_CHUNK_CHAR_BUDGET = 1400
+
+_HEADING_RE = None  # compiled lazily below to keep re import local
+
+
+def _split_sections(body: str) -> list[str]:
+    """Split a markdown body at heading lines, fence-aware.
+
+    Each section is a heading line plus everything under it; text before the
+    first heading is its own section. Heading-looking lines inside ``` / ~~~
+    code fences do not split — a commented `# step` in a recorded shell snippet
+    is content, not structure.
+    """
+    import re
+    global _HEADING_RE
+    if _HEADING_RE is None:
+        _HEADING_RE = re.compile(r"^#{1,6}\s")
+
+    sections: list[list[str]] = [[]]
+    in_fence = False
+    for line in body.splitlines():
+        if line.lstrip().startswith(("```", "~~~")):
+            in_fence = not in_fence
+        if not in_fence and _HEADING_RE.match(line) and sections[-1]:
+            sections.append([line])
+        else:
+            sections[-1].append(line)
+    return ["\n".join(chunk).strip() for chunk in sections if any(s.strip() for s in chunk)]
+
+
+def _split_paragraphs(section: str, budget: int) -> list[str]:
+    """Break one oversize section into budget-sized pieces at paragraph
+    boundaries, hard-splitting only a single paragraph that alone exceeds the
+    budget (a monster table or code block — split at line boundaries so no
+    content is dropped, which is the entire point of this module)."""
+    pieces: list[str] = []
+    current = ""
+    for para in section.split("\n\n"):
+        while len(para) > budget:
+            # Hard split: prefer the last newline inside the budget so lines
+            # stay whole, then the last space so words do; a clean char cut is
+            # the final resort for one enormous unbroken token.
+            cut = para.rfind("\n", 0, budget)
+            if cut <= 0:
+                cut = para.rfind(" ", 0, budget)
+            if cut <= 0:
+                cut = budget
+            head, para = para[:cut], para[cut:].lstrip("\n ")
+            if current:
+                pieces.append(current)
+                current = ""
+            pieces.append(head.strip())
+        if not para.strip():
+            continue
+        candidate = f"{current}\n\n{para}" if current else para
+        if len(candidate) > budget and current:
+            pieces.append(current)
+            current = para
+        else:
+            current = candidate
+    if current:
+        pieces.append(current)
+    return pieces
+
+
+def chunk_document(title: str | None, body: str | None) -> list[str]:
+    """The document(s) a record is embedded AS — one string per chunk.
+
+    The contract every retrieval surface builds on:
+
+    - A record that fits the model window yields EXACTLY ONE chunk, identical
+      to the historical `title\\nbody` shape — snippets and reference notes,
+      the corpus's sharpest records, are byte-for-byte unaffected.
+    - A longer record is split at markdown heading boundaries (fence-aware),
+      small neighbouring sections merged, oversize sections split at paragraph
+      boundaries, so every chunk fits the window. NOTHING is dropped: every
+      line of the body lands in some chunk.
+    - Every chunk is prefixed with the record's title — each vector carries
+      its own topical anchor, the property that makes snippets discriminative
+      (#2485). Pieces sub-split from one section also repeat that section's
+      heading line, so "which part of which topic" survives the split.
+    - An empty record yields [] (callers gate on falsiness to skip embedding).
+
+    Bump CHUNKER_VERSION when changing anything observable here.
+    """
+    single = embedding_text(title, body)
+    if not single:
+        return []
+    if len(single) <= _CHUNK_CHAR_BUDGET:
+        return [single]
+
+    title = title or ""
+    # Budget for section content, net of the title prefix added to every chunk.
+    budget = max(200, _CHUNK_CHAR_BUDGET - len(title) - 1)
+
+    # Merge small adjacent sections upward so tiny sections don't each spend a
+    # vector, then split anything still over budget at paragraph boundaries.
+    merged: list[str] = []
+    for section in _split_sections(body or ""):
+        if merged and len(merged[-1]) + 2 + len(section) <= budget:
+            merged[-1] = f"{merged[-1]}\n\n{section}"
+        else:
+            merged.append(section)
+
+    chunks: list[str] = []
+    for section in merged:
+        if len(section) <= budget:
+            chunks.append(embedding_text(title, section))
+            continue
+        pieces = _split_paragraphs(section, budget)
+        first_line = section.split("\n", 1)[0]
+        heading = first_line if first_line.lstrip().startswith("#") else ""
+        for i, piece in enumerate(pieces):
+            # Repeat the section heading on continuation pieces so each vector
+            # still knows what topic it is part of.
+            if i > 0 and heading and not piece.startswith(heading):
+                piece = f"{heading}\n{piece}"
+            chunks.append(embedding_text(title, piece))
+    return chunks
+
+
 async def upsert_note_embedding(note_id: int, user_id: int, text: str) -> None:
     """Generate and persist an embedding for a note. Safe to fire-and-forget."""
     if not text or not text.strip():