bvandeusen/FabledScribe
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases 53 Wiki Activity

fix(embeddings): no chunk is a heading alone (#4784)

Browse Source 
Judging live auto-inject menus (#4772) found "## Work log — 2026-08-21",
"## Findings worth carrying forward" and "Two dev→main PRs this session."
each embedded as a whole chunk. Text about nothing embeds close to
everything, so they took ranks 1–3 on vague queries and pushed real records
down.

Three ways the chunker made them, each closed:
- _split_paragraphs flushed a heading on its own when the paragraph under it
  was long. A thin lead is now carried into the paragraph that follows, and
  a hard cut never lands in the first quarter of the budget, where the
  newline it finds closes a heading.
- A heading with no body (above its ### parts) became a section of its own
  when the section before it was full. A thin section now takes the next.
- A short lead-in or a short last section stood alone. The lead-in joins
  what follows; a thin tail joins what precedes it.

A continuation piece now repeats the LAST heading before it, since one
section can now hold several. CHUNKER_VERSION 2 → 3, so the startup backfill
re-embeds; tuned dials will report their shape as changed, which is true.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-10-03 21:25:37 -04:00
co-authored by Claude Opus 5.5
parent 5d0e97576b
commit 78fa01b025
2 changed files with 167 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/scribe/services/embeddings.py
+83 -15
View File
@@ -333,7 +333,10 @@ def document_title(
# narrower wording would have read as "nothing to re-embed".
#
# 1 → 2: work logs joined the task document.
CHUNKER_VERSION = 2
# 2 → 3: no chunk is a heading alone (#4784). A content-free chunk embeds
# near the centre of the space and outranked real records on any
# vague query — "## Work log — 2026-08-21" took rank 1.
CHUNKER_VERSION = 3
# The public name of the space every score lives in, and the two facts that
@@ -404,9 +407,45 @@ async def rows_off_the_live_model(table) -> int:
# the exact data loss this exists to end.
_CHUNK_CHAR_BUDGET = 1400
# The least a chunk may say beyond its headings (#4784). Below this a chunk is
# a heading, a date or a one-line lead-in, and it embeds as text about nothing
# in particular — which is close to everything, so it outranks real records on
# any vague query. Such a piece is joined to its neighbour instead of being
# given a vector of its own.
_MIN_CHUNK_CONTENT = 160
_HEADING_RE = None # compiled lazily below to keep re import local
def _heading_re():
import re
global _HEADING_RE
if _HEADING_RE is None:
_HEADING_RE = re.compile(r"^#{1,6}\s")
return _HEADING_RE
def _is_thin(text: str) -> bool:
"""Says too little to be a chunk on its own: under `_MIN_CHUNK_CONTENT`
once its heading lines are set aside."""
rx = _heading_re()
said = "\n".join(ln for ln in text.splitlines() if not rx.match(ln)).strip()
return len(said) < _MIN_CHUNK_CONTENT
def _last_heading(text: str, default: str = "") -> str:
"""The last heading line outside a code fence, or `default`."""
rx = _heading_re()
in_fence = False
found = default
for line in text.splitlines():
if line.lstrip().startswith(("```", "~~~")):
in_fence = not in_fence
elif not in_fence and rx.match(line):
found = line
return found
def _split_sections(body: str) -> list[str]:
"""Split a markdown body at heading lines, fence-aware.
@@ -415,17 +454,13 @@ def _split_sections(body: str) -> list[str]:
code fences do not split — a commented `# step` in a recorded shell snippet
is content, not structure.
"""
import re
global _HEADING_RE
if _HEADING_RE is None:
_HEADING_RE = re.compile(r"^#{1,6}\s")
rx = _heading_re()
sections: list[list[str]] = [[]]
in_fence = False
for line in body.splitlines():
if line.lstrip().startswith(("```", "~~~")):
in_fence = not in_fence
if not in_fence and _HEADING_RE.match(line) and sections[-1]:
if not in_fence and rx.match(line) and sections[-1]:
sections.append([line])
else:
sections[-1].append(line)
@@ -436,17 +471,27 @@ def _split_paragraphs(section: str, budget: int) -> list[str]:
"""Break one oversize section into budget-sized pieces at paragraph
boundaries, hard-splitting only a single paragraph that alone exceeds the
budget (a monster table or code block — split at line boundaries so no
content is dropped, which is the entire point of this module)."""
content is dropped, which is the entire point of this module).
No piece is thin (#4784). A heading, or a lead-in line, that would be left
on its own is carried into the paragraph after it; a thin tail joins the
piece before it."""
pieces: list[str] = []
current = ""
# A hard cut never lands in the first quarter of the budget: the newline
# it would find there is the one closing a heading or a lead-in, and
# cutting at it is how a heading became a chunk of its own.
lo = budget // 4
for para in section.split("\n\n"):
if current and _is_thin(current):
para, current = f"{current}\n\n{para}", ""
while len(para) > budget:
# Hard split: prefer the last newline inside the budget so lines
# stay whole, then the last space so words do; a clean char cut is
# the final resort for one enormous unbroken token.
cut = para.rfind("\n", 0, budget)
cut = para.rfind("\n", lo, budget)
if cut <= 0:
cut = para.rfind(" ", 0, budget)
cut = para.rfind(" ", lo, budget)
if cut <= 0:
cut = budget
head, para = para[:cut], para[cut:].lstrip("\n ")
@@ -462,6 +507,19 @@ def _split_paragraphs(section: str, budget: int) -> list[str]:
current = para
else:
current = candidate
if current and pieces and _is_thin(current):
# A thin tail joins the piece before it. If that piece is full, its
# last paragraph moves down to keep the tail company instead.
last = pieces.pop()
if len(last) + 2 + len(current) <= budget:
current = f"{last}\n\n{current}"
else:
head, sep, tail = last.rpartition("\n\n")
if sep and len(tail) + 2 + len(current) <= budget:
pieces.append(head)
current = f"{tail}\n\n{current}"
else:
pieces.append(last)
if current:
pieces.append(current)
return pieces
@@ -499,12 +557,20 @@ def chunk_document(title: str | None, body: str | None) -> list[str]:
# Merge small adjacent sections upward so tiny sections don't each spend a
# vector, then split anything still over budget at paragraph boundaries.
# A THIN section takes the next one in whatever its size (#4784): a heading
# with nothing under it ("## Findings" above its "### …" parts) or a lone
# lead-in line would otherwise stand as a chunk, and the split below keeps
# it attached to the first paragraph that follows.
merged: list[str] = []
for section in _split_sections(body or ""):
if merged and len(merged[-1]) + 2 + len(section) <= budget:
if merged and (_is_thin(merged[-1])
or len(merged[-1]) + 2 + len(section) <= budget):
merged[-1] = f"{merged[-1]}\n\n{section}"
else:
merged.append(section)
if len(merged) > 1 and _is_thin(merged[-1]):
tail = merged.pop()
merged[-1] = f"{merged[-1]}\n\n{tail}"
chunks: list[str] = []
for section in merged:
@@ -512,13 +578,15 @@ def chunk_document(title: str | None, body: str | None) -> list[str]:
chunks.append(embedding_text(title, section))
continue
pieces = _split_paragraphs(section, budget)
first_line = section.split("\n", 1)[0]
heading = first_line if first_line.lstrip().startswith("#") else ""
heading = ""
for i, piece in enumerate(pieces):
# Repeat the section heading on continuation pieces so each vector
# still knows what topic it is part of.
# Repeat the heading a continuation piece sits under so each vector
# still knows what topic it is part of. The LAST heading so far,
# not the section's first: a thin heading merged forward (#4784)
# means one section can hold several.
if i > 0 and heading and not piece.startswith(heading):
piece = f"{heading}\n{piece}"
heading = _last_heading(piece, heading)
chunks.append(embedding_text(title, piece))
return chunks
tests/test_chunking.py
+84
View File
@@ -8,8 +8,11 @@ untouched), and long records lose NOTHING — every line of the body lands in
some chunk, each chunk inside the window budget, each carrying the title as
its topical anchor.
"""
import re
from scribe.services.embeddings import (
_CHUNK_CHAR_BUDGET,
_MIN_CHUNK_CONTENT,
chunk_document,
embedding_text,
)
@@ -136,6 +139,87 @@ def test_a_monster_single_paragraph_is_hard_split_not_dropped():
assert total_words == 2000
# --- no chunk is a heading alone (#4784) -------------------------------------
#
# Found by judging live auto-inject menus: "## Work log — 2026-08-21",
# "## Findings worth carrying forward" and "Two dev→main PRs this session."
# were each a whole chunk, and took ranks 1–3 on vague queries — text about
# nothing embeds close to everything. Each case below is one of those shapes.
_HEADING = re.compile(r"^#{1,6}\s")
def _said(chunk: str, title: str) -> str:
"""A chunk's text once its title prefix and heading lines are set aside."""
body = chunk[len(title) + 1:] if chunk.startswith(title + "\n") else chunk
return "\n".join(ln for ln in body.splitlines() if not _HEADING.match(ln)).strip()
def _assert_nothing_thin_and_nothing_lost(title: str, body: str) -> list[str]:
chunks = chunk_document(title, body)
assert len(chunks) > 1, "the case must be long enough to split"
for chunk in chunks:
assert len(_said(chunk, title)) >= _MIN_CHUNK_CONTENT, (
f"a chunk says almost nothing: {chunk[:120]!r}")
assert len(chunk) <= _CHUNK_CHAR_BUDGET + len(title) + 1 + 80
joined = "\n".join(chunks)
for line in (ln.strip() for ln in body.splitlines()):
# A line longer than a chunk is cut across two, so only its ends can
# be looked for whole.
for part in ([line] if len(line) < 400 else [line[:60], line[-60:]]):
assert part in joined, f"content dropped: {part[:60]!r}"
return chunks
def test_a_heading_over_one_long_paragraph_is_not_a_chunk_of_its_own():
"""A work log whose entry is one long paragraph: the split used to flush
the heading as its own piece before cutting the paragraph."""
entry = " ".join(["The deploy was verified against the live instance."] * 40)
_assert_nothing_thin_and_nothing_lost(
"Step 6 — webhook drift-flagging",
"The step body.\n\n## Work log — 2026-08-16\n\n" + entry,
)
def test_a_heading_with_nothing_under_it_joins_what_follows():
"""'## Findings' directly above its '### …' parts is a section with no
body, and stood alone whenever the section before it was full."""
body = (
# Five paragraphs fill the section before it, so the heading cannot
# merge backwards and has to be carried forward.
"## Context\n\n" + _long_section("context", paragraphs=5)
+ "\n\n## Findings worth carrying forward\n\n"
+ "### One\n\n" + _long_section("one", paragraphs=5)
+ "\n\n### Two\n\n" + _long_section("two", paragraphs=5)
)
_assert_nothing_thin_and_nothing_lost("A milestone dev-log", body)
def test_a_lone_lead_in_line_joins_the_section_after_it():
_assert_nothing_thin_and_nothing_lost(
"2026-06-10 — UI batch",
"Two dev→main PRs this session.\n\n## PR 89\n\n"
+ _long_section("pr89", paragraphs=8),
)
def test_a_thin_last_section_joins_the_one_before_it():
_assert_nothing_thin_and_nothing_lost(
"T",
"## Body\n\n" + _long_section("body", paragraphs=8) + "\n\n## Next\n\nTBD.",
)
def test_a_continuation_piece_repeats_the_heading_it_sits_under():
"""A thin heading merged forward puts two headings in one section; the
pieces of the second must carry the second, not the first."""
body = "## Parent\n\n## Child\n\n" + _long_section("child", paragraphs=40)
chunks = chunk_document("T", body)
assert len(chunks) > 2
for chunk in chunks[1:]:
assert "## Child" in chunk
# --- the read path: best chunk wins (#280 step 4) ----------------------------