Supersession (steps 1–4) — corrections demote, state lives on Systems #101
@@ -258,7 +258,13 @@ async def find_duplicate_note(
|
|||||||
|
|
||||||
# --- Signal 3: semantic similarity (only with a substantial body) ---
|
# --- Signal 3: semantic similarity (only with a substantial body) ---
|
||||||
if body and len(body.strip()) >= _MIN_BODY_FOR_SEMANTIC:
|
if body and len(body.strip()) >= _MIN_BODY_FOR_SEMANTIC:
|
||||||
query = f"{title}\n{body}".strip()
|
# Built by the SAME function the corpus was embedded with. This one is
|
||||||
|
# the copy that mattered most and was easiest to miss: it is a QUERY
|
||||||
|
# document, compared against embedded ones. Shaped differently from the
|
||||||
|
# corpus it searches, the gate degrades silently — it still returns
|
||||||
|
# neighbours, just less apt ones, and no signal says the query and the
|
||||||
|
# index stopped agreeing (found by the guard in test_embedding_text).
|
||||||
|
query = embeddings_svc.embedding_text(title, body)
|
||||||
# Scope the semantic check the same way as the title check: a record in
|
# Scope the semantic check the same way as the title check: a record in
|
||||||
# project P compares only to P; a project-less (orphan) record compares
|
# project P compares only to P; a project-less (orphan) record compares
|
||||||
# only to other orphans (orphan_only), NOT across every project — without
|
# only to other orphans (orphan_only), NOT across every project — without
|
||||||
|
|||||||
@@ -86,6 +86,31 @@ def _cosine_similarity(a: list[float], b: list[float]) -> float:
|
|||||||
return dot / (mag_a * mag_b)
|
return dot / (mag_a * mag_b)
|
||||||
|
|
||||||
|
|
||||||
|
def embedding_text(title: str | None, body: str | None) -> str:
|
||||||
|
"""The document a record is embedded AS.
|
||||||
|
|
||||||
|
One definition, deliberately. This was written out three times — the write
|
||||||
|
path (`notes.embed_note`), the recurring-task spawn, and the startup
|
||||||
|
backfill — and identical copies of a formatting rule are three chances to
|
||||||
|
change one and not the others. The spawn path is the dangerous one: a
|
||||||
|
recurring task embedded to a different shape than everything else would be
|
||||||
|
ranked against a corpus it doesn't match, and nothing would report it.
|
||||||
|
|
||||||
|
It is also a PRECONDITION for changing the shape at all (#2486). Measured,
|
||||||
|
a dev-log's vector separates from five unrelated dev-logs by 0.023 while a
|
||||||
|
snippet's separates by 0.153 — the difference being that a snippet states
|
||||||
|
its purpose twice in a short document, so the purpose dominates. Testing an
|
||||||
|
alternative shape against three copies would mean testing a shape that is
|
||||||
|
not the one in production.
|
||||||
|
|
||||||
|
Whether `title\\n{body}` is the RIGHT shape is the open question. That it is
|
||||||
|
one shape is what makes the question answerable.
|
||||||
|
"""
|
||||||
|
title = title or ""
|
||||||
|
body = body or ""
|
||||||
|
return f"{title}\n{body}".strip() if body else title
|
||||||
|
|
||||||
|
|
||||||
async def upsert_note_embedding(note_id: int, user_id: int, text: str) -> None:
|
async def upsert_note_embedding(note_id: int, user_id: int, text: str) -> None:
|
||||||
"""Generate and persist an embedding for a note. Safe to fire-and-forget."""
|
"""Generate and persist an embedding for a note. Safe to fire-and-forget."""
|
||||||
if not text or not text.strip():
|
if not text or not text.strip():
|
||||||
@@ -248,7 +273,7 @@ async def backfill_note_embeddings() -> None:
|
|||||||
logger.info("Embedding backfill: generating embeddings for %d notes", len(notes_to_embed))
|
logger.info("Embedding backfill: generating embeddings for %d notes", len(notes_to_embed))
|
||||||
success = 0
|
success = 0
|
||||||
for note_id, user_id, title, body in notes_to_embed:
|
for note_id, user_id, title, body in notes_to_embed:
|
||||||
text = f"{title}\n{body}".strip() if body else (title or "")
|
text = embedding_text(title, body)
|
||||||
if not text:
|
if not text:
|
||||||
continue
|
continue
|
||||||
await upsert_note_embedding(note_id, user_id, text)
|
await upsert_note_embedding(note_id, user_id, text)
|
||||||
|
|||||||
@@ -30,13 +30,13 @@ def embed_note(note) -> None:
|
|||||||
index refresh. No running loop (unit tests, scripts) is an ordinary case,
|
index refresh. No running loop (unit tests, scripts) is an ordinary case,
|
||||||
not an error.
|
not an error.
|
||||||
"""
|
"""
|
||||||
text = f"{note.title}\n{note.body}".strip() if note.body else (note.title or "")
|
|
||||||
if not text:
|
|
||||||
return
|
|
||||||
try:
|
try:
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|
||||||
from scribe.services.embeddings import upsert_note_embedding
|
from scribe.services.embeddings import embedding_text, upsert_note_embedding
|
||||||
|
text = embedding_text(note.title, note.body)
|
||||||
|
if not text:
|
||||||
|
return
|
||||||
asyncio.create_task(upsert_note_embedding(note.id, note.user_id, text))
|
asyncio.create_task(upsert_note_embedding(note.id, note.user_id, text))
|
||||||
except RuntimeError:
|
except RuntimeError:
|
||||||
pass # no running loop — a sync caller, not a failure
|
pass # no running loop — a sync caller, not a failure
|
||||||
|
|||||||
@@ -103,7 +103,7 @@ async def spawn_recurring_tasks() -> int:
|
|||||||
|
|
||||||
Returns the number of tasks spawned.
|
Returns the number of tasks spawned.
|
||||||
"""
|
"""
|
||||||
from scribe.services.embeddings import upsert_note_embedding
|
from scribe.services.embeddings import embedding_text, upsert_note_embedding
|
||||||
from scribe.services.notes import create_note
|
from scribe.services.notes import create_note
|
||||||
|
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
@@ -139,7 +139,7 @@ async def spawn_recurring_tasks() -> int:
|
|||||||
milestone_id=task.milestone_id,
|
milestone_id=task.milestone_id,
|
||||||
recurrence_rule=task.recurrence_rule,
|
recurrence_rule=task.recurrence_rule,
|
||||||
)
|
)
|
||||||
text = f"{child.title}\n{child.body}".strip() if child.body else (child.title or "")
|
text = embedding_text(child.title, child.body)
|
||||||
if text:
|
if text:
|
||||||
asyncio.create_task(upsert_note_embedding(child.id, task.user_id, text))
|
asyncio.create_task(upsert_note_embedding(child.id, task.user_id, text))
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
"""The document shape a record is embedded as, and the guard that keeps it one.
|
||||||
|
|
||||||
|
WHY THIS EXISTS
|
||||||
|
|
||||||
|
`f"{title}\\n{body}"` was written out three times — the write path, the
|
||||||
|
recurring-task spawn, and the startup backfill. Identical copies of a formatting
|
||||||
|
rule are three chances to change one and not the others, and the spawn path is
|
||||||
|
the dangerous one: a recurring task embedded to a different shape than the rest
|
||||||
|
of the corpus is ranked against documents it doesn't match, and nothing reports
|
||||||
|
it. A wrong vector returns results; it just returns the wrong ones.
|
||||||
|
|
||||||
|
It is also the precondition for #2486. A dev-log's vector separates from five
|
||||||
|
unrelated dev-logs by 0.023 where a snippet separates by 0.153, and the leading
|
||||||
|
explanation is shape — a snippet states its purpose twice in a short document.
|
||||||
|
Testing an alternative against three copies would mean testing a shape that
|
||||||
|
isn't the one in production.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ast
|
||||||
|
import pathlib
|
||||||
|
|
||||||
|
from scribe.services.embeddings import embedding_text
|
||||||
|
|
||||||
|
SERVICES = pathlib.Path(__file__).resolve().parents[1] / "src" / "scribe"
|
||||||
|
|
||||||
|
|
||||||
|
def test_title_and_body_are_joined_by_a_newline():
|
||||||
|
assert embedding_text("A title", "A body") == "A title\nA body"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_bodyless_record_embeds_as_its_title_alone():
|
||||||
|
"""Not "title\\n" — the trailing separator would be a token's worth of noise
|
||||||
|
on the shortest documents, which are the ones least able to spare it."""
|
||||||
|
assert embedding_text("Just a title", "") == "Just a title"
|
||||||
|
assert embedding_text("Just a title", None) == "Just a title"
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_empty_record_yields_an_empty_string():
|
||||||
|
"""Callers gate on falsiness to skip embedding entirely, so this must be
|
||||||
|
empty rather than a stray newline."""
|
||||||
|
assert embedding_text("", "") == ""
|
||||||
|
assert embedding_text(None, None) == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_surrounding_whitespace_is_stripped():
|
||||||
|
assert embedding_text(" A title ", " A body ") == "A title \n A body"
|
||||||
|
|
||||||
|
|
||||||
|
def test_nothing_else_builds_the_embedding_document_itself():
|
||||||
|
"""The guard. A fourth copy is how the first three happened.
|
||||||
|
|
||||||
|
Source inspection, because this is the shape no behavioural test catches:
|
||||||
|
an inlined copy produces the same string today and diverges silently the day
|
||||||
|
the shape changes. Matches the f-string pattern itself rather than a
|
||||||
|
variable name, so a copy that renames its locals is still caught.
|
||||||
|
"""
|
||||||
|
offenders = []
|
||||||
|
for path in SERVICES.rglob("*.py"):
|
||||||
|
source = path.read_text()
|
||||||
|
for node in ast.walk(ast.parse(source)):
|
||||||
|
if not isinstance(node, ast.JoinedStr):
|
||||||
|
continue
|
||||||
|
# An f-string whose literal parts are exactly a newline, with a
|
||||||
|
# substitution either side: `f"{x}\n{y}"`.
|
||||||
|
literals = [
|
||||||
|
v.value for v in node.values
|
||||||
|
if isinstance(v, ast.Constant) and isinstance(v.value, str)
|
||||||
|
]
|
||||||
|
subs = [v for v in node.values if isinstance(v, ast.FormattedValue)]
|
||||||
|
if literals == ["\n"] and len(subs) == 2:
|
||||||
|
offenders.append(f"{path.relative_to(SERVICES)}:{node.lineno}")
|
||||||
|
|
||||||
|
# embeddings.py holds the one definition.
|
||||||
|
offenders = [o for o in offenders if not o.startswith("services/embeddings.py")]
|
||||||
|
assert not offenders, (
|
||||||
|
f"these build the embedding document inline instead of calling "
|
||||||
|
f"embedding_text(): {offenders}. One definition — an inlined copy is "
|
||||||
|
f"ranked against a corpus it no longer matches the moment the shape "
|
||||||
|
f"changes, and nothing reports it (#2486)."
|
||||||
|
)
|
||||||
Reference in New Issue
Block a user