From cdb49b2f9f880bc63724533b5020fcdc63d5ea08 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 21 Oct 2026 20:17:09 -0400 Subject: [PATCH] docs(6205): dedup comments state the reason, not the instance that taught it Issue, milestone and rule citations and dated measurement headers are removed; calibration figures stay. Corrected the lesson-gate comment, which said lessons are not in _REPORT_KINDS and so have no duplicate-report backstop. They are, and the comment now says so. Co-Authored-By: Claude Opus 5.5 --- src/scribe/services/dedup.py | 150 +++++++++++++++++------------------ 1 file changed, 73 insertions(+), 77 deletions(-) diff --git a/src/scribe/services/dedup.py b/src/scribe/services/dedup.py index 208f0865..e8baebe3 100644 --- a/src/scribe/services/dedup.py +++ b/src/scribe/services/dedup.py @@ -16,9 +16,9 @@ Two signals, both scoped to the same owner + project + kind: 1. Normalized-title exact match — cheap, always checked. 2. Semantic similarity (cosine ≥ _SEMANTIC_THRESHOLD) — only when the incoming body is substantial. Short/title-only embeddings sit in a tight neighborhood - and false-positive (the pre-pivot lesson: "Lore: Shell 0" vs - "Lore: Reinitialization 0" matched at 0.91 with no body), so we gate it on - a minimum body length. + and false-positive (two title-only records like "Lore: Shell 0" and + "Lore: Reinitialization 0" match at 0.91), so we gate it on a minimum + body length. """ from __future__ import annotations @@ -48,10 +48,10 @@ logger = logging.getLogger(__name__) _MIN_BODY_FOR_SEMANTIC = 200 # Cosine threshold for "this is the same thing, reworded." Deliberately high to # keep false positives rare (a hard block with a force-override is unforgiving of -# noise). Matches the 0.90 the pre-pivot dedup settled on. +# noise). _SEMANTIC_THRESHOLD = 0.90 -# SNIPPETS ARE MEASURED DIFFERENTLY, and #2518 is why. A snippet's embedded +# SNIPPETS ARE MEASURED DIFFERENTLY. A snippet's embedded # document is mostly PROSE ABOUT the code — name, when-to-reach-for-it, # signature, the comments explaining the choice — with the artefact itself a # minority of the text. Two measurements on the same corpus: @@ -61,8 +61,8 @@ _SEMANTIC_THRESHOLD = 0.90 # under a different name <0.90 a literal copy, ALLOWED THROUGH # # The second is the one that settles it. Identical code at an identical -# repo·path·symbol sailed past the gate because the description differed, while -# two deliberately-parallel variants were refused because theirs did not. The +# repo·path·symbol passes a semantic gate because the description differs, +# while two deliberately-parallel variants are refused because theirs do not. The # arm is not mis-tuned; it is reading the wrong field, and no threshold fixes # that — lowering it blocks more siblings, raising it allows more copies. # @@ -74,37 +74,36 @@ _SNIPPET_SEMANTIC_THRESHOLD = 0.96 # A LESSON is measured the same way, for the first of those reasons and not the # second. Its document is `{what} — {trigger}` over a body that opens by -# restating the trigger (milestone 385 step 3) — the same prose-about-the-thing +# restating the trigger — the same prose-about-the-thing # shape, so two GENUINELY DIFFERENT lessons about one area ("CI cannot see this -# class of failure") land in the same sibling band that refused .btn-danger -# against .btn-danger-outline at 0.92. +# class of failure") land in the same sibling band as .btn-danger against +# .btn-danger-outline at 0.92. # # Its own constant rather than reusing the snippet's, because the two are -# separate facts that happen to coincide: this number is INHERITED from #2518's -# measurement of a structurally analogous corpus, not measured on lessons — -# there are none yet to measure. When there are, this moves without dragging -# snippets with it. +# separate facts that happen to coincide: this number is INHERITED from the +# snippet measurement of a structurally analogous corpus, not measured on +# lessons, and can move without dragging snippets with it. # # The trade-off differs and is worth naming. A snippet has structural signals # (code, repo·path·symbol) to catch the literal copy a high bar lets through; a -# lesson has none, and is not in `_REPORT_KINDS` either, so the duplicate report -# is not a backstop for it yet. What remains is the exact-title check, which -# still fires. That is the right way round for a gate that BLOCKS: a false +# lesson has none. What catches a lesson copy is the exact-title check at +# write time and the duplicate report (lessons are in `_REPORT_KINDS`) +# afterwards. That is the right way round for a gate that BLOCKS: a false # positive refuses a real lesson outright, while a false negative leaves two # records that can still be merged by hand. _LESSON_SEMANTIC_THRESHOLD = 0.96 -# NOTES AND TASKS BLOCK ONLY A COPY, and surface the rest (#4306). Measured -# 2026-09-22 with find_duplicate_records(note, 0.85): of the 74 note pairs at or -# above the old 0.90 bar, almost all were DISTINCT siblings — consecutive -# dev-logs (0.90–0.94), sub-notes of one design (0.90–0.94), research parts -# (0.90–0.97), lore entries (0.95–0.98). The one clear copy sat at 0.997. A -# block in that band refused the next dev-log and taught force=true, the same -# finding #4134 made for rules. So only the copy band blocks; below it, a -# near match is shown on the create reply for the session to judge. +# NOTES AND TASKS BLOCK ONLY A COPY, and surface the rest. Measured with +# find_duplicate_records(note, 0.85), almost every note pair at or above 0.90 +# is a DISTINCT sibling — consecutive dev-logs (0.90–0.94), sub-notes of one +# design (0.90–0.94), research parts (0.90–0.97), lore entries (0.95–0.98) — +# while a clear copy sits near 0.997. A block in that band refuses the next +# dev-log and teaches force=true, as it does for rules. So only the copy band +# blocks; below it, a near match is shown on the create reply for the session +# to judge. _NOTE_COPY_THRESHOLD = 0.98 # Where a near match starts being worth reading. The measured pair counts -# climb steeply under 0.87 (36 pairs at 0.87 against 200 capped at 0.85), and +# climb steeply under 0.87 (roughly five times as many at 0.85), and # the reply lists at most _NOTE_OVERLAP_LIMIT, so this is a cost floor — what # decides whether a match matters is the session reading it. _NOTE_OVERLAP_FLOOR = 0.87 @@ -113,7 +112,7 @@ _NOTE_OVERLAP_LIMIT = 3 # gate was not part of the measurement, so it keeps the general bar. _COPY_BAND_TYPES = {"note"} -# The constants above are the DEFAULTS; each is a setting (rule 25, #4385), +# The constants above are the DEFAULTS; each is a setting, # because how alike two distinct records get depends on how uniform a corpus # is, which nobody can know from here. GATE_THRESHOLD_KEYS = { @@ -138,7 +137,7 @@ _GATE_MIN_BLOCK = 0.80 # may go lower — but not so low that the three slots fill with noise. _GATE_MIN_OVERLAP = 0.70 -# The gate queries per CHUNK of the candidate (#280) — this caps how many +# The gate queries per CHUNK of the candidate — this caps how many # searches one save may cost. Eight chunks ≈ five thousand words of candidate; # a duplicate hiding past that is the duplicate report's job to find, not a # reason to stall the write path. @@ -210,7 +209,7 @@ async def _find_snippet_by_structure( """Exact-identity duplicate of an incoming snippet, or None. Two signals, both index-served off `notes.data` and both CERTAIN rather than - probabilistic — which is what the semantic arm could not be (#2518): + probabilistic — which is what the semantic arm cannot be: location the same named thing in the same file. Requires BOTH path and symbol: a path alone is a directory of many artefacts, and @@ -341,15 +340,15 @@ async def find_duplicate_note( `code` and `locations` are the snippet's structured fields. They are ignored for every other kind, and passing them is what lets the gate compare - ARTEFACTS rather than descriptions of artefacts (#2518). + ARTEFACTS rather than descriptions of artefacts. `data` is the candidate's structured mirror. For a snippet or lesson it - carries the trigger, which the TITLE no longer does (milestone 427): the + carries the trigger, which the stored TITLE does not: the title check compares names, and the semantic check rebuilds the embedded document from `data`. - `overlaps`, when given, is filled with the near matches below the copy band - (#4306) — from the SAME searches, so asking costs nothing extra. Only the + `overlaps`, when given, is filled with the near matches below the copy + band — from the SAME searches, so asking costs nothing extra. Only the kinds in _COPY_BAND_TYPES collect them. The caller creates the record and returns them with `note_overlap_response`. """ @@ -394,16 +393,15 @@ async def find_duplicate_note( # --- Signal 3: semantic similarity (only with a substantial body) --- if body and len(body.strip()) >= _MIN_BODY_FOR_SEMANTIC: - # Query with the SAME chunker the corpus was embedded with (#280). This - # was the copy that mattered most and was easiest to miss: these are + # Query with the SAME chunker the corpus was embedded with. These are # QUERY documents, compared against embedded ones — shaped differently - # from the corpus, the gate degrades silently. Chunking also makes the - # gate see what the whole-document query diluted: a long candidate that - # duplicates an existing record IN ONE SECTION now matches on that + # from the corpus, the gate degrades silently. Chunking also lets the + # gate see what a whole-document query would dilute: a long candidate + # that duplicates an existing record IN ONE SECTION matches on that # section. Capped so one pathological paste can't turn a save into # dozens of searches — a duplicate past the cap is the duplicate # report's job, not the gate's. - # The EMBEDDED title (milestone 427): a snippet or lesson is stored + # The EMBEDDED title: a snippet or lesson is stored # under its name and embedded under `name — trigger`, so the query # document is built the way the corpus was, from `data`. doc_title = embeddings_svc.document_title(title, note_type, data, body) @@ -427,7 +425,7 @@ async def find_duplicate_note( # else's record would refuse their write and point them at # something they may not be able to edit. scope="own", - # NOT demoted by supersession (#278). A superseded record is + # NOT demoted by supersession. A superseded record is # still a duplicate of what you are about to write — the claim # is that it is no longer CURRENT, not that it is gone. Demoting # it here would let the same note be recorded a second time, and @@ -456,7 +454,7 @@ async def find_duplicate_note( def note_overlap_response(overlaps: list[NoteOverlap], kind: str) -> dict: """The keys a note or task create adds to its reply when an existing record - reads closely like the one just written (#4306). Empty when none. + reads closely like the one just written. Empty when none. The judgement is the session's: the embedding cannot tell a restatement from the next dev-log in a series, and a reader can in one look.""" @@ -500,10 +498,11 @@ def batch_overlap_response(per_record: dict[int, list[NoteOverlap]]) -> dict: } -# --- corpus-wide near-duplicate report (#2088) ------------------------------- +# --- corpus-wide near-duplicate report --------------------------------------- # The gate above PREVENTS a new duplicate; merge_snippets CURES one you point it -# at. Neither FINDS the duplicates already sitting in the record — someone had to -# notice them by hand, which is the exact failure the Drafter exists to remove. +# at. Neither FINDS the duplicates already sitting in the record — without this, +# someone has to notice them by hand, the exact failure the Drafter exists to +# remove. # # WHY OWN SNIPPETS ONLY. merge_snippets requires every record to share the # target's owner (cross-owner merge is out of scope), so a report that surfaced @@ -511,21 +510,20 @@ def batch_overlap_response(per_record: dict[int, list[NoteOverlap]]) -> dict: # scope here is set by what the operator can actually act on, not by what they # can see. # -# WHY THE FLOOR IS PER-KIND. Chunked embeddings (#280) changed what a pair -# score MEANS for multi-chunk records: a note-pair's similarity is its closest -# chunk pair, so any family of related long records — a dev-log run, a research -# fan-out — clears a floor that whole-document vectors used to dilute below it. -# Measured on the live corpus (2026-08-09): at 0.82 the note/task reports -# saturate the pair cap with related-but-distinct families, while 0.93+ returns -# the genuinely-alike records. Snippets are single-chunk (short by nature), so -# their similarity scale never shifted and they keep the old floor. +# WHY THE FLOOR IS PER-KIND. With chunked embeddings a note-pair's +# similarity is its closest chunk pair, so any family of related long records +# — a dev-log run, a research fan-out — clears a floor that whole-document +# vectors would dilute below it. Measured: at 0.82 the note/task reports +# saturate the pair cap with related-but-distinct families, while 0.93+ +# returns the genuinely-alike records. Snippets are single-chunk (short by +# nature), so their scale is the whole-document one and their floor is lower. # # Snippets sit BELOW the 0.90 write gate — the report catches what the gate # lets through. Notes/tasks sit ABOVE it, and that is not a contradiction: the # gate compares a new record against best-matching chunks too, but it blocks a # WRITE and must stay forgiving, while the report proposes a REVIEW and at # chunk grain 0.90 would still drown it in families. Each is a setting rather -# than a constant (rule #25) because the right value depends on how uniform a +# than a constant because the right value depends on how uniform a # corpus is, and nobody can guess that from here. DUPLICATE_THRESHOLD_KEYS = { @@ -622,13 +620,13 @@ def _drop_sibling_pairs( apart, and merging them would destroy a distinction someone made on purpose. Without this, a design system's button family reports as a single merge set: - eight recipes, every direct pair over the floor, top score 0.92 (#2518). + eight recipes, every direct pair over the floor, top score 0.92. They resemble each other because variants of one component are SUPPOSED to — same selector prefix, same token families, deliberately parallel prose. The similarity is read correctly; it just does not mean "duplicate". THE COST, stated plainly: a helper genuinely recorded twice under two names - — `debounce` and `useDebouncedRef` — is no longer reported. That is real + — `debounce` and `useDebouncedRef` — is not reported. That is real recall lost. It is the better trade because the report is a merge PROPOSAL: a missed pair costs a duplicate nobody was going to notice anyway, while a wrong set invites an operator to collapse a component family in one click. @@ -651,10 +649,10 @@ def _drop_sibling_pairs( # What a duplicate group should become, per kind. Snippets merge losslessly — # one helper, every call site folded into the survivor. Notes and tasks do NOT -# merge: folding two records destroys what each actually said, which is why -# consolidated_at was dropped rather than built (#2483). The report can detect -# the cluster and name the options; deciding which applies is the reader's job, -# because it turns on what the records SAY, not how alike they score (#2547). +# merge: folding two records destroys what each actually said. The report can +# detect the cluster and name the options; deciding which applies is the +# reader's job, because it turns on what the records SAY, not how alike they +# score. _KIND_SUGGESTION = { "snippet": ( "Same reusable thing recorded more than once → merge_snippets(target, " @@ -759,7 +757,7 @@ async def find_duplicate_records( left_note = aliased(Note, name="left_note") right_note = aliased(Note, name="right_note") distance = left.embedding.cosine_distance(right.embedding) - # Chunk grain (#280): a note-pair's similarity is its closest CHUNK pair — + # Chunk grain: a note-pair's similarity is its closest CHUNK pair — # two records duplicate each other where their most similar sections do, # which is the honest definition when one section of a long note restates # another record. GROUP BY collapses the chunk cross-product to one row @@ -929,11 +927,10 @@ async def find_duplicate_rule( project (a project rule) — the one signal certain enough to BLOCK on. Fail-open like find_duplicate_note. - This is not the only duplicate signal for rules. It said so until #4134 — - "rules aren't a semantic-retrieval surface" — which stopped being true - when rules were embedded (rule_document, semantic_search_rules), and a - title is the field LEAST likely to collide when someone is deliberately - writing a second record about the same moment. find_overlapping_rules is + This is not the only duplicate signal for rules: rules are embedded + (rule_document, semantic_search_rules), and a title is the field LEAST + likely to collide when someone is deliberately writing a second record + about the same moment. find_overlapping_rules is the meaning half; it surfaces rather than blocks, for the reason recorded above _RULE_OVERLAP_FLOOR.""" norm = " ".join((title or "").split()).lower() @@ -958,7 +955,7 @@ async def find_duplicate_rule( -# --- rule / preference overlap (#4134) ---------------------------------------- +# --- rule / preference overlap ----------------------------------------------- # Rules and preferences are one table and one ranking: every hook arm searches # them with no `kind` filter. So a preference that restates a rule is not a # harmless near-copy — when only the preference places, a session receives @@ -966,10 +963,9 @@ async def find_duplicate_rule( # gate above cannot see it: a second record about the same moment is exactly # the case where someone chose a different title. # -# WHY THIS SURFACES INSTEAD OF BLOCKING. Measured 2026-09-21 on bge-small-en- -# v1.5, querying with the gate's own rule_document shape across 16 sampled -# records (10 preferences, 6 rules) and reading the nearest OTHER record (#4134 -# has the ids): +# WHY THIS SURFACES INSTEAD OF BLOCKING. Measured on bge-small-en-v1.5, +# querying with the gate's own rule_document shape across 16 sampled records +# (10 preferences, 6 rules) and reading the nearest OTHER record: # # a preference rewording an existing rule 0.850 # nearest distinct neighbours, 16 samples 0.672 – 0.853 @@ -1018,7 +1014,7 @@ async def find_overlapping_rules( Queried with rule_document — the exact shape the corpus is embedded as — so the score compares like with like. Both kinds, because the harm is - across them (#4134). + across them. Scope follows the new record's home. A project rule is compared with global rules plus that project's own, the set it will rank against. A @@ -1079,7 +1075,7 @@ def overlap_response(overlaps: list[RuleOverlap], new_kind: str) -> dict: } -# --- the plan gate (milestone 415) ------------------------------------------- +# --- the plan gate ----------------------------------------------------------- # A session asked "what work is open?" that cannot see an existing plan makes a # second one: a new milestone beside the one that already covers the work, or # loose tasks beside it. Each copy then collects its own steps, and neither @@ -1095,14 +1091,14 @@ def overlap_response(overlaps: list[RuleOverlap], new_kind: str) -> dict: # project is exactly what adding steps to its existing plan needs, so a caller # who reaches this gate can act on whatever it returns. # -# Its own threshold, as a setting (rule 25), and lower than the note gate's +# Its own threshold, as a setting, and lower than the note gate's # 0.90 because plan documents are shaped differently: a milestone is embedded # as title, description and plan (embeddings.milestone_document), while the # candidate usually has no description and carries its steps instead, so even # a faithful rewording never scores like a copy. # -# Measured after the first deploy (2026-09-15, bge-small-en-v1.5, #4079): three -# plans reworded from existing active milestones scored 0.83, 0.85 and 0.87 +# Measured on bge-small-en-v1.5: three plans reworded from existing active +# milestones scored 0.83, 0.85 and 0.87 # against the milestone they restated; the nearest DIFFERENT plan for each, and # a distinct plan on a neighbouring topic, scored 0.72-0.77. At 0.90 the # semantic arm matched nothing, leaving only the title arm. 0.80 sits in the @@ -1133,7 +1129,7 @@ def plan_candidate_text( The steps belong in it: a plan passed with steps and no design is still recognisable by them, and what its steps say is most of what makes two plans the same plan. Each step is (title, body), joined by embedding_text - like every other record that becomes embedded text (#2486). + like every other record that becomes embedded text. """ parts = [(description or "").strip(), (body or "").strip()] parts += [embeddings_svc.embedding_text(t, b) for t, b in (steps or [])] @@ -1158,7 +1154,7 @@ async def find_matching_plan( if not project_id: return None - # Rule 78, before either arm: a match names a milestone, and a caller who + # Access first, before either arm: a match names a milestone, and a caller who # cannot read the project must not learn its plans by guessing titles. try: if not await can_read_project(user_id, project_id):