fix(dedup): a rule or preference create surfaces what it overlaps by meaning (#4134)

find_duplicate_rule was title-only, on the stated premise that rules are not
a semantic-retrieval surface - false since rules were embedded. A preference
restating a rule under another title passed untouched, and since both kinds
share one ranking, the weaker label could arrive alone.

find_overlapping_rules queries semantic_search_rules with the rule_document
shape, both kinds, in the scope the new record ranks in (global: every rule
the caller owns; project: global + that project). All three MCP create doors
call it before creating and return overlaps + overlap_note on the reply.

It advises rather than blocks, on measurement: across 16 sampled records the
nearest DISTINCT neighbour reached 0.853, while a true rewording scored 0.850.
No threshold separates the bands, so the floor (0.80) sits below the
restatement and the author judges. The stale docstring is corrected.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01821k5B3Ysecp9fNYs92Kuy

src/scribe/mcp/tools/rulebooks.py

@@ -489,13 +489,20 @@ async def create_rule(
         order_index: Display order within the topic (default 0).
         force: Bypass the near-duplicate gate. By default, a title-identical rule
             already in this topic BLOCKS creation and returns its id so you update
-            it instead. Set true only for a genuinely distinct rule.
+            it instead. Set true only for a genuinely distinct rule. A rule or
+            preference that answers the same MOMENT under another title does
+            not block: the create goes through and the reply carries
+            `overlaps` and `overlap_note` — read the top one and decide.
     """
     uid = current_user_id()
     if not force:
         dup = await dedup_svc.find_duplicate_rule(title, topic_id=topic_id)
         if dup is not None:
             return dedup_svc.duplicate_response(dup, "rule")
+    # Before the create, so the new rule cannot find itself (#4134).
+    overlaps = await dedup_svc.find_overlapping_rules(
+        uid, title, statement, when_to_apply,
+    )
     rule = await rulebooks_svc.create_rule(
         topic_id=topic_id, user_id=uid,
         title=title, statement=statement, when_to_apply=when_to_apply,
@@ -503,7 +510,9 @@ async def create_rule(
         why=why, how_to_apply=how_to_apply, order_index=order_index,
         verify_with=verify_with, expires_when=expires_when,
     )
-    return await rulebooks_svc.rule_detail(uid, rule, system_ids)
+    data = await rulebooks_svc.rule_detail(uid, rule, system_ids)
+    data.update(dedup_svc.overlap_response(overlaps, "rule"))
+    return data
 
 
 async def create_project_rule(
@@ -582,7 +591,9 @@ async def create_project_rule(
         order_index: Display order within the project's rule list (default 0).
         force: Bypass the near-duplicate gate. By default, a title-identical rule
             already on this project BLOCKS creation and returns its id so you
-            update it instead. Set true only for a genuinely distinct rule.
+            update it instead. Set true only for a genuinely distinct rule. An
+            overlap by meaning never blocks; it arrives as `overlaps` and
+            `overlap_note` on the reply — see create_rule.
     """
     uid = current_user_id()
     derived_title = title.strip() or statement.strip().split(".")[0][:50]
@@ -590,6 +601,9 @@ async def create_project_rule(
         dup = await dedup_svc.find_duplicate_rule(derived_title, project_id=project_id)
         if dup is not None:
             return dedup_svc.duplicate_response(dup, "rule")
+    overlaps = await dedup_svc.find_overlapping_rules(
+        uid, derived_title, statement, when_to_apply, project_id=project_id,
+    )
     rule = await rulebooks_svc.create_project_rule(
         project_id=project_id, user_id=uid,
         title=derived_title, statement=statement, when_to_apply=when_to_apply,
@@ -597,7 +611,9 @@ async def create_project_rule(
         why=why, how_to_apply=how_to_apply, order_index=order_index,
         verify_with=verify_with, expires_when=expires_when,
     )
-    return await rulebooks_svc.rule_detail(uid, rule, system_ids)
+    data = await rulebooks_svc.rule_detail(uid, rule, system_ids)
+    data.update(dedup_svc.overlap_response(overlaps, "rule"))
+    return data
 
 
 async def update_rule(
@@ -784,7 +800,10 @@ async def create_preference(
             a preference could only be filed after the fact (#4249).
         force: Bypass the near-duplicate gate. For a genuinely distinct
             preference, not for one that is "mostly" different — a mostly
-            different preference is an update.
+            different preference is an update. A RULE that already answers
+            this moment comes back as `overlaps` / `overlap_note` on the reply
+            rather than blocking; if it says the same thing, the preference is
+            the weaker copy of it and should go.
     """
     uid = current_user_id()
     if not when_to_apply.strip():
@@ -803,13 +822,18 @@ async def create_preference(
         dup = await dedup_svc.find_duplicate_rule(title, topic_id=topic_id)
         if dup is not None:
             return dedup_svc.duplicate_response(dup, "rule")
+    overlaps = await dedup_svc.find_overlapping_rules(
+        uid, title, statement, when_to_apply,
+    )
     rule = await rulebooks_svc.create_rule(
         topic_id=topic_id, user_id=uid,
         title=title, statement=statement, when_to_apply=when_to_apply,
         kind="preference", arose_from_id=arose_from_id,
         why=why, how_to_apply=how_to_apply, order_index=order_index,
     )
-    return await rulebooks_svc.rule_detail(uid, rule, system_ids)
+    data = await rulebooks_svc.rule_detail(uid, rule, system_ids)
+    data.update(dedup_svc.overlap_response(overlaps, "preference"))
+    return data
 
 
 async def update_preference(

src/scribe/services/dedup.py

@@ -772,10 +772,17 @@ async def find_duplicate_rule(
     topic_id: int | None = None,
     project_id: int | None = None,
 ) -> DuplicateMatch | None:
-    """Title-based near-duplicate of a rule, scoped to the same topic (a rulebook
-    rule) or the same project (a project rule). Rules aren't a semantic-retrieval
-    surface, so a normalized-title match is the right (and only) signal. Fail-open
-    like find_duplicate_note."""
+    """Title-identical rule in the same topic (a rulebook rule) or the same
+    project (a project rule) — the one signal certain enough to BLOCK on.
+    Fail-open like find_duplicate_note.
+
+    This is not the only duplicate signal for rules. It said so until #4134 —
+    "rules aren't a semantic-retrieval surface" — which stopped being true
+    when rules were embedded (rule_document, semantic_search_rules), and a
+    title is the field LEAST likely to collide when someone is deliberately
+    writing a second record about the same moment. find_overlapping_rules is
+    the meaning half; it surfaces rather than blocks, for the reason recorded
+    above _RULE_OVERLAP_FLOOR."""
     norm = " ".join((title or "").split()).lower()
     if not norm or (topic_id is None and project_id is None):
         return None
@@ -797,6 +804,128 @@ async def find_duplicate_rule(
     return None
 
 
+
+# --- rule / preference overlap (#4134) ----------------------------------------
+# Rules and preferences are one table and one ranking: every hook arm searches
+# them with no `kind` filter. So a preference that restates a rule is not a
+# harmless near-copy — when only the preference places, a session receives
+# binding guidance labelled "preference" and treats it as optional. The title
+# gate above cannot see it: a second record about the same moment is exactly
+# the case where someone chose a different title.
+#
+# WHY THIS SURFACES INSTEAD OF BLOCKING. Measured 2026-09-21 on bge-small-en-
+# v1.5, querying with the gate's own rule_document shape across 16 sampled
+# records (10 preferences, 6 rules) and reading the nearest OTHER record (#4134
+# has the ids):
+#
+#   a preference rewording an existing rule                          0.850
+#   nearest distinct neighbours, 16 samples                0.672 – 0.853
+#     "when to delegate" beside "never delegate writing"             0.853
+#     "work lands on the working branch" beside "nothing reaches
+#      the release branch unasked"                                   0.850
+#     "let each action finish" beside "poll CI yourself"             0.847
+#
+# The two bands overlap: records that are deliberately distinct about one
+# moment — a rule for what must happen beside a rule for what must not — sit
+# exactly where a true restatement does. No threshold separates them, so a
+# block would refuse legitimate records and teach force=true on every create.
+# What the embedding CAN say reliably is "these answer the same moment", and
+# whether they say the same THING is a reading, which is the author's. So the
+# create goes through and carries the records it overlaps, with what to do if
+# they are the same.
+#
+# 0.80 is the floor because the one measured true duplicate sat at 0.850 and a
+# floor at the edge of it would miss the next, slightly looser rewording;
+# 6 of the 16 distinct neighbours also clear it, which is the cost, paid in
+# one line on the create's reply rather than in a refused write. Retune with
+# the embedder, not the corpus.
+_RULE_OVERLAP_FLOOR = 0.80
+_RULE_OVERLAP_LIMIT = 3
+
+
+@dataclass
+class RuleOverlap:
+    """An existing rule or preference that answers the same moment."""
+    id: int
+    title: str
+    kind: str          # "rule" | "preference"
+    project_id: int | None
+    similarity: float
+
+
+async def find_overlapping_rules(
+    user_id: int,
+    title: str,
+    statement: str,
+    when_to_apply: str,
+    *,
+    project_id: int | None = None,
+) -> list[RuleOverlap]:
+    """Existing rules AND preferences whose trigger reads as this one's.
+
+    Queried with rule_document — the exact shape the corpus is embedded as —
+    so the score compares like with like. Both kinds, because the harm is
+    across them (#4134).
+
+    Scope follows the new record's home. A project rule is compared with
+    global rules plus that project's own, the set it will rank against. A
+    global record (project_id None) applies everywhere, so it is compared with
+    every rule the caller owns: a global rule restating one project's rule is
+    the same overlap, arriving in that project.
+
+    Run BEFORE the create, so the new record cannot match itself. Never
+    raises: an overlap is advice, and a create must not depend on it.
+    """
+    doc_title, doc_body = embeddings_svc.rule_document(title, statement, when_to_apply)
+    query = "\n\n".join(p for p in (doc_title, doc_body) if p)
+    # The note gate's floor, for the same reason: a short document sits in a
+    # tight neighbourhood and resembles everything.
+    if len(query.strip()) < _MIN_BODY_FOR_SEMANTIC:
+        return []
+    try:
+        hits = await embeddings_svc.semantic_search_rules(
+            user_id, query, limit=_RULE_OVERLAP_LIMIT,
+            threshold=_RULE_OVERLAP_FLOOR,
+            project_id=project_id, everywhere=project_id is None,
+        )
+    except Exception:
+        logger.debug("rule overlap check skipped", exc_info=True)
+        return []
+    return [
+        RuleOverlap(
+            id=rule.id, title=rule.title, kind=rule.kind or "rule",
+            project_id=rule.project_id, similarity=round(score, 3),
+        )
+        for score, rule in hits
+    ]
+
+
+def overlap_response(overlaps: list[RuleOverlap], new_kind: str) -> dict:
+    """The keys a rule/preference create adds to its reply when the record it
+    just wrote answers the same moment as an existing one. Empty when none."""
+    if not overlaps:
+        return {}
+    top = overlaps[0]
+    named = "; ".join(
+        f'{o.kind} {o.id} "{o.title}" ({o.similarity})' for o in overlaps
+    )
+    return {
+        "overlaps": [
+            {"id": o.id, "title": o.title, "kind": o.kind,
+             "project_id": o.project_id, "similarity": o.similarity}
+            for o in overlaps
+        ],
+        "overlap_note": (
+            f"Created — and it answers the same moment as: {named}. Read "
+            f"{top.kind} {top.id} now. If it says the same thing, fold what is "
+            f"new into it (update_{top.kind}) and delete this {new_kind}: two "
+            f"records ranked together split one instruction, and the weaker "
+            f"one can arrive alone. If they say different things about one "
+            f"moment, keep both — that is common and fine."
+        ),
+    }
+
+
 # --- the plan gate (milestone 415) -------------------------------------------
 # A session asked "what work is open?" that cannot see an existing plan makes a
 # second one: a new milestone beside the one that already covers the work, or