fix(dedup): a rule or preference create surfaces what it overlaps by meaning (#4134)

find_duplicate_rule was title-only, on the stated premise that rules are not
a semantic-retrieval surface - false since rules were embedded. A preference
restating a rule under another title passed untouched, and since both kinds
share one ranking, the weaker label could arrive alone.

find_overlapping_rules queries semantic_search_rules with the rule_document
shape, both kinds, in the scope the new record ranks in (global: every rule
the caller owns; project: global + that project). All three MCP create doors
call it before creating and return overlaps + overlap_note on the reply.

It advises rather than blocks, on measurement: across 16 sampled records the
nearest DISTINCT neighbour reached 0.853, while a true rewording scored 0.850.
No threshold separates the bands, so the floor (0.80) sits below the
restatement and the author judges. The stale docstring is corrected.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01821k5B3Ysecp9fNYs92Kuy

src/scribe/services/dedup.py

@@ -772,10 +772,17 @@ async def find_duplicate_rule(
     topic_id: int | None = None,
     project_id: int | None = None,
 ) -> DuplicateMatch | None:
-    """Title-based near-duplicate of a rule, scoped to the same topic (a rulebook
-    rule) or the same project (a project rule). Rules aren't a semantic-retrieval
-    surface, so a normalized-title match is the right (and only) signal. Fail-open
-    like find_duplicate_note."""
+    """Title-identical rule in the same topic (a rulebook rule) or the same
+    project (a project rule) — the one signal certain enough to BLOCK on.
+    Fail-open like find_duplicate_note.
+
+    This is not the only duplicate signal for rules. It said so until #4134 —
+    "rules aren't a semantic-retrieval surface" — which stopped being true
+    when rules were embedded (rule_document, semantic_search_rules), and a
+    title is the field LEAST likely to collide when someone is deliberately
+    writing a second record about the same moment. find_overlapping_rules is
+    the meaning half; it surfaces rather than blocks, for the reason recorded
+    above _RULE_OVERLAP_FLOOR."""
     norm = " ".join((title or "").split()).lower()
     if not norm or (topic_id is None and project_id is None):
         return None
@@ -797,6 +804,128 @@ async def find_duplicate_rule(
     return None
 
 
+
+# --- rule / preference overlap (#4134) ----------------------------------------
+# Rules and preferences are one table and one ranking: every hook arm searches
+# them with no `kind` filter. So a preference that restates a rule is not a
+# harmless near-copy — when only the preference places, a session receives
+# binding guidance labelled "preference" and treats it as optional. The title
+# gate above cannot see it: a second record about the same moment is exactly
+# the case where someone chose a different title.
+#
+# WHY THIS SURFACES INSTEAD OF BLOCKING. Measured 2026-09-21 on bge-small-en-
+# v1.5, querying with the gate's own rule_document shape across 16 sampled
+# records (10 preferences, 6 rules) and reading the nearest OTHER record (#4134
+# has the ids):
+#
+#   a preference rewording an existing rule                          0.850
+#   nearest distinct neighbours, 16 samples                0.672 – 0.853
+#     "when to delegate" beside "never delegate writing"             0.853
+#     "work lands on the working branch" beside "nothing reaches
+#      the release branch unasked"                                   0.850
+#     "let each action finish" beside "poll CI yourself"             0.847
+#
+# The two bands overlap: records that are deliberately distinct about one
+# moment — a rule for what must happen beside a rule for what must not — sit
+# exactly where a true restatement does. No threshold separates them, so a
+# block would refuse legitimate records and teach force=true on every create.
+# What the embedding CAN say reliably is "these answer the same moment", and
+# whether they say the same THING is a reading, which is the author's. So the
+# create goes through and carries the records it overlaps, with what to do if
+# they are the same.
+#
+# 0.80 is the floor because the one measured true duplicate sat at 0.850 and a
+# floor at the edge of it would miss the next, slightly looser rewording;
+# 6 of the 16 distinct neighbours also clear it, which is the cost, paid in
+# one line on the create's reply rather than in a refused write. Retune with
+# the embedder, not the corpus.
+_RULE_OVERLAP_FLOOR = 0.80
+_RULE_OVERLAP_LIMIT = 3
+
+
+@dataclass
+class RuleOverlap:
+    """An existing rule or preference that answers the same moment."""
+    id: int
+    title: str
+    kind: str          # "rule" | "preference"
+    project_id: int | None
+    similarity: float
+
+
+async def find_overlapping_rules(
+    user_id: int,
+    title: str,
+    statement: str,
+    when_to_apply: str,
+    *,
+    project_id: int | None = None,
+) -> list[RuleOverlap]:
+    """Existing rules AND preferences whose trigger reads as this one's.
+
+    Queried with rule_document — the exact shape the corpus is embedded as —
+    so the score compares like with like. Both kinds, because the harm is
+    across them (#4134).
+
+    Scope follows the new record's home. A project rule is compared with
+    global rules plus that project's own, the set it will rank against. A
+    global record (project_id None) applies everywhere, so it is compared with
+    every rule the caller owns: a global rule restating one project's rule is
+    the same overlap, arriving in that project.
+
+    Run BEFORE the create, so the new record cannot match itself. Never
+    raises: an overlap is advice, and a create must not depend on it.
+    """
+    doc_title, doc_body = embeddings_svc.rule_document(title, statement, when_to_apply)
+    query = "\n\n".join(p for p in (doc_title, doc_body) if p)
+    # The note gate's floor, for the same reason: a short document sits in a
+    # tight neighbourhood and resembles everything.
+    if len(query.strip()) < _MIN_BODY_FOR_SEMANTIC:
+        return []
+    try:
+        hits = await embeddings_svc.semantic_search_rules(
+            user_id, query, limit=_RULE_OVERLAP_LIMIT,
+            threshold=_RULE_OVERLAP_FLOOR,
+            project_id=project_id, everywhere=project_id is None,
+        )
+    except Exception:
+        logger.debug("rule overlap check skipped", exc_info=True)
+        return []
+    return [
+        RuleOverlap(
+            id=rule.id, title=rule.title, kind=rule.kind or "rule",
+            project_id=rule.project_id, similarity=round(score, 3),
+        )
+        for score, rule in hits
+    ]
+
+
+def overlap_response(overlaps: list[RuleOverlap], new_kind: str) -> dict:
+    """The keys a rule/preference create adds to its reply when the record it
+    just wrote answers the same moment as an existing one. Empty when none."""
+    if not overlaps:
+        return {}
+    top = overlaps[0]
+    named = "; ".join(
+        f'{o.kind} {o.id} "{o.title}" ({o.similarity})' for o in overlaps
+    )
+    return {
+        "overlaps": [
+            {"id": o.id, "title": o.title, "kind": o.kind,
+             "project_id": o.project_id, "similarity": o.similarity}
+            for o in overlaps
+        ],
+        "overlap_note": (
+            f"Created — and it answers the same moment as: {named}. Read "
+            f"{top.kind} {top.id} now. If it says the same thing, fold what is "
+            f"new into it (update_{top.kind}) and delete this {new_kind}: two "
+            f"records ranked together split one instruction, and the weaker "
+            f"one can arrive alone. If they say different things about one "
+            f"moment, keep both — that is common and fine."
+        ),
+    }
+
+
 # --- the plan gate (milestone 415) -------------------------------------------
 # A session asked "what work is open?" that cannot see an existing plan makes a
 # second one: a new milestone beside the one that already covers the work, or