"""Write-time near-duplicate detection — the update-over-create gate. Goal: stop a second near-identical row from being created when an existing one should be UPDATED instead. Duplicates bloat the store and, worse, get surfaced by semantic search (RAG) later as competing/stale copies that then have to be reconciled. This is the enforcement half of the instruction-level "prefer updating over creating" reflex. OPT-IN by design: the interactive create paths (MCP create tools + REST create routes) run this gate; internal/programmatic creates do NOT (e.g. a recurring task spawning its next instance, or a bulk import — those legitimately repeat a title and must not be blocked). Callers that want the gate call find_duplicate_* themselves and act on a hit; nothing here mutates. Two signals, both scoped to the same owner + project + kind: 1. Normalized-title exact match — cheap, always checked. 2. Semantic similarity (cosine ≥ _SEMANTIC_THRESHOLD) — only when the incoming body is substantial. Short/title-only embeddings sit in a tight neighborhood and false-positive (the pre-pivot lesson: "Lore: Shell 0" vs "Lore: Reinitialization 0" matched at 0.91 with no body), so we gate it on a minimum body length. """ from __future__ import annotations import logging from dataclasses import dataclass from sqlalchemy import func, select from scribe.models import async_session from scribe.models.note import Note from scribe.models.rulebook import Rule from scribe.services import embeddings as embeddings_svc logger = logging.getLogger(__name__) # Run the semantic check only when the incoming body has at least this many # characters — below it, embeddings are dominated by the title and false-positive. _MIN_BODY_FOR_SEMANTIC = 200 # Cosine threshold for "this is the same thing, reworded." Deliberately high to # keep false positives rare (a hard block with a force-override is unforgiving of # noise). Matches the 0.90 the pre-pivot dedup settled on. _SEMANTIC_THRESHOLD = 0.90 @dataclass class DuplicateMatch: """An existing record judged a near-duplicate of an incoming create.""" id: int title: str similarity: float # 1.0 for an exact normalized-title match reason: str # "title" | "semantic" def duplicate_response(dup: "DuplicateMatch", kind: str) -> dict: """Standard 'blocked — update instead' payload returned by a create tool when the gate finds a near-duplicate. `kind` is 'note' or 'task' (drives the update_ hint).""" return { "duplicate": True, "existing_id": dup.id, "existing_title": dup.title, "similarity": dup.similarity, "match": dup.reason, "message": ( f'A {dup.reason}-similar {kind} already exists (id {dup.id}: ' f'"{dup.title}"). Prefer UPDATING it (update_{kind}) over creating a ' f"near-duplicate. If this really is a distinct {kind}, retry with " f"force=true." ), } async def find_duplicate_note( user_id: int, title: str, body: str = "", project_id: int | None = None, is_task: bool | None = None, note_type: str = "note", ) -> DuplicateMatch | None: """Best near-duplicate of (title, body) within the same owner + project + kind, or None. Title match first (cheap, exact), then semantic when the body is long enough to be meaningful. Never raises — embedder failure degrades to title-only (callers should still be able to create).""" norm = " ".join((title or "").split()).lower() # --- Signal 1: normalized-title exact match (same scope) --- # Fail-open: a dedup-check failure (DB down, etc.) must never block a # legitimate create — degrade to "no duplicate found" and let it through. if norm: try: async with async_session() as session: stmt = select(Note).where( Note.user_id == user_id, Note.deleted_at.is_(None), Note.note_type == note_type, func.lower(func.trim(Note.title)) == norm, ) if project_id is not None: stmt = stmt.where(Note.project_id == project_id) else: stmt = stmt.where(Note.project_id.is_(None)) if is_task is True: stmt = stmt.where(Note.status.isnot(None)) elif is_task is False: stmt = stmt.where(Note.status.is_(None)) existing = (await session.execute(stmt.limit(1))).scalars().first() if existing is not None: return DuplicateMatch(existing.id, existing.title, 1.0, "title") except Exception: logger.debug("dedup title check skipped — query failed", exc_info=True) return None # --- Signal 2: semantic similarity (only with a substantial body) --- if body and len(body.strip()) >= _MIN_BODY_FOR_SEMANTIC: query = f"{title}\n{body}".strip() # Scope the semantic check the same way as the title check: a record in # project P compares only to P; a project-less (orphan) record compares # only to other orphans (orphan_only), NOT across every project — without # this, semantic_search_notes applies no project filter when project_id # is None and would match an orphan note against any project's notes. hits = await embeddings_svc.semantic_search_notes( user_id, query, project_id=project_id, is_task=is_task, orphan_only=(project_id is None), limit=3, threshold=_SEMANTIC_THRESHOLD, # Owner-only, deliberately: this gate BLOCKS a create and tells the # caller to update the match instead. Matching someone else's record # would refuse their write and point them at something they may not # be able to edit. scope="own", ) for score, note in hits: # semantic_search_notes doesn't filter note_type — enforce it here so # a note doesn't shadow a task of the same wording, etc. if note.note_type == note_type: return DuplicateMatch(note.id, note.title, round(score, 3), "semantic") return None async def find_duplicate_rule( title: str, topic_id: int | None = None, project_id: int | None = None, ) -> DuplicateMatch | None: """Title-based near-duplicate of a rule, scoped to the same topic (a rulebook rule) or the same project (a project rule). Rules aren't a semantic-retrieval surface, so a normalized-title match is the right (and only) signal. Fail-open like find_duplicate_note.""" norm = " ".join((title or "").split()).lower() if not norm or (topic_id is None and project_id is None): return None try: async with async_session() as session: stmt = select(Rule).where( Rule.deleted_at.is_(None), func.lower(func.trim(Rule.title)) == norm, ) if topic_id is not None: stmt = stmt.where(Rule.topic_id == topic_id) else: stmt = stmt.where(Rule.project_id == project_id) existing = (await session.execute(stmt.limit(1))).scalars().first() if existing is not None: return DuplicateMatch(existing.id, existing.title, 1.0, "title") except Exception: logger.debug("dedup rule title check skipped — query failed", exc_info=True) return None