FabledScribe/src/scribe/services/dedup.py

"""Write-time near-duplicate detection — the update-over-create gate.

Goal: stop a second near-identical row from being created when an existing one
should be UPDATED instead. Duplicates bloat the store and, worse, get surfaced
by semantic search (RAG) later as competing/stale copies that then have to be
reconciled. This is the enforcement half of the instruction-level "prefer
updating over creating" reflex.

OPT-IN by design: the interactive create paths (MCP create tools + REST create
routes) run this gate; internal/programmatic creates do NOT (e.g. a recurring
task spawning its next instance, or a bulk import — those legitimately repeat a
title and must not be blocked). Callers that want the gate call find_duplicate_*
themselves and act on a hit; nothing here mutates.

Two signals, both scoped to the same owner + project + kind:
  1. Normalized-title exact match — cheap, always checked.
  2. Semantic similarity (cosine ≥ _SEMANTIC_THRESHOLD) — only when the incoming
     body is substantial. Short/title-only embeddings sit in a tight neighborhood
     and false-positive (the pre-pivot lesson: "Lore: Shell 0" vs
     "Lore: Reinitialization 0" matched at 0.91 with no body), so we gate it on
     a minimum body length.
"""
from __future__ import annotations

import logging
from dataclasses import dataclass

from sqlalchemy import func, select

from scribe.models import async_session
from scribe.models.note import Note
from scribe.models.rulebook import Rule
from scribe.services import embeddings as embeddings_svc

logger = logging.getLogger(__name__)

# Run the semantic check only when the incoming body has at least this many
# characters — below it, embeddings are dominated by the title and false-positive.
_MIN_BODY_FOR_SEMANTIC = 200
# Cosine threshold for "this is the same thing, reworded." Deliberately high to
# keep false positives rare (a hard block with a force-override is unforgiving of
# noise). Matches the 0.90 the pre-pivot dedup settled on.
_SEMANTIC_THRESHOLD = 0.90


@dataclass
class DuplicateMatch:
    """An existing record judged a near-duplicate of an incoming create."""
    id: int
    title: str
    similarity: float  # 1.0 for an exact normalized-title match
    reason: str        # "title" | "semantic"


def duplicate_response(dup: "DuplicateMatch", kind: str) -> dict:
    """Standard 'blocked — update instead' payload returned by a create tool
    when the gate finds a near-duplicate. `kind` is 'note' or 'task' (drives the
    update_<kind> hint)."""
    return {
        "duplicate": True,
        "existing_id": dup.id,
        "existing_title": dup.title,
        "similarity": dup.similarity,
        "match": dup.reason,
        "message": (
            f'A {dup.reason}-similar {kind} already exists (id {dup.id}: '
            f'"{dup.title}"). Prefer UPDATING it (update_{kind}) over creating a '
            f"near-duplicate. If this really is a distinct {kind}, retry with "
            f"force=true."
        ),
    }


async def find_duplicate_note(
    user_id: int,
    title: str,
    body: str = "",
    project_id: int | None = None,
    is_task: bool | None = None,
    note_type: str = "note",
) -> DuplicateMatch | None:
    """Best near-duplicate of (title, body) within the same owner + project +
    kind, or None. Title match first (cheap, exact), then semantic when the body
    is long enough to be meaningful. Never raises — embedder failure degrades to
    title-only (callers should still be able to create)."""
    norm = " ".join((title or "").split()).lower()

    # --- Signal 1: normalized-title exact match (same scope) ---
    # Fail-open: a dedup-check failure (DB down, etc.) must never block a
    # legitimate create — degrade to "no duplicate found" and let it through.
    if norm:
        try:
            async with async_session() as session:
                stmt = select(Note).where(
                    Note.user_id == user_id,
                    Note.deleted_at.is_(None),
                    Note.note_type == note_type,
                    func.lower(func.trim(Note.title)) == norm,
                )
                if project_id is not None:
                    stmt = stmt.where(Note.project_id == project_id)
                else:
                    stmt = stmt.where(Note.project_id.is_(None))
                if is_task is True:
                    stmt = stmt.where(Note.status.isnot(None))
                elif is_task is False:
                    stmt = stmt.where(Note.status.is_(None))
                existing = (await session.execute(stmt.limit(1))).scalars().first()
                if existing is not None:
                    return DuplicateMatch(existing.id, existing.title, 1.0, "title")
        except Exception:
            logger.debug("dedup title check skipped — query failed", exc_info=True)
            return None

    # --- Signal 2: semantic similarity (only with a substantial body) ---
    if body and len(body.strip()) >= _MIN_BODY_FOR_SEMANTIC:
        query = f"{title}\n{body}".strip()
        # Scope the semantic check the same way as the title check: a record in
        # project P compares only to P; a project-less (orphan) record compares
        # only to other orphans (orphan_only), NOT across every project — without
        # this, semantic_search_notes applies no project filter when project_id
        # is None and would match an orphan note against any project's notes.
        hits = await embeddings_svc.semantic_search_notes(
            user_id, query, project_id=project_id, is_task=is_task,
            orphan_only=(project_id is None),
            limit=3, threshold=_SEMANTIC_THRESHOLD,
            # Owner-only, deliberately: this gate BLOCKS a create and tells the
            # caller to update the match instead. Matching someone else's record
            # would refuse their write and point them at something they may not
            # be able to edit.
            scope="own",
        )
        for score, note in hits:
            # semantic_search_notes doesn't filter note_type — enforce it here so
            # a note doesn't shadow a task of the same wording, etc.
            if note.note_type == note_type:
                return DuplicateMatch(note.id, note.title, round(score, 3), "semantic")

    return None


async def find_duplicate_rule(
    title: str,
    topic_id: int | None = None,
    project_id: int | None = None,
) -> DuplicateMatch | None:
    """Title-based near-duplicate of a rule, scoped to the same topic (a rulebook
    rule) or the same project (a project rule). Rules aren't a semantic-retrieval
    surface, so a normalized-title match is the right (and only) signal. Fail-open
    like find_duplicate_note."""
    norm = " ".join((title or "").split()).lower()
    if not norm or (topic_id is None and project_id is None):
        return None
    try:
        async with async_session() as session:
            stmt = select(Rule).where(
                Rule.deleted_at.is_(None),
                func.lower(func.trim(Rule.title)) == norm,
            )
            if topic_id is not None:
                stmt = stmt.where(Rule.topic_id == topic_id)
            else:
                stmt = stmt.where(Rule.project_id == project_id)
            existing = (await session.execute(stmt.limit(1))).scalars().first()
            if existing is not None:
                return DuplicateMatch(existing.id, existing.title, 1.0, "title")
    except Exception:
        logger.debug("dedup rule title check skipped — query failed", exc_info=True)
    return None