FabledGrimoire/tests/test_services_dedup.py

"""Unit tests for the write-time near-duplicate gate (services/dedup.py)."""
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from scribe.services.dedup import (
    PLAN_MATCH_DEFAULT_THRESHOLD,
    DuplicateMatch,
    duplicate_response,
    find_duplicate_note,
    find_duplicate_rule,
    find_matching_plan,
    get_plan_match_threshold,
    plan_candidate_text,
    plan_match_response,
)
from tests.helpers import fake_note, make_mock_session, session_returning


@pytest.mark.asyncio
async def test_title_exact_match_returns_title_duplicate():
    note = fake_note(id=10, title="Setup CI")
    with patch("scribe.services.dedup.async_session",
               return_value=session_returning(note)):
        # whitespace/case differences are normalized away
        dup = await find_duplicate_note(7, "  setup   ci ", project_id=2, is_task=True)
    assert dup is not None
    assert dup.id == 10
    assert dup.reason == "title"
    assert dup.similarity == 1.0


@pytest.mark.asyncio
async def test_short_body_skips_semantic_check():
    sem = AsyncMock()
    with patch("scribe.services.dedup.async_session",
               return_value=session_returning(None)), \
         patch("scribe.services.dedup.embeddings_svc.semantic_search_notes", sem):
        dup = await find_duplicate_note(7, "Unique", body="too short", project_id=2)
    assert dup is None
    sem.assert_not_called()  # body under _MIN_BODY_FOR_SEMANTIC


@pytest.mark.asyncio
async def test_semantic_match_when_body_substantial():
    # A note blocks only in the copy band.
    hit = fake_note(id=20, title="Existing", note_type="note")
    sem = AsyncMock(return_value=[(0.99, hit)])
    with patch("scribe.services.dedup.async_session",
               return_value=session_returning(None)), \
         patch("scribe.services.dedup.embeddings_svc.semantic_search_notes", sem):
        dup = await find_duplicate_note(
            7, "Title", body="x" * 250, project_id=2, is_task=False, note_type="note",
        )
    assert dup is not None
    assert dup.id == 20
    assert dup.reason == "semantic"
    assert dup.similarity == 0.99


@pytest.mark.asyncio
async def test_a_near_note_is_surfaced_not_blocked():
    """Sibling notes — the next dev-log, another part of one design — measure
    0.90–0.98, so a match there is shown for the session to judge
    instead of refusing the write."""
    from scribe.services.dedup import _NOTE_OVERLAP_FLOOR

    hit = fake_note(id=20, title="Dev-log day 3", note_type="note")
    sem = AsyncMock(return_value=[(0.93, hit)])
    overlaps: list = []
    with patch("scribe.services.dedup.async_session",
               return_value=session_returning(None)), \
         patch("scribe.services.dedup.embeddings_svc.semantic_search_notes", sem):
        dup = await find_duplicate_note(
            7, "Dev-log day 4", body="x" * 250, project_id=2, is_task=False,
            note_type="note", overlaps=overlaps,
        )
    assert dup is None
    assert [(o.id, o.similarity) for o in overlaps] == [(20, 0.93)]
    # Asked at the overlap floor, so the one search serves both answers.
    assert sem.await_args.kwargs["threshold"] == _NOTE_OVERLAP_FLOOR


@pytest.mark.asyncio
async def test_a_record_matching_in_two_chunks_is_one_overlap():
    para = ("A paragraph long enough for the chunker to keep as its own "
            "section of real content in this test body. ") * 4
    body = "\n\n".join(f"## Part {i}\n\n{para} (p{i})" for i in range(8))
    hit = fake_note(id=31, title="Design part one", note_type="note")
    sem = AsyncMock(return_value=[(0.9, hit)])
    overlaps: list = []
    with patch("scribe.services.dedup.async_session",
               return_value=session_returning(None)), \
         patch("scribe.services.dedup.embeddings_svc.semantic_search_notes", sem):
        await find_duplicate_note(
            7, "Design part two", body=body, project_id=2, note_type="note",
            overlaps=overlaps,
        )
    assert [o.id for o in overlaps] == [31]


@pytest.mark.asyncio
async def test_without_an_overlaps_list_the_gate_asks_at_the_copy_band():
    """A caller that only wants the block (create_process) does not pay for
    a wider search it will not read."""
    from scribe.services.dedup import _NOTE_COPY_THRESHOLD

    sem = AsyncMock(return_value=[])
    with patch("scribe.services.dedup.async_session",
               return_value=session_returning(None)), \
         patch("scribe.services.dedup.embeddings_svc.semantic_search_notes", sem):
        await find_duplicate_note(7, "T", body="x" * 250, note_type="note")
    assert sem.await_args.kwargs["threshold"] == _NOTE_COPY_THRESHOLD


def test_the_overlap_reply_leaves_the_judgement_to_the_reader():
    from scribe.services.dedup import NoteOverlap, note_overlap_response

    assert note_overlap_response([], "note") == {}
    out = note_overlap_response([NoteOverlap(9, "Dev-log day 3", 0.93)], "task")
    assert out["overlaps"] == [{"id": 9, "title": "Dev-log day 3", "similarity": 0.93}]
    assert "update_task" in out["overlap_note"]
    assert "keep both" in out["overlap_note"]


def test_a_batch_overlap_names_its_record():
    from scribe.services.dedup import NoteOverlap, batch_overlap_response

    assert batch_overlap_response({}) == {}
    out = batch_overlap_response({2: [NoteOverlap(9, "X", 0.9)]})
    assert out["overlaps"] == [{"record": 2, "id": 9, "title": "X", "similarity": 0.9}]


@pytest.mark.asyncio
async def test_gate_catches_a_duplicate_hiding_in_a_later_chunk():
    """A long candidate that duplicates an existing record in ONE SECTION is
    caught; a whole-document query would dilute exactly the section that
    matters. The gate
    queries once per chunk and any chunk's hit blocks."""
    para = ("This section restates an existing decision in enough words to be "
            "a real paragraph of content for the chunker to keep. ") * 4
    body = "\n\n".join(f"## Topic {i}\n\n{para} (t{i})" for i in range(8))

    from scribe.services.embeddings import chunk_document
    n_chunks = len(chunk_document("Title", body))
    assert n_chunks > 1, "test body must actually chunk"

    hit = fake_note(id=30, title="The existing decision", note_type="note")
    # Every chunk misses except the LAST one the gate will ask about.
    sem = AsyncMock(side_effect=[[] for _ in range(n_chunks - 1)] + [[(0.99, hit)]])
    with patch("scribe.services.dedup.async_session",
               return_value=session_returning(None)), \
         patch("scribe.services.dedup.embeddings_svc.semantic_search_notes", sem):
        dup = await find_duplicate_note(
            7, "Title", body=body, project_id=2, is_task=False, note_type="note",
        )
    assert dup is not None and dup.id == 30
    assert sem.await_count == n_chunks


@pytest.mark.asyncio
async def test_semantic_match_of_other_note_type_is_ignored():
    other = fake_note(id=21, title="X", note_type="process")
    sem = AsyncMock(return_value=[(0.97, other)])
    with patch("scribe.services.dedup.async_session",
               return_value=session_returning(None)), \
         patch("scribe.services.dedup.embeddings_svc.semantic_search_notes", sem):
        dup = await find_duplicate_note(7, "Title", body="x" * 250, note_type="note")
    assert dup is None  # type mismatch must not block


@pytest.mark.asyncio
async def test_rule_title_match_in_topic():
    rule = fake_note(id=47, title="Honor the multi-user sharing ACL")
    with patch("scribe.services.dedup.async_session",
               return_value=session_returning(rule)):
        dup = await find_duplicate_rule(
            "honor the   multi-user sharing acl", topic_id=7,
        )
    assert dup is not None
    assert dup.id == 47
    assert dup.reason == "title"


@pytest.mark.asyncio
async def test_rule_requires_a_scope():
    # No topic_id and no project_id → nothing to scope to → no match, no query.
    sess = AsyncMock()
    with patch("scribe.services.dedup.async_session", return_value=sess):
        dup = await find_duplicate_rule("anything")
    assert dup is None
    sess.__aenter__.assert_not_called()


def test_duplicate_response_shape():
    dm = DuplicateMatch(id=5, title="Foo", similarity=1.0, reason="title")
    r = duplicate_response(dm, "task")
    assert r["duplicate"] is True
    assert r["existing_id"] == 5
    assert r["match"] == "title"
    assert "force=true" in r["message"]
    assert "update_task" in r["message"]


# --- snippet structural identity ---------------------------------------------
#
# A snippet's rendered DOCUMENT is mostly prose about the code, and comparing
# it fails in both directions at once: two deliberately-parallel variants
# refused at 0.92, while a verbatim re-record under a different name scores
# below 0.90 and is created. These tests pin the structural signals used
# instead.


def _session_sequence(results):
    """A mocked async_session() whose successive execute() calls yield `results`.

    The single-result helper above can't express this: the structural check runs
    a location query and then a code query, and the whole point is that they
    answer differently.
    """
    s = make_mock_session()
    wrapped = []
    for note in results:
        r = MagicMock()
        r.scalars.return_value.first.return_value = note
        wrapped.append(r)
    s.execute = AsyncMock(side_effect=wrapped)
    return s


@pytest.mark.asyncio
async def test_same_location_is_a_duplicate_however_it_is_described():
    """The measured false NEGATIVE: identical code at an identical
    repo·path·symbol was created because the prose around it differed."""
    existing = fake_note(id=30, title=".btn-primary — a page's main action", note_type="snippet")
    sem = AsyncMock()
    with patch("scribe.services.dedup.async_session",
               return_value=_session_sequence([None, existing])), \
         patch("scribe.services.dedup.embeddings_svc.semantic_search_notes", sem):
        dup = await find_duplicate_note(
            7, "primaryButton — something else entirely", body="x" * 400,
            project_id=2, is_task=False, note_type="snippet",
            code=".btn-primary { color: red; }",
            locations=[{"repo": "Grimoire", "path": "a/b.css", "symbol": ".btn-primary"}],
        )
    assert dup is not None
    assert dup.reason == "location"
    assert dup.similarity == 1.0
    # Structural identity is certain, so it must not be diluted by asking the
    # embedder for a second opinion.
    sem.assert_not_called()


@pytest.mark.asyncio
async def test_identical_code_is_a_duplicate_at_a_different_location():
    existing = fake_note(id=31, title="group_pairs", note_type="snippet")
    with patch("scribe.services.dedup.async_session",
               return_value=_session_sequence([None, None, existing])), \
         patch("scribe.services.dedup.embeddings_svc.semantic_search_notes",
               AsyncMock(return_value=[])):
        dup = await find_duplicate_note(
            7, "unionFind", body="x" * 400, project_id=2, is_task=False,
            note_type="snippet", code="def f():\n    return 1",
            locations=[{"repo": "Grimoire", "path": "z.py", "symbol": "f"}],
        )
    assert dup is not None
    assert dup.reason == "code"


@pytest.mark.asyncio
async def test_a_location_without_a_symbol_is_not_an_identity():
    """A path alone is a DIRECTORY of artefacts. Matching on it would refuse
    every second snippet recorded from one file — which is exactly the corpus
    the button recipes form."""
    session = _session_sequence([None])
    sem = AsyncMock(return_value=[])
    with patch("scribe.services.dedup.async_session", return_value=session), \
         patch("scribe.services.dedup.embeddings_svc.semantic_search_notes", sem):
        dup = await find_duplicate_note(
            7, "Some recipe", body="x" * 400, project_id=2, is_task=False,
            note_type="snippet", code="",
            locations=[{"repo": "Grimoire", "path": "a/b.css", "symbol": ""}],
        )
    assert dup is None
    # Exactly one query — the title check. With no symbol and no code there is
    # nothing to match structurally, and the sequence above would raise
    # StopIteration if a second query were issued.
    assert session.execute.await_count == 1


@pytest.mark.asyncio
async def test_snippets_use_the_raised_semantic_threshold():
    """Variants of one component legitimately reach 0.92. The semantic arm has
    to sit above that band or it refuses the corpus it exists to protect."""
    from scribe.services.dedup import (
        _SEMANTIC_THRESHOLD,
        _SNIPPET_SEMANTIC_THRESHOLD,
    )
    sem = AsyncMock(return_value=[])
    with patch("scribe.services.dedup.async_session",
               return_value=_session_sequence([None, None, None])), \
         patch("scribe.services.dedup.embeddings_svc.semantic_search_notes", sem):
        await find_duplicate_note(
            7, "A recipe", body="x" * 400, project_id=2, is_task=False,
            note_type="snippet", code="x",
            locations=[{"repo": "R", "path": "p", "symbol": "s"}],
        )
    assert sem.await_args.kwargs["threshold"] == _SNIPPET_SEMANTIC_THRESHOLD
    assert _SNIPPET_SEMANTIC_THRESHOLD > 0.92, (
        "the observed sibling band tops out at 0.92 (.btn-danger vs "
        ".btn-danger-outline); a threshold at or below it blocks legitimate "
        "variants again"
    )
    assert _SNIPPET_SEMANTIC_THRESHOLD > _SEMANTIC_THRESHOLD


def test_sibling_variants_are_not_reported_as_merge_candidates():
    """The measured false POSITIVE: eight button recipes, every direct pair over
    the floor, proposed as ONE merge set."""
    from scribe.services.dedup import _drop_sibling_pairs

    records = {
        1: {"locations": [{"repo": "S", "path": "c.css", "symbol": ".btn-primary"}],
            "code_sha": "aaa"},
        2: {"locations": [{"repo": "S", "path": "c.css", "symbol": ".btn-secondary"}],
            "code_sha": "bbb"},
    }
    assert _drop_sibling_pairs([(1, 2, 0.87)], records) == []


def test_identical_code_still_reports_even_with_different_symbols():
    """The filter keys on "the author named these apart", but a shared code
    fingerprint overrides that — the same code under two names IS the
    copy-paste the report exists to surface."""
    from scribe.services.dedup import _drop_sibling_pairs

    records = {
        1: {"locations": [{"repo": "S", "path": "a.py", "symbol": "debounce"}],
            "code_sha": "same"},
        2: {"locations": [{"repo": "S", "path": "b.py", "symbol": "useDebounced"}],
            "code_sha": "same"},
    }
    assert _drop_sibling_pairs([(1, 2, 0.9)], records) == [(1, 2, 0.9)]


def test_unnamed_snippets_still_report():
    """A snippet with no recorded symbol made no identity claim, so the filter
    must not protect it — re-recording without a location is a common way to
    duplicate."""
    from scribe.services.dedup import _drop_sibling_pairs

    records = {1: {"code_sha": "aaa"}, 2: {"code_sha": "bbb"}}
    assert _drop_sibling_pairs([(1, 2, 0.9)], records) == [(1, 2, 0.9)]


def test_duplicate_response_names_what_matched_for_structural_hits():
    """A structural hit is certain, so the message must not hedge with
    "similar" — and it points at merge, which is what two records of one
    artefact actually need."""
    r = duplicate_response(
        DuplicateMatch(id=9, title=".btn-primary", similarity=1.0, reason="location"),
        "snippet",
    )
    assert "repo · path · symbol" in r["message"]
    assert "merge_snippets" in r["message"]
    assert "similar" not in r["message"]

    r = duplicate_response(
        DuplicateMatch(id=9, title="x", similarity=1.0, reason="code"), "snippet",
    )
    assert "identical code" in r["message"]


# --- the generalised report -------------------------------------------------


@pytest.mark.asyncio
async def test_report_refuses_an_unknown_kind():
    """A typo'd kind must fail loudly, not scan snippets by default — the
    caller asked a question about a kind that doesn't exist, and answering a
    different question instead is how wrong conclusions get confident."""
    from scribe.services.dedup import find_duplicate_records

    with pytest.raises(ValueError, match="kind must be one of"):
        await find_duplicate_records(7, kind="rule")


def test_kind_clauses_split_notes_from_tasks_on_status():
    """Tasks are notes with a status, not a note_type of their own. A report
    that mixed them would propose folding a to-do into a write-up."""
    from scribe.models.note import Note
    from scribe.services.dedup import _kind_clauses

    note_sql = " AND ".join(str(c) for c in _kind_clauses("note", Note))
    task_sql = " AND ".join(str(c) for c in _kind_clauses("task", Note))
    snip_sql = " AND ".join(str(c) for c in _kind_clauses("snippet", Note))

    assert "status IS NULL" in note_sql
    assert "status IS NOT NULL" in task_sql
    assert "note_type" in snip_sql and "status" not in snip_sql


def test_every_kind_has_a_suggestion_and_none_proposes_merging_notes():
    """The suggestion is the report's point: what to DO differs by what the
    records are, and 'merge' is only ever the answer for snippets — folding two
    notes destroys what each said, which is why consolidated_at was dropped
    rather than built."""
    from scribe.services.dedup import _KIND_SUGGESTION, _REPORT_KINDS

    for kind in _REPORT_KINDS:
        assert _KIND_SUGGESTION.get(kind), f"no suggestion for {kind}"
        if kind != "snippet":
            # The property, not a spot-check: a sixth kind added to the
            # report inherits the bar without anyone editing this test.
            assert "merge_snippets" not in _KIND_SUGGESTION[kind]
    assert "merge" in _KIND_SUGGESTION["snippet"]
    assert "NOT merge" in _KIND_SUGGESTION["note"]
    assert "supersedes" in _KIND_SUGGESTION["note"]


# ── the plan gate ─────────────────────────────────────────────────────────────


@pytest.mark.asyncio
async def test_plan_title_match_short_circuits_the_semantic_arm():
    ms = MagicMock(id=415, title="Plan gate")
    sem = AsyncMock()
    with patch("scribe.services.dedup.can_read_project", AsyncMock(return_value=True)), \
         patch("scribe.services.dedup.async_session", return_value=session_returning(ms)), \
         patch("scribe.services.dedup.embeddings_svc.semantic_search_milestones", sem):
        dup = await find_matching_plan(7, 2, " plan  GATE", "x" * 300)
    assert (dup.id, dup.reason, dup.similarity) == (415, "title", 1.0)
    sem.assert_not_awaited()


@pytest.mark.asyncio
async def test_plan_semantic_arm_asks_for_active_plans_in_the_project_at_the_setting():
    ms = MagicMock(id=9, title="Metadata")
    sem = AsyncMock(return_value=[(0.912345, ms)])
    with patch("scribe.services.dedup.can_read_project", AsyncMock(return_value=True)), \
         patch("scribe.services.dedup.async_session", return_value=session_returning(None)), \
         patch("scribe.services.dedup.embeddings_svc.semantic_search_milestones", sem), \
         patch("scribe.services.settings.get_setting", AsyncMock(return_value="0.8")):
        dup = await find_matching_plan(7, 2, "Book metadata", "x" * 300)
    assert (dup.id, dup.reason, dup.similarity) == (9, "semantic", 0.912)
    kw = sem.call_args.kwargs
    assert (kw["project_id"], kw["status"], kw["threshold"]) == (2, "active", 0.8)


@pytest.mark.asyncio
async def test_plan_gate_fails_open():
    boom = MagicMock(side_effect=RuntimeError("db down"))
    with patch("scribe.services.dedup.can_read_project", AsyncMock(return_value=True)), \
         patch("scribe.services.dedup.async_session", boom):
        assert await find_matching_plan(7, 2, "Anything", "x" * 300) is None
    with patch("scribe.services.dedup.can_read_project", AsyncMock(side_effect=RuntimeError)):
        assert await find_matching_plan(7, 2, "Anything", "x" * 300) is None
    assert await find_matching_plan(7, 0, "No project") is None


@pytest.mark.asyncio
async def test_plan_gate_says_nothing_about_a_project_the_caller_cannot_read():
    ms = MagicMock(id=415, title="Their plan")
    session = MagicMock(return_value=session_returning(ms))
    with patch("scribe.services.dedup.can_read_project", AsyncMock(return_value=False)), \
         patch("scribe.services.dedup.async_session", session):
        assert await find_matching_plan(8, 2, "Their plan") is None
    session.assert_not_called()


@pytest.mark.asyncio
async def test_a_bad_threshold_setting_falls_back_to_the_default():
    with patch("scribe.services.settings.get_setting", AsyncMock(return_value="lots")):
        assert await get_plan_match_threshold(7) == PLAN_MATCH_DEFAULT_THRESHOLD
    with patch("scribe.services.settings.get_setting", AsyncMock(return_value="7")):
        assert await get_plan_match_threshold(7) == 1.0


def test_plan_candidate_text_carries_the_steps():
    text = plan_candidate_text(description=None, body="  design ",
                               steps=[("Step one", None), ("Step two", "with a body"), (None, None)])
    assert text == "design\n\nStep one\n\nStep two\nwith a body"


def test_plan_match_response_points_at_adding_steps_not_a_second_plan():
    out = plan_match_response(DuplicateMatch(12, "Metadata", 0.93, "semantic"),
                              {"total": 5, "completed": 2, "description": "providers"})
    assert out["duplicate"] is True and out["existing_id"] == 12
    assert out["existing_milestone"] == {
        "id": 12, "title": "Metadata", "description": "providers", "total": 5, "completed": 2,
    }
    for phrase in ("create_records(milestone_id=12", "get_milestone(12)", "force=true",
                   "2 of 5 steps"):
        assert phrase in out["message"]