feat(snippets): near-duplicate finder — surface the sets worth merging

#231's premise was unifying reusable things already scattered as one-offs.
The create gate PREVENTS a new duplicate and merge_snippets CURES one you
point it at, but nothing FOUND the duplicates already in the record —
someone had to notice them by hand, which is the exact failure the Drafter
exists to remove.

One indexed self-join over note_embeddings, not an N² Python scan:
pgvector's cosine distance is the same operator semantic search uses, so a
similarity floor is a distance ceiling and the work stays in Postgres.
`left.note_id < right.note_id` yields each unordered pair once and drops
the self-pair that would otherwise dominate the ranking.

Pairs are collapsed into merge SETS by connected components. Transitive on
purpose: A~B plus B~C puts all three together even when A and C don't
directly clear the bar, which is what merge actually does (it folds every
source into one survivor). The cost is that a chain of mild resemblances
can rope in a member that isn't really alike — so the UI presents a set as
a proposal, shows the members, and never merges without a confirm.

Two scope decisions worth naming:

- OWN snippets only. merge_snippets requires one owner across the set, so
  surfacing someone else's would propose a merge that cannot be performed.
  The report is bounded by what the operator can act on, not what they can
  see.

- Threshold defaults to 0.82, LOOSER than the write gate's 0.90, and is a
  setting rather than a constant (rule #25). The gate blocks a create and
  has to be unforgiving of noise; this only suggests a merge under review,
  so it must reach further or it would never surface the pairs the gate
  already let through — which are precisely the ones that accumulated.

Fixes a real bug in the merge flow while wiring the UI: selectedList
filtered the selection against the CURRENT PAGE, and doMerge derives its
source ids from that list. A corpus-wide suggested group with off-page
members would have rendered incomplete and silently merged only the
visible subset. A group under review is now the authority for that list.

Refs #2088

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UaYUaouG9jjhATyuxCKrQs

tests/test_mcp_tool_snippets.py

@@ -229,4 +229,5 @@ def test_register_attaches_all_tools():
     assert set(names) == {
         "list_snippets", "create_snippet", "get_snippet", "update_snippet",
         "delete_snippet", "merge_snippets", "verify_snippet",
+        "find_duplicate_snippets",
     }

tests/test_routes_snippets.py

@@ -20,7 +20,7 @@ def test_snippet_handlers_callable():
     for name in (
         "list_snippets_route", "create_snippet_route", "get_snippet_route",
         "update_snippet_route", "delete_snippet_route", "merge_snippet_route",
-        "verify_snippet_route",
+        "verify_snippet_route", "duplicate_snippets_route",
     ):
         assert callable(getattr(routes, name))
 
@@ -68,6 +68,11 @@ def test_agent_and_web_surfaces_stay_at_parity():
     assert "verification" in inspect.signature(tools.list_snippets).parameters
     assert "verification" in inspect.getsource(routes.list_snippets_route)
 
+    # The near-duplicate report (#2088) reaches both. The web side can only hand
+    # a group to the merge flow if it can get the groups in the first place.
+    assert callable(getattr(tools, "find_duplicate_snippets", None))
+    assert callable(getattr(routes, "duplicate_snippets_route", None))
+
 
 def test_project_scoping_reaches_every_caller():
     """A snippet search has to be narrowable to one project from both surfaces."""

tests/test_snippet_duplicates.py

@@ -0,0 +1,99 @@
+"""Tests for the near-duplicate finder (#2088).
+
+The SQL half needs a database and lives in the integration lane; what's covered
+here is the grouping rule — which is where the interesting decisions are — plus
+the fail-open contract and the threshold resolution.
+"""
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from scribe.services import dedup as dedup_svc
+from scribe.services.dedup import (
+    DUPLICATE_DEFAULT_THRESHOLD,
+    get_duplicate_threshold,
+    group_pairs,
+)
+
+
+# --- grouping -------------------------------------------------------------
+
+
+def test_a_simple_pair_becomes_one_set():
+    assert group_pairs([(1, 2, 0.9)]) == [[1, 2]]
+
+
+def test_grouping_is_transitive():
+    """A~B and B~C puts all three in one set even though A and C never cleared
+    the bar together. That mirrors merge, which folds every source into one
+    survivor — three overlapping pairs would just be the same chore, unsorted."""
+    assert group_pairs([(1, 2, 0.9), (2, 3, 0.9)]) == [[1, 2, 3]]
+
+
+def test_disjoint_clusters_stay_separate():
+    groups = group_pairs([(1, 2, 0.9), (3, 4, 0.9)])
+    assert groups == [[1, 2], [3, 4]]
+
+
+def test_largest_cluster_comes_first():
+    """The most tangled thing is the most worth fixing."""
+    groups = group_pairs([(5, 6, 0.9), (1, 2, 0.9), (2, 3, 0.9), (3, 4, 0.9)])
+    assert groups[0] == [1, 2, 3, 4]
+
+
+def test_output_is_stable_across_input_order():
+    """A report that reshuffles between runs is one nobody can work through."""
+    a = group_pairs([(1, 2, 0.9), (2, 3, 0.9), (7, 8, 0.9)])
+    b = group_pairs([(7, 8, 0.9), (2, 3, 0.9), (1, 2, 0.9)])
+    assert a == b
+
+
+def test_no_pairs_means_no_groups():
+    assert group_pairs([]) == []
+
+
+def test_a_node_never_forms_a_group_with_itself():
+    """The SQL uses `note_id <` so this shouldn't arrive, but a singleton set
+    would render as a "duplicate" of nothing and offer an impossible merge."""
+    assert group_pairs([(1, 1, 1.0)]) == []
+
+
+# --- threshold ------------------------------------------------------------
+
+
+async def test_threshold_falls_back_to_the_default_when_unset():
+    with patch("scribe.services.settings.get_setting",
+               AsyncMock(return_value=str(DUPLICATE_DEFAULT_THRESHOLD))):
+        assert await get_duplicate_threshold(1) == DUPLICATE_DEFAULT_THRESHOLD
+
+
+async def test_a_garbage_setting_falls_back_rather_than_raising():
+    with patch("scribe.services.settings.get_setting",
+               AsyncMock(return_value="not-a-number")):
+        assert await get_duplicate_threshold(1) == DUPLICATE_DEFAULT_THRESHOLD
+
+
+@pytest.mark.parametrize("stored, expected", [("2.5", 1.0), ("-3", 0.0)])
+async def test_threshold_is_clamped_to_the_valid_range(stored, expected):
+    with patch("scribe.services.settings.get_setting", AsyncMock(return_value=stored)):
+        assert await get_duplicate_threshold(1) == expected
+
+
+def test_the_report_threshold_is_looser_than_the_write_gate():
+    """The gate BLOCKS a create and must be unforgiving of noise; this only
+    suggests a merge the operator reviews, so it has to reach further or it
+    would never surface the pairs the gate already let through."""
+    assert DUPLICATE_DEFAULT_THRESHOLD < dedup_svc._SEMANTIC_THRESHOLD
+
+
+# --- fail-open ------------------------------------------------------------
+
+
+async def test_a_failed_scan_returns_an_empty_report_not_an_error():
+    """A suggestion feature must not be able to break the page it decorates."""
+    with (
+        patch.object(dedup_svc, "get_duplicate_threshold", AsyncMock(return_value=0.8)),
+        patch.object(dedup_svc, "async_session", side_effect=RuntimeError("boom")),
+    ):
+        out = await dedup_svc.find_duplicate_snippets(1)
+    assert out == {"groups": [], "pairs": [], "threshold": 0.8}