Merge pull request 'Per-kind duplicate-report floors — notes/tasks default 0.93' (#105) from dev into main

frontend/src/views/SettingsView.vue

@@ -27,10 +27,14 @@ const kbWritePathEnabled = ref(true);
 // code embeddings sit on a much higher similarity floor than prose, so 0.55 let
 // unrelated code through (#2223). Shares top-k, not the threshold.
 const kbWritePathThreshold = ref("0.68");
-// Near-duplicate report floor. Deliberately looser than the 0.90 write-time
-// gate: that one BLOCKS a create and must be unforgiving of noise, this one only
-// suggests a merge the operator reviews (services/dedup.py).
-const kbDuplicateThreshold = ref("0.82");
+// Near-duplicate report floors, one per record kind (services/dedup.py).
+// Snippets are single-chunk, so their floor sits below the 0.90 write-time
+// gate and catches what it lets through. Notes/tasks are scored at chunk
+// grain (#280) — related families clear 0.90 easily — so their floor sits
+// above the gate to keep the report pointed at genuinely-alike records.
+const kbDupThresholdSnippet = ref("0.82");
+const kbDupThresholdNote = ref("0.93");
+const kbDupThresholdTask = ref("0.93");
 const savingKbInject = ref(false);
 const kbInjectSaved = ref(false);
 
@@ -76,16 +80,20 @@ async function saveRetention() {
 async function saveKbInject() {
   const t = Math.min(1, Math.max(0, Number(kbInjectThreshold.value) || 0));
   const k = Math.min(10, Math.max(1, Math.floor(Number(kbInjectTopK.value) || 1)));
-  // `|| 0.82` not `|| 0`: an unparseable value here should fall back to the
-  // default, not to 0 — a 0 floor would report every snippet as a duplicate of
-  // every other one.
-  const dupT = Math.min(1, Math.max(0, Number(kbDuplicateThreshold.value) || 0.82));
-  // Same `|| default` reasoning as dupT: falling back to 0 would surface every
+  // `|| default` not `|| 0`: an unparseable value here should fall back to the
+  // per-kind default, not to 0 — a 0 floor would report every record as a
+  // duplicate of every other one.
+  const dupSnip = Math.min(1, Math.max(0, Number(kbDupThresholdSnippet.value) || 0.82));
+  const dupNote = Math.min(1, Math.max(0, Number(kbDupThresholdNote.value) || 0.93));
+  const dupTask = Math.min(1, Math.max(0, Number(kbDupThresholdTask.value) || 0.93));
+  // Same `|| default` reasoning: falling back to 0 would surface every
   // snippet in the corpus on every edit, which is the failure this knob fixes.
   const wpT = Math.min(1, Math.max(0, Number(kbWritePathThreshold.value) || 0.68));
   kbInjectThreshold.value = String(t);
   kbInjectTopK.value = String(k);
-  kbDuplicateThreshold.value = String(dupT);
+  kbDupThresholdSnippet.value = String(dupSnip);
+  kbDupThresholdNote.value = String(dupNote);
+  kbDupThresholdTask.value = String(dupTask);
   kbWritePathThreshold.value = String(wpT);
   savingKbInject.value = true;
   kbInjectSaved.value = false;
@@ -99,7 +107,9 @@ async function saveKbInject() {
       // measurements that split them.
       kb_writepath_enabled: kbWritePathEnabled.value ? 'true' : 'false',
       kb_writepath_threshold: String(wpT),
-      kb_duplicate_threshold: String(dupT),
+      kb_duplicate_threshold_snippet: String(dupSnip),
+      kb_duplicate_threshold_note: String(dupNote),
+      kb_duplicate_threshold_task: String(dupTask),
     });
     kbInjectSaved.value = true;
     setTimeout(() => (kbInjectSaved.value = false), 2000);
@@ -487,8 +497,14 @@ onMounted(async () => {
   if (allSettings.kb_writepath_threshold !== undefined) {
     kbWritePathThreshold.value = allSettings.kb_writepath_threshold;
   }
-  if (allSettings.kb_duplicate_threshold !== undefined) {
-    kbDuplicateThreshold.value = allSettings.kb_duplicate_threshold;
+  if (allSettings.kb_duplicate_threshold_snippet !== undefined) {
+    kbDupThresholdSnippet.value = allSettings.kb_duplicate_threshold_snippet;
+  }
+  if (allSettings.kb_duplicate_threshold_note !== undefined) {
+    kbDupThresholdNote.value = allSettings.kb_duplicate_threshold_note;
+  }
+  if (allSettings.kb_duplicate_threshold_task !== undefined) {
+    kbDupThresholdTask.value = allSettings.kb_duplicate_threshold_task;
   }
   if (allSettings.notify_task_reminders !== undefined) {
     notifyTaskReminders.value = allSettings.notify_task_reminders !== "false";
@@ -1267,10 +1283,10 @@ function formatUserDate(iso: string): string {
              is for (#274). -->
 
         <div class="field">
-          <label for="kb-duplicate-threshold">Near-duplicate report threshold</label>
+          <label for="kb-duplicate-threshold-snippet">Near-duplicate report threshold — snippets</label>
           <input
-            id="kb-duplicate-threshold"
-            v-model="kbDuplicateThreshold"
+            id="kb-duplicate-threshold-snippet"
+            v-model="kbDupThresholdSnippet"
             type="number"
             min="0"
             max="1"
@@ -1285,6 +1301,46 @@ function formatUserDate(iso: string): string {
             you review — it never acts on its own.
           </p>
         </div>
+
+        <div class="field">
+          <label for="kb-duplicate-threshold-note">Near-duplicate report threshold — notes</label>
+          <input
+            id="kb-duplicate-threshold-note"
+            v-model="kbDupThresholdNote"
+            type="number"
+            min="0"
+            max="1"
+            step="0.01"
+            class="input"
+            style="max-width: 8rem"
+          />
+          <p class="field-hint">
+            The floor for the Knowledge page's note report. Notes are compared
+            section by section, so related records — a run of dev-logs, notes on
+            one topic — score high without being duplicates. Stricter than the
+            snippet floor on purpose; lower it to browse related families rather
+            than hunt true duplicates.
+          </p>
+        </div>
+
+        <div class="field">
+          <label for="kb-duplicate-threshold-task">Near-duplicate report threshold — tasks</label>
+          <input
+            id="kb-duplicate-threshold-task"
+            v-model="kbDupThresholdTask"
+            type="number"
+            min="0"
+            max="1"
+            step="0.01"
+            class="input"
+            style="max-width: 8rem"
+          />
+          <p class="field-hint">
+            Same as the note floor, for the task report. Step tasks from different
+            milestones ("Verify on CI") legitimately resemble each other, so this
+            stays strict to keep the report pointed at work opened twice.
+          </p>
+        </div>
         <div class="actions">
           <button class="btn-primary" @click="saveKbInject" :disabled="savingKbInject">
             {{ savingKbInject ? 'Saving…' : 'Save' }}

src/scribe/services/dedup.py

@@ -318,30 +318,45 @@ async def find_duplicate_note(
 # scope here is set by what the operator can actually act on, not by what they
 # can see.
 #
-# WHY A LOWER THRESHOLD THAN THE GATE. The gate BLOCKS a write at 0.90 and has to
-# be unforgiving of noise. This report only makes a suggestion the operator
-# reviews, so it can afford to be looser and catch the pairs the gate lets
-# through — which are precisely the ones that accumulated. It is a setting rather
+# WHY THE FLOOR IS PER-KIND. Chunked embeddings (#280) changed what a pair
+# score MEANS for multi-chunk records: a note-pair's similarity is its closest
+# chunk pair, so any family of related long records — a dev-log run, a research
+# fan-out — clears a floor that whole-document vectors used to dilute below it.
+# Measured on the live corpus (2026-08-09): at 0.82 the note/task reports
+# saturate the pair cap with related-but-distinct families, while 0.93+ returns
+# the genuinely-alike records. Snippets are single-chunk (short by nature), so
+# their similarity scale never shifted and they keep the old floor.
+#
+# Snippets sit BELOW the 0.90 write gate — the report catches what the gate
+# lets through. Notes/tasks sit ABOVE it, and that is not a contradiction: the
+# gate compares a new record against best-matching chunks too, but it blocks a
+# WRITE and must stay forgiving, while the report proposes a REVIEW and at
+# chunk grain 0.90 would still drown it in families. Each is a setting rather
 # than a constant (rule #25) because the right value depends on how uniform a
 # corpus is, and nobody can guess that from here.
 
-DUPLICATE_THRESHOLD_KEY = "kb_duplicate_threshold"
-DUPLICATE_DEFAULT_THRESHOLD = 0.82
+DUPLICATE_THRESHOLD_KEYS = {
+    "snippet": "kb_duplicate_threshold_snippet",
+    "note": "kb_duplicate_threshold_note",
+    "task": "kb_duplicate_threshold_task",
+}
+DUPLICATE_DEFAULT_THRESHOLDS = {"snippet": 0.82, "note": 0.93, "task": 0.93}
 # Hard cap on returned pairs. A pathologically uniform corpus is O(n²) pairs, and
 # a report nobody can read is not a report.
 _MAX_DUPLICATE_PAIRS = 200
 
 
-async def get_duplicate_threshold(user_id: int) -> float:
-    """The user's near-duplicate similarity floor, clamped to [0, 1]."""
+async def get_duplicate_threshold(user_id: int, kind: str = "snippet") -> float:
+    """The user's near-duplicate similarity floor for `kind`, clamped to [0, 1]."""
     from scribe.services.settings import get_setting
 
+    default = DUPLICATE_DEFAULT_THRESHOLDS[kind]
     try:
         value = float(await get_setting(
-            user_id, DUPLICATE_THRESHOLD_KEY, str(DUPLICATE_DEFAULT_THRESHOLD)
+            user_id, DUPLICATE_THRESHOLD_KEYS[kind], str(default)
         ))
     except (TypeError, ValueError):
-        value = DUPLICATE_DEFAULT_THRESHOLD
+        value = default
     return min(1.0, max(0.0, value))
 
 
@@ -503,7 +518,10 @@ async def find_duplicate_records(
     """
     if kind not in _REPORT_KINDS:
         raise ValueError(f"kind must be one of {_REPORT_KINDS}, not {kind!r}")
-    floor = await get_duplicate_threshold(user_id) if threshold is None else threshold
+    floor = (
+        await get_duplicate_threshold(user_id, kind)
+        if threshold is None else threshold
+    )
     floor = min(1.0, max(0.0, floor))
     max_distance = min(2.0, max(0.0, 1.0 - floor))
 

tests/test_snippet_duplicates.py

@@ -10,7 +10,7 @@ import pytest
 
 from scribe.services import dedup as dedup_svc
 from scribe.services.dedup import (
-    DUPLICATE_DEFAULT_THRESHOLD,
+    DUPLICATE_DEFAULT_THRESHOLDS,
     get_duplicate_threshold,
     group_pairs,
 )
@@ -61,29 +61,52 @@ def test_a_node_never_forms_a_group_with_itself():
 # --- threshold ------------------------------------------------------------
 
 
-async def test_threshold_falls_back_to_the_default_when_unset():
+@pytest.mark.parametrize("kind", ["snippet", "note", "task"])
+async def test_threshold_falls_back_to_the_per_kind_default_when_unset(kind):
+    default = DUPLICATE_DEFAULT_THRESHOLDS[kind]
     with patch("scribe.services.settings.get_setting",
-               AsyncMock(return_value=str(DUPLICATE_DEFAULT_THRESHOLD))):
-        assert await get_duplicate_threshold(1) == DUPLICATE_DEFAULT_THRESHOLD
+               AsyncMock(return_value=str(default))):
+        assert await get_duplicate_threshold(1, kind) == default
+
+
+async def test_each_kind_reads_its_own_settings_key():
+    """Tuning the note floor must not move the snippet report — the whole
+    point of splitting the setting is that the kinds calibrate independently."""
+    get = AsyncMock(return_value="0.5")
+    with patch("scribe.services.settings.get_setting", get):
+        await get_duplicate_threshold(1, "note")
+    key, default = get.await_args.args[1], get.await_args.args[2]
+    assert key == "kb_duplicate_threshold_note"
+    assert default == str(DUPLICATE_DEFAULT_THRESHOLDS["note"])
 
 
 async def test_a_garbage_setting_falls_back_rather_than_raising():
     with patch("scribe.services.settings.get_setting",
                AsyncMock(return_value="not-a-number")):
-        assert await get_duplicate_threshold(1) == DUPLICATE_DEFAULT_THRESHOLD
+        assert (await get_duplicate_threshold(1, "snippet")
+                == DUPLICATE_DEFAULT_THRESHOLDS["snippet"])
 
 
 @pytest.mark.parametrize("stored, expected", [("2.5", 1.0), ("-3", 0.0)])
 async def test_threshold_is_clamped_to_the_valid_range(stored, expected):
     with patch("scribe.services.settings.get_setting", AsyncMock(return_value=stored)):
-        assert await get_duplicate_threshold(1) == expected
+        assert await get_duplicate_threshold(1, "snippet") == expected
 
 
-def test_the_report_threshold_is_looser_than_the_write_gate():
-    """The gate BLOCKS a create and must be unforgiving of noise; this only
-    suggests a merge the operator reviews, so it has to reach further or it
-    would never surface the pairs the gate already let through."""
-    assert DUPLICATE_DEFAULT_THRESHOLD < dedup_svc._SEMANTIC_THRESHOLD
+def test_the_snippet_report_threshold_is_looser_than_the_write_gate():
+    """The gate BLOCKS a create and must be unforgiving of noise; the snippet
+    report only suggests a merge the operator reviews, so it has to reach
+    further or it would never surface the pairs the gate already let through.
+    Snippet-only on purpose: note/task floors sit ABOVE the gate because chunk
+    grain (#280) lifts related families over it — see the module comment."""
+    assert DUPLICATE_DEFAULT_THRESHOLDS["snippet"] < dedup_svc._SEMANTIC_THRESHOLD
+
+
+def test_the_note_and_task_floors_are_stricter_than_the_snippet_floor():
+    """At chunk grain, 0.82 is a related-families view; the report's default
+    must point at genuinely-alike records (measured 2026-08-09, note #2551)."""
+    assert DUPLICATE_DEFAULT_THRESHOLDS["note"] == 0.93
+    assert DUPLICATE_DEFAULT_THRESHOLDS["task"] == 0.93
 
 
 # --- fail-open ------------------------------------------------------------