From 21a701c57270cf33f3433004a44237f2de6d694d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 15 Sep 2026 14:53:14 -0400 Subject: [PATCH] fix(planning): the plan gate's default drops to 0.80, where reworded plans actually score (#4079) Measured live after deploy: three plans reworded from existing active milestones scored 0.83-0.87 against the milestone they restated, and the nearest distinct plans 0.72-0.77. At the inherited 0.90 the semantic arm matched nothing, so only an identical title was caught. 0.80 sits in the gap. The Settings form's default moves with it (test_settings_defaults_agree). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01821k5B3Ysecp9fNYs92Kuy --- frontend/src/views/SettingsView.vue | 4 ++-- src/scribe/services/dedup.py | 24 +++++++++++++++--------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/frontend/src/views/SettingsView.vue b/frontend/src/views/SettingsView.vue index 2647c35..73fa61c 100644 --- a/frontend/src/views/SettingsView.vue +++ b/frontend/src/views/SettingsView.vue @@ -105,7 +105,7 @@ const kbDupThresholdSnippet = ref("0.82"); const kbDupThresholdNote = ref("0.93"); const kbDupThresholdTask = ref("0.93"); // PLAN_MATCH_DEFAULT_THRESHOLD in services/dedup.py. -const kbPlanMatchThreshold = ref("0.90"); +const kbPlanMatchThreshold = ref("0.80"); const savingKbInject = ref(false); const kbInjectSaved = ref(false); @@ -159,7 +159,7 @@ async function saveKbInject() { const dupTask = Math.min(1, Math.max(0, Number(kbDupThresholdTask.value) || 0.93)); // Same `|| default` guard: a floor of 0 would hand back an existing plan // for every new one, and no plan could be started without force. - const planT = Math.min(1, Math.max(0, Number(kbPlanMatchThreshold.value) || 0.9)); + const planT = Math.min(1, Math.max(0, Number(kbPlanMatchThreshold.value) || 0.8)); // Same `|| default` reasoning: falling back to 0 would surface every // snippet in the corpus on every edit, which is the failure this knob fixes. const wpT = Math.min(1, Math.max(0, Number(kbWritePathThreshold.value) || 0.68)); diff --git a/src/scribe/services/dedup.py b/src/scribe/services/dedup.py index dfa0cc9..ce93e8d 100644 --- a/src/scribe/services/dedup.py +++ b/src/scribe/services/dedup.py @@ -739,16 +739,22 @@ async def find_duplicate_rule( # project is exactly what adding steps to its existing plan needs, so a caller # who reaches this gate can act on whatever it returns. # -# Its own threshold, as a setting (rule 25). The note gate's 0.90 is where it -# starts, because it asks the same question ("the same thing, reworded") with -# the same embedder. Plan documents are shaped differently, though: a milestone -# is embedded as title, description and plan (embeddings.milestone_document), -# while the candidate usually has no description and carries its steps -# instead. That difference has not been measured, and a gate that blocks on -# noise teaches sessions to pass force=true every time, which is worse than no -# gate. So the default is conservative and the operator can lower it. +# Its own threshold, as a setting (rule 25), and lower than the note gate's +# 0.90 because plan documents are shaped differently: a milestone is embedded +# as title, description and plan (embeddings.milestone_document), while the +# candidate usually has no description and carries its steps instead, so even +# a faithful rewording never scores like a copy. +# +# Measured after the first deploy (2026-09-15, bge-small-en-v1.5, #4079): three +# plans reworded from existing active milestones scored 0.83, 0.85 and 0.87 +# against the milestone they restated; the nearest DIFFERENT plan for each, and +# a distinct plan on a neighbouring topic, scored 0.72-0.77. At 0.90 the +# semantic arm matched nothing, leaving only the title arm. 0.80 sits in the +# gap. A gate that blocks on noise teaches sessions to pass force=true every +# time, which is why it is not lower; four samples is thin, which is why the +# operator can move it. Retune alongside the embedder, not the corpus. PLAN_MATCH_THRESHOLD_KEY = "kb_plan_match_threshold" -PLAN_MATCH_DEFAULT_THRESHOLD = 0.90 +PLAN_MATCH_DEFAULT_THRESHOLD = 0.80 async def get_plan_match_threshold(user_id: int) -> float: