The two new retrieval bars get their Settings controls #154

Merged
bvandeusen merged 1 commits from dev into main 2026-09-11 20:58:50 -04:00
2 changed files with 162 additions and 8 deletions
+74 -8
View File
@@ -87,6 +87,15 @@ const kbWritePathEnabled = ref(true);
// unrelated code through (#2223). Shares top-k, not the threshold. // unrelated code through (#2223). Shares top-k, not the threshold.
const kbWritePathThreshold = ref("0.68"); const kbWritePathThreshold = ref("0.68");
const kbRuleHintThreshold = ref("0.72"); const kbRuleHintThreshold = ref("0.72");
// The two ACT arms no longer share a bar (#3853). A write-path query is a code
// payload; a pre-tool query is a shell command, often under a dozen words —
// less text, less signal, lower scores for the same relevance. Measured at one
// shared 0.72 the two behaved like different subsystems: the write-path arm
// spoke on 37% of its calls, the command arm on 2% of 11,768.
const kbToolRuleThreshold = ref("0.68");
// And the prompt boundary is a third query shape again — the operator's own
// prose rather than anything a tool produced (#3852).
const kbPromptRuleThreshold = ref("0.72");
// Near-duplicate report floors, one per record kind (services/dedup.py). // Near-duplicate report floors, one per record kind (services/dedup.py).
// Snippets are single-chunk, so their floor sits below the 0.90 write-time // Snippets are single-chunk, so their floor sits below the 0.90 write-time
// gate and catches what it lets through. Notes/tasks are scored at chunk // gate and catches what it lets through. Notes/tasks are scored at chunk
@@ -153,6 +162,10 @@ async function saveKbInject() {
// fires on every write, so a fallback of 0 would attach a standing rule to // fires on every write, so a fallback of 0 would attach a standing rule to
// every edit in the session. // every edit in the session.
const rhT = Math.min(1, Math.max(0, Number(kbRuleHintThreshold.value) || 0.72)); const rhT = Math.min(1, Math.max(0, Number(kbRuleHintThreshold.value) || 0.72));
// Same `|| default` guard, and the same reason: this arm fires before every
// Bash call, so a fallback of 0 would put a rule in front of every command.
const trT = Math.min(1, Math.max(0, Number(kbToolRuleThreshold.value) || 0.68));
const prT = Math.min(1, Math.max(0, Number(kbPromptRuleThreshold.value) || 0.72));
kbInjectThreshold.value = String(t); kbInjectThreshold.value = String(t);
kbInjectTopK.value = String(k); kbInjectTopK.value = String(k);
kbDupThresholdSnippet.value = String(dupSnip); kbDupThresholdSnippet.value = String(dupSnip);
@@ -160,6 +173,8 @@ async function saveKbInject() {
kbDupThresholdTask.value = String(dupTask); kbDupThresholdTask.value = String(dupTask);
kbWritePathThreshold.value = String(wpT); kbWritePathThreshold.value = String(wpT);
kbRuleHintThreshold.value = String(rhT); kbRuleHintThreshold.value = String(rhT);
kbToolRuleThreshold.value = String(trT);
kbPromptRuleThreshold.value = String(prT);
savingKbInject.value = true; savingKbInject.value = true;
kbInjectSaved.value = false; kbInjectSaved.value = false;
try { try {
@@ -176,6 +191,11 @@ async function saveKbInject() {
// in services/plugin_context.py for why rules cannot share the // in services/plugin_context.py for why rules cannot share the
// code threshold any more than code could share the prose one. // code threshold any more than code could share the prose one.
kb_rulehint_threshold: String(rhT), kb_rulehint_threshold: String(rhT),
// A FOURTH and FIFTH bar, and they are separate keys on purpose: the
// whole finding of #3853 is that one number cannot serve arms whose
// queries are different shapes. Moving one must not move the others.
kb_toolrule_threshold: String(trT),
kb_promptrule_threshold: String(prT),
kb_duplicate_threshold_snippet: String(dupSnip), kb_duplicate_threshold_snippet: String(dupSnip),
kb_duplicate_threshold_note: String(dupNote), kb_duplicate_threshold_note: String(dupNote),
kb_duplicate_threshold_task: String(dupTask), kb_duplicate_threshold_task: String(dupTask),
@@ -624,6 +644,12 @@ onMounted(async () => {
if (allSettings.kb_rulehint_threshold !== undefined) { if (allSettings.kb_rulehint_threshold !== undefined) {
kbRuleHintThreshold.value = allSettings.kb_rulehint_threshold; kbRuleHintThreshold.value = allSettings.kb_rulehint_threshold;
} }
if (allSettings.kb_toolrule_threshold !== undefined) {
kbToolRuleThreshold.value = allSettings.kb_toolrule_threshold;
}
if (allSettings.kb_promptrule_threshold !== undefined) {
kbPromptRuleThreshold.value = allSettings.kb_promptrule_threshold;
}
if (allSettings.kb_writepath_threshold !== undefined) { if (allSettings.kb_writepath_threshold !== undefined) {
kbWritePathThreshold.value = allSettings.kb_writepath_threshold; kbWritePathThreshold.value = allSettings.kb_writepath_threshold;
} }
@@ -1482,14 +1508,54 @@ async function deleteUser(userId: number) {
style="max-width: 8rem" style="max-width: 8rem"
/> />
<p class="field-hint"> <p class="field-hint">
The same hint can mention a standing rule whose trigger resembles what's The same hint can mention a standing rule whose trigger resembles the
being written — only rules marked <em>conditional</em>, since always-on code being written. <em>Every</em> rule is eligible nothing is
ones are already loaded. Stricter again than the threshold above, because preloaded any more, so this is the only way a rule reaches a write.
there are far fewer rules than snippets: with a small set, something Stricter than the threshold above, because there are far fewer rules
always ranks first, so the bar has to carry more of the judgement. than snippets: with a small set something always ranks first, so the
Raise it if rules keep arriving unread; lower it if a rule you needed bar has to carry more of the judgement. Raise it if rules keep
never showed up. Settings → check the pull-through in arriving unread; lower it if a rule you needed never showed up.
<code>retrieval_telemetry</code> to see which is happening. </p>
</div>
<div class="field">
<label for="kb-toolrule-threshold">Command confidence threshold (01)</label>
<input
id="kb-toolrule-threshold"
v-model="kbToolRuleThreshold"
type="number"
min="0"
max="1"
step="0.01"
class="fs-input input"
style="max-width: 8rem"
/>
<p class="field-hint">
The bar for a rule surfacing before a <em>command</em> runs, where the
query is the command text rather than code. Lower than the one above
on purpose: a shell command is short, so it scores lower for the same
relevance at a shared bar this arm spoke on 2% of calls against the
write path's 37%. Raise it if commands attract rules that do not
apply; lower it if a <code>git push</code> arrives with nothing.
</p>
</div>
<div class="field">
<label for="kb-promptrule-threshold">Prompt confidence threshold (01)</label>
<input
id="kb-promptrule-threshold"
v-model="kbPromptRuleThreshold"
type="number"
min="0"
max="1"
step="0.01"
class="fs-input input"
style="max-width: 8rem"
/>
<p class="field-hint">
The bar for a rule or preference surfacing against <em>what you just
said</em>, before anything is done. It is the only moment that reaches
a rule about how to answer rather than how to act, so it runs once a
turn rather than once a tool call. A third query shape — your prose,
not a command or a file — which is why it carries its own number.
</p> </p>
</div> </div>
<!-- A design system belongs to a PROJECT, and the picker for it lives on <!-- A design system belongs to a PROJECT, and the picker for it lives on
+88
View File
@@ -0,0 +1,88 @@
"""The Settings UI shows the default the server actually uses (#3927).
WHY THIS EXISTS
A retrieval threshold lives in two places by necessity: a Python constant the
arm reads when no row is stored, and a Vue `ref` the Settings form shows when
the operator has never touched it. Neither can import the other.
So the form's initial value is a CLAIM about the server's behaviour, and it is
the kind of claim that rots quietly. Retune the Python constant and the input
keeps rendering the old number — the operator reads it as the bar in force,
sees no reason to change anything, and the form has misinformed them about the
one fact it exists to convey. Nothing errors, nothing looks wrong, and the
value they are shown is simply not the value being applied.
WHAT THIS PINS
The RELATIONSHIP, never the number: for each threshold, the Vue ref's initial
string equals the Python default. Retuning either stays free as long as both
move — which is the point, because these are tuning values and #3853 moved one
of them the day this was written.
Both sides are read from SOURCE rather than imported. The Vue file cannot be
imported at all, and reading the Python constant through an import would tie
this to module-load side effects it has no interest in.
It cannot check that the form WRITES the right key — that is behaviour, and
the keys are asserted where they are built. It catches the drift that has no
other alarm.
"""
from __future__ import annotations
import pathlib
import re
import pytest
ROOT = pathlib.Path(__file__).resolve().parents[1]
_PY = ROOT / "src" / "scribe" / "services" / "plugin_context.py"
_VUE = ROOT / "frontend" / "src" / "views" / "SettingsView.vue"
# (python constant, vue ref). Hand-written because the pairing is an editorial
# fact — the names do not share a convention either side could derive — but
# every entry is asserted to EXIST on both sides, so a rename fails loudly
# here rather than silently dropping that threshold from the check.
_PAIRS = (
("AUTOINJECT_DEFAULT_THRESHOLD", "kbInjectThreshold"),
("WRITEPATH_DEFAULT_THRESHOLD", "kbWritePathThreshold"),
("RULEHINT_DEFAULT_THRESHOLD", "kbRuleHintThreshold"),
("TOOLRULE_DEFAULT_THRESHOLD", "kbToolRuleThreshold"),
("PROMPTRULE_DEFAULT_THRESHOLD", "kbPromptRuleThreshold"),
)
def _python_default(name: str) -> float:
m = re.search(rf"^{re.escape(name)}\s*=\s*([0-9.]+)\s*$",
_PY.read_text(), re.M)
assert m, (
f"{name} is no longer a bare module-level float in "
f"services/plugin_context.py. If it moved or was renamed, update "
f"_PAIRS; if it was retired, drop its row — leaving it here checks "
f"nothing while looking like coverage."
)
return float(m.group(1))
def _vue_default(ref_name: str) -> float:
m = re.search(rf"const {re.escape(ref_name)} = ref\(\"([0-9.]+)\"\)",
_VUE.read_text())
assert m, (
f"{ref_name} is no longer a `ref(\"<number>\")` in SettingsView.vue. "
f"If the control was renamed, update _PAIRS; if it was removed, the "
f"setting has lost its UI and that is the thing to fix (rule 25)."
)
return float(m.group(1))
@pytest.mark.parametrize(("constant", "ref_name"), _PAIRS,
ids=[p[0] for p in _PAIRS])
def test_the_form_shows_the_default_the_server_uses(constant, ref_name):
"""An untouched control must render the bar actually in force."""
server, form = _python_default(constant), _vue_default(ref_name)
assert form == server, (
f"SettingsView shows {form} for {ref_name} while the server defaults "
f"to {server} ({constant}). An operator who has never set this reads "
f"the form as the value in force, so the two must move together — "
f"retune both, or neither."
)