From 6240652dce25df13f4f54a69e7658fb2da0b198e Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 17 Sep 2026 11:36:22 -0400 Subject: [PATCH] feat(retrieval): the operator can see what was tuned, and every write is recorded (#4102) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The other half of the bargain in milestone 416 step 4. The model moves these dials; this is what makes that reviewable rather than merely automatic. THE HOLE THIS CLOSES Every retrieval floor is an ordinary settings key, and `/api/settings` accepts any key at all. A floor written through it landed correctly and recorded nothing — a tuning history with holes in it, which is worse than no history because it reads as complete. So the generic endpoint now routes registry-owned keys through `set_dial` instead of writing them as plain rows. ROUTED, not refused: refusing would only work for callers that had been updated, while this way the form, a script, and an old client all leave the trail, and there is no version of "forgot to use the other endpoint". Clearing a control is written as an explicit set back to the shipped default, because the operator reverting something is the single most important move this history can record. `set_dial` now also refuses to record a no-op. The Settings form re-sends every field on every save, so without that one press of Save would write six rows saying the operator set six dials to the numbers they were already on — and a history nobody can skim is one nobody reads. WHAT THE OPERATOR GETS `/api/retrieval/surfaces`, `/surfaces/` and `/tuning-history`, with `actor` fixed server-side rather than taken from the payload: a payload-supplied actor would let a model claim to be the operator, and "did I do this, or did the session?" is the first question this list is asked. In Settings: the five missing BUDGETS (until now only auto-inject had one, so the only control over a noisy surface was to raise its bar — which discards that surface's best candidates along with its worst), and a "What has been tuned" panel showing each change, who made it, and the reason given. The operator's own changes are marked. The MCP tool demands a reason; these endpoints do not. That asymmetry is deliberate and stated in routes/retrieval.py: the requirement exists to make the MODEL read the records before moving a number on someone else's behalf, and the operator is that someone — a mandatory justification box on every control would be friction charged to the one participant who owes no explanation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01821k5B3Ysecp9fNYs92Kuy --- frontend/src/views/SettingsView.vue | 267 ++++++++++++++++++++++ src/scribe/app.py | 2 + src/scribe/routes/retrieval.py | 106 +++++++++ src/scribe/routes/settings.py | 36 +++ src/scribe/services/retrieval_surfaces.py | 20 ++ src/scribe/services/retrieval_tuning.py | 18 ++ tests/test_routes_retrieval_tuning.py | 133 +++++++++++ 7 files changed, 582 insertions(+) create mode 100644 src/scribe/routes/retrieval.py create mode 100644 tests/test_routes_retrieval_tuning.py diff --git a/frontend/src/views/SettingsView.vue b/frontend/src/views/SettingsView.vue index 5ecaa00..79920f2 100644 --- a/frontend/src/views/SettingsView.vue +++ b/frontend/src/views/SettingsView.vue @@ -97,6 +97,20 @@ const kbToolRuleThreshold = ref("0.68"); // prose rather than anything a tool produced (#3852). const kbPromptRuleThreshold = ref("0.72"); const kbReportPrefThreshold = ref("0.72"); +// The BUDGETS, one per arm (#4102). Until this step only auto-inject had one +// and every other arm's ceiling was a module constant nobody could reach — so +// the only control an operator had over a noisy surface was to raise its bar, +// which discards the surface's best candidates along with its worst. A budget +// keeps the top of the ranking and drops the tail, which is what was wanted. +const kbWritePathTopK = ref("3"); +const kbRuleHintTopK = ref("5"); +const kbToolRuleTopK = ref("5"); +const kbPromptRuleTopK = ref("3"); +const kbReportPrefTopK = ref("3"); +// What has been changed about retrieval, newest first — the review surface for +// changes the model made on the operator's behalf (#4102). +const tuningEvents = ref([]); +const loadingTuning = ref(false); // Near-duplicate report floors, one per record kind (services/dedup.py). // Snippets are single-chunk, so their floor sits below the 0.90 write-time // gate and catches what it lets through. Notes/tasks are scored at chunk @@ -149,6 +163,37 @@ async function saveRetention() { } } +// One entry in the retrieval tuning trail (#4102). Mirrors +// RetrievalTuningEvent.to_dict() — `old_value` is null for the first change to +// a surface, which is a different event from moving off a value somebody chose +// and renders differently below. +interface TuningEvent { + id: number; + created_at: string | null; + surface: string; + dial: string; + old_value: number | null; + new_value: number; + actor: string; + reason: string; +} + +async function loadTuningHistory() { + loadingTuning.value = true; + try { + const res = await apiGet<{ events: TuningEvent[] }>( + "/api/retrieval/tuning-history?limit=25", + ); + tuningEvents.value = res.events ?? []; + } catch { + // A history that cannot be read is not worth a toast on page load — the + // panel says so itself, and the settings above are still usable. + tuningEvents.value = []; + } finally { + loadingTuning.value = false; + } +} + async function saveKbInject() { const t = Math.min(1, Math.max(0, Number(kbInjectThreshold.value) || 0)); const k = Math.min(10, Math.max(1, Math.floor(Number(kbInjectTopK.value) || 1))); @@ -173,6 +218,22 @@ async function saveKbInject() { const trT = Math.min(1, Math.max(0, Number(kbToolRuleThreshold.value) || 0.68)); const prT = Math.min(1, Math.max(0, Number(kbPromptRuleThreshold.value) || 0.72)); const rpT = Math.min(1, Math.max(0, Number(kbReportPrefThreshold.value) || 0.72)); + // The budgets, clamped the way the server clamps them: a whole number in + // [1, 10]. Never 0 — an arm turned off is turned off by its switch, and a + // budget of zero would run the search, log the retrieval and render nothing, + // which reads in the telemetry exactly like a bar nothing cleared. + const asK = (v: string, d: number) => + Math.min(10, Math.max(1, Math.floor(Number(v) || d))); + const wpK = asK(kbWritePathTopK.value, 3); + const rhK = asK(kbRuleHintTopK.value, 5); + const trK = asK(kbToolRuleTopK.value, 5); + const prK = asK(kbPromptRuleTopK.value, 3); + const rpK = asK(kbReportPrefTopK.value, 3); + kbWritePathTopK.value = String(wpK); + kbRuleHintTopK.value = String(rhK); + kbToolRuleTopK.value = String(trK); + kbPromptRuleTopK.value = String(prK); + kbReportPrefTopK.value = String(rpK); kbInjectThreshold.value = String(t); kbInjectTopK.value = String(k); kbDupThresholdSnippet.value = String(dupSnip); @@ -211,6 +272,15 @@ async function saveKbInject() { // and a constant that lands under the bar is a dead arm, not a quiet // one (#3860). kb_reportpref_threshold: String(rpT), + // The budgets. Every one of these keys is recognised by the server as a + // retrieval dial, so this save is recorded in the tuning history as a + // change the OPERATOR made — which is the one entry the model must not + // quietly tune back. + kb_writepath_top_k: String(wpK), + kb_rulehint_top_k: String(rhK), + kb_toolrule_top_k: String(trK), + kb_promptrule_top_k: String(prK), + kb_reportpref_top_k: String(rpK), kb_duplicate_threshold_snippet: String(dupSnip), kb_duplicate_threshold_note: String(dupNote), kb_duplicate_threshold_task: String(dupTask), @@ -218,6 +288,10 @@ async function saveKbInject() { }); kbInjectSaved.value = true; setTimeout(() => (kbInjectSaved.value = false), 2000); + // The save just appended to the history it sits above, so re-read it — + // otherwise the panel shows a trail that is stale by exactly the change + // the operator is looking at it to confirm. + await loadTuningHistory(); } catch { toastStore.show('Failed to save auto-inject settings', 'error'); } finally { @@ -672,6 +746,27 @@ onMounted(async () => { if (allSettings.kb_writepath_threshold !== undefined) { kbWritePathThreshold.value = allSettings.kb_writepath_threshold; } + if (allSettings.kb_writepath_top_k !== undefined) { + kbWritePathTopK.value = allSettings.kb_writepath_top_k; + } else if (allSettings.kb_autoinject_top_k !== undefined) { + // The write path shared auto-inject's ceiling until it was given its own + // key, so an install that tuned the shared one must keep seeing that value + // here — the server falls back the same way. + kbWritePathTopK.value = allSettings.kb_autoinject_top_k; + } + if (allSettings.kb_rulehint_top_k !== undefined) { + kbRuleHintTopK.value = allSettings.kb_rulehint_top_k; + } + if (allSettings.kb_toolrule_top_k !== undefined) { + kbToolRuleTopK.value = allSettings.kb_toolrule_top_k; + } + if (allSettings.kb_promptrule_top_k !== undefined) { + kbPromptRuleTopK.value = allSettings.kb_promptrule_top_k; + } + if (allSettings.kb_reportpref_top_k !== undefined) { + kbReportPrefTopK.value = allSettings.kb_reportpref_top_k; + } + await loadTuningHistory(); if (allSettings.kb_duplicate_threshold_snippet !== undefined) { kbDupThresholdSnippet.value = allSettings.kb_duplicate_threshold_snippet; } @@ -1517,6 +1612,20 @@ async function deleteUser(userId: number) { location, not by resemblance.

+
+ + +

How many snippets and issues one edit may be offered (1–10). Reach for this rather than the threshold when the hint feels long: lowering it keeps the best matches and drops the tail, while raising the bar above throws away good matches along with weak ones.

+
+
+ + +

How many standing rules one edit may be shown (1–10).

+
git push arrives with nothing.

+
+ + +

How many standing rules one command may be shown (1–10). This is the busiest arm there is — it fires before every command — so its budget is the one most worth keeping small.

+
+
+ + +

How many rules or preferences one message may be shown (1–10).

+
+
+ + +

How many preferences a finished task may be shown (1–10).

+
+ +
+

What has been tuned

+

+ Claude adjusts the bars and budgets above from what each arm actually + surfaced and refused, and has to state a reason to change one. Nothing + here needs your attention as a matter of course — it is here so that + when a surface behaves oddly, why it is set the way it is can be read + rather than guessed at. Anything you change yourself is recorded the + same way, and is the one entry Claude will not quietly move back. +

+

Loading…

+

+ Nothing has been changed yet — every surface is on the value Scribe + shipped. Those are starting points measured against one corpus with + one embedding model, not answers, so expect this to fill. +

+
    +
  • +
    + {{ ev.surface }} + {{ ev.dial }} + + + + {{ ev.new_value }} + + + {{ ev.actor === 'human' ? 'you' : 'Claude' }} + + {{ fmtDate(ev.created_at) }} +
    +

    {{ ev.reason }}

    +
    • +
+
+