diff --git a/alembic/versions/0103_retrieval_tuning_events.py b/alembic/versions/0103_retrieval_tuning_events.py new file mode 100644 index 0000000..7dc69ac --- /dev/null +++ b/alembic/versions/0103_retrieval_tuning_events.py @@ -0,0 +1,75 @@ +"""retrieval_tuning_events — why a floor is where it is (#4102) + +Revision ID: 0103 +Revises: 0102 +Create Date: 2026-09-17 + +Milestone 416 stops shipping similarity thresholds as values somebody has to +defend, and hands the adjustment to the model that reads the surface's own +telemetry. The operator's decision: + + "the floor should be chosen and adjusted by the model using it… the user + should be able to touch it but the model should be the thing handling it 9 + times out of 10." + +The number itself already has a home — the generic settings table. What has no +home is the ARGUMENT, and once the values move on their own the argument is the +part an operator needs: what changed, from what to what, who moved it, and on +what evidence. This table is that trail, and it is what makes the delegation +reviewable rather than merely automatic. + +Nothing is backfilled. A surface with no rows here is sitting on its shipped +starting point, which is a true and useful thing for the history to say. +""" +import sqlalchemy as sa +from alembic import op + +revision = "0103" +down_revision = "0102" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "retrieval_tuning_events", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + # FK-free, like retrieval_logs and app_logs: the record of why a number + # is where it is must outlive the account that moved it. + sa.Column("user_id", sa.Integer(), nullable=True), + # Also the surface's `retrieval_logs.source`, so a change can be read + # next to what the change did. + sa.Column("surface", sa.Text(), nullable=False), + sa.Column("dial", sa.Text(), nullable=False), + # Nullable: the first change to a surface has no stored predecessor. It + # moved off the shipped starting point, which is a different event from + # moving off a value somebody chose. + sa.Column("old_value", sa.Float(), nullable=True), + sa.Column("new_value", sa.Float(), nullable=False), + sa.Column( + "actor", sa.Text(), nullable=False, server_default=sa.text("'model'") + ), + # Non-null here; non-BLANK is enforced at the service boundary, because + # a column that merely forbids NULL is satisfied by "" and a required + # field that accepts "" is a formality. + sa.Column("reason", sa.Text(), nullable=False), + ) + # The only read this table has: one surface's history, newest first. + op.create_index( + "ix_retrieval_tuning_surface_created", + "retrieval_tuning_events", + ["surface", sa.text("created_at DESC")], + ) + + +def downgrade() -> None: + op.drop_index( + "ix_retrieval_tuning_surface_created", table_name="retrieval_tuning_events" + ) + op.drop_table("retrieval_tuning_events") diff --git a/frontend/src/views/SettingsView.vue b/frontend/src/views/SettingsView.vue index 5ecaa00..8b1b452 100644 --- a/frontend/src/views/SettingsView.vue +++ b/frontend/src/views/SettingsView.vue @@ -97,6 +97,20 @@ const kbToolRuleThreshold = ref("0.68"); // prose rather than anything a tool produced (#3852). const kbPromptRuleThreshold = ref("0.72"); const kbReportPrefThreshold = ref("0.72"); +// The BUDGETS, one per arm (#4102). Until this step only auto-inject had one +// and every other arm's ceiling was a module constant nobody could reach — so +// the only control an operator had over a noisy surface was to raise its bar, +// which discards the surface's best candidates along with its worst. A budget +// keeps the top of the ranking and drops the tail, which is what was wanted. +const kbWritePathTopK = ref("3"); +const kbRuleHintTopK = ref("5"); +const kbToolRuleTopK = ref("5"); +const kbPromptRuleTopK = ref("3"); +const kbReportPrefTopK = ref("3"); +// What has been changed about retrieval, newest first — the review surface for +// changes the model made on the operator's behalf (#4102). +const tuningEvents = ref([]); +const loadingTuning = ref(false); // Near-duplicate report floors, one per record kind (services/dedup.py). // Snippets are single-chunk, so their floor sits below the 0.90 write-time // gate and catches what it lets through. Notes/tasks are scored at chunk @@ -149,6 +163,37 @@ async function saveRetention() { } } +// One entry in the retrieval tuning trail (#4102). Mirrors +// RetrievalTuningEvent.to_dict() — `old_value` is null for the first change to +// a surface, which is a different event from moving off a value somebody chose +// and renders differently below. +interface TuningEvent { + id: number; + created_at: string | null; + surface: string; + dial: string; + old_value: number | null; + new_value: number; + actor: string; + reason: string; +} + +async function loadTuningHistory() { + loadingTuning.value = true; + try { + const res = await apiGet<{ events: TuningEvent[] }>( + "/api/retrieval/tuning-history?limit=25", + ); + tuningEvents.value = res.events ?? []; + } catch { + // A history that cannot be read is not worth a toast on page load — the + // panel says so itself, and the settings above are still usable. + tuningEvents.value = []; + } finally { + loadingTuning.value = false; + } +} + async function saveKbInject() { const t = Math.min(1, Math.max(0, Number(kbInjectThreshold.value) || 0)); const k = Math.min(10, Math.max(1, Math.floor(Number(kbInjectTopK.value) || 1))); @@ -173,6 +218,22 @@ async function saveKbInject() { const trT = Math.min(1, Math.max(0, Number(kbToolRuleThreshold.value) || 0.68)); const prT = Math.min(1, Math.max(0, Number(kbPromptRuleThreshold.value) || 0.72)); const rpT = Math.min(1, Math.max(0, Number(kbReportPrefThreshold.value) || 0.72)); + // The budgets, clamped the way the server clamps them: a whole number in + // [1, 10]. Never 0 — an arm turned off is turned off by its switch, and a + // budget of zero would run the search, log the retrieval and render nothing, + // which reads in the telemetry exactly like a bar nothing cleared. + const asK = (v: string, d: number) => + Math.min(10, Math.max(1, Math.floor(Number(v) || d))); + const wpK = asK(kbWritePathTopK.value, 3); + const rhK = asK(kbRuleHintTopK.value, 5); + const trK = asK(kbToolRuleTopK.value, 5); + const prK = asK(kbPromptRuleTopK.value, 3); + const rpK = asK(kbReportPrefTopK.value, 3); + kbWritePathTopK.value = String(wpK); + kbRuleHintTopK.value = String(rhK); + kbToolRuleTopK.value = String(trK); + kbPromptRuleTopK.value = String(prK); + kbReportPrefTopK.value = String(rpK); kbInjectThreshold.value = String(t); kbInjectTopK.value = String(k); kbDupThresholdSnippet.value = String(dupSnip); @@ -211,6 +272,15 @@ async function saveKbInject() { // and a constant that lands under the bar is a dead arm, not a quiet // one (#3860). kb_reportpref_threshold: String(rpT), + // The budgets. Every one of these keys is recognised by the server as a + // retrieval dial, so this save is recorded in the tuning history as a + // change the OPERATOR made — which is the one entry the model must not + // quietly tune back. + kb_writepath_top_k: String(wpK), + kb_rulehint_top_k: String(rhK), + kb_toolrule_top_k: String(trK), + kb_promptrule_top_k: String(prK), + kb_reportpref_top_k: String(rpK), kb_duplicate_threshold_snippet: String(dupSnip), kb_duplicate_threshold_note: String(dupNote), kb_duplicate_threshold_task: String(dupTask), @@ -218,6 +288,10 @@ async function saveKbInject() { }); kbInjectSaved.value = true; setTimeout(() => (kbInjectSaved.value = false), 2000); + // The save just appended to the history it sits above, so re-read it — + // otherwise the panel shows a trail that is stale by exactly the change + // the operator is looking at it to confirm. + await loadTuningHistory(); } catch { toastStore.show('Failed to save auto-inject settings', 'error'); } finally { @@ -672,6 +746,27 @@ onMounted(async () => { if (allSettings.kb_writepath_threshold !== undefined) { kbWritePathThreshold.value = allSettings.kb_writepath_threshold; } + if (allSettings.kb_writepath_top_k !== undefined) { + kbWritePathTopK.value = allSettings.kb_writepath_top_k; + } else if (allSettings.kb_autoinject_top_k !== undefined) { + // The write path shared auto-inject's ceiling until it was given its own + // key, so an install that tuned the shared one must keep seeing that value + // here — the server falls back the same way. + kbWritePathTopK.value = allSettings.kb_autoinject_top_k; + } + if (allSettings.kb_rulehint_top_k !== undefined) { + kbRuleHintTopK.value = allSettings.kb_rulehint_top_k; + } + if (allSettings.kb_toolrule_top_k !== undefined) { + kbToolRuleTopK.value = allSettings.kb_toolrule_top_k; + } + if (allSettings.kb_promptrule_top_k !== undefined) { + kbPromptRuleTopK.value = allSettings.kb_promptrule_top_k; + } + if (allSettings.kb_reportpref_top_k !== undefined) { + kbReportPrefTopK.value = allSettings.kb_reportpref_top_k; + } + await loadTuningHistory(); if (allSettings.kb_duplicate_threshold_snippet !== undefined) { kbDupThresholdSnippet.value = allSettings.kb_duplicate_threshold_snippet; } @@ -1510,13 +1605,27 @@ async function deleteUser(userId: number) { Stricter than the prompt threshold above on purpose. Any two pieces of code look somewhat alike — shared keywords, indentation, structure — so resemblance scores start higher for code than for prose, and a bar tuned - for prompts flags unrelated code as prior art. Lower this if genuine - duplicates go unnoticed; raise it if you're being offered snippets that - have nothing to do with what's being written. Snippets recorded at the + for prompts flags unrelated code as prior art. Claude keeps this + one current from what the arm actually surfaced and refused; set it + yourself if you disagree with where it has landed. Snippets recorded at the exact file are always shown regardless — those are prior art by location, not by resemblance.

+
+ + +

How many snippets and issues one edit may be offered (1–10). Reach for this rather than the threshold when the hint feels long: lowering it keeps the best matches and drops the tail, while raising the bar above throws away good matches along with weak ones.

+
+
+ + +

How many standing rules one edit may be shown (1–10).

+
git push arrives with nothing. + write path's 37%.

+
+ + +

How many standing rules one command may be shown (1–10). This is the busiest arm there is — it fires before every command — so its budget is the one most worth keeping small.

+
+
+ + +

How many rules or preferences one message may be shown (1–10).

+
, looked up when a task closes. Unlike every other bar here, the question this arm asks never changes — so its score is fixed by your preferences alone, and it will either always find one - or never find one. If you have written a preference for report shape - and it is not arriving, lower this; there is no run of calls that - will reveal the problem on its own. + or never find one, and no run of calls will reveal a dead one on its + own. That is why this arm is worth looking up in the panel below when + a report preference never seems to arrive.

+
+ + +

How many preferences a finished task may be shown (1–10).

+
+ +
+

What has been tuned

+

+ Claude adjusts the bars and budgets above from what each arm actually + surfaced and refused, and has to state a reason to change one. Nothing + here needs your attention as a matter of course — it is here so that + when a surface behaves oddly, why it is set the way it is can be read + rather than guessed at. Anything you change yourself is recorded the + same way, and is the one entry Claude will not quietly move back. +

+

Loading…

+

+ Nothing has been changed yet — every surface is on the value Scribe + shipped. Those are starting points measured against one corpus with + one embedding model, not answers, so expect this to fill. +

+ +
+