Every ledger is cleared on a compact, and a retrieval floor becomes something the model maintains #163
@@ -97,6 +97,20 @@ const kbToolRuleThreshold = ref("0.68");
|
|||||||
// prose rather than anything a tool produced (#3852).
|
// prose rather than anything a tool produced (#3852).
|
||||||
const kbPromptRuleThreshold = ref("0.72");
|
const kbPromptRuleThreshold = ref("0.72");
|
||||||
const kbReportPrefThreshold = ref("0.72");
|
const kbReportPrefThreshold = ref("0.72");
|
||||||
|
// The BUDGETS, one per arm (#4102). Until this step only auto-inject had one
|
||||||
|
// and every other arm's ceiling was a module constant nobody could reach — so
|
||||||
|
// the only control an operator had over a noisy surface was to raise its bar,
|
||||||
|
// which discards the surface's best candidates along with its worst. A budget
|
||||||
|
// keeps the top of the ranking and drops the tail, which is what was wanted.
|
||||||
|
const kbWritePathTopK = ref("3");
|
||||||
|
const kbRuleHintTopK = ref("5");
|
||||||
|
const kbToolRuleTopK = ref("5");
|
||||||
|
const kbPromptRuleTopK = ref("3");
|
||||||
|
const kbReportPrefTopK = ref("3");
|
||||||
|
// What has been changed about retrieval, newest first — the review surface for
|
||||||
|
// changes the model made on the operator's behalf (#4102).
|
||||||
|
const tuningEvents = ref<TuningEvent[]>([]);
|
||||||
|
const loadingTuning = ref(false);
|
||||||
// Near-duplicate report floors, one per record kind (services/dedup.py).
|
// Near-duplicate report floors, one per record kind (services/dedup.py).
|
||||||
// Snippets are single-chunk, so their floor sits below the 0.90 write-time
|
// Snippets are single-chunk, so their floor sits below the 0.90 write-time
|
||||||
// gate and catches what it lets through. Notes/tasks are scored at chunk
|
// gate and catches what it lets through. Notes/tasks are scored at chunk
|
||||||
@@ -149,6 +163,37 @@ async function saveRetention() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One entry in the retrieval tuning trail (#4102). Mirrors
|
||||||
|
// RetrievalTuningEvent.to_dict() — `old_value` is null for the first change to
|
||||||
|
// a surface, which is a different event from moving off a value somebody chose
|
||||||
|
// and renders differently below.
|
||||||
|
interface TuningEvent {
|
||||||
|
id: number;
|
||||||
|
created_at: string | null;
|
||||||
|
surface: string;
|
||||||
|
dial: string;
|
||||||
|
old_value: number | null;
|
||||||
|
new_value: number;
|
||||||
|
actor: string;
|
||||||
|
reason: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadTuningHistory() {
|
||||||
|
loadingTuning.value = true;
|
||||||
|
try {
|
||||||
|
const res = await apiGet<{ events: TuningEvent[] }>(
|
||||||
|
"/api/retrieval/tuning-history?limit=25",
|
||||||
|
);
|
||||||
|
tuningEvents.value = res.events ?? [];
|
||||||
|
} catch {
|
||||||
|
// A history that cannot be read is not worth a toast on page load — the
|
||||||
|
// panel says so itself, and the settings above are still usable.
|
||||||
|
tuningEvents.value = [];
|
||||||
|
} finally {
|
||||||
|
loadingTuning.value = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function saveKbInject() {
|
async function saveKbInject() {
|
||||||
const t = Math.min(1, Math.max(0, Number(kbInjectThreshold.value) || 0));
|
const t = Math.min(1, Math.max(0, Number(kbInjectThreshold.value) || 0));
|
||||||
const k = Math.min(10, Math.max(1, Math.floor(Number(kbInjectTopK.value) || 1)));
|
const k = Math.min(10, Math.max(1, Math.floor(Number(kbInjectTopK.value) || 1)));
|
||||||
@@ -173,6 +218,22 @@ async function saveKbInject() {
|
|||||||
const trT = Math.min(1, Math.max(0, Number(kbToolRuleThreshold.value) || 0.68));
|
const trT = Math.min(1, Math.max(0, Number(kbToolRuleThreshold.value) || 0.68));
|
||||||
const prT = Math.min(1, Math.max(0, Number(kbPromptRuleThreshold.value) || 0.72));
|
const prT = Math.min(1, Math.max(0, Number(kbPromptRuleThreshold.value) || 0.72));
|
||||||
const rpT = Math.min(1, Math.max(0, Number(kbReportPrefThreshold.value) || 0.72));
|
const rpT = Math.min(1, Math.max(0, Number(kbReportPrefThreshold.value) || 0.72));
|
||||||
|
// The budgets, clamped the way the server clamps them: a whole number in
|
||||||
|
// [1, 10]. Never 0 — an arm turned off is turned off by its switch, and a
|
||||||
|
// budget of zero would run the search, log the retrieval and render nothing,
|
||||||
|
// which reads in the telemetry exactly like a bar nothing cleared.
|
||||||
|
const asK = (v: string, d: number) =>
|
||||||
|
Math.min(10, Math.max(1, Math.floor(Number(v) || d)));
|
||||||
|
const wpK = asK(kbWritePathTopK.value, 3);
|
||||||
|
const rhK = asK(kbRuleHintTopK.value, 5);
|
||||||
|
const trK = asK(kbToolRuleTopK.value, 5);
|
||||||
|
const prK = asK(kbPromptRuleTopK.value, 3);
|
||||||
|
const rpK = asK(kbReportPrefTopK.value, 3);
|
||||||
|
kbWritePathTopK.value = String(wpK);
|
||||||
|
kbRuleHintTopK.value = String(rhK);
|
||||||
|
kbToolRuleTopK.value = String(trK);
|
||||||
|
kbPromptRuleTopK.value = String(prK);
|
||||||
|
kbReportPrefTopK.value = String(rpK);
|
||||||
kbInjectThreshold.value = String(t);
|
kbInjectThreshold.value = String(t);
|
||||||
kbInjectTopK.value = String(k);
|
kbInjectTopK.value = String(k);
|
||||||
kbDupThresholdSnippet.value = String(dupSnip);
|
kbDupThresholdSnippet.value = String(dupSnip);
|
||||||
@@ -211,6 +272,15 @@ async function saveKbInject() {
|
|||||||
// and a constant that lands under the bar is a dead arm, not a quiet
|
// and a constant that lands under the bar is a dead arm, not a quiet
|
||||||
// one (#3860).
|
// one (#3860).
|
||||||
kb_reportpref_threshold: String(rpT),
|
kb_reportpref_threshold: String(rpT),
|
||||||
|
// The budgets. Every one of these keys is recognised by the server as a
|
||||||
|
// retrieval dial, so this save is recorded in the tuning history as a
|
||||||
|
// change the OPERATOR made — which is the one entry the model must not
|
||||||
|
// quietly tune back.
|
||||||
|
kb_writepath_top_k: String(wpK),
|
||||||
|
kb_rulehint_top_k: String(rhK),
|
||||||
|
kb_toolrule_top_k: String(trK),
|
||||||
|
kb_promptrule_top_k: String(prK),
|
||||||
|
kb_reportpref_top_k: String(rpK),
|
||||||
kb_duplicate_threshold_snippet: String(dupSnip),
|
kb_duplicate_threshold_snippet: String(dupSnip),
|
||||||
kb_duplicate_threshold_note: String(dupNote),
|
kb_duplicate_threshold_note: String(dupNote),
|
||||||
kb_duplicate_threshold_task: String(dupTask),
|
kb_duplicate_threshold_task: String(dupTask),
|
||||||
@@ -218,6 +288,10 @@ async function saveKbInject() {
|
|||||||
});
|
});
|
||||||
kbInjectSaved.value = true;
|
kbInjectSaved.value = true;
|
||||||
setTimeout(() => (kbInjectSaved.value = false), 2000);
|
setTimeout(() => (kbInjectSaved.value = false), 2000);
|
||||||
|
// The save just appended to the history it sits above, so re-read it —
|
||||||
|
// otherwise the panel shows a trail that is stale by exactly the change
|
||||||
|
// the operator is looking at it to confirm.
|
||||||
|
await loadTuningHistory();
|
||||||
} catch {
|
} catch {
|
||||||
toastStore.show('Failed to save auto-inject settings', 'error');
|
toastStore.show('Failed to save auto-inject settings', 'error');
|
||||||
} finally {
|
} finally {
|
||||||
@@ -672,6 +746,27 @@ onMounted(async () => {
|
|||||||
if (allSettings.kb_writepath_threshold !== undefined) {
|
if (allSettings.kb_writepath_threshold !== undefined) {
|
||||||
kbWritePathThreshold.value = allSettings.kb_writepath_threshold;
|
kbWritePathThreshold.value = allSettings.kb_writepath_threshold;
|
||||||
}
|
}
|
||||||
|
if (allSettings.kb_writepath_top_k !== undefined) {
|
||||||
|
kbWritePathTopK.value = allSettings.kb_writepath_top_k;
|
||||||
|
} else if (allSettings.kb_autoinject_top_k !== undefined) {
|
||||||
|
// The write path shared auto-inject's ceiling until it was given its own
|
||||||
|
// key, so an install that tuned the shared one must keep seeing that value
|
||||||
|
// here — the server falls back the same way.
|
||||||
|
kbWritePathTopK.value = allSettings.kb_autoinject_top_k;
|
||||||
|
}
|
||||||
|
if (allSettings.kb_rulehint_top_k !== undefined) {
|
||||||
|
kbRuleHintTopK.value = allSettings.kb_rulehint_top_k;
|
||||||
|
}
|
||||||
|
if (allSettings.kb_toolrule_top_k !== undefined) {
|
||||||
|
kbToolRuleTopK.value = allSettings.kb_toolrule_top_k;
|
||||||
|
}
|
||||||
|
if (allSettings.kb_promptrule_top_k !== undefined) {
|
||||||
|
kbPromptRuleTopK.value = allSettings.kb_promptrule_top_k;
|
||||||
|
}
|
||||||
|
if (allSettings.kb_reportpref_top_k !== undefined) {
|
||||||
|
kbReportPrefTopK.value = allSettings.kb_reportpref_top_k;
|
||||||
|
}
|
||||||
|
await loadTuningHistory();
|
||||||
if (allSettings.kb_duplicate_threshold_snippet !== undefined) {
|
if (allSettings.kb_duplicate_threshold_snippet !== undefined) {
|
||||||
kbDupThresholdSnippet.value = allSettings.kb_duplicate_threshold_snippet;
|
kbDupThresholdSnippet.value = allSettings.kb_duplicate_threshold_snippet;
|
||||||
}
|
}
|
||||||
@@ -1517,6 +1612,20 @@ async function deleteUser(userId: number) {
|
|||||||
location, not by resemblance.
|
location, not by resemblance.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="field">
|
||||||
|
<label for="kb-writepath-topk">Max prior-art lines per edit</label>
|
||||||
|
<input
|
||||||
|
id="kb-writepath-topk"
|
||||||
|
v-model="kbWritePathTopK"
|
||||||
|
type="number"
|
||||||
|
min="1"
|
||||||
|
max="10"
|
||||||
|
step="1"
|
||||||
|
class="fs-input input"
|
||||||
|
style="max-width: 8rem"
|
||||||
|
/>
|
||||||
|
<p class="field-hint">How many snippets and issues one edit may be offered (1–10). Reach for this rather than the threshold when the hint feels long: lowering it keeps the best matches and drops the tail, while raising the bar above throws away good matches along with weak ones.</p>
|
||||||
|
</div>
|
||||||
<div class="field">
|
<div class="field">
|
||||||
<label for="kb-rulehint-threshold">Standing-rule confidence threshold (0–1)</label>
|
<label for="kb-rulehint-threshold">Standing-rule confidence threshold (0–1)</label>
|
||||||
<input
|
<input
|
||||||
@@ -1539,6 +1648,20 @@ async function deleteUser(userId: number) {
|
|||||||
arriving unread; lower it if a rule you needed never showed up.
|
arriving unread; lower it if a rule you needed never showed up.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="field">
|
||||||
|
<label for="kb-rulehint-topk">Max rules per edit</label>
|
||||||
|
<input
|
||||||
|
id="kb-rulehint-topk"
|
||||||
|
v-model="kbRuleHintTopK"
|
||||||
|
type="number"
|
||||||
|
min="1"
|
||||||
|
max="10"
|
||||||
|
step="1"
|
||||||
|
class="fs-input input"
|
||||||
|
style="max-width: 8rem"
|
||||||
|
/>
|
||||||
|
<p class="field-hint">How many standing rules one edit may be shown (1–10).</p>
|
||||||
|
</div>
|
||||||
<div class="field">
|
<div class="field">
|
||||||
<label for="kb-toolrule-threshold">Command confidence threshold (0–1)</label>
|
<label for="kb-toolrule-threshold">Command confidence threshold (0–1)</label>
|
||||||
<input
|
<input
|
||||||
@@ -1560,6 +1683,20 @@ async function deleteUser(userId: number) {
|
|||||||
apply; lower it if a <code>git push</code> arrives with nothing.
|
apply; lower it if a <code>git push</code> arrives with nothing.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="field">
|
||||||
|
<label for="kb-toolrule-topk">Max rules per command</label>
|
||||||
|
<input
|
||||||
|
id="kb-toolrule-topk"
|
||||||
|
v-model="kbToolRuleTopK"
|
||||||
|
type="number"
|
||||||
|
min="1"
|
||||||
|
max="10"
|
||||||
|
step="1"
|
||||||
|
class="fs-input input"
|
||||||
|
style="max-width: 8rem"
|
||||||
|
/>
|
||||||
|
<p class="field-hint">How many standing rules one command may be shown (1–10). This is the busiest arm there is — it fires before every command — so its budget is the one most worth keeping small.</p>
|
||||||
|
</div>
|
||||||
<div class="field">
|
<div class="field">
|
||||||
<label for="kb-promptrule-threshold">Prompt confidence threshold (0–1)</label>
|
<label for="kb-promptrule-threshold">Prompt confidence threshold (0–1)</label>
|
||||||
<input
|
<input
|
||||||
@@ -1580,6 +1717,20 @@ async function deleteUser(userId: number) {
|
|||||||
not a command or a file — which is why it carries its own number.
|
not a command or a file — which is why it carries its own number.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="field">
|
||||||
|
<label for="kb-promptrule-topk">Max rules per prompt</label>
|
||||||
|
<input
|
||||||
|
id="kb-promptrule-topk"
|
||||||
|
v-model="kbPromptRuleTopK"
|
||||||
|
type="number"
|
||||||
|
min="1"
|
||||||
|
max="10"
|
||||||
|
step="1"
|
||||||
|
class="fs-input input"
|
||||||
|
style="max-width: 8rem"
|
||||||
|
/>
|
||||||
|
<p class="field-hint">How many rules or preferences one message may be shown (1–10).</p>
|
||||||
|
</div>
|
||||||
<div class="field">
|
<div class="field">
|
||||||
<label for="kb-reportpref-threshold">Completion-report confidence threshold (0–1)</label>
|
<label for="kb-reportpref-threshold">Completion-report confidence threshold (0–1)</label>
|
||||||
<input
|
<input
|
||||||
@@ -1602,6 +1753,61 @@ async function deleteUser(userId: number) {
|
|||||||
will reveal the problem on its own.
|
will reveal the problem on its own.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="field">
|
||||||
|
<label for="kb-reportpref-topk">Max report preferences</label>
|
||||||
|
<input
|
||||||
|
id="kb-reportpref-topk"
|
||||||
|
v-model="kbReportPrefTopK"
|
||||||
|
type="number"
|
||||||
|
min="1"
|
||||||
|
max="10"
|
||||||
|
step="1"
|
||||||
|
class="fs-input input"
|
||||||
|
style="max-width: 8rem"
|
||||||
|
/>
|
||||||
|
<p class="field-hint">How many preferences a finished task may be shown (1–10).</p>
|
||||||
|
</div>
|
||||||
|
<!-- THE REVIEW SURFACE (#4102). These numbers are maintained by the
|
||||||
|
model that uses them: it reads which records each bar refused and
|
||||||
|
moves the dial with the argument attached. This panel is the other
|
||||||
|
half of that bargain — a change made on your behalf is one you can
|
||||||
|
read, disagree with, and set back by hand above. -->
|
||||||
|
<div class="tuning-history">
|
||||||
|
<h4 class="tuning-history-title">What has been tuned</h4>
|
||||||
|
<p class="field-hint">
|
||||||
|
Claude adjusts the bars and budgets above from what each arm actually
|
||||||
|
surfaced and refused, and has to state a reason to change one. Nothing
|
||||||
|
here needs your attention as a matter of course — it is here so that
|
||||||
|
when a surface behaves oddly, why it is set the way it is can be read
|
||||||
|
rather than guessed at. Anything you change yourself is recorded the
|
||||||
|
same way, and is the one entry Claude will not quietly move back.
|
||||||
|
</p>
|
||||||
|
<p v-if="loadingTuning" class="field-hint">Loading…</p>
|
||||||
|
<p v-else-if="!tuningEvents.length" class="field-hint">
|
||||||
|
Nothing has been changed yet — every surface is on the value Scribe
|
||||||
|
shipped. Those are starting points measured against one corpus with
|
||||||
|
one embedding model, not answers, so expect this to fill.
|
||||||
|
</p>
|
||||||
|
<ul v-else class="tuning-list">
|
||||||
|
<li v-for="ev in tuningEvents" :key="ev.id" class="tuning-item">
|
||||||
|
<div class="tuning-item-head">
|
||||||
|
<span class="tuning-surface">{{ ev.surface }}</span>
|
||||||
|
<span class="tuning-dial">{{ ev.dial }}</span>
|
||||||
|
<span class="tuning-move">
|
||||||
|
<template v-if="ev.old_value !== null">{{ ev.old_value }} →</template>
|
||||||
|
<template v-else>set to</template>
|
||||||
|
{{ ev.new_value }}
|
||||||
|
</span>
|
||||||
|
<span class="tuning-actor" :class="{ 'is-human': ev.actor === 'human' }">
|
||||||
|
{{ ev.actor === 'human' ? 'you' : 'Claude' }}
|
||||||
|
</span>
|
||||||
|
<span v-if="ev.created_at" class="tuning-when">{{ fmtDate(ev.created_at) }}</span>
|
||||||
|
</div>
|
||||||
|
<p class="tuning-reason">{{ ev.reason }}</p>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- A design system belongs to a PROJECT, and the picker for it lives on
|
<!-- A design system belongs to a PROJECT, and the picker for it lives on
|
||||||
the project. There was a setting here that designated the system
|
the project. There was a setting here that designated the system
|
||||||
this install's own interface was built from; it only ever described
|
this install's own interface was built from; it only ever described
|
||||||
@@ -3848,4 +4054,65 @@ async function deleteUser(userId: number) {
|
|||||||
.area-admin-form { display: flex; flex-direction: column; gap: 0.5rem; flex: 1; }
|
.area-admin-form { display: flex; flex-direction: column; gap: 0.5rem; flex: 1; }
|
||||||
.area-admin-create { margin-top: var(--fs-space-4); }
|
.area-admin-create { margin-top: var(--fs-space-4); }
|
||||||
.area-admin-actions { display: flex; gap: 0.4rem; }
|
.area-admin-actions { display: flex; gap: 0.4rem; }
|
||||||
|
|
||||||
|
/* The retrieval tuning trail (#4102). Reads as a record, not a control panel:
|
||||||
|
the operator is reviewing what was done, and the reason is the part worth
|
||||||
|
reading, so it gets the full-width line under a compact header row. */
|
||||||
|
.tuning-history {
|
||||||
|
margin-top: var(--fs-space-5);
|
||||||
|
padding-top: var(--fs-space-4);
|
||||||
|
border-top: 1px solid var(--fs-border-color);
|
||||||
|
}
|
||||||
|
.tuning-history-title {
|
||||||
|
margin: 0 0 var(--fs-space-2);
|
||||||
|
font-size: 0.95rem;
|
||||||
|
color: var(--fs-text-primary);
|
||||||
|
}
|
||||||
|
.tuning-list {
|
||||||
|
list-style: none;
|
||||||
|
margin: var(--fs-space-3) 0 0;
|
||||||
|
padding: 0;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: var(--fs-space-2);
|
||||||
|
}
|
||||||
|
.tuning-item {
|
||||||
|
padding: var(--fs-space-3);
|
||||||
|
background: var(--fs-surface-raised);
|
||||||
|
border: 1px solid var(--fs-border-color);
|
||||||
|
border-radius: var(--fs-radius-md);
|
||||||
|
}
|
||||||
|
.tuning-item-head {
|
||||||
|
display: flex;
|
||||||
|
align-items: baseline;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: var(--fs-space-2);
|
||||||
|
font-size: 0.85rem;
|
||||||
|
}
|
||||||
|
.tuning-surface {
|
||||||
|
font-family: var(--fs-font-mono);
|
||||||
|
color: var(--fs-text-primary);
|
||||||
|
}
|
||||||
|
.tuning-dial,
|
||||||
|
.tuning-move {
|
||||||
|
color: var(--fs-text-secondary);
|
||||||
|
}
|
||||||
|
.tuning-actor {
|
||||||
|
color: var(--fs-text-tertiary);
|
||||||
|
border: 1px solid var(--fs-border-color);
|
||||||
|
border-radius: var(--fs-radius-sm);
|
||||||
|
padding: 0 0.4rem;
|
||||||
|
}
|
||||||
|
/* The operator's own changes are marked, because "did I do this, or did the
|
||||||
|
session?" is the first question this list is asked. */
|
||||||
|
.tuning-actor.is-human {
|
||||||
|
color: var(--fs-accent);
|
||||||
|
border-color: var(--fs-accent);
|
||||||
|
}
|
||||||
|
.tuning-when { margin-left: auto; color: var(--fs-text-tertiary); }
|
||||||
|
.tuning-reason {
|
||||||
|
margin: var(--fs-space-2) 0 0;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: var(--fs-text-secondary);
|
||||||
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from scribe.routes.notes import notes_bp
|
|||||||
from scribe.routes.milestones import milestones_bp
|
from scribe.routes.milestones import milestones_bp
|
||||||
from scribe.routes.task_logs import task_logs_bp
|
from scribe.routes.task_logs import task_logs_bp
|
||||||
from scribe.routes.projects import projects_bp
|
from scribe.routes.projects import projects_bp
|
||||||
|
from scribe.routes.retrieval import retrieval_bp
|
||||||
from scribe.routes.settings import settings_bp
|
from scribe.routes.settings import settings_bp
|
||||||
from scribe.routes.tasks import tasks_bp
|
from scribe.routes.tasks import tasks_bp
|
||||||
from scribe.routes.groups import groups_bp
|
from scribe.routes.groups import groups_bp
|
||||||
@@ -79,6 +80,7 @@ def create_app() -> Quart:
|
|||||||
app.register_blueprint(milestones_bp)
|
app.register_blueprint(milestones_bp)
|
||||||
app.register_blueprint(notes_bp)
|
app.register_blueprint(notes_bp)
|
||||||
app.register_blueprint(projects_bp)
|
app.register_blueprint(projects_bp)
|
||||||
|
app.register_blueprint(retrieval_bp)
|
||||||
app.register_blueprint(settings_bp)
|
app.register_blueprint(settings_bp)
|
||||||
app.register_blueprint(task_logs_bp)
|
app.register_blueprint(task_logs_bp)
|
||||||
app.register_blueprint(tasks_bp)
|
app.register_blueprint(tasks_bp)
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
"""The operator's view of retrieval tuning — what the dials are, and who moved them (#4102).
|
||||||
|
|
||||||
|
WHY THIS EXISTS SEPARATELY FROM /api/settings
|
||||||
|
|
||||||
|
The numbers themselves are ordinary settings rows and could be written through
|
||||||
|
the generic KV endpoint. They must not be, and that is the whole point of this
|
||||||
|
blueprint: a floor changed through `/api/settings` moves silently, leaving the
|
||||||
|
history saying nothing happened. From milestone 416 those dials are moved by
|
||||||
|
the model on the operator's behalf, so a trail with holes in it is worse than
|
||||||
|
no trail — it reads as complete.
|
||||||
|
|
||||||
|
So every change to a retrieval dial goes through `set_dial`, from the UI as
|
||||||
|
much as from the MCP tool, and the only difference is the `actor` recorded.
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from quart import Blueprint, jsonify, request
|
||||||
|
|
||||||
|
from scribe.auth import get_current_user_id, login_required
|
||||||
|
from scribe.services.retrieval_tuning import set_dial, current_settings, tuning_history
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
retrieval_bp = Blueprint("retrieval", __name__, url_prefix="/api/retrieval")
|
||||||
|
|
||||||
|
# What is recorded when the operator changes a dial in Settings and says
|
||||||
|
# nothing about why.
|
||||||
|
#
|
||||||
|
# The MCP tool REFUSES a blank reason, and this endpoint does not, which is a
|
||||||
|
# deliberate asymmetry rather than an oversight. The requirement exists to make
|
||||||
|
# the model look at the records before it moves a number on someone else's
|
||||||
|
# behalf. The operator IS that someone: they are the audience the trail is
|
||||||
|
# written for, they cannot be uninformed about their own decision, and a
|
||||||
|
# mandatory justification textarea on every control would be friction charged
|
||||||
|
# to the one participant who owes no explanation (rule 24).
|
||||||
|
#
|
||||||
|
# The event is still written, because "the operator set this by hand" is the
|
||||||
|
# single most useful thing the history can tell a later session — it is the one
|
||||||
|
# entry the model must not quietly tune back.
|
||||||
|
_OPERATOR_DEFAULT_REASON = "Set directly in Settings by the operator."
|
||||||
|
|
||||||
|
|
||||||
|
@retrieval_bp.route("/surfaces", methods=["GET"])
|
||||||
|
@login_required
|
||||||
|
async def get_surfaces_route():
|
||||||
|
"""Every tunable surface: its live floor and budget, what it asks and over
|
||||||
|
what, and the reason each dial was last moved."""
|
||||||
|
uid = get_current_user_id()
|
||||||
|
surfaces = await current_settings(uid)
|
||||||
|
return jsonify({"surfaces": surfaces, "total": len(surfaces)})
|
||||||
|
|
||||||
|
|
||||||
|
@retrieval_bp.route("/surfaces/<surface>", methods=["PUT"])
|
||||||
|
@login_required
|
||||||
|
async def tune_surface_route(surface: str):
|
||||||
|
"""Move one dial, recorded as a human change.
|
||||||
|
|
||||||
|
`actor` is fixed here rather than taken from the payload: this endpoint is
|
||||||
|
reached with a session cookie from the Settings form, so the actor is known
|
||||||
|
and accepting a claim about it would let the one field a reviewer relies on
|
||||||
|
be set to anything.
|
||||||
|
"""
|
||||||
|
uid = get_current_user_id()
|
||||||
|
data = await request.get_json()
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return jsonify({"error": "Expected a JSON object"}), 400
|
||||||
|
if "dial" not in data or "value" not in data:
|
||||||
|
return jsonify({"error": "dial and value are required"}), 400
|
||||||
|
|
||||||
|
try:
|
||||||
|
value = float(data["value"])
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return jsonify({"error": f"value must be a number, got {data['value']!r}"}), 400
|
||||||
|
|
||||||
|
reason = str(data.get("reason") or "").strip() or _OPERATOR_DEFAULT_REASON
|
||||||
|
try:
|
||||||
|
result = await set_dial(
|
||||||
|
uid, surface, str(data["dial"]), value, reason=reason, actor="human",
|
||||||
|
)
|
||||||
|
except ValueError as e:
|
||||||
|
# Unknown surface, unknown dial — the service names the alternatives,
|
||||||
|
# so the message is worth passing through rather than flattening.
|
||||||
|
return jsonify({"error": str(e)}), 400
|
||||||
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
|
@retrieval_bp.route("/tuning-history", methods=["GET"])
|
||||||
|
@login_required
|
||||||
|
async def tuning_history_route():
|
||||||
|
"""What has been changed about retrieval on this install, newest first.
|
||||||
|
|
||||||
|
The review surface. Unscoped by default because the operator's question
|
||||||
|
here is "what has been done on my behalf", not "why is this one arm set
|
||||||
|
like that" — the per-surface scoping is for the model.
|
||||||
|
"""
|
||||||
|
uid = get_current_user_id()
|
||||||
|
surface = request.args.get("surface") or None
|
||||||
|
try:
|
||||||
|
limit = int(request.args.get("limit", 50))
|
||||||
|
except ValueError:
|
||||||
|
return jsonify({"error": "limit must be a whole number"}), 400
|
||||||
|
try:
|
||||||
|
events = await tuning_history(uid, surface=surface, limit=limit)
|
||||||
|
except ValueError as e:
|
||||||
|
return jsonify({"error": str(e)}), 400
|
||||||
|
return jsonify({"events": events, "total": len(events)})
|
||||||
@@ -9,6 +9,8 @@ from quart import Blueprint, jsonify, request
|
|||||||
|
|
||||||
from scribe.auth import login_required, get_current_user_id
|
from scribe.auth import login_required, get_current_user_id
|
||||||
from scribe.config import Config
|
from scribe.config import Config
|
||||||
|
from scribe.services.retrieval_surfaces import dial_for_key, get_surface
|
||||||
|
from scribe.services.retrieval_tuning import set_dial
|
||||||
from scribe.services.settings import (
|
from scribe.services.settings import (
|
||||||
SECRET_MASK, delete_setting, get_all_settings, get_setting, set_settings_batch,
|
SECRET_MASK, delete_setting, get_all_settings, get_setting, set_settings_batch,
|
||||||
)
|
)
|
||||||
@@ -25,6 +27,13 @@ settings_bp = Blueprint("settings", __name__, url_prefix="/api/settings")
|
|||||||
# (forge_token left with 0078: forge credentials are keyring rows now, #2778.)
|
# (forge_token left with 0078: forge credentials are keyring rows now, #2778.)
|
||||||
_SECRET_KEYS = frozenset({"smtp_password", "forge_webhook_secret"})
|
_SECRET_KEYS = frozenset({"smtp_password", "forge_webhook_secret"})
|
||||||
|
|
||||||
|
# What the tuning history records for a dial changed through this form. The MCP
|
||||||
|
# tool refuses a blank reason; the operator is not asked for one, because the
|
||||||
|
# requirement exists to make the MODEL read the records before moving a number
|
||||||
|
# on someone else's behalf — and the operator is that someone. See
|
||||||
|
# routes/retrieval.py, which states the asymmetry in full.
|
||||||
|
_SETTINGS_FORM_REASON = "Changed in Settings by the operator."
|
||||||
|
|
||||||
|
|
||||||
def _masked(settings: dict) -> dict:
|
def _masked(settings: dict) -> dict:
|
||||||
return {
|
return {
|
||||||
@@ -52,6 +61,33 @@ async def update_settings_route():
|
|||||||
to_save = {}
|
to_save = {}
|
||||||
for k, v in data.items():
|
for k, v in data.items():
|
||||||
str_v = str(v)
|
str_v = str(v)
|
||||||
|
# A retrieval dial is never written as a plain settings row, wherever
|
||||||
|
# the request came from (#4102). The value would land correctly and the
|
||||||
|
# tuning history would say nothing happened — and since milestone 416
|
||||||
|
# those dials are moved by the model on the operator's behalf, a
|
||||||
|
# history with holes in it is worse than none: it reads as complete.
|
||||||
|
#
|
||||||
|
# Routed rather than refused on purpose. Refusing would work only for
|
||||||
|
# callers that had been updated; this way every caller that ever writes
|
||||||
|
# one of these keys — this form, a script, an old client — leaves the
|
||||||
|
# trail, and there is no version of "forgot to use the other endpoint".
|
||||||
|
dial = dial_for_key(k)
|
||||||
|
if dial:
|
||||||
|
surface, which = dial
|
||||||
|
s = get_surface(surface)
|
||||||
|
# A CLEARED control means "back to the shipped starting point", and
|
||||||
|
# that is a change like any other — it is the operator reverting
|
||||||
|
# something, which is the single most important move this history
|
||||||
|
# can record. So it is written as an explicit set to the default
|
||||||
|
# rather than deleted, which would leave the same value behind and
|
||||||
|
# no record of anyone having chosen it.
|
||||||
|
default = s.floor_default if which == "floor" else s.budget_default
|
||||||
|
try:
|
||||||
|
await set_dial(uid, surface, which, float(str_v or default),
|
||||||
|
reason=_SETTINGS_FORM_REASON, actor="human")
|
||||||
|
except (TypeError, ValueError) as e:
|
||||||
|
return jsonify({"error": f"{k}: {e}"}), 400
|
||||||
|
continue
|
||||||
# A masked secret round-tripping through a client is "unchanged", not
|
# A masked secret round-tripping through a client is "unchanged", not
|
||||||
# a request to store the mask over the real credential.
|
# a request to store the mask over the real credential.
|
||||||
if k in _SECRET_KEYS and str_v == SECRET_MASK:
|
if k in _SECRET_KEYS and str_v == SECRET_MASK:
|
||||||
|
|||||||
@@ -213,6 +213,26 @@ def get_surface(name: str) -> Surface:
|
|||||||
) from None
|
) from None
|
||||||
|
|
||||||
|
|
||||||
|
def dial_for_key(key: str) -> tuple[str, str] | None:
|
||||||
|
"""Which `(surface, dial)` a settings key belongs to, or None.
|
||||||
|
|
||||||
|
The registry read backwards, and it exists for one caller: the generic
|
||||||
|
`/api/settings` endpoint, which accepts any key at all. Without this, a
|
||||||
|
floor written through that endpoint moves with no event recorded, and the
|
||||||
|
tuning history says nothing happened — a trail with holes in it, which is
|
||||||
|
worse than no trail because it reads as complete.
|
||||||
|
|
||||||
|
Derived rather than listed so a seventh surface is covered the moment it is
|
||||||
|
added here, which is the only way this stays true.
|
||||||
|
"""
|
||||||
|
for surface in SURFACES.values():
|
||||||
|
if key == surface.floor_key:
|
||||||
|
return (surface.name, "floor")
|
||||||
|
if key == surface.budget_key:
|
||||||
|
return (surface.name, "budget")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
async def floor_for(user_id: int, name: str) -> float:
|
async def floor_for(user_id: int, name: str) -> float:
|
||||||
"""This install's current floor for a surface, clamped to [0, 1]."""
|
"""This install's current floor for a surface, clamped to [0, 1]."""
|
||||||
s = get_surface(name)
|
s = get_surface(name)
|
||||||
|
|||||||
@@ -162,6 +162,21 @@ async def set_dial(
|
|||||||
applied = float(min(MAX_BUDGET, max(1, int(float(value)))))
|
applied = float(min(MAX_BUDGET, max(1, int(float(value)))))
|
||||||
key, stored = s.budget_key, str(int(applied))
|
key, stored = s.budget_key, str(int(applied))
|
||||||
|
|
||||||
|
# A change to the value it already has is not a change, and must not be
|
||||||
|
# written. The Settings form re-sends every field on every save, so without
|
||||||
|
# this the history fills with rows saying the operator set six dials to the
|
||||||
|
# numbers they were already on — and a history nobody can skim is one
|
||||||
|
# nobody reads, which costs the surface its entire purpose.
|
||||||
|
#
|
||||||
|
# Reported rather than silently skipped, so a caller that expected to move
|
||||||
|
# something learns that it did not.
|
||||||
|
if abs(applied - old) < 1e-9:
|
||||||
|
return {
|
||||||
|
"surface": surface, "dial": dial, "previous": old,
|
||||||
|
"applied": applied, "clamped": abs(applied - float(value)) > 1e-9,
|
||||||
|
"reason": text, "actor": actor, "unchanged": True,
|
||||||
|
}
|
||||||
|
|
||||||
await set_setting(user_id, key, stored)
|
await set_setting(user_id, key, stored)
|
||||||
async with async_session() as session:
|
async with async_session() as session:
|
||||||
session.add(RetrievalTuningEvent(
|
session.add(RetrievalTuningEvent(
|
||||||
@@ -181,6 +196,9 @@ async def set_dial(
|
|||||||
"clamped": abs(applied - float(value)) > 1e-9,
|
"clamped": abs(applied - float(value)) > 1e-9,
|
||||||
"reason": text,
|
"reason": text,
|
||||||
"actor": actor,
|
"actor": actor,
|
||||||
|
# Always present, both ways round: a caller that has to test for the
|
||||||
|
# key's absence to learn the answer will eventually forget to.
|
||||||
|
"unchanged": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,133 @@
|
|||||||
|
"""The operator's half of the tuning surface (#4102).
|
||||||
|
|
||||||
|
WHY THIS EXISTS
|
||||||
|
|
||||||
|
The model moves these dials — that is the operator's decision for milestone
|
||||||
|
416 — so the browser's job is no longer "set the number". It is to show what
|
||||||
|
was set, by whom, and on what argument, and to let the operator disagree.
|
||||||
|
|
||||||
|
The load-bearing guard here is the last one. `/api/settings` is a generic
|
||||||
|
key-value endpoint that accepts any key at all, and every retrieval floor IS an
|
||||||
|
ordinary settings key. A floor written straight through it would land correctly
|
||||||
|
and record nothing — a tuning history with holes in it, which is worse than no
|
||||||
|
history because it reads as complete. So the generic endpoint routes those keys
|
||||||
|
through `set_dial`, and that routing is asserted rather than remembered.
|
||||||
|
"""
|
||||||
|
import inspect
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
def test_retrieval_blueprint_registered():
|
||||||
|
from scribe.routes.retrieval import retrieval_bp
|
||||||
|
assert retrieval_bp.name == "retrieval"
|
||||||
|
assert retrieval_bp.url_prefix == "/api/retrieval"
|
||||||
|
|
||||||
|
|
||||||
|
def test_retrieval_blueprint_registered_in_app():
|
||||||
|
from scribe.app import create_app
|
||||||
|
app = create_app()
|
||||||
|
assert "retrieval" in app.blueprints
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_endpoint_is_reachable_on_the_app():
|
||||||
|
"""Handlers existing is not the same as them being routed."""
|
||||||
|
from scribe.app import create_app
|
||||||
|
app = create_app()
|
||||||
|
rules = {
|
||||||
|
str(r.rule) for r in app.url_map.iter_rules()
|
||||||
|
if r.endpoint.startswith("retrieval.")
|
||||||
|
}
|
||||||
|
assert rules == {
|
||||||
|
"/api/retrieval/surfaces",
|
||||||
|
"/api/retrieval/surfaces/<surface>",
|
||||||
|
"/api/retrieval/tuning-history",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_browser_and_the_agent_call_the_same_service():
|
||||||
|
"""Rule 33 parity. Two callers, one service — or the two surfaces drift and
|
||||||
|
the guard that a reason is required exists on only one of them."""
|
||||||
|
from scribe.mcp.tools import retrieval_tuning as tool
|
||||||
|
from scribe.routes import retrieval as routes
|
||||||
|
from scribe.services import retrieval_tuning as svc
|
||||||
|
|
||||||
|
assert routes.set_dial is svc.set_dial
|
||||||
|
assert tool.tuning_svc is svc
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_route_does_not_take_the_actor_from_the_caller():
|
||||||
|
"""`actor` is the one field a reviewer leans on to answer "did I do this,
|
||||||
|
or did the session?". A payload-supplied actor would let a model claim to
|
||||||
|
be the operator, which turns the column into decoration."""
|
||||||
|
from scribe.routes import retrieval as routes
|
||||||
|
|
||||||
|
src = inspect.getsource(routes.tune_surface_route)
|
||||||
|
assert 'actor="human"' in src
|
||||||
|
assert 'data.get("actor"' not in src and 'data["actor"]' not in src
|
||||||
|
|
||||||
|
|
||||||
|
# ── the hole the generic settings endpoint would otherwise leave ────────────
|
||||||
|
|
||||||
|
def test_every_registry_key_is_recognised_as_a_dial():
|
||||||
|
"""Derived from the registry, both directions. A seventh surface added
|
||||||
|
without a row here is a floor that can be written silently again."""
|
||||||
|
from scribe.services.retrieval_surfaces import SURFACES, dial_for_key
|
||||||
|
|
||||||
|
for name, surface in SURFACES.items():
|
||||||
|
assert dial_for_key(surface.floor_key) == (name, "floor")
|
||||||
|
assert dial_for_key(surface.budget_key) == (name, "budget")
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_ordinary_setting_is_not_mistaken_for_a_dial():
|
||||||
|
"""The interception must be narrow. A false positive here would send an
|
||||||
|
unrelated setting through a service that parses it as a float and rejects
|
||||||
|
the save."""
|
||||||
|
from scribe.services.retrieval_surfaces import dial_for_key
|
||||||
|
|
||||||
|
for key in ("smtp_password", "kb_autoinject_enabled", "theme",
|
||||||
|
"kb_planmatch_threshold", ""):
|
||||||
|
assert dial_for_key(key) is None, key
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_settings_endpoint_routes_a_dial_through_the_recorder():
|
||||||
|
"""THE GUARD. Asserted on the source because the alternative is a full
|
||||||
|
request-context round trip for a branch whose whole content is "which
|
||||||
|
function gets called" — and because what must not regress is precisely
|
||||||
|
that this module reaches for `set_dial` at all."""
|
||||||
|
from scribe.routes import settings as routes
|
||||||
|
|
||||||
|
src = inspect.getsource(routes.update_settings_route)
|
||||||
|
assert "dial_for_key" in src, (
|
||||||
|
"the generic settings endpoint no longer recognises retrieval dials — "
|
||||||
|
"a floor written through it now moves with no event recorded, and the "
|
||||||
|
"tuning history will say nothing happened"
|
||||||
|
)
|
||||||
|
assert "set_dial" in src
|
||||||
|
assert 'actor="human"' in src
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_setting_a_dial_to_the_value_it_already_has_records_nothing():
|
||||||
|
"""The Settings form re-sends every field on every save. Without this, one
|
||||||
|
press of Save writes six rows saying the operator set six dials to the
|
||||||
|
numbers they were already on — and a history nobody can skim is one nobody
|
||||||
|
reads."""
|
||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
|
from scribe.services import retrieval_tuning as rt
|
||||||
|
from tests.helpers import make_mock_session
|
||||||
|
|
||||||
|
session = make_mock_session()
|
||||||
|
with patch.object(rt, "async_session", MagicMock(return_value=session)), \
|
||||||
|
patch.object(rt, "set_setting", AsyncMock()) as setter, \
|
||||||
|
patch.object(rt, "floor_for", AsyncMock(return_value=0.72)), \
|
||||||
|
patch.object(rt, "budget_for", AsyncMock(return_value=3)):
|
||||||
|
out = await rt.set_dial(1, "prompt_rule", "floor", 0.72,
|
||||||
|
reason="re-saved the settings form untouched")
|
||||||
|
|
||||||
|
assert out["unchanged"] is True
|
||||||
|
session.add.assert_not_called()
|
||||||
|
# And the setting is left alone too: rewriting the same value would bump
|
||||||
|
# whatever timestamp the row carries for no reason.
|
||||||
|
setter.assert_not_called()
|
||||||
Reference in New Issue
Block a user