Every ledger is cleared on a compact, and a retrieval floor becomes something the model maintains (#163)
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 10s
CI & Build / integration (push) Successful in 48s
CI & Build / TypeScript typecheck (push) Successful in 53s
CI & Build / Python tests (push) Successful in 1m32s
CI & Build / Build & push image (push) Successful in 16s

This commit was merged in pull request #163.
This commit is contained in:
2026-09-17 12:01:42 -04:00
34 changed files with 2966 additions and 324 deletions
@@ -0,0 +1,75 @@
"""retrieval_tuning_events — why a floor is where it is (#4102)
Revision ID: 0103
Revises: 0102
Create Date: 2026-09-17
Milestone 416 stops shipping similarity thresholds as values somebody has to
defend, and hands the adjustment to the model that reads the surface's own
telemetry. The operator's decision:
"the floor should be chosen and adjusted by the model using it… the user
should be able to touch it but the model should be the thing handling it 9
times out of 10."
The number itself already has a home — the generic settings table. What has no
home is the ARGUMENT, and once the values move on their own the argument is the
part an operator needs: what changed, from what to what, who moved it, and on
what evidence. This table is that trail, and it is what makes the delegation
reviewable rather than merely automatic.
Nothing is backfilled. A surface with no rows here is sitting on its shipped
starting point, which is a true and useful thing for the history to say.
"""
import sqlalchemy as sa
from alembic import op
revision = "0103"
down_revision = "0102"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"retrieval_tuning_events",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
# FK-free, like retrieval_logs and app_logs: the record of why a number
# is where it is must outlive the account that moved it.
sa.Column("user_id", sa.Integer(), nullable=True),
# Also the surface's `retrieval_logs.source`, so a change can be read
# next to what the change did.
sa.Column("surface", sa.Text(), nullable=False),
sa.Column("dial", sa.Text(), nullable=False),
# Nullable: the first change to a surface has no stored predecessor. It
# moved off the shipped starting point, which is a different event from
# moving off a value somebody chose.
sa.Column("old_value", sa.Float(), nullable=True),
sa.Column("new_value", sa.Float(), nullable=False),
sa.Column(
"actor", sa.Text(), nullable=False, server_default=sa.text("'model'")
),
# Non-null here; non-BLANK is enforced at the service boundary, because
# a column that merely forbids NULL is satisfied by "" and a required
# field that accepts "" is a formality.
sa.Column("reason", sa.Text(), nullable=False),
)
# The only read this table has: one surface's history, newest first.
op.create_index(
"ix_retrieval_tuning_surface_created",
"retrieval_tuning_events",
["surface", sa.text("created_at DESC")],
)
def downgrade() -> None:
op.drop_index(
"ix_retrieval_tuning_surface_created", table_name="retrieval_tuning_events"
)
op.drop_table("retrieval_tuning_events")
+277 -10
View File
@@ -97,6 +97,20 @@ const kbToolRuleThreshold = ref("0.68");
// prose rather than anything a tool produced (#3852).
const kbPromptRuleThreshold = ref("0.72");
const kbReportPrefThreshold = ref("0.72");
// The BUDGETS, one per arm (#4102). Until this step only auto-inject had one
// and every other arm's ceiling was a module constant nobody could reach — so
// the only control an operator had over a noisy surface was to raise its bar,
// which discards the surface's best candidates along with its worst. A budget
// keeps the top of the ranking and drops the tail, which is what was wanted.
const kbWritePathTopK = ref("3");
const kbRuleHintTopK = ref("5");
const kbToolRuleTopK = ref("5");
const kbPromptRuleTopK = ref("3");
const kbReportPrefTopK = ref("3");
// What has been changed about retrieval, newest first — the review surface for
// changes the model made on the operator's behalf (#4102).
const tuningEvents = ref<TuningEvent[]>([]);
const loadingTuning = ref(false);
// Near-duplicate report floors, one per record kind (services/dedup.py).
// Snippets are single-chunk, so their floor sits below the 0.90 write-time
// gate and catches what it lets through. Notes/tasks are scored at chunk
@@ -149,6 +163,37 @@ async function saveRetention() {
}
}
// One entry in the retrieval tuning trail (#4102). Mirrors
// RetrievalTuningEvent.to_dict() — `old_value` is null for the first change to
// a surface, which is a different event from moving off a value somebody chose
// and renders differently below.
interface TuningEvent {
id: number;
created_at: string | null;
surface: string;
dial: string;
old_value: number | null;
new_value: number;
actor: string;
reason: string;
}
async function loadTuningHistory() {
loadingTuning.value = true;
try {
const res = await apiGet<{ events: TuningEvent[] }>(
"/api/retrieval/tuning-history?limit=25",
);
tuningEvents.value = res.events ?? [];
} catch {
// A history that cannot be read is not worth a toast on page load — the
// panel says so itself, and the settings above are still usable.
tuningEvents.value = [];
} finally {
loadingTuning.value = false;
}
}
async function saveKbInject() {
const t = Math.min(1, Math.max(0, Number(kbInjectThreshold.value) || 0));
const k = Math.min(10, Math.max(1, Math.floor(Number(kbInjectTopK.value) || 1)));
@@ -173,6 +218,22 @@ async function saveKbInject() {
const trT = Math.min(1, Math.max(0, Number(kbToolRuleThreshold.value) || 0.68));
const prT = Math.min(1, Math.max(0, Number(kbPromptRuleThreshold.value) || 0.72));
const rpT = Math.min(1, Math.max(0, Number(kbReportPrefThreshold.value) || 0.72));
// The budgets, clamped the way the server clamps them: a whole number in
// [1, 10]. Never 0 — an arm turned off is turned off by its switch, and a
// budget of zero would run the search, log the retrieval and render nothing,
// which reads in the telemetry exactly like a bar nothing cleared.
const asK = (v: string, d: number) =>
Math.min(10, Math.max(1, Math.floor(Number(v) || d)));
const wpK = asK(kbWritePathTopK.value, 3);
const rhK = asK(kbRuleHintTopK.value, 5);
const trK = asK(kbToolRuleTopK.value, 5);
const prK = asK(kbPromptRuleTopK.value, 3);
const rpK = asK(kbReportPrefTopK.value, 3);
kbWritePathTopK.value = String(wpK);
kbRuleHintTopK.value = String(rhK);
kbToolRuleTopK.value = String(trK);
kbPromptRuleTopK.value = String(prK);
kbReportPrefTopK.value = String(rpK);
kbInjectThreshold.value = String(t);
kbInjectTopK.value = String(k);
kbDupThresholdSnippet.value = String(dupSnip);
@@ -211,6 +272,15 @@ async function saveKbInject() {
// and a constant that lands under the bar is a dead arm, not a quiet
// one (#3860).
kb_reportpref_threshold: String(rpT),
// The budgets. Every one of these keys is recognised by the server as a
// retrieval dial, so this save is recorded in the tuning history as a
// change the OPERATOR made — which is the one entry the model must not
// quietly tune back.
kb_writepath_top_k: String(wpK),
kb_rulehint_top_k: String(rhK),
kb_toolrule_top_k: String(trK),
kb_promptrule_top_k: String(prK),
kb_reportpref_top_k: String(rpK),
kb_duplicate_threshold_snippet: String(dupSnip),
kb_duplicate_threshold_note: String(dupNote),
kb_duplicate_threshold_task: String(dupTask),
@@ -218,6 +288,10 @@ async function saveKbInject() {
});
kbInjectSaved.value = true;
setTimeout(() => (kbInjectSaved.value = false), 2000);
// The save just appended to the history it sits above, so re-read it —
// otherwise the panel shows a trail that is stale by exactly the change
// the operator is looking at it to confirm.
await loadTuningHistory();
} catch {
toastStore.show('Failed to save auto-inject settings', 'error');
} finally {
@@ -672,6 +746,27 @@ onMounted(async () => {
if (allSettings.kb_writepath_threshold !== undefined) {
kbWritePathThreshold.value = allSettings.kb_writepath_threshold;
}
if (allSettings.kb_writepath_top_k !== undefined) {
kbWritePathTopK.value = allSettings.kb_writepath_top_k;
} else if (allSettings.kb_autoinject_top_k !== undefined) {
// The write path shared auto-inject's ceiling until it was given its own
// key, so an install that tuned the shared one must keep seeing that value
// here — the server falls back the same way.
kbWritePathTopK.value = allSettings.kb_autoinject_top_k;
}
if (allSettings.kb_rulehint_top_k !== undefined) {
kbRuleHintTopK.value = allSettings.kb_rulehint_top_k;
}
if (allSettings.kb_toolrule_top_k !== undefined) {
kbToolRuleTopK.value = allSettings.kb_toolrule_top_k;
}
if (allSettings.kb_promptrule_top_k !== undefined) {
kbPromptRuleTopK.value = allSettings.kb_promptrule_top_k;
}
if (allSettings.kb_reportpref_top_k !== undefined) {
kbReportPrefTopK.value = allSettings.kb_reportpref_top_k;
}
await loadTuningHistory();
if (allSettings.kb_duplicate_threshold_snippet !== undefined) {
kbDupThresholdSnippet.value = allSettings.kb_duplicate_threshold_snippet;
}
@@ -1510,13 +1605,27 @@ async function deleteUser(userId: number) {
Stricter than the prompt threshold above on purpose. Any two pieces of
code look somewhat alike shared keywords, indentation, structure so
resemblance scores start higher for code than for prose, and a bar tuned
for prompts flags unrelated code as prior art. Lower this if genuine
duplicates go unnoticed; raise it if you're being offered snippets that
have nothing to do with what's being written. Snippets recorded at the
for prompts flags unrelated code as prior art. Claude keeps this
one current from what the arm actually surfaced and refused; set it
yourself if you disagree with where it has landed. Snippets recorded at the
exact file are always shown regardless those are prior art by
location, not by resemblance.
</p>
</div>
<div class="field">
<label for="kb-writepath-topk">Max prior-art lines per edit</label>
<input
id="kb-writepath-topk"
v-model="kbWritePathTopK"
type="number"
min="1"
max="10"
step="1"
class="fs-input input"
style="max-width: 8rem"
/>
<p class="field-hint">How many snippets and issues one edit may be offered (110). Reach for this rather than the threshold when the hint feels long: lowering it keeps the best matches and drops the tail, while raising the bar above throws away good matches along with weak ones.</p>
</div>
<div class="field">
<label for="kb-rulehint-threshold">Standing-rule confidence threshold (01)</label>
<input
@@ -1535,10 +1644,25 @@ async function deleteUser(userId: number) {
preloaded any more, so this is the only way a rule reaches a write.
Stricter than the threshold above, because there are far fewer rules
than snippets: with a small set something always ranks first, so the
bar has to carry more of the judgement. Raise it if rules keep
arriving unread; lower it if a rule you needed never showed up.
bar has to carry more of the judgement. If rules arrive unread, or one
you needed never showed up, that is Claude's to notice and correct
from the telemetry — and the budget below is usually the better lever.
</p>
</div>
<div class="field">
<label for="kb-rulehint-topk">Max rules per edit</label>
<input
id="kb-rulehint-topk"
v-model="kbRuleHintTopK"
type="number"
min="1"
max="10"
step="1"
class="fs-input input"
style="max-width: 8rem"
/>
<p class="field-hint">How many standing rules one edit may be shown (110).</p>
</div>
<div class="field">
<label for="kb-toolrule-threshold">Command confidence threshold (01)</label>
<input
@@ -1556,10 +1680,23 @@ async function deleteUser(userId: number) {
query is the command text rather than code. Lower than the one above
on purpose: a shell command is short, so it scores lower for the same
relevance — at a shared bar this arm spoke on 2% of calls against the
write path's 37%. Raise it if commands attract rules that do not
apply; lower it if a <code>git push</code> arrives with nothing.
write path's 37%.
</p>
</div>
<div class="field">
<label for="kb-toolrule-topk">Max rules per command</label>
<input
id="kb-toolrule-topk"
v-model="kbToolRuleTopK"
type="number"
min="1"
max="10"
step="1"
class="fs-input input"
style="max-width: 8rem"
/>
<p class="field-hint">How many standing rules one command may be shown (110). This is the busiest arm there is it fires before every command so its budget is the one most worth keeping small.</p>
</div>
<div class="field">
<label for="kb-promptrule-threshold">Prompt confidence threshold (01)</label>
<input
@@ -1580,6 +1717,20 @@ async function deleteUser(userId: number) {
not a command or a file which is why it carries its own number.
</p>
</div>
<div class="field">
<label for="kb-promptrule-topk">Max rules per prompt</label>
<input
id="kb-promptrule-topk"
v-model="kbPromptRuleTopK"
type="number"
min="1"
max="10"
step="1"
class="fs-input input"
style="max-width: 8rem"
/>
<p class="field-hint">How many rules or preferences one message may be shown (110).</p>
</div>
<div class="field">
<label for="kb-reportpref-threshold">Completion-report confidence threshold (01)</label>
<input
@@ -1597,11 +1748,66 @@ async function deleteUser(userId: number) {
written</em>, looked up when a task closes. Unlike every other bar
here, the question this arm asks never changes so its score is
fixed by your preferences alone, and it will either always find one
or never find one. If you have written a preference for report shape
and it is not arriving, lower this; there is no run of calls that
will reveal the problem on its own.
or never find one, and no run of calls will reveal a dead one on its
own. That is why this arm is worth looking up in the panel below when
a report preference never seems to arrive.
</p>
</div>
<div class="field">
<label for="kb-reportpref-topk">Max report preferences</label>
<input
id="kb-reportpref-topk"
v-model="kbReportPrefTopK"
type="number"
min="1"
max="10"
step="1"
class="fs-input input"
style="max-width: 8rem"
/>
<p class="field-hint">How many preferences a finished task may be shown (110).</p>
</div>
<!-- THE REVIEW SURFACE (#4102). These numbers are maintained by the
model that uses them: it reads which records each bar refused and
moves the dial with the argument attached. This panel is the other
half of that bargain a change made on your behalf is one you can
read, disagree with, and set back by hand above. -->
<div class="tuning-history">
<h4 class="tuning-history-title">What has been tuned</h4>
<p class="field-hint">
Claude adjusts the bars and budgets above from what each arm actually
surfaced and refused, and has to state a reason to change one. Nothing
here needs your attention as a matter of course it is here so that
when a surface behaves oddly, why it is set the way it is can be read
rather than guessed at. Anything you change yourself is recorded the
same way, and is the one entry Claude will not quietly move back.
</p>
<p v-if="loadingTuning" class="field-hint">Loading</p>
<p v-else-if="!tuningEvents.length" class="field-hint">
Nothing has been changed yet every surface is on the value Scribe
shipped. Those are starting points measured against one corpus with
one embedding model, not answers, so expect this to fill.
</p>
<ul v-else class="tuning-list">
<li v-for="ev in tuningEvents" :key="ev.id" class="tuning-item">
<div class="tuning-item-head">
<span class="tuning-surface">{{ ev.surface }}</span>
<span class="tuning-dial">{{ ev.dial }}</span>
<span class="tuning-move">
<template v-if="ev.old_value !== null">{{ ev.old_value }} </template>
<template v-else>set to</template>
{{ ev.new_value }}
</span>
<span class="tuning-actor" :class="{ 'is-human': ev.actor === 'human' }">
{{ ev.actor === 'human' ? 'you' : 'Claude' }}
</span>
<span v-if="ev.created_at" class="tuning-when">{{ fmtDate(ev.created_at) }}</span>
</div>
<p class="tuning-reason">{{ ev.reason }}</p>
</li>
</ul>
</div>
<!-- A design system belongs to a PROJECT, and the picker for it lives on
the project. There was a setting here that designated the system
this install's own interface was built from; it only ever described
@@ -3848,4 +4054,65 @@ async function deleteUser(userId: number) {
.area-admin-form { display: flex; flex-direction: column; gap: 0.5rem; flex: 1; }
.area-admin-create { margin-top: var(--fs-space-4); }
.area-admin-actions { display: flex; gap: 0.4rem; }
/* The retrieval tuning trail (#4102). Reads as a record, not a control panel:
the operator is reviewing what was done, and the reason is the part worth
reading, so it gets the full-width line under a compact header row. */
.tuning-history {
margin-top: var(--fs-space-5);
padding-top: var(--fs-space-4);
border-top: 1px solid var(--fs-border-color);
}
.tuning-history-title {
margin: 0 0 var(--fs-space-2);
font-size: 0.95rem;
color: var(--fs-text-primary);
}
.tuning-list {
list-style: none;
margin: var(--fs-space-3) 0 0;
padding: 0;
display: flex;
flex-direction: column;
gap: var(--fs-space-2);
}
.tuning-item {
padding: var(--fs-space-3);
background: var(--fs-surface-raised);
border: 1px solid var(--fs-border-color);
border-radius: var(--fs-radius-md);
}
.tuning-item-head {
display: flex;
align-items: baseline;
flex-wrap: wrap;
gap: var(--fs-space-2);
font-size: 0.85rem;
}
.tuning-surface {
font-family: var(--fs-font-mono);
color: var(--fs-text-primary);
}
.tuning-dial,
.tuning-move {
color: var(--fs-text-secondary);
}
.tuning-actor {
color: var(--fs-text-tertiary);
border: 1px solid var(--fs-border-color);
border-radius: var(--fs-radius-sm);
padding: 0 0.4rem;
}
/* The operator's own changes are marked, because "did I do this, or did the
session?" is the first question this list is asked. */
.tuning-actor.is-human {
color: var(--fs-accent);
border-color: var(--fs-accent);
}
.tuning-when { margin-left: auto; color: var(--fs-text-tertiary); }
.tuning-reason {
margin: var(--fs-space-2) 0 0;
font-size: 0.85rem;
color: var(--fs-text-secondary);
}
</style>
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "scribe",
"description": "Scribe for Claude Code: connects the scribe MCP server, adds the hooks that deliver live project state and relevant records at the right moment, ships the shared client-neutral Scribe skills (using-scribe, writing-plans, reporting-back, systematic-debugging, verification, brainstorming, reusing-code, shape-accounting), and syncs your saved Scribe Processes as skills (/scribe:sync).",
"version": "2026.09.16.2102",
"version": "2026.09.17.0111",
"author": {
"name": "Bryan Van Deusen"
},
+44
View File
@@ -308,6 +308,50 @@ scribe_held_query() {
return 0
}
# Drop EVERY per-session ledger, matched by convention rather than listed (#4101).
#
# A LIST IS THE BUG. Until now the compact/clear branch named its files one at
# a time, and it named two of the five: `.rules.ids` and `.opened.ids` were
# cleared while `.ids` (notes), `.sync.ids` (shape signals) and `.derive.ids`
# survived. So milestone 386's defect — "a compaction destroys the context but
# not the ledger, so the most applicable records become permanently
# unreachable mid-session" — was fixed for rules and left standing on the note
# and snippet surfaces, which are the ones that fire most often.
#
# Nobody decided that. The list was written when rules were the only ledger
# that mattered and was never revisited when the others arrived, which is what
# a hand-maintained list of "things to remember to clean up" does. Adding three
# more `rm` lines would rebuild the same trap for the sixth ledger.
#
# So the rule is the NAME: a per-session ledger is `<sid>[.<kind>].ids`, and
# everything matching that goes. A new ledger following the convention is
# covered the day it is written, by nobody. One that does not follow it is a
# deliberate exception and has to say so.
#
# Scoped to `.ids` rather than `<sid>.*` so a marker that is REWRITTEN on
# compact rather than discarded can still live in these directories without
# being swept away by a glob that was never told about it. `<sid>.unreached`
# is the live example: it records that the instance could not be reached, not
# what the session holds, and #2932 needs it to outlive a compaction.
#
# TWO DIRECTORIES, WHICH IS ITS OWN LESSON. The first cut of this swept only
# `scribe-priorart` — every ledger named in the hooks was there, so the list
# looked complete. `scribe_autoinject.sh` keeps its note ledger in
# `scribe-autoinject`, so the arm that fires most (598 calls in five days) was
# the one the clear could not reach, and the fix read as finished. Hence the
# roster here rather than a path at the call site: one place to add to, and
# the tests read THIS string rather than a copy of it.
SCRIBE_LEDGER_DIRS="scribe-priorart scribe-autoinject"
scribe_clear_session_ledgers() {
local sid="$1" dir
[ -n "$sid" ] || return 0
for dir in $SCRIBE_LEDGER_DIRS; do
rm -f "${TMPDIR:-/tmp}/$dir/$sid"*.ids 2>/dev/null || true
done
return 0
}
# ---------------------------------------------------------------------------
# WHICH PROJECT IS THIS DIRECTORY'S? (#4085)
#
+36 -17
View File
@@ -77,9 +77,11 @@ source=$(printf '%s' "$event" | jq -r '.source // empty' 2>/dev/null) || source=
# this state are the ones that fire most often, which is to say the ones that
# apply most.
#
# The session id survives a compaction — the etag marker further down is
# rewritten on `compact` and keyed by session_id, which is only meaningful if
# the id is stable — so the stale ledger is genuinely found again, not orphaned.
# The session id survives a compaction — the ledgers are keyed by it and are
# still found under the same name afterwards, so a stale ledger is genuinely
# reached again rather than orphaned. (This used to point at an etag marker as
# the evidence for that; milestone 394 removed the preload the etag described,
# and `plugin_context.py` records its retirement.)
#
# CLEARED ON THE SOURCES THAT DESTROY CONTEXT, AND ONLY THOSE:
#
@@ -93,25 +95,42 @@ source=$(printf '%s' "$event" | jq -r '.source // empty' 2>/dev/null) || source=
# fork carries the conversation, so if it inherits the id the ledger
# is accurate, and if it gets a new one the file is empty anyway.
#
# ONLY the rules ledger. The same directory holds .ids / .sync.ids /
# .derive.ids for the note arms. Whether a surfaced NOTE should return after a
# compaction is a different question with a different answer, and leaving those
# alone is a decision rather than an oversight.
# EVERY LEDGER, AND THE NOTE ARMS NEEDED IT MOST (#4101).
#
# This used to clear the two rule ledgers by name and say, in a comment, that
# leaving `.ids` / `.sync.ids` / `.derive.ids` alone was "a decision rather than
# an oversight". Reading the note arms says otherwise, on two counts:
#
# - They are HARD exclusions. `exclude_ids` goes into `semantic_search_notes`
# itself, so a surfaced note is removed from the result set — it is not
# rendered as a reference the way #3750 made a repeated rule. There is no
# weaker form for it to fall back to.
# - They never AGE. #3751 gave the rules ledger a TTL precisely because
# salience decays without a context event; the note channels were left on a
# flat read.
#
# Hard plus permanent plus never cleared means a note surfaced in the first
# minute of a session is unreachable for the rest of it, through any number of
# compactions. That is milestone 386's original defect, alive on the arms that
# fire most often, and nothing about it was decided.
#
# THE LIST WAS THE BUG, so the fix is not a longer list. `scribe_clear_session_
# ledgers` matches on the naming convention — a per-session ledger is
# `<sid>[.<kind>].ids` — which covers all five and covers the sixth on the day
# it is written. Best-effort, like every other filesystem touch in these hooks:
# a ledger that cannot be removed costs a repeated exclusion, never a session.
#
# NOT swept: `<sid>.unreached`, which records that the instance was unreachable
# rather than what the session holds, and survives on purpose. The directories
# swept are `scribe_defs.sh`'s `SCRIBE_LEDGER_DIRS` — plural, because the
# auto-inject arm keeps its ledger somewhere else and a single-directory sweep
# missed exactly the arm that fires most.
case "$source" in
compact|clear)
sid=$(printf '%s' "$event" | jq -r '.session_id // empty' 2>/dev/null) || sid=""
if [ -n "$sid" ]; then
safe_sid=$(printf '%s' "$sid" | tr -c 'A-Za-z0-9._-' '_')
# Best-effort, like every other filesystem touch in these hooks: a ledger
# that cannot be removed costs a repeated exclusion, never a session.
rm -f "${TMPDIR:-/tmp}/scribe-priorart/${safe_sid}.rules.ids" 2>/dev/null || true
# BOTH ledgers, for one reason (#4100). `.opened.ids` records what the
# session read; a compaction is exactly the event that takes it away
# again. Clearing the naming ledger while keeping this one would leave
# the surfacing arms telling a freshly-summarised session "you opened
# it earlier" about a rule that is no longer anywhere in its context —
# a more confident version of the claim this milestone removed.
rm -f "${TMPDIR:-/tmp}/scribe-priorart/${safe_sid}.opened.ids" 2>/dev/null || true
scribe_clear_session_ledgers "$safe_sid"
fi
;;
esac
+2
View File
@@ -14,6 +14,7 @@ from scribe.routes.notes import notes_bp
from scribe.routes.milestones import milestones_bp
from scribe.routes.task_logs import task_logs_bp
from scribe.routes.projects import projects_bp
from scribe.routes.retrieval import retrieval_bp
from scribe.routes.settings import settings_bp
from scribe.routes.tasks import tasks_bp
from scribe.routes.groups import groups_bp
@@ -79,6 +80,7 @@ def create_app() -> Quart:
app.register_blueprint(milestones_bp)
app.register_blueprint(notes_bp)
app.register_blueprint(projects_bp)
app.register_blueprint(retrieval_bp)
app.register_blueprint(settings_bp)
app.register_blueprint(task_logs_bp)
app.register_blueprint(tasks_bp)
+9
View File
@@ -131,6 +131,12 @@ _READ_ONLY_TOOLS = frozenset({
# looked at names that had one (#3191). rule_history records a pull the way
# the getters above do.
"rules_due_for_verification", "rule_history",
# What each retrieval surface's floor and budget currently are, and what
# has been changed about them (#4102). Both pure reads; `tune_retrieval` is
# the write and is deliberately NOT here. Read access matters more than
# usual for these two: a session that cannot see the bar in force, or the
# reason it was last moved, is a session that will move it again blind.
"retrieval_surfaces", "retrieval_tuning_history",
})
# Every tool that WRITES, by name. Nothing reads this set at runtime — a tool
@@ -166,6 +172,9 @@ _WRITE_TOOLS = frozenset({
"create_rule", "create_project_rule", "update_rule", "move_rule", "delete_rule",
"create_preference", "update_preference",
"relate_rules", "unrelate_rules", "mark_rule_verified",
# retrieval tuning — a write in both senses: it moves the number the arm
# reads, and it appends the reason to the audit trail (#4102).
"tune_retrieval",
# trash
"restore", "purge_trash",
})
+3 -2
View File
@@ -5,14 +5,15 @@ to a FastMCP instance. `register_all(mcp)` is the single entry point called
from `mcp.server.build_mcp_server`.
"""
from scribe.mcp.tools import (
design_systems, milestones, notes, processes, projects, recent, repos, rulebooks, search, shapes,
snippets, systems, tags, tasks, trash,
design_systems, milestones, notes, processes, projects, recent, repos, retrieval_tuning,
rulebooks, search, shapes, snippets, systems, tags, tasks, trash,
)
def register_all(mcp) -> None:
"""Register every tool module's tools on the given FastMCP instance."""
search.register(mcp)
retrieval_tuning.register(mcp)
notes.register(mcp)
tasks.register(mcp)
projects.register(mcp)
+115
View File
@@ -0,0 +1,115 @@
"""Reading and moving a retrieval surface's floor and budget (#4102).
The write half of the loop `retrieval_telemetry` opens. That tool says what a
bar did; these say what the bar IS, and let it be changed with the argument
attached.
"""
from scribe.mcp._context import current_user_id
from scribe.services import retrieval_tuning as tuning_svc
async def retrieval_surfaces() -> dict:
"""Every retrieval surface Scribe pushes on, with its floor and its budget.
Read this before changing either, and read it beside
`retrieval_telemetry(days=…, near_miss_samples=5)` — this tool says what is
in force, that one says what it did.
Each surface carries what it ASKS, what corpus it asks OVER, and how often
it FIRES, because a floor cannot be moved sensibly without all three. An arm
firing before every Bash call is spending attention on a scale an arm firing
once a turn is not, and the same number means different things to a query
that is a shell command, a code payload, or an operator's sentence.
`floor` is a COST floor: is this worth ranking at all. It is not a relevance
judgement — relevance is decided by the reader, which is the only
participant that can read a record's trigger against the actual situation,
and every injected line says so out loud ("read it before deciding it does
not apply").
`budget` is what actually binds. It is how many lines this surface may spend
on one injection, and it is the control to reach for when a surface feels
noisy — lowering a budget removes the weakest candidates, while raising a
floor removes whichever candidates happen to sit under a number.
`last_change` carries the reason last given for each dial, or is absent when
the surface is still on its shipped starting point. Those starting points
are starting points: they were measured against one corpus with one
embedding model and cannot be right for another install by construction.
That is why this tool exists rather than a better set of defaults.
"""
return {"surfaces": await tuning_svc.current_settings(current_user_id())}
async def tune_retrieval(
surface: str, dial: str, value: float, reason: str, actor: str = "model",
) -> dict:
"""Move one surface's floor or budget, recording why.
DO NOT CALL THIS FROM A PERCENTILE ALONE. `retrieval_telemetry` gives
`near_misses.p90` — the mass sitting just under the bar — and that number
says nothing about whether the mass is RELEVANT. The two have been measured
disagreeing: one surface logged 69 consecutive declines with the refused
record 0.0006 under its bar, and every percentile said "lower it". The
refused record turned out to be a rule about interpreting a REQUEST, matched
against a query about report layout — a false positive. Lowering would have
attached that rule to every completion report ever written. The statistic
and the correct action pointed in opposite directions, and only opening the
record could tell.
So the procedure is: `retrieval_telemetry(near_miss_samples=5)`, then
`get_rule` / `get_note` the `record_id`s it names, and decide whether those
records SHOULD have surfaced for those queries. Then move the dial, and say
in `reason` what you read and what it showed.
`reason` is required and must be substantive. It is the guardrail, not
bookkeeping: it is what lets the operator review a change they did not make,
disagree with it, and revert it. A number that moved with no stated basis is
one nobody can audit — including you, next week.
WHICH DIAL. Reach for `budget` when the complaint is volume, and `floor`
when the complaint is quality. Raising a floor to quieten a surface throws
away its best candidates along with its worst, because a floor cannot tell
rank from relevance; lowering a budget keeps the top of the ranking and
drops the tail, which is usually what was wanted.
Args:
surface: the surface name from `retrieval_surfaces` — also its
`retrieval_telemetry` source, so the two always name the same arm.
dial: "floor" or "budget".
value: the new value. A floor is clamped to [0, 1], a budget to a whole
number in [1, 10]. The result says whether a clamp bit, which
matters: believing you set a value the registry corrected means
reading the next telemetry as evidence about a bar never in force.
reason: what you read and what it showed. Required.
actor: "model" (default) or "human" — pass "human" only when relaying a
value the operator chose themselves, so the audit trail can tell
a change they made from one made on their behalf.
"""
return await tuning_svc.set_dial(
current_user_id(), surface, dial, value, reason=reason, actor=actor,
)
async def retrieval_tuning_history(surface: str = "", limit: int = 20) -> dict:
"""What has been changed about retrieval on this install, newest first.
Reach for it before moving a dial that has been moved before — the previous
reason is the argument the next change has to answer, and a surface that has
been walked up and down repeatedly is evidence the floor is not the problem.
Args:
surface: restrict to one surface; omit for everything.
limit: how many events, default 20, capped at 200.
"""
return {
"events": await tuning_svc.tuning_history(
current_user_id(), surface=surface or None, limit=limit,
)
}
def register(mcp) -> None:
mcp.tool(name="retrieval_surfaces")(retrieval_surfaces)
mcp.tool(name="tune_retrieval")(tune_retrieval)
mcp.tool(name="retrieval_tuning_history")(retrieval_tuning_history)
+7 -5
View File
@@ -209,11 +209,13 @@ async def retrieval_telemetry(
) -> dict:
"""What the retrieval telemetry says about YOUR surfaces, over a window.
The read half of the loop the ranker's thresholds are meant to be tuned
from (#2975). Reach for it before changing a similarity threshold, a top-k,
or deciding whether a reranker is worth building — the alternative is
hand-probing the live instance, which is how the last such decision had to
be made.
The read half of the tuning loop, whose write half is `tune_retrieval`
(#2975, #4102). Reach for it before moving any floor or budget, and read
the records it names rather than its percentiles alone: this readout has
been measured pointing the WRONG WAY — 69 consecutive declines where every
percentile said "lower the bar" and the refused record was a false positive
— so `near_miss_samples=5` and opening the ids it returns is the step that
separates a real miss from a bar doing its job.
Three readouts, from the three tables built for them:
+1
View File
@@ -27,6 +27,7 @@ from scribe.models.password_reset import PasswordResetToken # noqa: E402, F401
from scribe.models.invitation import InvitationToken # noqa: E402, F401
from scribe.models.embedding import MilestoneEmbedding, NoteEmbedding, RuleEmbedding # noqa: E402, F401
from scribe.models.retrieval_log import RetrievalLog # noqa: E402, F401
from scribe.models.retrieval_tuning import RetrievalTuningEvent # noqa: E402, F401
from scribe.models.note_usage import NoteUsageEvent # noqa: E402, F401
from scribe.models.rule_usage import RuleUsageEvent # noqa: E402, F401
from scribe.models.project import Project # noqa: E402, F401
+97
View File
@@ -0,0 +1,97 @@
from datetime import datetime, timezone
from sqlalchemy import DateTime, Float, Index, Integer, Text
from sqlalchemy.orm import Mapped, mapped_column
from scribe.models import Base
class RetrievalTuningEvent(Base):
"""One row per change to a retrieval surface's floor or budget (#4102).
WHY A TABLE AND NOT JUST THE SETTING
The setting holds the number. It cannot hold the argument, and from this
step onward the argument is the interesting part: the operator's decision is
that the model moves these values, *"9 times out of 10"*, without being
asked each time. Delegation like that is only safe if it leaves a trail the
operator can read afterwards — what changed, from what to what, and on what
evidence — and can disagree with.
So `reason` is not decoration and not optional. It is the guardrail: a model
that has to state why is a model that has to have looked, and a reason that
turns out to be wrong is a reason someone can point at. A bare number in a
settings row records that something moved and nothing about whether it
should have.
WHY `actor` IS NOT JUST `user_id`
Both a person and a model act as the same user — the MCP tools run under the
operator's own id — so the id cannot distinguish them. That distinction is
exactly what an operator scanning this table wants first: "did I do this, or
did the session?" A change the operator made themselves needs no review; one
made on their behalf might.
FK-FREE ON user_id, like `RetrievalLog` and `AppLog` above it: the history of
how a surface was tuned should outlive the account that tuned it, and a
deleted user must not cascade away the record of why a number is where it is.
"""
__tablename__ = "retrieval_tuning_events"
id: Mapped[int] = mapped_column(primary_key=True)
# Declared here rather than via CreatedAtMixin because the index below
# orders on `created_at.desc()`, which needs the column object in this class
# body — a mixin's column is not in scope there.
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
)
user_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
# The surface's registry name, which is also its `retrieval_logs.source`.
# That identity is what lets a reader put a change next to what the change
# did; `tests/test_retrieval_surfaces.py` asserts it rather than trusting it.
surface: Mapped[str] = mapped_column(Text, nullable=False)
# "floor" or "budget". Text rather than an enum because rule 36 makes a new
# CHECK value a migration, and this is a young surface that may well grow a
# third tunable — a constraint bought nothing here except a future migration.
dial: Mapped[str] = mapped_column(Text, nullable=False)
# Both stored as float so one pair of columns serves both dials; a budget is
# a whole number and reads back as one. `old_value` is nullable because the
# first change to a surface has no stored predecessor — it moved off the
# shipped starting point, which is a different event from moving off a value
# somebody chose, and worth being able to tell apart.
old_value: Mapped[float | None] = mapped_column(Float, nullable=True)
new_value: Mapped[float] = mapped_column(Float, nullable=False)
# "model" | "human". See the class docstring: the user id cannot tell these
# apart, and the difference is the first thing a reviewer wants.
actor: Mapped[str] = mapped_column(Text, nullable=False, default="model")
# REQUIRED at the service boundary, not merely non-null here. A column that
# is technically non-null is satisfied by "", which is how a required field
# becomes a formality; the service refuses a blank one.
reason: Mapped[str] = mapped_column(Text, nullable=False)
__table_args__ = (
# The only read this table has: one surface's history, newest first.
Index(
"ix_retrieval_tuning_surface_created",
"surface",
created_at.desc(),
),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"created_at": self.created_at.isoformat() if self.created_at else None,
"surface": self.surface,
"dial": self.dial,
"old_value": self.old_value,
"new_value": self.new_value,
"actor": self.actor,
"reason": self.reason,
}
+106
View File
@@ -0,0 +1,106 @@
"""The operator's view of retrieval tuning — what the dials are, and who moved them (#4102).
WHY THIS EXISTS SEPARATELY FROM /api/settings
The numbers themselves are ordinary settings rows and could be written through
the generic KV endpoint. They must not be, and that is the whole point of this
blueprint: a floor changed through `/api/settings` moves silently, leaving the
history saying nothing happened. From milestone 416 those dials are moved by
the model on the operator's behalf, so a trail with holes in it is worse than
no trail — it reads as complete.
So every change to a retrieval dial goes through `set_dial`, from the UI as
much as from the MCP tool, and the only difference is the `actor` recorded.
"""
import logging
from quart import Blueprint, jsonify, request
from scribe.auth import get_current_user_id, login_required
from scribe.services.retrieval_tuning import set_dial, current_settings, tuning_history
logger = logging.getLogger(__name__)
retrieval_bp = Blueprint("retrieval", __name__, url_prefix="/api/retrieval")
# What is recorded when the operator changes a dial in Settings and says
# nothing about why.
#
# The MCP tool REFUSES a blank reason, and this endpoint does not, which is a
# deliberate asymmetry rather than an oversight. The requirement exists to make
# the model look at the records before it moves a number on someone else's
# behalf. The operator IS that someone: they are the audience the trail is
# written for, they cannot be uninformed about their own decision, and a
# mandatory justification textarea on every control would be friction charged
# to the one participant who owes no explanation (rule 24).
#
# The event is still written, because "the operator set this by hand" is the
# single most useful thing the history can tell a later session — it is the one
# entry the model must not quietly tune back.
_OPERATOR_DEFAULT_REASON = "Set directly in Settings by the operator."
@retrieval_bp.route("/surfaces", methods=["GET"])
@login_required
async def get_surfaces_route():
"""Every tunable surface: its live floor and budget, what it asks and over
what, and the reason each dial was last moved."""
uid = get_current_user_id()
surfaces = await current_settings(uid)
return jsonify({"surfaces": surfaces, "total": len(surfaces)})
@retrieval_bp.route("/surfaces/<surface>", methods=["PUT"])
@login_required
async def tune_surface_route(surface: str):
"""Move one dial, recorded as a human change.
`actor` is fixed here rather than taken from the payload: this endpoint is
reached with a session cookie from the Settings form, so the actor is known
and accepting a claim about it would let the one field a reviewer relies on
be set to anything.
"""
uid = get_current_user_id()
data = await request.get_json()
if not isinstance(data, dict):
return jsonify({"error": "Expected a JSON object"}), 400
if "dial" not in data or "value" not in data:
return jsonify({"error": "dial and value are required"}), 400
try:
value = float(data["value"])
except (TypeError, ValueError):
return jsonify({"error": f"value must be a number, got {data['value']!r}"}), 400
reason = str(data.get("reason") or "").strip() or _OPERATOR_DEFAULT_REASON
try:
result = await set_dial(
uid, surface, str(data["dial"]), value, reason=reason, actor="human",
)
except ValueError as e:
# Unknown surface, unknown dial — the service names the alternatives,
# so the message is worth passing through rather than flattening.
return jsonify({"error": str(e)}), 400
return jsonify(result)
@retrieval_bp.route("/tuning-history", methods=["GET"])
@login_required
async def tuning_history_route():
"""What has been changed about retrieval on this install, newest first.
The review surface. Unscoped by default because the operator's question
here is "what has been done on my behalf", not "why is this one arm set
like that" — the per-surface scoping is for the model.
"""
uid = get_current_user_id()
surface = request.args.get("surface") or None
try:
limit = int(request.args.get("limit", 50))
except ValueError:
return jsonify({"error": "limit must be a whole number"}), 400
try:
events = await tuning_history(uid, surface=surface, limit=limit)
except ValueError as e:
return jsonify({"error": str(e)}), 400
return jsonify({"events": events, "total": len(events)})
+36
View File
@@ -9,6 +9,8 @@ from quart import Blueprint, jsonify, request
from scribe.auth import login_required, get_current_user_id
from scribe.config import Config
from scribe.services.retrieval_surfaces import dial_for_key, get_surface
from scribe.services.retrieval_tuning import set_dial
from scribe.services.settings import (
SECRET_MASK, delete_setting, get_all_settings, get_setting, set_settings_batch,
)
@@ -25,6 +27,13 @@ settings_bp = Blueprint("settings", __name__, url_prefix="/api/settings")
# (forge_token left with 0078: forge credentials are keyring rows now, #2778.)
_SECRET_KEYS = frozenset({"smtp_password", "forge_webhook_secret"})
# What the tuning history records for a dial changed through this form. The MCP
# tool refuses a blank reason; the operator is not asked for one, because the
# requirement exists to make the MODEL read the records before moving a number
# on someone else's behalf — and the operator is that someone. See
# routes/retrieval.py, which states the asymmetry in full.
_SETTINGS_FORM_REASON = "Changed in Settings by the operator."
def _masked(settings: dict) -> dict:
return {
@@ -52,6 +61,33 @@ async def update_settings_route():
to_save = {}
for k, v in data.items():
str_v = str(v)
# A retrieval dial is never written as a plain settings row, wherever
# the request came from (#4102). The value would land correctly and the
# tuning history would say nothing happened — and since milestone 416
# those dials are moved by the model on the operator's behalf, a
# history with holes in it is worse than none: it reads as complete.
#
# Routed rather than refused on purpose. Refusing would work only for
# callers that had been updated; this way every caller that ever writes
# one of these keys — this form, a script, an old client — leaves the
# trail, and there is no version of "forgot to use the other endpoint".
dial = dial_for_key(k)
if dial:
surface, which = dial
s = get_surface(surface)
# A CLEARED control means "back to the shipped starting point", and
# that is a change like any other — it is the operator reverting
# something, which is the single most important move this history
# can record. So it is written as an explicit set to the default
# rather than deleted, which would leave the same value behind and
# no record of anyone having chosen it.
default = s.floor_default if which == "floor" else s.budget_default
try:
await set_dial(uid, surface, which, float(str_v or default),
reason=_SETTINGS_FORM_REASON, actor="human")
except (TypeError, ValueError) as e:
return jsonify({"error": f"{k}: {e}"}), 400
continue
# A masked secret round-tripping through a client is "unchanged", not
# a request to store the mask over the real credential.
if k in _SECRET_KEYS and str_v == SECRET_MASK:
+83 -1
View File
@@ -13,6 +13,7 @@ from scribe.models.rule_version import RuleVersion
from scribe.models.design_system import DesignSystem, DesignToken
from scribe.models.note_usage import NoteUsageEvent
from scribe.models.rule_usage import RuleUsageEvent
from scribe.models.retrieval_tuning import RetrievalTuningEvent
from scribe.models.canonical_system import CanonicalSystem
from scribe.models.rulebook import RuleRelation, rule_systems as rule_systems_t
from scribe.models.code_shape import CodeShape, CodeShapeEvent, CodeShapeUse
@@ -67,8 +68,14 @@ logger = logging.getLogger(__name__)
# topic_suppressions with their tables (milestone 414): a rule's scope is its
# home now. Older archives carrying those sections still restore — the keys are
# simply not read — as do the subscribe_rulebooks inception choices they hold.
# v16 (2026-09) added retrieval_tuning_events (milestone 416): the REASON each
# retrieval floor and budget is where it is. `settings` already carried the
# numbers, so leaving this behind would restore six moved dials with the
# argument for them silently dropped — and from this step on those dials are
# moved by the model, which is exactly the case where the operator needs the
# argument to review.
# Bump when the serialized schema changes.
BACKUP_VERSION = 15
BACKUP_VERSION = 16
# Every table this backup carries, by its REAL name. Paired with _NOT_INCLUDED
# below, these two lists must together account for the entire schema — which is
@@ -100,6 +107,12 @@ _BACKED_UP = [
# accumulated, so a restore that dropped it would silently reset the
# measurement to zero while everything still looked fine.
"rule_usage_events",
# v16 (2026-09): why each retrieval floor and budget is where it is
# (milestone 416). The values live in `settings` and already travelled; the
# argument for them had nowhere to go. Now that the model moves these dials
# on the operator's behalf, a restore that kept the numbers and dropped the
# reasons would leave an install tuned by nobody it can name.
"retrieval_tuning_events",
]
# Tables intentionally NOT in the backup, surfaced in the payload so the gap is
@@ -189,6 +202,9 @@ _COLUMN_EXCLUSIONS: dict[str, set[str]] = {
"note_usage_events": {"id"},
# Same as the note twin: the surrogate key is re-issued on insert.
"rule_usage_events": {"id"},
# Same again — and everything else travels, because each remaining column
# is part of the argument: what moved, from what, to what, by whom, why.
"retrieval_tuning_events": {"id"},
"design_systems": {"deleted_at", "deleted_batch_id", "created_at", "updated_at"},
"design_tokens": {"deleted_at", "deleted_batch_id", "created_at", "updated_at"},
"repo_bindings": {"id", "created_at", "updated_at"},
@@ -329,6 +345,25 @@ def _rule_usage_event_rows(rows) -> list[dict]:
]
def _retrieval_tuning_event_rows(rows) -> list[dict]:
"""The record of why a retrieval dial is where it is (#4102).
Not `to_dict()`: that method serves the MCP reader, which already knows
whose install it is asking about, so it omits `user_id`. A restore has to
remap it, and a row that arrived without it would have to be dropped or
guessed at.
"""
return [
{
"user_id": r.user_id, "surface": r.surface, "dial": r.dial,
"old_value": r.old_value, "new_value": r.new_value,
"actor": r.actor, "reason": r.reason,
"created_at": r.created_at.isoformat() if r.created_at else None,
}
for r in rows
]
def _code_shape_rows(rows) -> list[dict]:
return [r.to_dict() for r in rows]
@@ -617,6 +652,12 @@ async def export_full_backup() -> dict:
rule_usage_events = (
await session.execute(select(RuleUsageEvent))
).scalars().all()
# Oldest first, so a restored history reads in the order the dials
# actually moved — the sequence IS the argument when a surface has been
# walked up and down.
retrieval_tuning_events = (await session.execute(
select(RetrievalTuningEvent).order_by(RetrievalTuningEvent.id)
)).scalars().all()
repo_bindings = (await session.execute(select(RepoBinding))).scalars().all()
code_shapes = (await session.execute(select(CodeShape))).scalars().all()
code_shape_events = (await session.execute(
@@ -661,6 +702,9 @@ async def export_full_backup() -> dict:
"design_tokens": _design_token_rows(design_tokens),
"note_usage_events": _usage_event_rows(usage_events),
"rule_usage_events": _rule_usage_event_rows(rule_usage_events),
"retrieval_tuning_events": _retrieval_tuning_event_rows(
retrieval_tuning_events
),
"repo_bindings": _repo_binding_rows(repo_bindings),
"note_supersessions": _note_supersession_rows(supersessions),
"code_shapes": _code_shape_rows(code_shapes),
@@ -795,6 +839,15 @@ async def export_user_backup(user_id: int) -> dict:
rule_usage_events = (await session.execute(
select(RuleUsageEvent).where(RuleUsageEvent.rule_id.in_(_rule_ids))
)).scalars().all() if _rule_ids else []
# Scoped on user_id, and here that IS the right column — unlike the
# rule usage events directly above. These record changes to this user's
# OWN retrieval settings, which is what `user_id` means on this table;
# there is no second owner to route around.
retrieval_tuning_events = (await session.execute(
select(RetrievalTuningEvent)
.where(RetrievalTuningEvent.user_id == user_id)
.order_by(RetrievalTuningEvent.id)
)).scalars().all()
rule_relations = (await session.execute(
select(RuleRelation).where(
RuleRelation.from_rule_id.in_(_rule_ids),
@@ -836,6 +889,9 @@ async def export_user_backup(user_id: int) -> dict:
"design_tokens": _design_token_rows(design_tokens),
"note_usage_events": _usage_event_rows(usage_events),
"rule_usage_events": _rule_usage_event_rows(rule_usage_events),
"retrieval_tuning_events": _retrieval_tuning_event_rows(
retrieval_tuning_events
),
"repo_bindings": _repo_binding_rows(repo_bindings),
"note_supersessions": _note_supersession_rows(supersessions),
"code_shapes": _code_shape_rows(code_shapes),
@@ -975,6 +1031,7 @@ async def _restore_v2(data: dict) -> dict:
"note_supersessions": 0, "code_shapes": 0, "code_shape_events": 0,
"code_shape_uses": 0, "canonical_systems": 0,
"rule_systems": 0, "rule_relations": 0, "rule_versions": 0,
"retrieval_tuning_events": 0,
}
async with async_session() as session:
@@ -1169,6 +1226,31 @@ async def _restore_v2(data: dict) -> dict:
session.add(Setting(user_id=mapped_uid, key=s_data["key"], value=s_data.get("value", "")))
stats["settings"] += 1
# 8b. Retrieval tuning history (v16) — restored beside the settings it
# explains, and for the same reason: the numbers above are the state,
# these rows are the argument for it. From milestone 416 those dials
# move on the operator's behalf, so an install restored with the values
# and without the reasons is one tuned by nobody it can name.
#
# No id remapping beyond the user: `surface` is a registry NAME, not a
# foreign key, which is what lets this history survive a restore into
# an install whose row ids all differ.
for t_data in data.get("retrieval_tuning_events", []):
mapped_uid = user_id_map.get(t_data.get("user_id") or 0)
if mapped_uid is None:
continue
session.add(RetrievalTuningEvent(
user_id=mapped_uid,
surface=t_data.get("surface", ""),
dial=t_data.get("dial", ""),
old_value=t_data.get("old_value"),
new_value=t_data.get("new_value"),
actor=t_data.get("actor") or "model",
reason=t_data.get("reason", ""),
created_at=_dt(t_data.get("created_at")),
))
stats["retrieval_tuning_events"] += 1
# 9. Rulebooks (v3)
for rb_data in data.get("rulebooks", []):
mapped_uid = user_id_map.get(rb_data.get("owner_user_id", 0))
+240 -133
View File
@@ -31,6 +31,12 @@ from scribe.services.embeddings import semantic_search_notes, semantic_search_ru
from scribe.services.note_usage import record_surfaced
from scribe.services.rule_usage import record_rule_surfaced
from scribe.services.supersession import superseded_ids
from scribe.services.retrieval_surfaces import (
MAX_BUDGET,
SURFACES,
budget_for,
floor_for,
)
from scribe.services.retrieval_telemetry import record_retrieval
from scribe.services.settings import get_setting
@@ -50,16 +56,25 @@ _GOAL_CHARS = 200
# Per-user settings (keys live in the generic settings table). The threshold is
# deliberately STRICTER than the pull-search default (embeddings
# DEFAULT_SIMILARITY_THRESHOLD = 0.45): an unsolicited per-turn inject must clear
# a higher bar than a search the agent chose to run. Defaults start conservative
# and are meant to be tuned from retrieval_logs (source='auto_inject') once data
# accrues — they're exposed in the Settings UI, no restart needed.
# a higher bar than a search the agent chose to run.
#
# The defaults below are STARTING POINTS, and correcting them is the model's
# job, not the operator's (#4102): `retrieval_surfaces` says what is in force,
# `retrieval_telemetry(near_miss_samples=N)` says what it refused, and
# `tune_retrieval` moves it with the reason attached. The operator can set any
# of them in Settings and their change is recorded the same way — but nobody
# has to read a log to get correct behaviour out of this.
AUTOINJECT_ENABLED_KEY = "kb_autoinject_enabled"
AUTOINJECT_THRESHOLD_KEY = "kb_autoinject_threshold"
AUTOINJECT_TOP_K_KEY = "kb_autoinject_top_k"
# The key and the value both live in the registry now (#4102); these names
# survive because the comments above them are where each number's measurement
# is recorded, and because the Settings UI agreement test pairs them with the
# Vue refs by name. One definition, two readable places.
AUTOINJECT_THRESHOLD_KEY = SURFACES["auto_inject"].floor_key
AUTOINJECT_TOP_K_KEY = SURFACES["auto_inject"].budget_key
AUTOINJECT_DEFAULT_ENABLED = True
AUTOINJECT_DEFAULT_THRESHOLD = 0.55
AUTOINJECT_DEFAULT_TOP_K = 3
AUTOINJECT_DEFAULT_THRESHOLD = SURFACES["auto_inject"].floor_default
AUTOINJECT_DEFAULT_TOP_K = SURFACES["auto_inject"].budget_default
# The write-path trigger (#2082) gets its own on/off switch, its own threshold,
# and shares only top-k. It originally shared the threshold too, on the argument
@@ -83,9 +98,9 @@ AUTOINJECT_DEFAULT_TOP_K = 3
# pull-through (#2085) once a real corpus accrues; a cross-encoder rerank
# (#1038) would subsume this bump.
WRITEPATH_ENABLED_KEY = "kb_writepath_enabled"
WRITEPATH_THRESHOLD_KEY = "kb_writepath_threshold"
WRITEPATH_THRESHOLD_KEY = SURFACES["write_path"].floor_key
WRITEPATH_DEFAULT_ENABLED = True
WRITEPATH_DEFAULT_THRESHOLD = 0.68
WRITEPATH_DEFAULT_THRESHOLD = SURFACES["write_path"].floor_default
# The standing-rule arm (milestone 307) gets its own bar — the split #2223 made
# one surface down, now made for the THIRD corpus. It inherited 0.68 above, and
@@ -133,8 +148,8 @@ WRITEPATH_DEFAULT_THRESHOLD = 0.68
# case 0.72 was calibrated on, and the telemetry says it is working — the
# write-path rule arm speaks on 37% of its calls and its refused mass sits at
# p50 0.6989, comfortably under the bar rather than piled against it.
RULEHINT_THRESHOLD_KEY = "kb_rulehint_threshold"
RULEHINT_DEFAULT_THRESHOLD = 0.72
RULEHINT_THRESHOLD_KEY = SURFACES["write_path_rule"].floor_key
RULEHINT_DEFAULT_THRESHOLD = SURFACES["write_path_rule"].floor_default
# THE COMMAND ARM'S OWN BAR, AND WHY IT IS NOT THE WRITE PATH'S (#3853).
#
@@ -193,8 +208,8 @@ RULEHINT_DEFAULT_THRESHOLD = 0.72
# eligible. Retrieval is ownership-scoped, not project-scoped. Scoping it
# would drop that ceiling and widen the 0.0115, which is the larger fix and
# the reason to settle project scoping before tuning this number twice.
TOOLRULE_THRESHOLD_KEY = "kb_toolrule_threshold"
TOOLRULE_DEFAULT_THRESHOLD = 0.68
TOOLRULE_THRESHOLD_KEY = SURFACES["pre_tool_rule"].floor_key
TOOLRULE_DEFAULT_THRESHOLD = SURFACES["pre_tool_rule"].floor_default
# A SET OF RULES PER ACT, NOT THE SINGLE BEST ONE (#3851).
#
@@ -223,7 +238,10 @@ TOOLRULE_DEFAULT_THRESHOLD = 0.68
# and nothing else, so a moment with one clearly-relevant rule still shows
# one, and a moment with four shows four. The corpus decides, not a constant.
# The cap survives as a ceiling on the worst case, not as the usual answer.
RULEHINT_LIMIT = 5
# NOW A DEFAULT RATHER THAN A CAP (#4102): both rule act-arms read their own
# budget from the registry, so this is the value an install starts at and not
# the value it is stuck with. The reasoning below is why 5 is where it starts.
RULEHINT_LIMIT = SURFACES["write_path_rule"].budget_default
# MEASURED, NOT REASONED — and the reasoning it replaced was wrong (#3851).
#
@@ -433,7 +451,11 @@ _CONCEPT_MIN_CHARS = 16
_AUTOINJECT_BAND = 0.10
# Hard ceiling on top-k regardless of the user's setting — this is an
# awareness menu (titles only), never a content dump.
_AUTOINJECT_MAX_TOP_K = 10
# The budget ceiling, now shared by every surface rather than owned by this
# one (#4102). It bounds the same thing everywhere — how many lines a single
# unsolicited injection may occupy — so one surface having a private ceiling
# was an accident of which arm got a configurable budget first.
_AUTOINJECT_MAX_TOP_K = MAX_BUDGET
# --- the prompt-boundary rule arm (#3852) ------------------------------------
#
@@ -446,7 +468,7 @@ _AUTOINJECT_MAX_TOP_K = 10
# The operator's message is the only query that exists before one is composed,
# and this arm is what runs against it. Until now that hook searched notes
# alone, so no rule had ever been retrieved against a thing the operator said.
PROMPTRULE_THRESHOLD_KEY = "kb_promptrule_threshold"
PROMPTRULE_THRESHOLD_KEY = SURFACES["prompt_rule"].floor_key
# INHERITED FROM THE ACT ARMS, AND NOT YET EARNED HERE. 0.72 was tuned against
# code and shell commands. An operator's prose is a different query shape
# against the same documents, and nothing yet says the two distributions line
@@ -458,7 +480,7 @@ PROMPTRULE_THRESHOLD_KEY = "kb_promptrule_threshold"
# front of a corpus that binds. Every call is logged under `prompt_rule` from
# the first deploy, so a few days of real traffic settles it — read
# `near_miss_samples` (#3807) before moving this, not the percentile alone.
PROMPTRULE_DEFAULT_THRESHOLD = 0.72
PROMPTRULE_DEFAULT_THRESHOLD = SURFACES["prompt_rule"].floor_default
# MORE THAN THE ACT ARMS' SINGLE SLOT, anchored on this hook's budget rather
# than theirs. RULEHINT_LIMIT is 1 because that arm fires before EVERY Bash
@@ -469,7 +491,7 @@ PROMPTRULE_DEFAULT_THRESHOLD = 0.72
# And a prompt genuinely contains more than one act. "Merge to main and then
# start on X" is two, governed by different rules; k=1 cannot serve that case
# at all, where the act arms never face it because a command is one thing.
PROMPTRULE_LIMIT = 3
PROMPTRULE_LIMIT = SURFACES["prompt_rule"].budget_default
# THE COMPLETION-REPORT ARM'S OWN BAR (services/reply_preferences.py).
#
@@ -487,8 +509,8 @@ PROMPTRULE_LIMIT = 3
# Kept at the prose arm's starting value rather than tuned: the split is what
# makes the two independently movable, and a default is a product decision
# that this install's corpus cannot settle (rule 115).
REPORTPREF_THRESHOLD_KEY = "kb_reportpref_threshold"
REPORTPREF_DEFAULT_THRESHOLD = 0.72
REPORTPREF_THRESHOLD_KEY = SURFACES["report_preference"].floor_key
REPORTPREF_DEFAULT_THRESHOLD = SURFACES["report_preference"].floor_default
def _slugify(text: str) -> str:
@@ -592,8 +614,17 @@ async def build_process_manifest(user_id: int) -> dict:
async def get_autoinject_config(user_id: int) -> dict:
"""Resolve a user's auto-inject settings, falling back to the defaults.
Returns {"enabled": bool, "threshold": float, "top_k": int}, clamped to
sane ranges (threshold to [0,1]; top_k to [1, _AUTOINJECT_MAX_TOP_K]).
Returns {"enabled": bool, "threshold": float, "top_k": int}.
THE TWO NUMBERS COME FROM THE REGISTRY NOW (#4102). They used to be read and
clamped here, and identically again in `get_writepath_config`, and again in
three rule arms, and once more in `reply_preferences`. That was
tolerable while the values were shipped constants. It stops being tolerable
once a tool is expected to MOVE them, because a tuning surface cannot be
consistent across arms that each spell their configuration differently.
`enabled` stays here: it is this surface's own switch, not a tunable number,
and the registry deliberately holds only the pair a floor-tuner touches.
"""
enabled_raw = await get_setting(
user_id, AUTOINJECT_ENABLED_KEY,
@@ -601,21 +632,11 @@ async def get_autoinject_config(user_id: int) -> dict:
)
enabled = enabled_raw.strip().lower() in ("true", "1", "yes", "on")
try:
threshold = float(await get_setting(
user_id, AUTOINJECT_THRESHOLD_KEY, str(AUTOINJECT_DEFAULT_THRESHOLD)))
except (TypeError, ValueError):
threshold = AUTOINJECT_DEFAULT_THRESHOLD
threshold = min(1.0, max(0.0, threshold))
try:
top_k = int(float(await get_setting(
user_id, AUTOINJECT_TOP_K_KEY, str(AUTOINJECT_DEFAULT_TOP_K))))
except (TypeError, ValueError):
top_k = AUTOINJECT_DEFAULT_TOP_K
top_k = min(_AUTOINJECT_MAX_TOP_K, max(1, top_k))
return {"enabled": enabled, "threshold": threshold, "top_k": top_k}
return {
"enabled": enabled,
"threshold": await floor_for(user_id, "auto_inject"),
"top_k": await budget_for(user_id, "auto_inject"),
}
def _record_kind(note) -> str:
@@ -644,7 +665,7 @@ async def _reserve_slot_for_reuse(
cfg: dict,
*,
project_id: int | None,
exclude_ids: set[int],
already: set[int],
) -> list:
"""Guarantee the reuse-shaped kinds one slot, if one clears threshold (#2246).
@@ -671,16 +692,23 @@ async def _reserve_slot_for_reuse(
top_k = cfg["top_k"]
_t0 = time.perf_counter()
_rep: dict = {}
# THE LEDGER IS NOT AN EXCLUSION HERE EITHER (#4101) — but what is already
# in `kept` still is, and the two are different claims. A record sitting in
# this call's own menu must not be shown twice in it; a record shown in an
# EARLIER call is exactly what the slot should be allowed to spend itself
# on, because a snippet that was relevant then and is relevant now is the
# reuse case rather than a duplicate of it.
reuse = await semantic_search_notes(
user_id, query,
limit=1,
threshold=cfg["threshold"],
project_id=project_id,
exclude_ids=exclude_ids | {int(n.id) for _s, n in kept},
exclude_ids={int(n.id) for _s, n in kept},
note_type=_REUSE_KINDS,
scope="browse",
report=_rep,
)
fresh_reuse = [(s, n) for s, n in reuse if int(n.id) not in already]
# A real semantic query competing for a menu slot — logged like the scored
# arm it displaces. Before this, the hit it PUSHED OUT was in
# retrieval_logs and the query that pushed it out was not, so the slot
@@ -689,16 +717,17 @@ async def _reserve_slot_for_reuse(
record_retrieval(
user_id=user_id, source="reuse_slot", query=query,
threshold=cfg["threshold"], limit=1, project_id=project_id,
is_task=None, results=reuse,
is_task=None, results=fresh_reuse,
best_available=_rep.get("best_available_score"),
best_available_id=_rep.get("best_available_id"),
searched=bool(_rep.get("searched", True)),
suppressed=len(reuse) - len(fresh_reuse),
duration_ms=(time.perf_counter() - _t0) * 1000.0,
)
# Verify the kind rather than trusting the query that asked for it, and
# dedup on top of exclude_ids. This slot exists FOR reuse kinds — a slot
# silently spent on something else is worse than no slot, because the line
# is indistinguishable from one that earned its place on score.
# dedup against this call's own menu. This slot exists FOR reuse kinds — a
# slot silently spent on something else is worse than no slot, because the
# line is indistinguishable from one that earned its place on score.
kept_ids = {int(n.id) for _s, n in kept}
fresh = [
(s, n) for s, n in reuse
@@ -726,7 +755,8 @@ async def build_autoinject_hint(
The four anti-bloat gates (see the module + milestone-93 design):
1. high-confidence threshold (stricter than pull) — set per-user;
2. margin gate — keep only hits within _AUTOINJECT_BAND of the top score;
3. session dedup — caller passes already-injected ids as `exclude_ids`;
3. session marking — caller passes already-injected ids as `exclude_ids`
and they are rendered again with `[seen]`, never withheld (#4101);
4. title-first payload — id + kind + title + score only, never bodies.
Disabled, blank-query, or nothing-clears-the-gates all return empty context,
so most turns inject nothing.
@@ -741,6 +771,24 @@ async def build_autoinject_hint(
if not cfg["enabled"] or not q:
return empty
# THE LEDGER LEAVES THE SEARCH (#4101). `exclude_ids` used to go into
# `semantic_search_notes` itself, so a record this session had already been
# shown was removed from the candidate set — which is #3750's defect, on the
# arm that fires most. Three things followed from it, none of them intended:
#
# - the second time a note was the best answer, the session got SILENCE,
# indistinguishable from "nothing matched";
# - a compaction made that permanent, because the ledger outlived the
# context it described (fixed one layer down in this same step);
# - and the score the search reported was measured against a candidate set
# the caller had already edited, so `best_available` could name a bar
# that turned nothing away (#3739).
#
# Now the ledger is a RENDERING fact, not a retrieval one: every repeat is
# still ranked, still shown, and carries a marker saying it was surfaced
# before. A note line is a title and a score — the repeat costs about as
# much as the comma in this sentence — so there is nothing here to save.
already = {int(i) for i in (exclude_ids or [])}
t0 = time.perf_counter()
_rep_ai: dict = {}
hits = await semantic_search_notes(
@@ -748,7 +796,6 @@ async def build_autoinject_hint(
limit=cfg["top_k"],
threshold=cfg["threshold"],
project_id=(project_id or None),
exclude_ids=set(exclude_ids or []),
# Injection is the one retrieval nobody asked for, so it takes the BROWSE
# scope: never a record shared one-to-one with the operator. What can
# still appear is a collaborator's note inside a shared project — legible
@@ -756,24 +803,37 @@ async def build_autoinject_hint(
scope="browse",
report=_rep_ai,
)
# `results=fresh` and `suppressed` together, on the rule arms' contract
# (#3752): a rendered repeat is not a new surfacing, so it stays out of the
# row's result set and is COUNTED instead. That keeps this source's surfaced
# set identical to its own log row (#3668) while making a zero-result call
# readable — `result_count == 0` with `suppressed_count > 0` is "everything
# that matched, this session has already seen", which is a different fact
# about the bar from "nothing cleared it" and used to be unreportable here.
fresh = [(s, n) for s, n in hits if int(n.id) not in already]
record_retrieval(
user_id=user_id, source="auto_inject", query=q,
threshold=cfg["threshold"], limit=cfg["top_k"],
project_id=(project_id or None), is_task=None, results=hits,
project_id=(project_id or None), is_task=None, results=fresh,
best_available=_rep_ai.get("best_available_score"),
best_available_id=_rep_ai.get("best_available_id"),
searched=bool(_rep_ai.get("searched", True)),
suppressed=len(hits) - len(fresh),
duration_ms=(time.perf_counter() - t0) * 1000.0,
)
if not hits:
return empty
# Margin gate: keep only hits close to the strongest one.
# Margin gate: keep only hits close to the strongest one. Computed over ALL
# hits, repeats included — the band measures distance from the top SCORE,
# and letting the ledger move that cutoff would make "you were shown this"
# change what counts as relevant, which is the axis independence the rule
# band keeps for the same reason.
top_score = hits[0][0]
kept = [(s, n) for s, n in hits if s >= top_score - _AUTOINJECT_BAND]
kept = await _reserve_slot_for_reuse(
user_id, q, kept, cfg, project_id=(project_id or None),
exclude_ids=set(exclude_ids or []),
already=already,
)
# A collaborator's note can reach this menu via a shared project, and the
@@ -786,10 +846,15 @@ async def build_autoinject_hint(
# "records", not "notes" — the menu can hold snippets, processes and tasks
# too, and the kind marker on each line is only legible if the header doesn't
# already claim they're all one thing.
# "injected once per session" was true and is not any more (#4101): a repeat
# is shown again with a marker rather than withheld, so the header must stop
# promising the old contract. It now says what the marker means instead,
# once, rather than each repeated line having to explain itself.
lines = [
"> Possibly relevant from your Scribe records — open any in full with "
"`get_note(id)`, or `get_snippet` / `get_process` for those kinds "
"(titles only; injected once per session):",
"(titles only; a line marked `seen` was surfaced earlier this session "
"and may no longer be in context):",
]
# A superseded record is DEMOTED, not removed (#278) — so one can still reach
# this menu, and when it does the reader has to be told. An agent handed
@@ -802,6 +867,13 @@ async def build_autoinject_hint(
note_ids.append(int(note.id))
title = (note.title or "(untitled)").replace("\n", " ").strip()
line = f"> - #{note.id} [{_record_kind(note)}] \"{title}\" ({score:.2f})"
# ONE WORD, NOT A SENTENCE, and deliberately not the rule arms' phrasing.
# A rule line says "before deciding it does not apply", which is the
# voice of a record that BINDS; a note binds nothing, and borrowing that
# tone would tell the reader a dev-log has authority it does not have.
# The header carries the meaning, so the line carries only the flag.
if int(note.id) in already:
line += " [seen]"
if int(note.id) in stale:
line += " — SUPERSEDED, a later record covers this; check that first"
if note.user_id != user_id:
@@ -813,7 +885,17 @@ async def build_autoinject_hint(
# menu the agent actually saw. retrieval_logs already holds the full
# candidate set for threshold tuning; conflating the two would make
# "surfaced" mean two different things depending on the surface (#2085).
record_surfaced(user_id=user_id, note_ids=note_ids, source="auto_inject")
#
# FRESH ONLY, which is the same cut the log row above takes (#4101). A
# repeat is rendered but is not a new surfacing, and counting it again would
# make this table disagree with `retrieval_logs` about the same call —
# #3668's identity, which is the cheapest true statement available about
# this pair of tables and is not worth a marker's convenience.
record_surfaced(
user_id=user_id,
note_ids=[i for i in note_ids if i not in already],
source="auto_inject",
)
return {"context": "\n".join(lines), "note_ids": note_ids, "config": cfg}
@@ -963,13 +1045,8 @@ async def build_prompt_rule_hint(
return out
try:
try:
threshold = float(await get_setting(
user_id, PROMPTRULE_THRESHOLD_KEY,
str(PROMPTRULE_DEFAULT_THRESHOLD)))
except (TypeError, ValueError):
threshold = PROMPTRULE_DEFAULT_THRESHOLD
threshold = min(1.0, max(0.0, threshold))
threshold = await floor_for(user_id, "prompt_rule")
limit = await budget_for(user_id, "prompt_rule")
t0 = time.perf_counter()
_rep: dict = {}
@@ -980,7 +1057,7 @@ async def build_prompt_rule_hint(
# sessions — this surface speaks unasked, and a whole-rulebook answer
# is only right for someone who asked the whole rulebook.
hits = await semantic_search_rules(
user_id, q, limit=PROMPTRULE_LIMIT, threshold=threshold,
user_id, q, limit=limit, threshold=threshold,
report=_rep, project_id=project_id or None,
)
duration_ms = (time.perf_counter() - t0) * 1000.0
@@ -996,7 +1073,7 @@ async def build_prompt_rule_hint(
# and unverified for this corpus, so the zero rows are the point.
record_retrieval(
user_id=user_id, source="prompt_rule", query=q,
threshold=threshold, limit=PROMPTRULE_LIMIT,
threshold=threshold, limit=limit,
project_id=project_id,
is_task=None, results=fresh, duration_ms=duration_ms,
best_available=_rep.get("best_available_score"),
@@ -1233,52 +1310,37 @@ def concept_query(code: str) -> str:
async def get_writepath_config(user_id: int) -> dict:
"""Write-path trigger settings: its own `enabled` and `threshold`, auto-inject's top_k.
"""Write-path trigger settings and the two rule arms' numbers alongside.
The threshold OVERRIDES the inherited auto-inject value — code embeddings
have a much higher similarity floor than prose, so the two surfaces need
different bars. See WRITEPATH_DEFAULT_THRESHOLD for the measurements (#2223).
top_k is still shared: "how many titles at once" means the same thing on
both surfaces, and nothing suggests they want different ceilings.
Three surfaces' worth of configuration arrives in one call because one hook
request drives all three arms. They are still three SURFACES with three
independent pairs, resolved from the registry (#4102).
The floors were split apart one at a time, each on its own measurement, and
those measurements are recorded where the defaults are: WRITEPATH (#2223 —
code embeddings sit on a much higher similarity floor than prose), RULEHINT
(a third corpus again), TOOLRULE (#3853 — a shell command is a different
query shape from a code payload and scores lower for the same relevance).
THE BUDGET IS NOW SPLIT TOO. `top_k` used to be auto-inject's outright, on
the argument that "how many titles at once" means the same thing on both
surfaces. It does not: this arm fires before every Write and Edit while
auto-inject fires once a turn, so the same number buys wildly different
amounts of attention. `write_path` inherits auto-inject's value when it has
none of its own, so no install that tuned the shared knob loses it.
"""
cfg = await get_autoinject_config(user_id)
enabled_raw = await get_setting(
user_id, WRITEPATH_ENABLED_KEY,
"true" if WRITEPATH_DEFAULT_ENABLED else "false",
)
try:
threshold = float(await get_setting(
user_id, WRITEPATH_THRESHOLD_KEY, str(WRITEPATH_DEFAULT_THRESHOLD)))
except (TypeError, ValueError):
threshold = WRITEPATH_DEFAULT_THRESHOLD
threshold = min(1.0, max(0.0, threshold))
try:
rule_threshold = float(await get_setting(
user_id, RULEHINT_THRESHOLD_KEY, str(RULEHINT_DEFAULT_THRESHOLD)))
except (TypeError, ValueError):
rule_threshold = RULEHINT_DEFAULT_THRESHOLD
rule_threshold = min(1.0, max(0.0, rule_threshold))
try:
tool_rule_threshold = float(await get_setting(
user_id, TOOLRULE_THRESHOLD_KEY, str(TOOLRULE_DEFAULT_THRESHOLD)))
except (TypeError, ValueError):
tool_rule_threshold = TOOLRULE_DEFAULT_THRESHOLD
tool_rule_threshold = min(1.0, max(0.0, tool_rule_threshold))
return {
**cfg,
"enabled": enabled_raw.strip().lower() in ("true", "1", "yes", "on"),
"threshold": threshold,
# Its own bar, for a third corpus — see RULEHINT_DEFAULT_THRESHOLD.
"rule_threshold": rule_threshold,
# And the COMMAND arm's own bar again, for the same reason one level
# down: a shell command is a different query shape from a code payload
# and scores lower for the same relevance (#3853). Separate keys, so an
# install can move one without the other — which is the whole finding.
"tool_rule_threshold": tool_rule_threshold,
"threshold": await floor_for(user_id, "write_path"),
"top_k": await budget_for(user_id, "write_path"),
"rule_threshold": await floor_for(user_id, "write_path_rule"),
"rule_top_k": await budget_for(user_id, "write_path_rule"),
"tool_rule_threshold": await floor_for(user_id, "pre_tool_rule"),
"tool_rule_top_k": await budget_for(user_id, "pre_tool_rule"),
}
def _rule_band(hits: list) -> list:
@@ -1468,12 +1530,16 @@ async def build_write_path_hint(
#2707): the record gets corrected in the session that has the context,
at the moment of change. Nearby and semantic hits stay the REUSE menu.
The two classes dedup on SEPARATE channels — `exclude_ids` (reuse) and
`exclude_sync_ids` (sync) — because they answer different questions: a
title shown as "consider reusing this" twenty turns ago must not silence
"you are editing the recorded file right now" (#2708).
The two classes track the session on SEPARATE channels — `exclude_ids`
(reuse) and `exclude_sync_ids` (sync) — because they answer different
questions: a title shown as "consider reusing this" twenty turns ago must
not silence "you are editing the recorded file right now" (#2708). The two
channels also now ACT differently, which is #4101: a reuse repeat is
rendered again with a `seen` marker, while the sync class still shows
once, because its claim is about an edit in progress rather than about a
record's continuing relevance and repeating it would be nagging.
Carries auto-inject's anti-bloat gates (margin, session dedup,
Carries auto-inject's anti-bloat gates (margin, session marking,
titles-never-bodies) plus the shared top-k cap across ALL arms — so a file
with a lot of recorded history can't turn one edit into a wall of text. Two
gates are its OWN, because code is not prose: a stricter similarity
@@ -1546,7 +1612,13 @@ async def build_write_path_hint(
# stay eligible here, which is the whole point of the split. Either way
# they join `seen`, so the reuse arms (where the directory query would
# surface them again) never re-list a record the sync block owns.
seen: set[int] = set(excluded)
#
# `seen` IS THIS CALL'S OWN MENU, and nothing else (#4101). It used to start
# from `excluded` — the session ledger — which folded two different claims
# into one variable: "already listed a few lines above" and "shown at some
# earlier point in the session". Only the first is a reason to stay quiet.
# The ledger is now a marker instead, on both reuse arms.
seen: set[int] = set()
synced: list[dict] = []
for item in here:
nid = int(item["id"])
@@ -1560,7 +1632,12 @@ async def build_write_path_hint(
if nid in seen:
continue
seen.add(nid)
placed.append(("nearby", item))
# MARKED HERE TOO, not just on the semantic arm, because these are one
# menu. A hint where some repeats carry `seen` and others are silently
# dropped — decided by which arm happened to find them — is worse than
# either rule applied consistently: the marker would read as a complete
# account of what the session has met before, and it would not be one.
placed.append(("nearby · seen" if nid in excluded else "nearby", item))
# The stamping feed's "actually pulled it" half (#2791). Read once, before
# the semantic arm, because the arm's query doubles as the resemblance
@@ -1592,16 +1669,19 @@ async def build_write_path_hint(
query = concept_query(query) or query
if remaining > 0 and query:
t0 = time.perf_counter()
# Pulled-and-seen ids stay in the query (as evidence) but never in
# the menu — the dedup contract holds, the resemblance still lands.
pulled_seen = seen & set(pulled)
# Pulled-and-already-listed ids stay in the query (as evidence for
# `resembles`) but never in the menu, so the limit has to cover them.
# `seen` is this call's own menu now, which is the only thing left that
# is a reason to withhold (#4101).
in_menu = seen
pulled_in_menu = in_menu & set(pulled)
_rep_wp: dict = {}
hits = await semantic_search_notes(
user_id, query,
limit=remaining + len(pulled_seen),
limit=remaining + len(pulled_in_menu),
threshold=cfg["threshold"],
project_id=scope_project,
exclude_ids=seen - pulled_seen,
exclude_ids=in_menu - set(pulled),
# Snippets AND recorded experience (#2246). This arm was
# snippets-only, which is auto-inject's mistake inverted: an issue
# saying "we tried this and it deadlocked", or a dev-log recording
@@ -1625,41 +1705,50 @@ async def build_write_path_hint(
int(note.id): float(score) for score, note in hits
if int(note.id) in pulled
}
shown = [(s, n) for s, n in hits if int(n.id) not in seen]
shown = [(s, n) for s, n in hits if int(n.id) not in in_menu]
# WHAT THIS ARM WITHHELD AFTER THE SEARCH ANSWERED, and the reason
# `best_available_score` cannot always be reported here (#3739 again,
# from the side its fix did not reach).
#
# This arm is the one note arm that filters TWICE. `exclude_ids` takes
# `seen - pulled_seen` into the search, but the pulled-and-seen ids stay
# in the query deliberately — `resembles` above needs them — and are
# dropped in the line above instead. So the score the search reported is
# PRE that drop while the row's `result_count` is POST it, and a record
# the session had already been shown could be logged as something the
# BAR turned away. Live proof on the first read after #3739 shipped:
# write_path's near-miss max was 0.822 while the lowest score it ever
# RETURNED was 0.6857 — a "rejection" that beat every acceptance.
#
# The suppression column cannot rescue it the way it does for the rule
# arms: this arm's count would be PARTIAL, covering only the drops made
# here and not the ones `exclude_ids` made inside the search, and a
# partial number under a name that reads as complete is the substitution
# this whole milestone exists to stop.
# the already-listed ids into the search, but the PULLED ones among them
# stay in the query deliberately — `resembles` above needs them — and
# are dropped in the line above instead. So the score the search
# reported is PRE that drop while the row's `result_count` is POST it,
# and a record already listed in this same menu could be logged as
# something the BAR turned away. Live proof on the first read after
# #3739 shipped: write_path's near-miss max was 0.822 while the lowest
# score it ever RETURNED was 0.6857 — a "rejection" that beat every
# acceptance.
#
# So the honest answer is null — "not measured on this call" — whenever
# this filter removed anything, because then the bar is not the only
# thing that turned something away and the reported score may belong to
# a record we withheld ourselves. Calls where nothing was dropped keep
# reporting it, which is most of them.
#
# THE LEDGER IS NO LONGER PART OF THIS (#4101), and that is why the
# suppression column can now be filled in where it could not before.
# The old objection was that a count here would be PARTIAL — covering
# the drops made in this function but not the ones `exclude_ids` made
# inside the search — and a partial number under a name that reads as
# complete is the substitution this milestone exists to stop. That was
# right while the ledger was one of the things `exclude_ids` carried.
# It no longer is: every ledger repeat comes back from the search and is
# rendered, so `suppressed` counts all of them and none are hidden
# inside the query. What `exclude_ids` still removes is this call's own
# menu, which is not suppression at all — those records ARE being shown,
# one block further up.
withheld_here = len(hits) - len(shown)
hits = shown[:remaining]
fresh = [(s, n) for s, n in hits if int(n.id) not in excluded]
record_retrieval(
user_id=user_id, source="write_path", query=query,
threshold=cfg["threshold"], limit=remaining,
# is_task is None, not False: this arm now returns issues too, and
# recording it as a notes-only retrieval would misdescribe the
# candidate set the threshold is being tuned against.
project_id=scope_project, is_task=None, results=hits,
project_id=scope_project, is_task=None, results=fresh,
best_available=(
None if withheld_here else _rep_wp.get("best_available_score")
),
@@ -1670,6 +1759,7 @@ async def build_write_path_hint(
None if withheld_here else _rep_wp.get("best_available_id")
),
searched=bool(_rep_wp.get("searched", True)),
suppressed=len(hits) - len(fresh),
duration_ms=(time.perf_counter() - t0) * 1000.0,
)
if hits:
@@ -1683,9 +1773,19 @@ async def build_write_path_hint(
# and an unlabelled line would be read as "here is code to
# reuse", which is the opposite of what it says.
kind = _record_kind(note)
scored.append((
marker = (
f"similar {score:.2f}" if kind == "snippet"
else f"similar {score:.2f} · {kind}",
else f"similar {score:.2f} · {kind}"
)
# The repeat marker rides the same dotted list as the kind, so a
# reference costs four characters and needs no second line
# (#4101). Same word as the auto-inject menu deliberately: a
# reader meeting `seen` on two different surfaces should not
# have to work out whether they mean the same thing.
if int(note.id) in excluded:
marker += " · seen"
scored.append((
marker,
{
"id": int(note.id), "title": note.title, "user_id": note.user_id,
# Carried so the line can disclose a cross-language hit
@@ -1811,7 +1911,8 @@ async def build_write_path_hint(
"`get_snippet(id)` for a snippet, `get_task(id)` for an issue, "
"`get_note(id)` otherwise. Reuse a snippet rather than writing a fresh "
"one-off; read an issue before repeating what it records "
"(titles only; shown once per session):"
"(titles only; a line marked `seen` was surfaced earlier this "
"session and may no longer be in context):"
)
# Say what a language tag MEANS, and only when one is actually on the menu.
# Without this the reader has to infer why "· python" is attached to a hit on
@@ -1848,7 +1949,13 @@ async def build_write_path_hint(
if sync_note_ids:
by_arm["write_path_sync"] = list(sync_note_ids)
for marker, item in menu:
arm = "write_path_place" if marker == "nearby" else "write_path_semantic"
# A rendered repeat is not a new surfacing (#4101) — same cut the log
# row takes, so this table and `retrieval_logs` keep agreeing about the
# same call (#3668).
if int(item["id"]) in excluded:
continue
arm = ("write_path_place" if marker.startswith("nearby")
else "write_path_semantic")
by_arm.setdefault(arm, []).append(int(item["id"]))
for arm, ids in by_arm.items():
record_surfaced(user_id=user_id, note_ids=ids, source=arm)
@@ -1882,7 +1989,7 @@ async def build_write_path_hint(
rule_t0 = time.perf_counter()
_rep_wpr: dict = {}
hits = await semantic_search_rules(
user_id, code or path, limit=RULEHINT_LIMIT,
user_id, code or path, limit=cfg["rule_top_k"],
threshold=cfg["rule_threshold"],
report=_rep_wpr, project_id=project_id or None,
)
@@ -1943,7 +2050,7 @@ async def build_write_path_hint(
# the same readout.
record_retrieval(
user_id=user_id, source="write_path_rule", query=code or path,
threshold=cfg["rule_threshold"], limit=RULEHINT_LIMIT,
threshold=cfg["rule_threshold"], limit=cfg["rule_top_k"],
project_id=project_id,
is_task=None, results=fresh, duration_ms=rule_ms,
best_available=_rep_wpr.get("best_available_score"),
@@ -2039,7 +2146,7 @@ async def build_tool_rule_hint(
t0 = time.perf_counter()
_rep_ptr: dict = {}
hits = await semantic_search_rules(
user_id, query, limit=RULEHINT_LIMIT,
user_id, query, limit=cfg["tool_rule_top_k"],
threshold=cfg["tool_rule_threshold"],
report=_rep_ptr, project_id=project_id or None,
)
@@ -2062,7 +2169,7 @@ async def build_tool_rule_hint(
# failure the arm was built to stop.
record_retrieval(
user_id=user_id, source="pre_tool_rule", query=query,
threshold=cfg["tool_rule_threshold"], limit=RULEHINT_LIMIT,
threshold=cfg["tool_rule_threshold"], limit=cfg["tool_rule_top_k"],
project_id=project_id,
is_task=None, results=fresh, duration_ms=duration_ms,
best_available=_rep_ptr.get("best_available_score"),
+15 -14
View File
@@ -61,13 +61,9 @@ import logging
import time
from scribe.services.embeddings import semantic_search_rules
from scribe.services.plugin_context import (
REPORTPREF_DEFAULT_THRESHOLD,
REPORTPREF_THRESHOLD_KEY,
)
from scribe.services.retrieval_surfaces import SURFACES, budget_for, floor_for
from scribe.services.retrieval_telemetry import record_retrieval
from scribe.services.rule_usage import record_rule_surfaced
from scribe.services.settings import get_setting
logger = logging.getLogger(__name__)
@@ -85,16 +81,20 @@ COMPLETION_QUERY = (
# A handful, not a menu. More than a few shape preferences for ONE kind of
# reply would contradict each other before they helped; the limit is here to
# keep one noisy corpus from turning a status change into a wall of text.
LIMIT = 3
# The STARTING budget, not the budget (#4102). Both numbers this arm runs on
# now come from the surface registry, so the model that reads this arm's
# telemetry can move either — which matters more here than anywhere else,
# because a fixed query makes this arm's score a constant and a floor a hair
# above it produces a dead arm no amount of traffic will ever reveal.
LIMIT = SURFACES["report_preference"].budget_default
async def _threshold(user_id: int) -> float:
try:
value = float(await get_setting(
user_id, REPORTPREF_THRESHOLD_KEY, str(REPORTPREF_DEFAULT_THRESHOLD)))
except (TypeError, ValueError):
value = REPORTPREF_DEFAULT_THRESHOLD
return min(1.0, max(0.0, value))
return await floor_for(user_id, SOURCE)
async def _limit(user_id: int) -> int:
return await budget_for(user_id, SOURCE)
async def completion_preferences(user_id: int, *, project_id: int | None = None) -> list[dict]:
@@ -115,16 +115,17 @@ async def completion_preferences(user_id: int, *, project_id: int | None = None)
"""
try:
threshold = await _threshold(user_id)
limit = await _limit(user_id)
report: dict = {}
t0 = time.perf_counter()
hits = await semantic_search_rules(
user_id, COMPLETION_QUERY, limit=LIMIT, threshold=threshold,
user_id, COMPLETION_QUERY, limit=limit, threshold=threshold,
kind="preference", report=report, project_id=project_id,
)
hits = [(score, rule) for score, rule in hits if rule.kind == "preference"]
record_retrieval(
user_id=user_id, source=SOURCE, query=COMPLETION_QUERY,
threshold=threshold, limit=LIMIT, project_id=project_id,
threshold=threshold, limit=limit, project_id=project_id,
is_task=None, results=hits,
best_available=report.get("best_available_score"),
best_available_id=report.get("best_available_id"),
+264
View File
@@ -0,0 +1,264 @@
"""One registry of the retrieval surfaces, and the two numbers each one has (#4102).
WHY THIS EXISTS
Six push arms each carried their own loose copy of the same shape: a settings
key, a default, and a limit that was usually a module constant nobody could
change. The read-and-clamp was written out separately in `plugin_context` (twice
over, for auto-inject and the write path), in three rule arms, and again as
`reply_preferences._threshold`. That was survivable while the numbers were
shipped constants an operator occasionally edited.
It stops being survivable once the numbers are meant to MOVE. The operator's
decision for this step:
"the floor should be chosen and adjusted by the model using it. we've come
back to something either fails or has to be looked at by the user we need a
model consistent surface for the adjustment of these floor values. the user
should be able to touch it but the model should be the thing handling it 9
times out of 10."
A tuning surface cannot be "model consistent" if every arm spells its own
configuration differently. So the arms stop owning their numbers and read them
from here instead, and the tuning tool, the routes and the Settings UI all
enumerate THIS table rather than hard-coding six special cases.
WHAT A FLOOR IS NOW, AND WHAT IT IS NOT
It is not a relevance judgement. Relevance is decided by the reader, which is
the only participant that can read a trigger against a situation — that is the
milestone's whole argument, and the injected line has always said so out loud
("read it before deciding it does not apply").
A floor answers the cheaper question: *is this worth ranking at all*. `k` is
what binds, and `k` is a BUDGET — how much of this surface's attention a
candidate list may spend. An arm that fires before every Bash call cannot
afford what an arm that fires once a turn can.
WHY THE DEFAULTS BELOW ARE STARTING POINTS AND NOT ANSWERS
A cosine score is a distance in BAAI/bge-small-en-v1.5's vector space, measured
against THIS corpus. It cannot transfer to an install with different records,
and rule 115 forbids defending a shipped default from this instance's
telemetry — which is what made the old design unbuildable: every number was a
guess everywhere except here, and there was no mechanism that could ever
improve it.
The mechanism is the fix. Scribe ships a starting point and the means to
correct it, so the values below carry the measurement that motivated them (see
the long comments in `plugin_context.py`, which are kept where they are because
they record how each number was first arrived at) without claiming to be right
for anybody else.
HOW A FLOOR SHOULD ACTUALLY BE MOVED
By reading the records the floor refused — `retrieval_telemetry(
near_miss_samples=N)` returns them by id — and never by the percentile alone.
That is not a style preference; it is the one case where the two disagreed and
was checked. `report_preference` logged 69 consecutive declines with the
refused score 0.0006 under the bar, and every percentile said "lower it".
Reading the refused record showed it was rule 77 "Extract intent from loose
phrasing", a false positive, so lowering the bar would have delivered that rule
on every completion report ever written. The statistic and the correct action
pointed in opposite directions, and only opening the record could tell.
"""
from __future__ import annotations
from dataclasses import dataclass
from scribe.services.settings import get_setting
# A budget nobody should be able to set past. Not a tuning value — a guard on
# the worst case, so a mistyped setting cannot turn a menu into a wall of text.
# Shared by every surface because it bounds the same thing everywhere: how many
# lines a single unsolicited injection may occupy.
MAX_BUDGET = 10
@dataclass(frozen=True)
class Surface:
"""One push arm's tunable pair, plus enough prose to tune it responsibly.
`asks` / `over` / `fires` are not documentation for this file — they are
rendered by the tuning tool and the Settings UI. A floor cannot be moved
sensibly by anyone, model or human, who does not know what the query is, what
corpus it runs against, or how often it costs something. Those three facts
are exactly what separates these arms from each other, and they were
previously recoverable only by reading `plugin_context.py`.
"""
name: str
"""The telemetry `source` value, and the join key.
MUST equal the string this arm passes to `record_retrieval`. Everything
useful about tuning depends on that identity: the tool that moves a floor
and the table that says what the floor did have to be talking about the same
arm. A test asserts it rather than a comment asking nicely.
"""
floor_key: str
floor_default: float
budget_key: str
budget_default: int
asks: str
over: str
fires: str
budget_falls_back_to: str = ""
"""A budget key to inherit when this surface has none of its own set.
Only `write_path` uses it, and only because it USED to share auto-inject's
`top_k` outright. Giving it a key without this would silently reset the
budget of every install that had tuned the shared one — a behaviour change
delivered as a default, which is the shape of regression nobody reports
because nothing looks broken.
"""
SURFACES: dict[str, Surface] = {
"auto_inject": Surface(
name="auto_inject",
floor_key="kb_autoinject_threshold",
floor_default=0.55,
budget_key="kb_autoinject_top_k",
budget_default=3,
asks="the operator's message, as they typed it",
over="notes, snippets, processes and issues",
fires="once per operator turn",
),
"write_path": Surface(
name="write_path",
floor_key="kb_writepath_threshold",
floor_default=0.68,
budget_key="kb_writepath_top_k",
budget_default=3,
budget_falls_back_to="kb_autoinject_top_k",
asks="the code being written, rewritten as a concept query",
over="snippets and recorded issues",
fires="before every Write and Edit",
),
"write_path_rule": Surface(
name="write_path_rule",
floor_key="kb_rulehint_threshold",
floor_default=0.72,
budget_key="kb_rulehint_top_k",
budget_default=5,
asks="the code being written, against rule triggers",
over="global rules plus the bound project's own",
fires="before every Write and Edit",
),
"pre_tool_rule": Surface(
name="pre_tool_rule",
floor_key="kb_toolrule_threshold",
floor_default=0.68,
budget_key="kb_toolrule_top_k",
budget_default=5,
asks="the command about to run, against rule triggers",
over="global rules plus the bound project's own",
fires="before every Bash call — the busiest arm there is",
),
"prompt_rule": Surface(
name="prompt_rule",
floor_key="kb_promptrule_threshold",
floor_default=0.72,
budget_key="kb_promptrule_top_k",
budget_default=3,
asks="the operator's message, against rule triggers",
over="global rules plus the bound project's own",
fires="once per operator turn",
),
"report_preference": Surface(
name="report_preference",
floor_key="kb_reportpref_threshold",
floor_default=0.72,
budget_key="kb_reportpref_top_k",
budget_default=3,
# THE ONE FIXED QUERY, and the reason this arm behaves unlike the rest.
# The others score something that varies per call; this one scores a
# constant string, so its top score for a given corpus is also a
# constant. A floor a hair above that constant is not a quiet arm, it is
# a dead one, and no amount of traffic will ever reveal it — which is
# precisely how this arm spent 69 calls declining the same record.
asks="a fixed question about how to lay out a completion report",
over="preferences",
fires="when a task finishes",
),
}
# Reserved slots are deliberately absent. `preference_slot` and `reuse_slot`
# borrow their parent arm's floor and are hard-limited to one hit each, because
# their entire purpose is to guarantee a single line to a kind of record that
# keeps losing a general score contest (#2246, #3894). A budget of "1" is the
# feature; exposing it as tunable would invite setting it to 0 and silently
# removing the guarantee.
def surface_names() -> list[str]:
"""Every tunable surface, in a stable order for menus and listings."""
return list(SURFACES)
def get_surface(name: str) -> Surface:
"""Look one up, refusing an unknown name loudly.
A typo'd surface must not be writable. Settings keys are free-form strings
in a generic table, so a tuning call naming `pretool_rule` would otherwise
write a key nothing ever reads — a change that appears to succeed, reports a
new value, and alters nothing.
"""
try:
return SURFACES[name]
except KeyError:
raise ValueError(
f"unknown retrieval surface {name!r}. Tunable surfaces are: "
+ ", ".join(surface_names())
) from None
def dial_for_key(key: str) -> tuple[str, str] | None:
"""Which `(surface, dial)` a settings key belongs to, or None.
The registry read backwards, and it exists for one caller: the generic
`/api/settings` endpoint, which accepts any key at all. Without this, a
floor written through that endpoint moves with no event recorded, and the
tuning history says nothing happened — a trail with holes in it, which is
worse than no trail because it reads as complete.
Derived rather than listed so a seventh surface is covered the moment it is
added here, which is the only way this stays true.
"""
for surface in SURFACES.values():
if key == surface.floor_key:
return (surface.name, "floor")
if key == surface.budget_key:
return (surface.name, "budget")
return None
async def floor_for(user_id: int, name: str) -> float:
"""This install's current floor for a surface, clamped to [0, 1]."""
s = get_surface(name)
try:
value = float(await get_setting(user_id, s.floor_key, str(s.floor_default)))
except (TypeError, ValueError):
value = s.floor_default
return min(1.0, max(0.0, value))
async def budget_for(user_id: int, name: str) -> int:
"""This install's current budget for a surface, clamped to [1, MAX_BUDGET].
The lower clamp is 1, never 0: a surface turned off is turned off by its
`enabled` switch, which says so. A budget of zero would be an arm that runs
a search, logs a retrieval, and renders nothing — indistinguishable in the
telemetry from a bar nothing cleared, which is the exact confusion this
milestone exists to remove.
"""
s = get_surface(name)
raw = await get_setting(user_id, s.budget_key, "")
if not raw and s.budget_falls_back_to:
raw = await get_setting(user_id, s.budget_falls_back_to, "")
try:
value = int(float(raw)) if raw else s.budget_default
except (TypeError, ValueError):
value = s.budget_default
return min(MAX_BUDGET, max(1, value))
+226
View File
@@ -0,0 +1,226 @@
"""Moving a retrieval surface's floor or budget, with the argument attached (#4102).
WHY THIS EXISTS
The operator's decision for milestone 416 step 4:
"the floor should be chosen and adjusted by the model using it. we've come
back to something either fails or has to be looked at by the user we need a
model consistent surface for the adjustment of these floor values. the user
should be able to touch it but the model should be the thing handling it 9
times out of 10."
`retrieval_surfaces` made the six arms describe their numbers the same way.
This module is the write half: one call that moves one dial on one surface,
records what it was, what it became, who moved it and why, and refuses to do
any of that without a reason.
WHY A REASON IS REQUIRED
Because the alternative was tried and measured. The milestone originally listed
self-tuning as a non-goal on the strength of one case: `report_preference`
logged 69 consecutive declines with the refused record 0.0006 under the bar, and
every percentile in the readout said "lower it". Reading the refused record
showed it was rule 77 "Extract intent from loose phrasing" — a false positive —
so lowering the bar would have attached that rule to every completion report
ever written. The statistic and the correct action pointed in opposite
directions.
What separated them was opening the record. A required `reason` is the cheapest
mechanism that makes that step happen: a caller who must write down why has to
have looked, and a caller who writes down something wrong has left the operator
a sentence to disagree with. A number moved silently leaves nothing.
WHAT THIS DELIBERATELY DOES NOT DO
It does not decide anything itself. There is no rule here that reads a
percentile and picks a value, and that absence is the design — the non-goal that
survived is *statistical* auto-tuning, precisely because the statistic was the
thing that was wrong. The judgement stays with the reader; this module only
makes the judgement recordable and reversible.
"""
from __future__ import annotations
import logging
from sqlalchemy import select
from scribe.models import async_session
from scribe.models.retrieval_tuning import RetrievalTuningEvent
from scribe.services.retrieval_surfaces import (
MAX_BUDGET,
budget_for,
floor_for,
get_surface,
surface_names,
)
from scribe.services.settings import set_setting
logger = logging.getLogger(__name__)
DIALS = ("floor", "budget")
# Long enough to say what was read and what it showed; short enough that nobody
# pastes a telemetry dump in. The number is not a measurement — it is the point
# at which "0.66" stops being an acceptable answer to "why".
_MIN_REASON_CHARS = 20
def _clean_reason(reason: str) -> str:
"""The guardrail, enforced here rather than by the column.
`reason` is NOT NULL in the schema, which "" satisfies. A required field
that accepts an empty string is a formality, and this one is the whole
mechanism — see the module docstring.
"""
text = (reason or "").strip()
if len(text) < _MIN_REASON_CHARS:
raise ValueError(
"reason is required, and has to say what you read. A floor moved "
"without a stated basis is a number nobody can review or revert. "
"Name the evidence: which surface's telemetry, and what the refused "
"records actually were — `retrieval_telemetry(near_miss_samples=N)` "
"returns them by id, and reading them is the step that separates a "
"real miss from a bar doing its job."
)
return text
async def current_settings(user_id: int) -> list[dict]:
"""Every tunable surface with its live pair and its last stated reason.
The read side of the tuning surface, and shaped for a reader who is about to
change something: the value, what the arm asks and over what corpus and how
often — because a floor cannot be moved sensibly without those three — and
the reason last given, so the next change argues with the last one instead
of overwriting it blind.
"""
out: list[dict] = []
async with async_session() as session:
for name in surface_names():
s = get_surface(name)
rows = (
await session.execute(
select(RetrievalTuningEvent)
.where(
RetrievalTuningEvent.surface == name,
RetrievalTuningEvent.user_id == user_id,
)
.order_by(RetrievalTuningEvent.created_at.desc())
.limit(len(DIALS))
)
).scalars().all()
last = {r.dial: r for r in rows}
out.append({
"surface": name,
"floor": await floor_for(user_id, name),
"budget": await budget_for(user_id, name),
"floor_default": s.floor_default,
"budget_default": s.budget_default,
"asks": s.asks,
"over": s.over,
"fires": s.fires,
# Absent rather than empty when a surface has never been moved:
# "still on the shipped starting point" is a real state and
# should not render as a blank reason somebody wrote.
"last_change": {
dial: last[dial].to_dict() for dial in DIALS if dial in last
},
})
return out
async def set_dial(
user_id: int,
surface: str,
dial: str,
value: float,
*,
reason: str,
actor: str = "model",
) -> dict:
"""Move one dial on one surface, recording the change and its argument.
Returns the applied value alongside the previous one, so a caller can see
that a clamp bit rather than assuming the number it sent is the number in
force — the failure being avoided is a tool reporting success for a value
the registry silently corrected.
"""
s = get_surface(surface) # refuses an unknown name
if dial not in DIALS:
raise ValueError(f"dial must be one of {DIALS}, got {dial!r}")
if actor not in ("model", "human"):
raise ValueError(f"actor must be 'model' or 'human', got {actor!r}")
text = _clean_reason(reason)
if dial == "floor":
old = await floor_for(user_id, surface)
applied = min(1.0, max(0.0, float(value)))
key, stored = s.floor_key, str(applied)
else:
old = float(await budget_for(user_id, surface))
applied = float(min(MAX_BUDGET, max(1, int(float(value)))))
key, stored = s.budget_key, str(int(applied))
# A change to the value it already has is not a change, and must not be
# written. The Settings form re-sends every field on every save, so without
# this the history fills with rows saying the operator set six dials to the
# numbers they were already on — and a history nobody can skim is one
# nobody reads, which costs the surface its entire purpose.
#
# Reported rather than silently skipped, so a caller that expected to move
# something learns that it did not.
if abs(applied - old) < 1e-9:
return {
"surface": surface, "dial": dial, "previous": old,
"applied": applied, "clamped": abs(applied - float(value)) > 1e-9,
"reason": text, "actor": actor, "unchanged": True,
}
await set_setting(user_id, key, stored)
async with async_session() as session:
session.add(RetrievalTuningEvent(
user_id=user_id, surface=surface, dial=dial,
old_value=old, new_value=applied, actor=actor, reason=text,
))
await session.commit()
return {
"surface": surface,
"dial": dial,
"previous": old,
"applied": applied,
# True when the registry corrected what was asked for. Said out loud
# because a caller that believes it set 1.4 will read the next
# telemetry as evidence about a bar that was never in force.
"clamped": abs(applied - float(value)) > 1e-9,
"reason": text,
"actor": actor,
# Always present, both ways round: a caller that has to test for the
# key's absence to learn the answer will eventually forget to.
"unchanged": False,
}
async def tuning_history(
user_id: int, *, surface: str | None = None, limit: int = 20
) -> list[dict]:
"""What has been moved, newest first — the operator's review surface.
Scoped to one surface when asked, because the question is almost always
"why is THIS arm set like this", and an unscoped list buries one surface's
two changes under another's twenty.
"""
if surface is not None:
get_surface(surface) # refuse a typo on the read too
async with async_session() as session:
q = (
select(RetrievalTuningEvent)
.where(RetrievalTuningEvent.user_id == user_id)
.order_by(RetrievalTuningEvent.created_at.desc())
.limit(max(1, min(int(limit), 200)))
)
if surface is not None:
q = q.where(RetrievalTuningEvent.surface == surface)
rows = (await session.execute(q)).scalars().all()
return [r.to_dict() for r in rows]
+29
View File
@@ -322,3 +322,32 @@ def http_sink(reply: bytes = b'{"context":"","note_ids":[]}'):
finally:
server.shutdown()
server.server_close()
def writepath_cfg(**over):
"""A complete `get_writepath_config` stand-in, built from the registry (#4102).
DERIVED, NOT LITERAL, and the reason is a failure mode this file already
warned about in prose without being able to prevent: the write-path hint
drives three arms, each of which reads its numbers out of the config dict
inside a fail-open `except`. A dict missing one key does not raise where a
reader would see it — the arm silently becomes a no-op, which is
indistinguishable from the arm working and finding nothing.
So the keys come from `retrieval_surfaces.SURFACES`. A seventh surface, or a
rename, changes this helper for free and cannot quietly disable an arm in
ten hand-written dicts that each looked complete on the day they were typed.
"""
from scribe.services.retrieval_surfaces import SURFACES
cfg = {
"enabled": True,
"threshold": SURFACES["write_path"].floor_default,
"top_k": SURFACES["write_path"].budget_default,
"rule_threshold": SURFACES["write_path_rule"].floor_default,
"rule_top_k": SURFACES["write_path_rule"].budget_default,
"tool_rule_threshold": SURFACES["pre_tool_rule"].floor_default,
"tool_rule_top_k": SURFACES["pre_tool_rule"].budget_default,
}
cfg.update(over)
return cfg
+241
View File
@@ -0,0 +1,241 @@
"""A repeat on a note arm is referenced, not withheld (#4101).
WHY THIS EXISTS
#3750 settled the argument for rules: a record the session was told about an
hour ago is not a record in front of the reader now, so the second time it is
the best answer the session gets the line again with a tail saying so — never
silence, which is indistinguishable from "nothing matched".
The note and snippet arms never got that fix. Worse, their ledger went into
`semantic_search_notes` as `exclude_ids`, so the repeat left the candidate set
entirely, which had three consequences:
- the session got silence the second time, on the arms that fire most;
- a compaction made it permanent, because the ledger outlived the context it
described (the same step fixes that one layer down);
- and `best_available` was measured against a candidate set the caller had
already edited, so the bar could be blamed for a record the caller withheld
(#3739, from the side its fix never reached).
WHAT THIS PINS
1. The ledger does not reach the search. Asserted on the call's kwargs,
because this is the difference between a reference and silence and every
behavioural assertion below rests on it.
2. The repeat is rendered, and marked. The wording is NOT the rule arms'
"before deciding it does not apply" is the voice of a record that binds,
and a dev-log borrowing it would claim authority it does not have — so
what is pinned is that the line appears and is distinguishable, not its
prose.
3. The telemetry splits the two (#3752 / #3668): the row's result set and the
surfaced table both take FRESH only, and the repeat is COUNTED in
`suppressed` instead. This is what makes a zero-result call readable —
`result_count == 0` with `suppressed_count > 0` says "everything that
matched, the session has already seen", which was unreportable here.
4. The deliberate exception: the write-path SYNC class still shows once. Its
claim is about an edit in progress, not about a record's continuing
relevance, and repeating it would be nagging rather than reminding.
"""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from tests.helpers import fake_note, writepath_cfg
REAL_CODE = '''def debounce(fn, wait=0.25):
"""Rate-limit a callback so it fires once after the last call."""
timer = None
def wrapped(*a, **kw):
nonlocal timer
if timer:
timer.cancel()
timer = threading.Timer(wait, fn, a, kw)
timer.start()
return wrapped
'''
def _wp_cfg(**over):
return writepath_cfg(**over)
def _snippet_item(nid, title, user_id=1):
return {"id": nid, "title": title, "user_id": user_id, "note_type": "snippet"}
# ── auto-inject ────────────────────────────────────────────────────────────
async def _autoinject(hits, exclude_ids, *, rec=None, surf=None):
from scribe.services import plugin_context as pc
search = AsyncMock(return_value=hits)
with patch.object(pc, "get_autoinject_config", AsyncMock(
return_value={"enabled": True, "threshold": 0.55, "top_k": 3})), \
patch.object(pc, "semantic_search_notes", search), \
patch.object(pc, "record_retrieval", rec or MagicMock()), \
patch.object(pc, "record_surfaced", surf or MagicMock()), \
patch.object(pc, "owner_names_for", AsyncMock(return_value={})), \
patch.object(pc, "superseded_ids", AsyncMock(return_value=set())):
out = await pc.build_autoinject_hint(
1, "postgres pool", project_id=2, exclude_ids=exclude_ids)
return out, search
@pytest.mark.asyncio
async def test_the_ledger_never_reaches_the_search():
"""The load-bearing one. A ledger inside the query IS the silence."""
hits = [(0.80, fake_note(id=11, title="Pool sizing decision", user_id=1))]
_out, search = await _autoinject(hits, [11])
# The menu arm runs first; the reuse slot's own call may legitimately
# exclude this call's menu, which is a different claim (see below).
menu_call = search.call_args_list[0]
assert not menu_call.kwargs.get("exclude_ids"), (
"the session ledger was passed into the search, so a repeat is "
"withheld rather than referenced"
)
@pytest.mark.asyncio
async def test_a_repeat_is_shown_again_and_marked():
hits = [(0.80, fake_note(id=11, title="Pool sizing decision", user_id=1)),
(0.78, fake_note(id=22, title="run_maintenance thresholds", user_id=1))]
out, _ = await _autoinject(hits, [11])
assert "#11" in out["context"] and "#22" in out["context"]
# Distinguishable, without pinning the word's neighbours in the sentence.
line_11 = next(ln for ln in out["context"].splitlines() if "#11" in ln)
line_22 = next(ln for ln in out["context"].splitlines() if "#22" in ln)
assert "seen" in line_11 and "seen" not in line_22
@pytest.mark.asyncio
async def test_the_repeat_is_counted_not_reported_as_a_result():
rec = MagicMock()
hits = [(0.80, fake_note(id=11, title="Pool sizing decision", user_id=1)),
(0.78, fake_note(id=22, title="run_maintenance thresholds", user_id=1))]
await _autoinject(hits, [11], rec=rec)
row = next(c.kwargs for c in rec.call_args_list
if c.kwargs["source"] == "auto_inject")
assert [int(n.id) for _s, n in row["results"]] == [22]
assert row["suppressed"] == 1
@pytest.mark.asyncio
async def test_a_call_where_everything_was_already_seen_is_readable():
"""The fact that could not be expressed before.
Previously this call logged zero results with no suppression count, so it
was indistinguishable from a bar nothing cleared — and the operator tuning
that bar would have been reading the wrong number.
"""
rec = MagicMock()
hits = [(0.80, fake_note(id=11, title="Pool sizing decision", user_id=1))]
out, _ = await _autoinject(hits, [11], rec=rec)
row = next(c.kwargs for c in rec.call_args_list
if c.kwargs["source"] == "auto_inject")
assert row["results"] == [] and row["suppressed"] == 1
# And the session still gets the line, which is the whole point.
assert "#11" in out["context"]
@pytest.mark.asyncio
async def test_a_repeat_is_not_recorded_as_a_fresh_surfacing():
"""#3668's identity: this table and the log row describe the same call."""
surf = MagicMock()
hits = [(0.80, fake_note(id=11, title="Pool sizing decision", user_id=1)),
(0.78, fake_note(id=22, title="run_maintenance thresholds", user_id=1))]
await _autoinject(hits, [11], surf=surf)
rows = [c.kwargs for c in surf.call_args_list
if c.kwargs.get("source") == "auto_inject"]
assert rows and rows[0]["note_ids"] == [22]
@pytest.mark.asyncio
async def test_the_header_no_longer_promises_once_per_session():
"""A contract stated in the prose is a contract, and this one changed.
Cheap to forget and invisible when wrong: the menu would carry a marker
the header had never explained, and a reader would have to guess.
"""
hits = [(0.80, fake_note(id=11, title="Pool sizing decision", user_id=1))]
out, _ = await _autoinject(hits, [])
header = out["context"].splitlines()[0]
assert "once per session" not in header
assert "seen" in header
# ── the write path ─────────────────────────────────────────────────────────
async def _write_path(hits, exclude_ids, *, here=(), rec=None, sync_exclude=()):
from scribe.services import plugin_context as pc
search = AsyncMock(return_value=hits)
listing = AsyncMock(side_effect=[(list(here), len(here)), ([], 0)])
with patch.object(pc, "get_writepath_config", AsyncMock(return_value=_wp_cfg())), \
patch.object(pc.snippets_svc, "list_snippets", listing), \
patch.object(pc, "semantic_search_notes", search), \
patch.object(pc, "record_retrieval", rec or MagicMock()), \
patch.object(pc, "record_surfaced", MagicMock()), \
patch.object(pc, "owner_names_for", AsyncMock(return_value={})):
out = await pc.build_write_path_hint(
1, "src/x.py", code=REAL_CODE,
exclude_ids=list(exclude_ids),
exclude_sync_ids=list(sync_exclude),
)
return out, search
@pytest.mark.asyncio
async def test_the_write_path_ledger_does_not_reach_its_search_either():
hits = [(0.83, fake_note(id=11, title="debounce helper", user_id=1))]
_out, search = await _write_path(hits, [11])
assert 11 not in (search.call_args.kwargs.get("exclude_ids") or set())
@pytest.mark.asyncio
async def test_a_write_path_repeat_is_rendered_with_a_marker():
hits = [(0.83, fake_note(id=11, title="debounce helper", user_id=1))]
out, _ = await _write_path(hits, [11])
assert "#11" in out["context"]
assert "seen" in next(ln for ln in out["context"].splitlines() if "#11" in ln)
@pytest.mark.asyncio
async def test_the_write_path_counts_its_repeat():
rec = MagicMock()
hits = [(0.83, fake_note(id=11, title="debounce helper", user_id=1)),
(0.80, fake_note(id=22, title="throttle helper", user_id=1))]
await _write_path(hits, [11], rec=rec)
row = next(c.kwargs for c in rec.call_args_list
if c.kwargs["source"] == "write_path")
assert [int(n.id) for _s, n in row["results"]] == [22]
assert row["suppressed"] == 1
@pytest.mark.asyncio
async def test_this_calls_own_menu_is_still_excluded_from_its_search():
"""The claim that stayed an exclusion, and must not be lost with the other.
A snippet already listed by PLACE in this same hint has nothing to gain
from a second line in it. That is same-call duplication, not a repeat
across calls, and the two were the same variable until this step.
"""
hits = []
_out, search = await _write_path(
hits, [], here=[_snippet_item(7, "records this file")])
assert 7 in (search.call_args.kwargs.get("exclude_ids") or set())
@pytest.mark.asyncio
async def test_the_sync_class_still_shows_only_once():
"""The deliberate exception, pinned so it reads as a decision.
The sync nudge says "you are editing the file this record describes, so
updating it is part of the edit". Repeated every write to the same file it
is nagging, and unlike a reuse suggestion it is not a claim whose relevance
can return — it either got acted on or it did not.
"""
out, _ = await _write_path(
[], [], here=[_snippet_item(7, "records this file")], sync_exclude=[7])
assert out["sync_note_ids"] == []
assert "#7" not in out["context"]
+2 -3
View File
@@ -20,7 +20,7 @@ from scribe.services.note_usage import (
record_surfaced,
usage_for_notes,
)
from tests.helpers import fake_note
from tests.helpers import fake_note, writepath_cfg
# --- recording ------------------------------------------------------------
@@ -154,8 +154,7 @@ async def test_unscored_location_arms_are_recorded(lookups, expected_source):
patch.object(
plugin_context,
"get_writepath_config",
AsyncMock(return_value={"enabled": True, "threshold": 0.55, "top_k": 3,
"rule_threshold": 0.72}),
AsyncMock(return_value=writepath_cfg(threshold=0.55)),
),
patch.object(
plugin_context.snippets_svc,
+187
View File
@@ -0,0 +1,187 @@
"""The surface registry, and the identity the whole tuning story rests on (#4102).
WHY THIS EXISTS
Six arms used to own their own numbers: a settings key, a default, and a limit
that was usually a module constant nobody could change. That survived while the
values were shipped constants. It stops surviving once the operator's decision
is that the numbers MOVE:
"the floor should be chosen and adjusted by the model using it… the user
should be able to touch it but the model should be the thing handling it 9
times out of 10."
A tuning surface cannot be model-consistent across arms that each spell their
configuration differently, so the arms now read `{floor, budget}` from one
registry.
WHAT THIS PINS
1. **A surface's name IS its telemetry source.** This is the load-bearing
one. The tool that moves a floor and the table that reports what the floor
did have to be naming the same arm, and nothing in the type system says so
— `record_retrieval(source="pre_tool_rule")` is a string literal in a
different file. Renaming one without the other produces a surface that can
be tuned and cannot be measured, or measured and not tuned, and both fail
silently.
2. **Keys are unique.** Two surfaces sharing a settings key is how
`report_preference` spent its first release moving whenever the prompt arm
was tuned (#3860) — one dial wearing two labels.
3. **An unknown surface is refused.** Settings keys are free-form strings in
a generic table, so a typo'd name would write a key nothing reads: a
change that appears to succeed, reports a new value, and alters nothing.
4. **The budget clamps at 1, never 0.** A zero-budget arm searches, logs a
retrieval and renders nothing — indistinguishable in the telemetry from a
bar nothing cleared, which is the exact confusion this milestone exists to
remove. "Off" is what the `enabled` switch is for.
5. **`write_path` inherits auto-inject's budget when it has none.** It used
to share that key outright; giving it its own without a fallback would
silently reset the budget on every install that had tuned the shared one.
"""
from unittest.mock import AsyncMock, patch
import pytest
from scribe.services import retrieval_surfaces as rs
SERVICES = ("plugin_context", "reply_preferences")
def _service_source(name: str) -> str:
from pathlib import Path
root = Path(__file__).resolve().parents[1] / "src" / "scribe" / "services"
return (root / f"{name}.py").read_text()
def test_every_surface_name_is_a_real_telemetry_source():
"""The join key, checked against the arms that emit it.
Asserted on the source text rather than by calling the arms, because what
can rot here is the literal: a surface renamed in the registry and not in
the `record_retrieval(source=…)` call still runs, still logs, and still
tunes — just against two different names, so the readout an operator uses
to justify a change describes a different arm from the one the change hits.
"""
blob = "\n".join(_service_source(n) for n in SERVICES)
# `report_preference` passes its name through a module constant rather than
# a literal, so that one name is satisfied by the constant holding it.
from scribe.services.reply_preferences import SOURCE
missing = [
s.name for s in rs.SURFACES.values()
if f'source="{s.name}"' not in blob and s.name != SOURCE
]
assert not missing, (
f"these surfaces can be tuned but never measured: {missing}. The "
"registry name must match the string the arm passes to record_retrieval."
)
def test_no_two_surfaces_share_a_settings_key():
"""One dial, one label. #3860 is what the other way costs."""
floors = [s.floor_key for s in rs.SURFACES.values()]
budgets = [s.budget_key for s in rs.SURFACES.values()]
assert len(set(floors)) == len(floors), f"duplicate floor key in {floors}"
assert len(set(budgets)) == len(budgets), f"duplicate budget key in {budgets}"
assert not (set(floors) & set(budgets)), "a floor key doubles as a budget key"
def test_every_surface_says_what_it_asks_over_what_and_how_often():
"""The prose is rendered, not decorative.
A floor cannot be moved responsibly by anyone — model or human — who does
not know the query shape, the corpus, or how often the arm costs something.
An empty string here reaches the tuning tool and the Settings UI as a blank.
"""
for s in rs.SURFACES.values():
for field in ("asks", "over", "fires"):
assert getattr(s, field).strip(), f"{s.name}.{field} is empty"
def test_an_unknown_surface_is_refused_rather_than_written():
with pytest.raises(ValueError) as e:
rs.get_surface("pretool_rule") # a real typo for pre_tool_rule
# The message has to name the alternatives, or the caller's next move is a
# second guess.
assert "pre_tool_rule" in str(e.value)
@pytest.mark.asyncio
@pytest.mark.parametrize("stored, expected", [
("0.8", 0.8),
("5", 1.0), # clamped, not believed
("-3", 0.0),
("banana", 0.55), # unparseable falls back to the surface's default
("", 0.55),
])
async def test_a_floor_is_clamped_to_the_unit_interval(stored, expected):
with patch.object(rs, "get_setting", AsyncMock(return_value=stored)):
assert await rs.floor_for(1, "auto_inject") == expected
@pytest.mark.asyncio
@pytest.mark.parametrize("stored, expected", [
("4", 4),
("999", rs.MAX_BUDGET),
("0", 1),
("-2", 1),
("banana", 3),
])
async def test_a_budget_is_clamped_with_a_floor_of_one(stored, expected):
"""Zero is the value that must not get through.
An arm with a budget of 0 runs its search, writes a `retrieval_logs` row
with `result_count == 0`, and renders nothing — which reads in the telemetry
exactly like a bar nothing cleared. Turning a surface off is the `enabled`
switch's job, and that one says so.
"""
with patch.object(rs, "get_setting", AsyncMock(return_value=stored)):
assert await rs.budget_for(1, "auto_inject") == expected
@pytest.mark.asyncio
async def test_the_write_path_budget_falls_back_to_auto_injects():
"""The migration this fallback exists for.
`write_path` had no budget key of its own — it read auto-inject's. An
install that had tuned that shared knob to 6 must not silently drop to the
new key's default the day this ships; nothing would look broken and the
operator would never know to look.
"""
async def _setting(_uid, key, default=""):
return {"kb_writepath_top_k": "", "kb_autoinject_top_k": "6"}.get(key, default)
with patch.object(rs, "get_setting", AsyncMock(side_effect=_setting)):
assert await rs.budget_for(1, "write_path") == 6
@pytest.mark.asyncio
async def test_its_own_budget_wins_once_it_is_set():
"""And the fallback must not become a permanent override."""
async def _setting(_uid, key, default=""):
return {"kb_writepath_top_k": "2", "kb_autoinject_top_k": "6"}.get(key, default)
with patch.object(rs, "get_setting", AsyncMock(side_effect=_setting)):
assert await rs.budget_for(1, "write_path") == 2
@pytest.mark.asyncio
async def test_the_write_path_config_carries_every_arm_it_drives():
"""One hook request drives three arms, and each reads its pair out of this.
A missing key does not raise where a reader would see it — the rule arms
fail open, so a KeyError becomes an empty hint, and the arm reads as "fired
and found nothing". That is the failure this asserts against, and it is the
reason tests build the dict from the registry rather than by hand.
"""
from scribe.services import plugin_context as pc
# Both modules read settings: `pc` for the enabled switch, `rs` for the
# pairs. Patching one and not the other reaches a real database.
with patch.object(pc, "get_setting", AsyncMock(return_value="")), \
patch.object(rs, "get_setting", AsyncMock(return_value="")):
cfg = await pc.get_writepath_config(1)
for key in ("threshold", "top_k", "rule_threshold", "rule_top_k",
"tool_rule_threshold", "tool_rule_top_k"):
assert key in cfg, f"{key} missing — its arm will silently no-op"
+199
View File
@@ -0,0 +1,199 @@
"""Moving a floor, and the reason that has to come with it (#4102).
WHY THIS EXISTS
The operator handed the dial to the model: *"the user should be able to touch it
but the model should be the thing handling it 9 times out of 10."* Everything
here guards the half of that sentence people skip — the operator still has to be
able to see what was done on their behalf and disagree with it.
WHAT THIS PINS
1. **A reason is required, and "" does not count.** The column is NOT NULL,
which an empty string satisfies; the service is where the requirement is
real. This is the whole guardrail: a caller who must write down why has to
have looked, and a caller who writes down something wrong leaves the
operator a sentence to argue with. A number that moved silently leaves
nothing.
2. **The change is recorded, with what it was before.** Without `old_value` a
history cannot answer "was this always like that", which is the first
question anyone asks of a surface behaving oddly.
3. **A clamp is reported, never swallowed.** A caller that believes it set 1.4
will read the next telemetry as evidence about a bar that was never in
force — the same class of error as #3739, one layer up.
4. **An unknown surface is refused before anything is written.** Settings keys
are free-form strings, so a typo'd surface would write a key nothing reads:
a change that reports success and alters nothing.
5. **The tool teaches the procedure that works**, not the one the numbers
suggest. Asserted on the docstring because the docstring IS the contract an
agent reads, and the failure it prevents is a model tuning from a
percentile — which has been measured pointing the wrong way.
"""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from scribe.services import retrieval_tuning as rt
from tests.helpers import make_mock_session
def _patches(floor=0.72, budget=3):
"""Patch everything `set_dial` touches except the thing under test."""
session = make_mock_session()
return session, (
patch.object(rt, "async_session", MagicMock(return_value=session)),
patch.object(rt, "set_setting", AsyncMock()),
patch.object(rt, "floor_for", AsyncMock(return_value=floor)),
patch.object(rt, "budget_for", AsyncMock(return_value=budget)),
)
@pytest.mark.asyncio
@pytest.mark.parametrize("reason", ["", " ", "noisy", "too high"])
async def test_a_change_without_a_real_reason_is_refused(reason):
"""The guardrail. "too high" is a restatement of the change, not a basis."""
session, ctx = _patches()
with ctx[0], ctx[1], ctx[2], ctx[3], pytest.raises(ValueError) as e:
await rt.set_dial(1, "prompt_rule", "floor", 0.66, reason=reason)
# The message has to name the tool that produces a real basis, or the
# caller's next move is a longer sentence rather than a look at the records.
assert "near_miss_samples" in str(e.value)
session.add.assert_not_called()
@pytest.mark.asyncio
async def test_nothing_is_written_when_the_reason_is_refused():
"""Refused BEFORE the setting is touched, not after.
Writing the value and then raising would leave the number moved and the
history empty — the exact state this table exists to make impossible.
"""
session, ctx = _patches()
with ctx[0], patch.object(rt, "set_setting", AsyncMock()) as setter, \
ctx[2], ctx[3]:
with pytest.raises(ValueError):
await rt.set_dial(1, "prompt_rule", "floor", 0.66, reason="x")
setter.assert_not_called()
@pytest.mark.asyncio
async def test_a_good_change_writes_the_setting_and_the_event():
session, ctx = _patches(floor=0.72)
reason = ("read prompt_rule's 5 highest declines: 3 were project rules for "
"another repo, so the bar is doing its job here")
with ctx[0], patch.object(rt, "set_setting", AsyncMock()) as setter, \
ctx[2], ctx[3]:
out = await rt.set_dial(1, "prompt_rule", "floor", 0.66, reason=reason)
setter.assert_awaited_once()
_uid, key, stored = setter.await_args.args
assert key == "kb_promptrule_threshold" and stored == "0.66"
event = session.add.call_args.args[0]
assert event.surface == "prompt_rule" and event.dial == "floor"
# The before-value is what makes the history answerable.
assert event.old_value == 0.72 and event.new_value == 0.66
assert event.reason == reason and event.actor == "model"
assert out["previous"] == 0.72 and out["applied"] == 0.66
assert out["clamped"] is False
@pytest.mark.asyncio
@pytest.mark.parametrize("dial, sent, applied", [
("floor", 1.4, 1.0),
("floor", -0.2, 0.0),
("budget", 99, 10),
("budget", 0, 1),
])
async def test_a_clamp_is_reported_rather_than_swallowed(dial, sent, applied):
"""Said out loud, because silence here poisons the next reading.
A caller that believes it set 1.4 treats the following week's telemetry as
evidence about a bar that never existed, and then moves the dial again to
fix a problem it invented.
"""
session, ctx = _patches()
reason = "checked the refused records for this surface and they were fine"
with ctx[0], ctx[1], ctx[2], ctx[3]:
out = await rt.set_dial(1, "auto_inject", dial, sent, reason=reason)
assert out["applied"] == applied
assert out["clamped"] is True
@pytest.mark.asyncio
async def test_an_unknown_surface_is_refused_before_anything_is_written():
session, ctx = _patches()
with ctx[0], patch.object(rt, "set_setting", AsyncMock()) as setter, \
ctx[2], ctx[3]:
with pytest.raises(ValueError):
await rt.set_dial(1, "pretool_rule", "floor", 0.6,
reason="a perfectly good reason that is long enough")
setter.assert_not_called()
session.add.assert_not_called()
@pytest.mark.asyncio
@pytest.mark.parametrize("dial", ["threshold", "limit", "", "FLOOR"])
async def test_an_unknown_dial_is_refused(dial):
"""The near-misses are the old vocabulary — `threshold` and `limit` are what
these were called before this step, so they are exactly what a stale caller
will send, and a silent no-op there would be indistinguishable from a
change that did not take."""
session, ctx = _patches()
with ctx[0], ctx[1], ctx[2], ctx[3], pytest.raises(ValueError):
await rt.set_dial(1, "auto_inject", dial, 0.6,
reason="a perfectly good reason that is long enough")
session.add.assert_not_called()
@pytest.mark.asyncio
async def test_a_human_change_is_distinguishable_from_the_models():
"""Both act as the same user, so the id cannot tell them apart — and "did I
do this, or did the session?" is the first question the history is asked."""
session, ctx = _patches()
with ctx[0], ctx[1], ctx[2], ctx[3]:
await rt.set_dial(1, "auto_inject", "floor", 0.6, actor="human",
reason="operator set this themselves in Settings")
assert session.add.call_args.args[0].actor == "human"
@pytest.mark.asyncio
async def test_an_unknown_actor_is_refused():
"""Free text here would make the column unreadable within a month."""
session, ctx = _patches()
with ctx[0], ctx[1], ctx[2], ctx[3], pytest.raises(ValueError):
await rt.set_dial(1, "auto_inject", "floor", 0.6, actor="agent",
reason="a perfectly good reason that is long enough")
def test_the_tool_teaches_reading_the_records_not_the_percentile():
"""The contract an agent actually reads (rule 167).
The failure this prevents is a model moving a bar because `near_misses.p90`
sat close to it. That has been measured pointing the wrong way — 69 declines
where every percentile said "lower it" and the refused record was a false
positive — so the docstring has to carry the method, not just the warning.
"""
from scribe.mcp.tools import retrieval_tuning as tool
doc = tool.tune_retrieval.__doc__
assert "near_miss_samples" in doc, "the tool does not name how to get records"
assert "PERCENTILE ALONE" in doc.upper()
# And the worked example, because an abstract warning loses to a number.
assert "69" in doc
def test_all_three_tools_are_registered():
from scribe.mcp.tools import retrieval_tuning as tool
names = []
class _MCP:
def tool(self, name):
names.append(name)
return lambda fn: fn
tool.register(_MCP())
assert names == [
"retrieval_surfaces", "tune_retrieval", "retrieval_tuning_history",
]
+133
View File
@@ -0,0 +1,133 @@
"""The operator's half of the tuning surface (#4102).
WHY THIS EXISTS
The model moves these dials — that is the operator's decision for milestone
416 — so the browser's job is no longer "set the number". It is to show what
was set, by whom, and on what argument, and to let the operator disagree.
The load-bearing guard here is the last one. `/api/settings` is a generic
key-value endpoint that accepts any key at all, and every retrieval floor IS an
ordinary settings key. A floor written straight through it would land correctly
and record nothing — a tuning history with holes in it, which is worse than no
history because it reads as complete. So the generic endpoint routes those keys
through `set_dial`, and that routing is asserted rather than remembered.
"""
import inspect
import pytest
def test_retrieval_blueprint_registered():
from scribe.routes.retrieval import retrieval_bp
assert retrieval_bp.name == "retrieval"
assert retrieval_bp.url_prefix == "/api/retrieval"
def test_retrieval_blueprint_registered_in_app():
from scribe.app import create_app
app = create_app()
assert "retrieval" in app.blueprints
def test_every_endpoint_is_reachable_on_the_app():
"""Handlers existing is not the same as them being routed."""
from scribe.app import create_app
app = create_app()
rules = {
str(r.rule) for r in app.url_map.iter_rules()
if r.endpoint.startswith("retrieval.")
}
assert rules == {
"/api/retrieval/surfaces",
"/api/retrieval/surfaces/<surface>",
"/api/retrieval/tuning-history",
}
def test_the_browser_and_the_agent_call_the_same_service():
"""Rule 33 parity. Two callers, one service — or the two surfaces drift and
the guard that a reason is required exists on only one of them."""
from scribe.mcp.tools import retrieval_tuning as tool
from scribe.routes import retrieval as routes
from scribe.services import retrieval_tuning as svc
assert routes.set_dial is svc.set_dial
assert tool.tuning_svc is svc
def test_the_route_does_not_take_the_actor_from_the_caller():
"""`actor` is the one field a reviewer leans on to answer "did I do this,
or did the session?". A payload-supplied actor would let a model claim to
be the operator, which turns the column into decoration."""
from scribe.routes import retrieval as routes
src = inspect.getsource(routes.tune_surface_route)
assert 'actor="human"' in src
assert 'data.get("actor"' not in src and 'data["actor"]' not in src
# ── the hole the generic settings endpoint would otherwise leave ────────────
def test_every_registry_key_is_recognised_as_a_dial():
"""Derived from the registry, both directions. A seventh surface added
without a row here is a floor that can be written silently again."""
from scribe.services.retrieval_surfaces import SURFACES, dial_for_key
for name, surface in SURFACES.items():
assert dial_for_key(surface.floor_key) == (name, "floor")
assert dial_for_key(surface.budget_key) == (name, "budget")
def test_an_ordinary_setting_is_not_mistaken_for_a_dial():
"""The interception must be narrow. A false positive here would send an
unrelated setting through a service that parses it as a float and rejects
the save."""
from scribe.services.retrieval_surfaces import dial_for_key
for key in ("smtp_password", "kb_autoinject_enabled", "theme",
"kb_planmatch_threshold", ""):
assert dial_for_key(key) is None, key
def test_the_settings_endpoint_routes_a_dial_through_the_recorder():
"""THE GUARD. Asserted on the source because the alternative is a full
request-context round trip for a branch whose whole content is "which
function gets called" — and because what must not regress is precisely
that this module reaches for `set_dial` at all."""
from scribe.routes import settings as routes
src = inspect.getsource(routes.update_settings_route)
assert "dial_for_key" in src, (
"the generic settings endpoint no longer recognises retrieval dials — "
"a floor written through it now moves with no event recorded, and the "
"tuning history will say nothing happened"
)
assert "set_dial" in src
assert 'actor="human"' in src
@pytest.mark.asyncio
async def test_setting_a_dial_to_the_value_it_already_has_records_nothing():
"""The Settings form re-sends every field on every save. Without this, one
press of Save writes six rows saying the operator set six dials to the
numbers they were already on — and a history nobody can skim is one nobody
reads."""
from unittest.mock import AsyncMock, MagicMock, patch
from scribe.services import retrieval_tuning as rt
from tests.helpers import make_mock_session
session = make_mock_session()
with patch.object(rt, "async_session", MagicMock(return_value=session)), \
patch.object(rt, "set_setting", AsyncMock()) as setter, \
patch.object(rt, "floor_for", AsyncMock(return_value=0.72)), \
patch.object(rt, "budget_for", AsyncMock(return_value=3)):
out = await rt.set_dial(1, "prompt_rule", "floor", 0.72,
reason="re-saved the settings form untouched")
assert out["unchanged"] is True
session.add.assert_not_called()
# And the setting is left alone too: rewriting the same value would bump
# whatever timestamp the row carries for no reason.
setter.assert_not_called()
+3 -15
View File
@@ -135,21 +135,9 @@ def test_an_event_with_nothing_usable_records_nothing_and_still_exits_zero(event
assert not led.exists()
# ── the two ledgers stay in step ───────────────────────────────────────────
def test_a_compaction_clears_both_ledgers():
"""The one that would be worst to get wrong. `.opened.ids` describes a
context the compaction just destroyed, so keeping it while clearing the
naming ledger would have the arms telling a freshly-summarised session
"you opened it earlier" about a rule now nowhere in its context.
"""
sh = (HOOKS / "scribe_session_context.sh").read_text()
block = sh.split("case \"$source\" in")[1].split("esac")[0]
assert "compact|clear)" in block
for led in (".rules.ids", ".opened.ids"):
assert re.search(rf"rm -f .*{re.escape(led)}", block), (
f"{led} survives a compaction that destroyed what it describes"
)
# What a compaction clears — including `.opened.ids`, whose claim is the one
# that would be worst to get wrong — is asserted against the running hook in
# tests/test_session_ledger_clear.py, so there is one home for it.
def test_the_recorder_is_registered_on_the_get_rule_tool():
+33 -37
View File
@@ -17,7 +17,8 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from tests.helpers import fake_note, fake_rule
from scribe.services import retrieval_surfaces as rs
from tests.helpers import fake_note, fake_rule, writepath_cfg
# The MCP tool layer reads its caller from a ContextVar the HTTP transport sets
# per request; a unit test has no request, so it binds the caller itself. The
@@ -59,16 +60,15 @@ def _arm_patches(pc, hits, recorder, prior_art=None, cfg=None, rule_search=None,
"""
return (
patch.object(pc, "get_writepath_config",
AsyncMock(return_value=cfg or {
"enabled": True, "threshold": 0.6,
"top_k": 3, "rule_threshold": 0.6,
# The command arm reads its OWN bar since #3853, and
# a stub missing this key does not fail where a
# reader would see it: the arm fails open, so the
# KeyError becomes an empty hint and every case in
# _ARMS reports the arm went silent instead.
"tool_rule_threshold": 0.6,
})),
# Every key, derived from the surface registry (#4102).
# A stub missing one does not fail where a reader would
# see it: the arm fails open, so the KeyError becomes an
# empty hint and every case in _ARMS reports the arm went
# silent instead.
AsyncMock(return_value=cfg or writepath_cfg(
threshold=0.6, rule_threshold=0.6,
tool_rule_threshold=0.6,
))),
patch.object(pc.snippets_svc, "list_snippets", AsyncMock(return_value=([], 0))),
patch.object(pc, "semantic_search_notes",
AsyncMock(return_value=_PRIOR_ART if prior_art is None
@@ -156,8 +156,7 @@ async def test_the_arm_searches_on_its_OWN_bar_not_the_code_one():
with ExitStack() as stack:
for ctx in _arm_patches(
pc, [], MagicMock(), rule_search=search,
cfg={"enabled": True, "threshold": 0.60,
"top_k": 3, "rule_threshold": 0.81},
cfg=writepath_cfg(threshold=0.60, top_k=3, rule_threshold=0.81),
):
stack.enter_context(ctx)
await pc.build_write_path_hint(
@@ -392,16 +391,15 @@ def test_every_rules_payload_caller_names_itself():
def _tool_patches(pc, hits, recorder, cfg=None, retrieval_log=None):
return (
patch.object(pc, "get_writepath_config",
AsyncMock(return_value=cfg or {
"enabled": True, "threshold": 0.6,
"top_k": 3, "rule_threshold": 0.6,
# The command arm reads its OWN bar since #3853, and
# a stub missing this key does not fail where a
# reader would see it: the arm fails open, so the
# KeyError becomes an empty hint and every case in
# _ARMS reports the arm went silent instead.
"tool_rule_threshold": 0.6,
})),
# Every key, derived from the surface registry (#4102).
# A stub missing one does not fail where a reader would
# see it: the arm fails open, so the KeyError becomes an
# empty hint and every case in _ARMS reports the arm went
# silent instead.
AsyncMock(return_value=cfg or writepath_cfg(
threshold=0.6, rule_threshold=0.6,
tool_rule_threshold=0.6,
))),
patch.object(pc, "semantic_search_rules", AsyncMock(return_value=hits)),
patch.object(pc, "record_retrieval", retrieval_log or MagicMock()),
patch.object(pc, "record_rule_surfaced", recorder),
@@ -433,9 +431,10 @@ async def _run_tool_arm(hits, recorder, command="curl -s https://git.example/api
def _prompt_patches(pc, hits, recorder, retrieval_log=None):
return (
# The arm reads its own threshold key rather than a shared config
# object — a third corpus with a bar nothing has yet tuned for it.
patch.object(pc, "get_setting", AsyncMock(return_value="0.6")),
# The arm reads its own floor and budget from the surface registry
# rather than a shared config object (#4102) — a third corpus, with a
# pair nothing has yet tuned for it.
patch.object(rs, "get_setting", AsyncMock(return_value="0.6")),
patch.object(pc, "semantic_search_rules", AsyncMock(return_value=hits)),
patch.object(pc, "record_retrieval", retrieval_log or MagicMock()),
patch.object(pc, "record_rule_surfaced", recorder),
@@ -469,7 +468,7 @@ async def test_the_prompt_arm_retrieves_against_what_the_operator_SAID():
))])
from scribe.services import plugin_context as pc
with ExitStack() as stack:
stack.enter_context(patch.object(pc, "get_setting", AsyncMock(return_value="0.6")))
stack.enter_context(patch.object(rs, "get_setting", AsyncMock(return_value="0.6")))
stack.enter_context(patch.object(pc, "semantic_search_rules", search))
stack.enter_context(patch.object(pc, "record_retrieval", MagicMock()))
stack.enter_context(patch.object(pc, "record_rule_surfaced", rec))
@@ -503,7 +502,7 @@ async def test_the_prompt_arm_says_nothing_when_asked_nothing():
log = MagicMock()
from scribe.services import plugin_context as pc
with ExitStack() as stack:
stack.enter_context(patch.object(pc, "get_setting", AsyncMock(return_value="0.6")))
stack.enter_context(patch.object(rs, "get_setting", AsyncMock(return_value="0.6")))
stack.enter_context(patch.object(pc, "semantic_search_rules", search))
stack.enter_context(patch.object(pc, "record_retrieval", log))
stack.enter_context(patch.object(pc, "record_rule_surfaced", MagicMock()))
@@ -1416,7 +1415,7 @@ async def _run_slot(general, preference, recorder=None, retrieval_log=None, **kw
from scribe.services import plugin_context as pc
rec = recorder or MagicMock()
with ExitStack() as stack:
stack.enter_context(patch.object(pc, "get_setting", AsyncMock(return_value="0.6")))
stack.enter_context(patch.object(rs, "get_setting", AsyncMock(return_value="0.6")))
stack.enter_context(patch.object(
pc, "semantic_search_rules", _search_by_kind(general, preference)))
stack.enter_context(patch.object(
@@ -1480,7 +1479,7 @@ async def test_the_slot_query_can_only_answer_with_a_preference():
from scribe.services import plugin_context as pc
search = _search_by_kind(_RULES_FILLING_THE_LIMIT, _PREF_HIT)
with ExitStack() as stack:
stack.enter_context(patch.object(pc, "get_setting", AsyncMock(return_value="0.6")))
stack.enter_context(patch.object(rs, "get_setting", AsyncMock(return_value="0.6")))
stack.enter_context(patch.object(pc, "semantic_search_rules", search))
stack.enter_context(patch.object(pc, "record_retrieval", MagicMock()))
stack.enter_context(patch.object(pc, "record_rule_surfaced", MagicMock()))
@@ -1617,8 +1616,7 @@ async def test_each_act_arm_searches_at_its_own_bar():
"""The split, where it actually takes effect."""
from scribe.services import plugin_context as pc
cfg = {"enabled": True, "threshold": 0.6, "top_k": 3,
"rule_threshold": 0.77, "tool_rule_threshold": 0.61}
cfg = writepath_cfg(threshold=0.6, top_k=3, rule_threshold=0.77, tool_rule_threshold=0.61)
search = AsyncMock(return_value=list(_THREE_HITS))
with ExitStack() as stack:
@@ -1646,8 +1644,7 @@ async def test_an_act_arm_reports_the_bar_it_actually_searched_at():
"""
from scribe.services import plugin_context as pc
cfg = {"enabled": True, "threshold": 0.6, "top_k": 3,
"rule_threshold": 0.77, "tool_rule_threshold": 0.61}
cfg = writepath_cfg(threshold=0.6, top_k=3, rule_threshold=0.77, tool_rule_threshold=0.61)
search = AsyncMock(return_value=list(_THREE_HITS))
log = MagicMock()
@@ -1712,8 +1709,7 @@ async def test_the_act_arms_scope_their_search_to_the_bound_project(bound, scope
global rules only, which the search spells as `project_id=None`."""
from scribe.services import plugin_context as pc
cfg = {"enabled": True, "threshold": 0.6, "top_k": 3,
"rule_threshold": 0.6, "tool_rule_threshold": 0.6}
cfg = writepath_cfg(threshold=0.6, top_k=3, rule_threshold=0.6, tool_rule_threshold=0.6)
tool_search = AsyncMock(return_value=[])
with ExitStack() as stack:
stack.enter_context(patch.object(
@@ -1726,7 +1722,7 @@ async def test_the_act_arms_scope_their_search_to_the_bound_project(bound, scope
prompt_search = AsyncMock(return_value=[])
with ExitStack() as stack:
stack.enter_context(patch.object(pc, "get_setting", AsyncMock(return_value="0.6")))
stack.enter_context(patch.object(rs, "get_setting", AsyncMock(return_value="0.6")))
stack.enter_context(patch.object(pc, "semantic_search_rules", prompt_search))
stack.enter_context(patch.object(pc, "record_retrieval", MagicMock()))
stack.enter_context(patch.object(pc, "record_rule_surfaced", MagicMock()))
+8 -2
View File
@@ -23,7 +23,7 @@ def test_backup_version_is_current():
(Named for the number it asserted until v10, which is exactly the drift a
name-carrying-a-value invites; it now says what it checks.)"""
assert backup.BACKUP_VERSION == 15
assert backup.BACKUP_VERSION == 16
def _exportable_note(**over):
@@ -134,6 +134,7 @@ def _column_guard_targets():
from scribe.models.note_supersession import NoteSupersession
from scribe.models.note_usage import NoteUsageEvent
from scribe.models.rule_usage import RuleUsageEvent
from scribe.models.retrieval_tuning import RetrievalTuningEvent
from scribe.models.note_version import NoteVersion
from scribe.models.rule_version import RuleVersion
from scribe.models.project import Project
@@ -164,6 +165,9 @@ def _column_guard_targets():
"rule_relations": (RuleRelation, backup._rule_relation_rows),
"note_usage_events": (NoteUsageEvent, backup._usage_event_rows),
"rule_usage_events": (RuleUsageEvent, backup._rule_usage_event_rows),
"retrieval_tuning_events": (
RetrievalTuningEvent, backup._retrieval_tuning_event_rows,
),
"design_systems": (DesignSystem, backup._design_system_rows),
"design_tokens": (DesignToken, backup._design_token_rows),
"repo_bindings": (RepoBinding, backup._repo_binding_rows),
@@ -340,7 +344,9 @@ async def test_export_full_backup_contains_every_declared_section():
"systems", "record_systems", "design_systems",
"design_tokens", "note_usage_events", "repo_bindings",
"note_supersessions", "code_shapes", "code_shape_events",
"code_shape_uses"):
"code_shape_uses",
# v16: the reasons beside the settings they explain.
"retrieval_tuning_events"):
assert key in out, f"missing export section: {key}"
assert out[key] == []
+12 -10
View File
@@ -1,7 +1,8 @@
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from tests.helpers import fake_note
from scribe.services import retrieval_surfaces as rs
from tests.helpers import fake_note, writepath_cfg
pytestmark = pytest.mark.usefixtures("_no_supersession")
@@ -17,7 +18,8 @@ async def test_get_autoinject_config_defaults_and_clamps():
from scribe.services import plugin_context as pc
# No settings stored → defaults.
with patch.object(pc, "get_setting", AsyncMock(side_effect=lambda uid, k, d: d)):
with patch.object(pc, "get_setting", AsyncMock(side_effect=lambda uid, k, d: d)), \
patch.object(rs, "get_setting", AsyncMock(side_effect=lambda uid, k, d="": d)):
cfg = await pc.get_autoinject_config(1)
assert cfg == {
"enabled": pc.AUTOINJECT_DEFAULT_ENABLED,
@@ -31,8 +33,12 @@ async def test_get_autoinject_config_defaults_and_clamps():
pc.AUTOINJECT_THRESHOLD_KEY: "5",
pc.AUTOINJECT_TOP_K_KEY: "999",
}
with patch.object(pc, "get_setting",
AsyncMock(side_effect=lambda uid, k, d: stored.get(k, d))):
# The switch comes from plugin_context, the two numbers from the registry.
def _side(uid, k, d=""):
return stored.get(k, d)
with patch.object(pc, "get_setting", AsyncMock(side_effect=_side)), \
patch.object(rs, "get_setting", AsyncMock(side_effect=_side)):
cfg = await pc.get_autoinject_config(1)
assert cfg["enabled"] is False
assert cfg["threshold"] == 1.0
@@ -437,9 +443,7 @@ async def test_write_path_semantic_arm_asks_for_experience_not_just_snippets():
search = AsyncMock(return_value=hits)
rec = MagicMock()
with patch.object(pc, "get_writepath_config",
AsyncMock(return_value={"enabled": True, "threshold": 0.6,
"top_k": 3,
"rule_threshold": 0.72})), \
AsyncMock(return_value=writepath_cfg(threshold=0.6))), \
patch.object(pc.snippets_svc, "list_snippets",
AsyncMock(return_value=([], 0))), \
patch.object(pc, "semantic_search_notes", search), \
@@ -470,9 +474,7 @@ async def test_write_path_labels_a_non_snippet_hit_with_its_kind():
hits = [(0.72, fake_note(id=9, title="debounce helper", user_id=1, note_type="snippet")),
(0.71, fake_note(id=7, title="Debounce dropped the trailing call", user_id=1, is_task=True, task_kind="issue"))]
with patch.object(pc, "get_writepath_config",
AsyncMock(return_value={"enabled": True, "threshold": 0.6,
"top_k": 3,
"rule_threshold": 0.72})), \
AsyncMock(return_value=writepath_cfg(threshold=0.6))), \
patch.object(pc.snippets_svc, "list_snippets",
AsyncMock(return_value=([], 0))), \
patch.object(pc, "semantic_search_notes", AsyncMock(return_value=hits)), \
+18 -5
View File
@@ -9,6 +9,9 @@ import re
from unittest.mock import AsyncMock, MagicMock, patch
M = "scribe.services.reply_preferences"
# Its two numbers are resolved through the registry now (#4102), so the
# settings reader to patch lives there rather than in the arm's own module.
RS = "scribe.services.retrieval_surfaces"
def _rule(rid, kind="preference"):
@@ -26,7 +29,7 @@ async def _run(hits, *, searched=True, raises=None):
search_mock = AsyncMock(side_effect=search)
with patch(f"{M}.semantic_search_rules", search_mock), \
patch(f"{M}.get_setting", AsyncMock(return_value="0.72")), \
patch(f"{RS}.get_setting", AsyncMock(return_value="0.72")), \
patch(f"{M}.record_retrieval") as logged, \
patch(f"{M}.record_rule_surfaced") as surfaced:
out = await completion_preferences(7, project_id=3)
@@ -93,7 +96,7 @@ def test_its_bar_is_its_own_key_not_the_prompt_arm_s(monkeypatch):
asked: list[str] = []
async def get_setting(user_id, key, default):
async def get_setting(user_id, key, default=""):
asked.append(key)
return default
@@ -101,16 +104,26 @@ def test_its_bar_is_its_own_key_not_the_prompt_arm_s(monkeypatch):
kw["report"].update({"searched": True})
return []
with patch(f"{M}.get_setting", AsyncMock(side_effect=get_setting)), \
with patch(f"{RS}.get_setting", AsyncMock(side_effect=get_setting)), \
patch(f"{M}.semantic_search_rules", AsyncMock(side_effect=search)), \
patch(f"{M}.record_retrieval"):
asyncio.run(completion_preferences(7))
assert asked == [plugin_context.REPORTPREF_THRESHOLD_KEY]
assert plugin_context.REPORTPREF_THRESHOLD_KEY != plugin_context.PROMPTRULE_THRESHOLD_KEY, (
# Both numbers are its own since #4102 — a floor AND a budget — so the arm
# asks for two keys, and neither may be the prompt arm's.
from scribe.services.retrieval_surfaces import get_surface
mine = get_surface("report_preference")
assert asked == [mine.floor_key, mine.budget_key]
theirs = get_surface("prompt_rule")
assert mine.floor_key != theirs.floor_key, (
"the two keys are the same string again, so the settings form has one "
"dial driving two arms — which is the defect, whatever the value is"
)
assert mine.floor_key == plugin_context.REPORTPREF_THRESHOLD_KEY, (
"the registry and the module constant disagree about this arm's key, "
"so the Settings form would write one and the arm would read the other"
)
def test_the_query_assumes_no_particular_domain():
+25 -11
View File
@@ -108,14 +108,29 @@ def test_the_rules_ledger_survives_exactly_when_the_context_does(tmp_path):
)
def test_only_the_rule_ledger_is_cleared_and_the_note_ledgers_are_left(tmp_path):
"""Scope, asserted rather than described.
def test_the_note_ledgers_are_cleared_too_now_that_a_repeat_is_shown(tmp_path):
"""The decision this test used to pin, reversed — and why it could be.
The same directory holds `.ids`, `.sync.ids` and `.derive.ids` for the note
arms. Whether a surfaced NOTE should come back after a compaction is a
different question with a different answer, and it is not being answered.
A `rm` glob over `<sid>.*` would pass every assertion in the test above
while silently deciding it.
It read: the note arms were deliberately left out of #3749, whether a
surfaced NOTE should come back after a compaction is a different question,
and a glob over `<sid>.*` would decide it by accident. That was right when
it was written, and the reason was the note arms' behaviour rather than the
note arms' nature: a note repeat was WITHHELD, so clearing their ledger
meant re-injecting whole menu lines the session had already been given.
Against that, leaving them was the cheaper mistake.
#4101 removed the premise. A note repeat is now rendered again with a
`seen` marker instead of dropped from the search, so the ledger no longer
decides whether a record is reachable — only whether its line says the
session has met it before. Across a compaction that marker becomes a false
statement: it tells a freshly-summarised session it has already seen a
record that is nowhere in its context. Keeping the ledger buys nothing and
asserts something untrue.
Kept as a named test rather than deleted, because the roster it guards
moved to tests/test_session_ledger_clear.py and what is worth keeping HERE
is the record that the answer changed, with the reason attached — a
deleted test takes the old decision's reasoning with it.
"""
env = _env(tmp_path)
sid = "sess-scope"
@@ -127,10 +142,9 @@ def test_only_the_rule_ledger_is_cleared_and_the_note_ledgers_are_left(tmp_path)
_fire("compact", sid, env)
assert not rules.exists(), "the rule ledger should have been cleared"
assert notes.exists() and sync.exists() and derive.exists(), (
"a note ledger was cleared too. The note arms were deliberately left "
"out of #3749 — clearing them is a decision about a different surface, "
"and a glob that takes them along makes it by accident."
assert not (notes.exists() or sync.exists() or derive.exists()), (
"a note ledger outlived the context it describes, so its records are "
"marked `seen` to a session that can no longer see them"
)
+228
View File
@@ -0,0 +1,228 @@
"""A compaction clears every session ledger, by convention not by list (#4101).
WHY THIS EXISTS
Milestone 386 established the claim: a ledger describes what a session HOLDS,
so the events that destroy context must destroy it too, or the records it names
become permanently unreachable mid-session. `scribe_session_context.sh`
implemented that — for the rules ledger, by name.
Five ledgers live in that directory and two were on the list. The note arms'
three (`.ids`, `.sync.ids`, `.derive.ids`) were left, under a comment asserting
this was "a decision rather than an oversight". It was not a decision, and the
arms left out are the ones where it costs most:
- their exclusions are HARD — `exclude_ids` is passed into
`semantic_search_notes` itself, so a surfaced note leaves the result set
entirely, with no weaker rendering to fall back to the way #3750 gave a
repeated rule one;
- and they never AGE — #3751's TTL was added to the rules ledger only.
Hard, permanent and never cleared: a note surfaced in a session's first minute
is unreachable for the rest of it, through any number of compactions.
WHAT THIS PINS
The list was the bug, so the fix cannot be a longer list and neither can the
test. Both sides are asserted:
1. BEHAVIOUR — the hook is run on a real `compact` event with all five
ledgers on disk, and all five are gone afterwards. Run rather than
grepped, because grepping for the names is the hand-maintained pattern
this step removes.
2. THE CONVENTION THE BEHAVIOUR RESTS ON — every per-session ledger any hook
builds is named `<sid>[.<kind>].ids`. That is what makes a sixth ledger
covered on the day it is written, and it is the assumption that would
rot silently, because a ledger named outside it simply never clears and
nothing says so.
And the negative: `<sid>.unreached` is not a ledger of held context but a
record that the instance could not be reached, and a glob that swept it away
would make a hook forget an outage it is meant to report (#2932).
"""
import json
import os
import re
import shutil
import subprocess
from pathlib import Path
import pytest
HOOKS = Path(__file__).resolve().parents[1] / "plugin" / "hooks"
SESSION_START = HOOKS / "scribe_session_context.sh"
DEFS = HOOKS / "scribe_defs.sh"
# Every ledger the hooks write today, and where. Named here so a failure READS
# as "this one survived", but never used to build the hook's own delete list —
# the convention tests below are what keep this roster honest as it grows.
LEDGERS = {
"scribe-priorart": (".ids", ".rules.ids", ".opened.ids", ".sync.ids",
".derive.ids"),
"scribe-autoinject": (".ids",),
}
def _swept_dirs() -> set[str]:
"""The directories the SHIPPED hook sweeps, read from the hook itself.
Read rather than restated: a test carrying its own copy of the roster would
agree with itself forever, which is precisely the failure that let the
auto-inject ledger sit in a directory nothing cleared.
"""
line = re.search(r'SCRIBE_LEDGER_DIRS="([^"]*)"', DEFS.read_text())
assert line, "SCRIBE_LEDGER_DIRS is gone; the clear has no roster"
return set(line.group(1).split())
def _run_session_start(source: str, tmp: Path) -> Path:
"""Run the SessionStart hook for real, with the ledger directories filled."""
for tool in ("bash", "jq"):
if shutil.which(tool) is None:
pytest.skip(f"hook runtime tool {tool!r} not installed")
for dirname, suffixes in LEDGERS.items():
state = tmp / dirname
state.mkdir(parents=True, exist_ok=True)
for suffix in suffixes:
(state / f"s1{suffix}").write_text("42\t1789600000\n")
# Not a ledger: an outage marker that must outlive the clear.
(tmp / "scribe-priorart" / "s1.unreached").write_text("1\n")
env = {"PATH": os.environ["PATH"], "HOME": str(tmp), "TMPDIR": str(tmp)}
out = subprocess.run(
["bash", str(SESSION_START)],
input=json.dumps({"session_id": "s1", "source": source}),
capture_output=True, text=True, env=env, timeout=60,
)
assert out.returncode == 0, out.stderr
return tmp
@pytest.mark.parametrize("source", ["compact", "clear"])
def test_a_context_destroying_source_clears_every_ledger(source, tmp_path):
"""The step, stated as behaviour: all of them, in both directories."""
root = _run_session_start(source, tmp_path)
survived = [f"{d}/s1{s}" for d, suffixes in LEDGERS.items()
for s in suffixes if (root / d / f"s1{s}").exists()]
assert not survived, (
f"{survived} survived a {source!r} that destroyed what they describe"
)
@pytest.mark.parametrize("source", ["startup", "resume"])
def test_a_source_that_kept_the_context_keeps_the_ledgers(source, tmp_path):
"""The mirror error, and the more expensive one.
`resume` genuinely restores the context, so the ledger still describes what
the session holds; clearing there would re-surface every record after a
restore that lost nothing. A blanket glob makes over-clearing cheap to
write, which is exactly why this direction needs a test of its own.
"""
root = _run_session_start(source, tmp_path)
for dirname, suffixes in LEDGERS.items():
for suffix in suffixes:
assert (root / dirname / f"s1{suffix}").exists(), (
f"{dirname}/s1{suffix} was cleared on {source!r}, which lost "
"no context"
)
def test_the_outage_marker_is_not_swept_with_them(tmp_path):
"""`.unreached` records that the instance was down, not what was surfaced.
Different lifetime, different question. #2932's whole point is that "we
checked and found nothing" and "we never managed to check" must stay
distinguishable, and a clear that took this file out would quietly answer
the second with the first.
"""
root = _run_session_start("compact", tmp_path)
assert (root / "scribe-priorart" / "s1.unreached").exists()
# ── the two assumptions the sweep rests on ─────────────────────────────────
#
# Both are checked against the hooks rather than trusted, because neither fails
# loudly: a ledger outside them simply never clears, on the arm whose author had
# no reason to know a convention existed.
def _dir_vars(text: str) -> dict[str, str]:
"""`var="${TMPDIR:-/tmp}/<name>"` → {var: name}, per script."""
return dict(re.findall(
r'(\w+)="\$\{TMPDIR:-/tmp\}/([A-Za-z0-9_-]+)"', text))
def _session_paths(text: str):
"""Every `$<var>/${safe_sid}<suffix>` a script composes."""
return re.findall(r'\$(\w+)"?/\$\{safe_sid\}([A-Za-z0-9_.]*)', text)
def test_every_directory_holding_a_ledger_is_one_the_clear_visits():
"""The failure that shipped once already, in the same step that fixed it.
The first cut swept `scribe-priorart` alone. Every ledger NAMED in the
hooks was there, so it read as complete — and `scribe_autoinject.sh` keeps
its note ledger in `scribe-autoinject`, which meant the arm that fires most
was the only one still carrying the bug. Nothing said so: the clear ran,
found nothing to remove, and exited 0.
"""
swept = _swept_dirs()
offenders = []
for script in sorted(HOOKS.glob("*.sh")):
text = script.read_text()
dirs = _dir_vars(text)
for var, suffix in _session_paths(text):
if not suffix.endswith(".ids"):
continue
where = dirs.get(var)
if where not in swept:
offenders.append(f"{script.name}: ${var} -> {where or '?'}")
assert not offenders, (
"these ledgers live in directories the compaction clear never visits, "
f"so they outlive the context they describe: {offenders}. Add the "
"directory to SCRIBE_LEDGER_DIRS in scribe_defs.sh."
)
def test_every_session_file_in_a_swept_directory_follows_the_convention():
"""The other half: the sweep matches `.ids`, so a ledger must be named it.
Stated as its own test because the two assumptions fail differently — this
one lets a ledger sit in the right directory and still never clear.
"""
# Session files in a swept directory that are NOT per-session ledgers.
NOT_LEDGERS = {".unreached"}
swept = _swept_dirs()
offenders = []
for script in sorted(HOOKS.glob("*.sh")):
text = script.read_text()
dirs = _dir_vars(text)
for var, suffix in _session_paths(text):
if dirs.get(var) not in swept:
continue # a different directory, a different question
if suffix in NOT_LEDGERS or suffix.endswith(".ids"):
continue
offenders.append(f"{script.name}: ${var}/${{safe_sid}}{suffix}")
assert not offenders, (
"these session files sit in a swept directory but clear on no "
f"compaction, because the sweep matches `.ids`: {offenders}"
)
def test_the_clear_is_derived_and_not_a_list_of_names():
"""The regression that would look like a fix.
Appending an `rm -f` per ledger passes every behavioural test above while
rebuilding the trap for the next one. The property worth keeping is that
the hook names no ledger at all.
"""
sh = SESSION_START.read_text()
block = sh.split('case "$source" in')[1].split("esac")[0]
assert "scribe_clear_session_ledgers" in block
for suffix in {s for suffixes in LEDGERS.values() for s in suffixes}:
assert suffix not in block, (
f"the clear names {suffix} again — a list, not a convention"
)
+64 -24
View File
@@ -20,9 +20,12 @@ string equals the Python default. Retuning either stays free as long as both
move — which is the point, because these are tuning values and #3853 moved one
of them the day this was written.
Both sides are read from SOURCE rather than imported. The Vue file cannot be
imported at all, and reading the Python constant through an import would tie
this to module-load side effects it has no interest in.
The Vue side is read from SOURCE, because that file cannot be imported at all.
The Python side is read from source for a loose constant and IMPORTED for a
registry surface — the registry is a plain table of frozen dataclasses with
nothing to trigger on import, and importing it means this guard checks the value
the server will actually resolve rather than a literal that happens to look
right.
It cannot check that the form WRITES the right key — that is behaviour, and
the keys are asserted where they are built. It catches the drift that has no
@@ -39,20 +42,29 @@ ROOT = pathlib.Path(__file__).resolve().parents[1]
_SERVICES = ROOT / "src" / "scribe" / "services"
_VUE = ROOT / "frontend" / "src" / "views" / "SettingsView.vue"
# (services module, python constant, vue ref). Hand-written because the
# pairing is an editorial fact — the names do not share a convention either
# side could derive — but every entry is asserted to EXIST on both sides, so a
# rename fails loudly here rather than silently dropping that threshold from
# the check.
_PAIRS = (
("plugin_context.py", "AUTOINJECT_DEFAULT_THRESHOLD", "kbInjectThreshold"),
("plugin_context.py", "WRITEPATH_DEFAULT_THRESHOLD", "kbWritePathThreshold"),
("plugin_context.py", "RULEHINT_DEFAULT_THRESHOLD", "kbRuleHintThreshold"),
("plugin_context.py", "TOOLRULE_DEFAULT_THRESHOLD", "kbToolRuleThreshold"),
("plugin_context.py", "PROMPTRULE_DEFAULT_THRESHOLD", "kbPromptRuleThreshold"),
("plugin_context.py", "REPORTPREF_DEFAULT_THRESHOLD", "kbReportPrefThreshold"),
# THE SIX RETRIEVAL FLOORS now live in one registry (#4102), so their side of
# the pairing is a SURFACE NAME rather than a module constant. That is strictly
# better for this guard: the surface name is also the telemetry source, so a row
# here names the same arm the readout does, and a floor that moved cannot be
# checked against a stale constant that happened to keep its old value.
_SURFACE_PAIRS = (
("auto_inject", "kbInjectThreshold"),
("write_path", "kbWritePathThreshold"),
("write_path_rule", "kbRuleHintThreshold"),
("pre_tool_rule", "kbToolRuleThreshold"),
("prompt_rule", "kbPromptRuleThreshold"),
("report_preference", "kbReportPrefThreshold"),
)
# (services module, python constant, vue ref) for the defaults that are NOT
# retrieval surfaces. Hand-written because the pairing is an editorial fact —
# the names share no convention either side could derive — and asserted to
# exist on both sides, so a rename fails loudly rather than silently dropping
# that threshold from the check.
_CONSTANT_PAIRS = (
# The plan gate (milestone 415): it blocks a create, so a form showing a
# looser bar than the one in force would be the more misleading drift.
# looser bar than the one in force would be the more misleading drift. Not
# a push surface, so it has no registry entry and keeps the older shape.
("dedup.py", "PLAN_MATCH_DEFAULT_THRESHOLD", "kbPlanMatchThreshold"),
)
@@ -80,14 +92,42 @@ def _vue_default(ref_name: str) -> float:
return float(m.group(1))
@pytest.mark.parametrize(("module", "constant", "ref_name"), _PAIRS,
ids=[p[1] for p in _PAIRS])
def test_the_form_shows_the_default_the_server_uses(module, constant, ref_name):
"""An untouched control must render the bar actually in force."""
server, form = _python_default(module, constant), _vue_default(ref_name)
def _assert_agrees(server: float, form: float, ref_name: str, what: str) -> None:
assert form == server, (
f"SettingsView shows {form} for {ref_name} while the server defaults "
f"to {server} ({constant}). An operator who has never set this reads "
f"the form as the value in force, so the two must move together — "
f"retune both, or neither."
f"to {server} ({what}). An operator who has never set this reads the "
f"form as the value in force, so the two must move together — retune "
f"both, or neither."
)
@pytest.mark.parametrize(("surface", "ref_name"), _SURFACE_PAIRS,
ids=[p[0] for p in _SURFACE_PAIRS])
def test_the_form_shows_the_floor_the_surface_uses(surface, ref_name):
"""An untouched control must render the bar actually in force."""
from scribe.services.retrieval_surfaces import get_surface
_assert_agrees(get_surface(surface).floor_default, _vue_default(ref_name),
ref_name, f"{surface} floor")
def test_every_tunable_surface_has_a_control():
"""Rule 25/27: a number an operator may need different has a UI or it is
not shipped. Derived from the registry, so a seventh surface arrives here
as a failure rather than as a setting only the model can reach."""
from scribe.services.retrieval_surfaces import surface_names
covered = {s for s, _ref in _SURFACE_PAIRS}
missing = [n for n in surface_names() if n not in covered]
assert not missing, (
f"these surfaces are tunable with no Settings control: {missing}. "
"Add the control and its row, or say in _SURFACE_PAIRS why it has none."
)
@pytest.mark.parametrize(("module", "constant", "ref_name"), _CONSTANT_PAIRS,
ids=[p[1] for p in _CONSTANT_PAIRS])
def test_the_form_shows_the_default_the_server_uses(module, constant, ref_name):
"""Same claim, for the defaults that are not retrieval surfaces."""
_assert_agrees(_python_default(module, constant), _vue_default(ref_name),
ref_name, constant)
+147 -34
View File
@@ -6,6 +6,7 @@ source, and the two ways this must stay silent. Plus the plugin hook contract
a PreToolUse hook that returns a permission decision would be able to block the
operator's edit, which this feature must never do.
"""
import contextlib
import json
import re
import subprocess
@@ -13,7 +14,30 @@ from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from tests.helpers import fake_note, http_sink
from tests.helpers import fake_note, http_sink, writepath_cfg
@contextlib.contextmanager
def _stored_settings(stored):
"""Patch BOTH readers, because one call now uses two (#4102).
`get_writepath_config` reads its `enabled` switch through
`plugin_context.get_setting` and all six tunable numbers through
`retrieval_surfaces.get_setting`. Patching only the first leaves the numbers
talking to a real database — which in a unit job is a connection error, and
in an integration job is worse: the test would pass or fail on whatever the
instance happened to have stored.
"""
from scribe.services import plugin_context as pc
from scribe.services import retrieval_surfaces as rs
def _side(uid, k, d=""):
return stored.get(k, d)
with patch.object(pc, "get_setting", AsyncMock(side_effect=_side)), \
patch.object(rs, "get_setting", AsyncMock(side_effect=_side)):
yield
PLUGIN = Path(__file__).resolve().parents[1] / "plugin"
HOOK = PLUGIN / "hooks" / "scribe_prior_art.sh"
@@ -31,9 +55,7 @@ def _cfg(**over):
# missing key raises inside its fail-open except and turns the arm into a
# silent no-op — which is indistinguishable from it working and finding
# nothing.
base = {"enabled": True, "threshold": 0.68, "top_k": 3, "rule_threshold": 0.72}
base.update(over)
return base
return writepath_cfg(**over)
# The semantic arm ignores payloads carrying less than WRITEPATH_MIN_CODE_CHARS
@@ -310,9 +332,23 @@ async def test_semantic_arm_only_asks_for_the_budget_the_place_arm_left():
@pytest.mark.asyncio
async def test_session_dedup_excludes_ids_from_the_reuse_arms():
async def test_session_dedup_marks_the_reuse_arms_rather_than_silencing_them():
"""exclude_ids governs the REUSE classes — nearby and semantic. (The sync
class has its own channel; see the tests above.)"""
class has its own channel; see the tests above.)
WHAT CHANGED AND WHY (#4101). This used to assert the opposite: the nearby
hit was dropped and its id pushed into the search's `exclude_ids`. That was
#3750's defect on the note arms — the second time a record was the best
answer, the session got silence, which reads exactly like "nothing is
recorded here". Now the repeat is rendered with a `seen` marker, and the
ledger never reaches the query, so the score the search reports describes
the bar rather than a candidate set the caller had already edited.
Marked on BOTH reuse arms, which is what this asserts: one menu with two
rules — `seen` on the semantic hits, silence on the nearby ones — would
make the marker read as a complete account of what the session has met,
when it would only cover half the lines.
"""
from scribe.services import plugin_context as pc
async def _listing(uid, **kw):
@@ -327,8 +363,15 @@ async def test_session_dedup_excludes_ids_from_the_reuse_arms():
patch.object(pc, "semantic_search_notes", search), \
patch.object(pc, "record_retrieval", MagicMock()):
out = await pc.build_write_path_hint(1, "src/x.py", code=REAL_CODE, exclude_ids=[12])
# The nearby hit was already surfaced this session → dropped, not repeated.
assert out["note_ids"] == []
assert out["note_ids"] == [12]
assert "seen" in next(
line for line in out["context"].splitlines() if "#12" in line
)
# #12 IS in this call's `exclude_ids`, and that is the other rule working:
# the place arm has just listed it, so the semantic arm must not list it
# again. What must never reach the query is the LEDGER, which is asserted
# on its own in tests/test_ledger_references_not_silence.py against a
# record this call has not otherwise rendered.
assert 12 in search.await_args.kwargs["exclude_ids"]
@@ -402,11 +445,10 @@ async def test_sync_surfacing_is_measured_under_its_own_usage_source():
@pytest.mark.asyncio
async def test_config_has_its_own_switch_and_threshold_but_shares_top_k():
async def test_config_has_its_own_switch_threshold_and_inherited_budget():
from scribe.services import plugin_context as pc
stored = {pc.WRITEPATH_ENABLED_KEY: "false"}
with patch.object(pc, "get_setting",
AsyncMock(side_effect=lambda uid, k, d: stored.get(k, d))):
with _stored_settings(stored):
cfg = await pc.get_writepath_config(1)
# Its own switch is off while auto-inject stays on...
assert cfg["enabled"] is False
@@ -415,8 +457,12 @@ async def test_config_has_its_own_switch_and_threshold_but_shares_top_k():
# made unrelated code — including `x = 1` at 0.58 — clear the bar.
assert cfg["threshold"] == pc.WRITEPATH_DEFAULT_THRESHOLD
assert cfg["threshold"] > pc.AUTOINJECT_DEFAULT_THRESHOLD
# ...and top_k is still shared: "how many titles at once" means the same
# thing on both surfaces.
# ...and the budget is INHERITED rather than shared (#4102). This arm has
# its own key now, because it fires before every Write and Edit while
# auto-inject fires once a turn, so the same number buys very different
# amounts of attention. Unset, it still reads auto-inject's — which is what
# stops the split from silently resetting an install that had tuned the
# knob when it was shared.
assert cfg["top_k"] == pc.AUTOINJECT_DEFAULT_TOP_K
@@ -428,8 +474,7 @@ async def test_writepath_threshold_is_operator_tunable_and_clamped():
async def _cfg_with(raw):
stored = {pc.WRITEPATH_THRESHOLD_KEY: raw}
with patch.object(pc, "get_setting",
AsyncMock(side_effect=lambda uid, k, d: stored.get(k, d))):
with _stored_settings(stored):
return await pc.get_writepath_config(1)
assert (await _cfg_with("0.9"))["threshold"] == 0.9
@@ -453,8 +498,7 @@ async def test_the_rule_arm_has_its_own_tunable_bar():
async def _cfg_with(raw):
stored = {pc.RULEHINT_THRESHOLD_KEY: raw}
with patch.object(pc, "get_setting",
AsyncMock(side_effect=lambda uid, k, d: stored.get(k, d))):
with _stored_settings(stored):
return await pc.get_writepath_config(1)
assert (await _cfg_with("0.8"))["rule_threshold"] == 0.8
@@ -473,8 +517,7 @@ async def test_the_two_write_path_bars_are_independent():
from scribe.services import plugin_context as pc
stored = {pc.WRITEPATH_THRESHOLD_KEY: "0.90", pc.RULEHINT_THRESHOLD_KEY: "0.61"}
with patch.object(pc, "get_setting",
AsyncMock(side_effect=lambda uid, k, d: stored.get(k, d))):
with _stored_settings(stored):
cfg = await pc.get_writepath_config(1)
assert cfg["threshold"] == 0.90
@@ -495,8 +538,7 @@ async def test_the_two_act_arms_read_independent_rule_bars():
from scribe.services import plugin_context as pc
stored = {pc.RULEHINT_THRESHOLD_KEY: "0.75", pc.TOOLRULE_THRESHOLD_KEY: "0.61"}
with patch.object(pc, "get_setting",
AsyncMock(side_effect=lambda uid, k, d: stored.get(k, d))):
with _stored_settings(stored):
cfg = await pc.get_writepath_config(1)
assert cfg["rule_threshold"] == 0.75
@@ -515,8 +557,7 @@ async def test_a_garbage_command_bar_falls_back_to_its_own_default():
from scribe.services import plugin_context as pc
stored = {pc.TOOLRULE_THRESHOLD_KEY: "banana"}
with patch.object(pc, "get_setting",
AsyncMock(side_effect=lambda uid, k, d: stored.get(k, d))):
with _stored_settings(stored):
cfg = await pc.get_writepath_config(1)
assert cfg["tool_rule_threshold"] == pc.TOOLRULE_DEFAULT_THRESHOLD
@@ -1264,8 +1305,16 @@ async def test_stamping_needs_named_shapes_and_a_recent_pull():
async def test_a_pulled_snippet_already_seen_is_evidence_not_menu():
"""The pulled-then-written flow IS the dedup-excluded flow: the hint offered
#7 earlier (so it sits in exclude_ids), the session pulled it, and now
writes code resembling it. #7 must be scored for this payload and handed
to the stamp as resemblance — without being re-listed in the menu."""
writes code resembling it. #7 must be scored for this payload and handed to
the stamp as resemblance.
THE MENU HALF INVERTED AT #4101. This asserted that #7 was evidence and NOT
menu — scored for the stamp, kept out of the lines. That followed from the
ledger being a hard exclusion; it is now a marker, so #7 is both, and the
`limit` no longer needs widening for it because nothing is dropped after
the search. What survives unchanged is the part the stamp depends on: the
pulled id stays in the query and its score reaches `resembles`.
"""
from scribe.services import plugin_context as pc
search = AsyncMock(return_value=[(0.91, fake_note(id=7, title="pulled", user_id=1, note_type="snippet")), (0.80, fake_note(id=8, title="fresh", user_id=1, note_type="snippet"))])
stamp = AsyncMock(return_value=[{
@@ -1283,15 +1332,20 @@ async def test_a_pulled_snippet_already_seen_is_evidence_not_menu():
1, "src/x.py", code=REAL_CODE, project_id=4, exclude_ids=[7],
stamp_shapes=[("sym", "debounce")], repo_key="git.example.com/a/b",
)
# The query kept #7 eligible (and widened the budget by one for it)...
# The query kept #7 eligible, and needs no extra budget for it now that
# nothing is dropped between the search and the menu.
kw = search.call_args.kwargs
assert 7 not in kw["exclude_ids"]
assert kw["limit"] == 4
# ...but the menu still honours the session dedup.
assert out["note_ids"] == [8]
assert "#7" not in "\n".join(
line for line in out["context"].splitlines() if "[similar" in line
)
assert kw["limit"] == 3
# ...and the menu carries it, marked.
seven = next(line for line in out["context"].splitlines() if "#7" in line)
assert "seen" in seven
# #8 (0.80) now falls OUTSIDE the margin band, because #7 (0.91) anchors it
# at 0.81 instead of being filtered out first. That is the point rather than
# a casualty: the band measures distance from the best answer, and the best
# answer here is #7 — suppressing it used to promote a materially weaker hit
# into a slot it had not earned, with nothing in the output saying so.
assert out["note_ids"] == [7]
# The stamp saw the pull and the resemblance score for this payload.
skw = stamp.call_args.kwargs
assert skw["pulled"] == {7: skw["pulled"][7]}
@@ -1670,9 +1724,47 @@ async def _write_path_row(rec, **kwargs):
async def test_a_record_this_arm_withheld_itself_is_not_a_near_miss():
"""The defect: the row's count is POST this arm's filter and the score was
captured PRE it, so a withheld record is indistinguishable from one the bar
rejected — while scoring higher than anything the bar ever let through."""
rejected — while scoring higher than anything the bar ever let through.
REACHED DIFFERENTLY SINCE #4101, and that is the news. The ledger used to
produce this case and no longer can: a repeat comes back from the search
and is rendered, so nothing is dropped and `best_available` describes the
bar alone. What still produces it is the one post-search filter left — a
PULLED record that this same call already listed by place. It stays in the
query because `resembles` needs its score, and is dropped from the menu
because it is already on it, which is exactly the pre/post split.
The property is unchanged and worth as much as it ever was; only the setup
that exhibits it moved. Written this way rather than deleted, because the
filter is still there and an arm that reported a score for it would be
making the same false claim about the bar.
"""
from scribe.services import plugin_context as pc
rec = MagicMock()
row = await _write_path_row(rec, exclude_ids=[7])
# Recorded at a sibling file → listed by PLACE in this same call, and
# pulled this session → kept in the query as resemblance evidence.
async def _listing(uid, **kw):
if kw["path"] == "src":
return ([_snippet_item(7, "already listed by place")], 1)
return ([], 0)
with patch.object(pc, "get_writepath_config", AsyncMock(return_value=_cfg())), \
patch.object(pc.snippets_svc, "list_snippets", _listing), \
patch.object(pc, "record_retrieval", rec), \
patch.object(pc, "owner_names_for", AsyncMock(return_value={})), \
patch.object(pc.shape_ledger_svc, "recent_pulls",
AsyncMock(return_value={7: _ts()})), \
patch.object(pc.shape_ledger_svc, "stamp_write_path_instances",
AsyncMock(return_value=[])), \
patch.object(pc, "semantic_search_notes",
_search_reporting(0.9, fake_note(
id=7, title="scored", user_id=1, note_type="snippet"))):
await pc.build_write_path_hint(
1, "src/x.py", code=REAL_CODE,
stamp_shapes=[("sym", "debounce")],
)
row = next(c for c in rec.call_args_list
if c.kwargs["source"] == "write_path")
assert row.kwargs["results"] == [], "the hit was withheld, so nothing shown"
assert row.kwargs["best_available"] is None, (
@@ -1682,6 +1774,27 @@ async def test_a_record_this_arm_withheld_itself_is_not_a_near_miss():
)
@pytest.mark.asyncio
async def test_a_ledger_repeat_no_longer_produces_that_case_at_all():
"""The other direction, and the reason the setup above had to move.
A record on the session ledger now clears the search, is rendered with a
marker, and is counted in `suppressed` — so nothing is withheld and the
reported score describes the bar. Pinned because the tempting way to keep
the old test passing would have been to null the score whenever the ledger
matched, which would delete the measurement #3670 was built for on exactly
the calls where the bar is most worth reading.
"""
rec = MagicMock()
row = await _write_path_row(rec, exclude_ids=[7])
assert row.kwargs["results"] == [], "a repeat is not a fresh result"
assert row.kwargs["suppressed"] == 1, "and it is counted rather than lost"
assert row.kwargs["best_available"] == 0.9, (
"nothing was withheld, so the reported score describes the bar"
)
@pytest.mark.asyncio
async def test_a_call_that_withheld_nothing_still_reports_what_the_bar_refused():
"""The other half, and what stops the fix being 'never report it'.