Every ledger is cleared on a compact, and a retrieval floor becomes something the model maintains (#163)
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 10s
CI & Build / integration (push) Successful in 48s
CI & Build / TypeScript typecheck (push) Successful in 53s
CI & Build / Python tests (push) Successful in 1m32s
CI & Build / Build & push image (push) Successful in 16s
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 10s
CI & Build / integration (push) Successful in 48s
CI & Build / TypeScript typecheck (push) Successful in 53s
CI & Build / Python tests (push) Successful in 1m32s
CI & Build / Build & push image (push) Successful in 16s
This commit was merged in pull request #163.
This commit is contained in:
@@ -0,0 +1,75 @@
|
|||||||
|
"""retrieval_tuning_events — why a floor is where it is (#4102)
|
||||||
|
|
||||||
|
Revision ID: 0103
|
||||||
|
Revises: 0102
|
||||||
|
Create Date: 2026-09-17
|
||||||
|
|
||||||
|
Milestone 416 stops shipping similarity thresholds as values somebody has to
|
||||||
|
defend, and hands the adjustment to the model that reads the surface's own
|
||||||
|
telemetry. The operator's decision:
|
||||||
|
|
||||||
|
"the floor should be chosen and adjusted by the model using it… the user
|
||||||
|
should be able to touch it but the model should be the thing handling it 9
|
||||||
|
times out of 10."
|
||||||
|
|
||||||
|
The number itself already has a home — the generic settings table. What has no
|
||||||
|
home is the ARGUMENT, and once the values move on their own the argument is the
|
||||||
|
part an operator needs: what changed, from what to what, who moved it, and on
|
||||||
|
what evidence. This table is that trail, and it is what makes the delegation
|
||||||
|
reviewable rather than merely automatic.
|
||||||
|
|
||||||
|
Nothing is backfilled. A surface with no rows here is sitting on its shipped
|
||||||
|
starting point, which is a true and useful thing for the history to say.
|
||||||
|
"""
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0103"
|
||||||
|
down_revision = "0102"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"retrieval_tuning_events",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
# FK-free, like retrieval_logs and app_logs: the record of why a number
|
||||||
|
# is where it is must outlive the account that moved it.
|
||||||
|
sa.Column("user_id", sa.Integer(), nullable=True),
|
||||||
|
# Also the surface's `retrieval_logs.source`, so a change can be read
|
||||||
|
# next to what the change did.
|
||||||
|
sa.Column("surface", sa.Text(), nullable=False),
|
||||||
|
sa.Column("dial", sa.Text(), nullable=False),
|
||||||
|
# Nullable: the first change to a surface has no stored predecessor. It
|
||||||
|
# moved off the shipped starting point, which is a different event from
|
||||||
|
# moving off a value somebody chose.
|
||||||
|
sa.Column("old_value", sa.Float(), nullable=True),
|
||||||
|
sa.Column("new_value", sa.Float(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"actor", sa.Text(), nullable=False, server_default=sa.text("'model'")
|
||||||
|
),
|
||||||
|
# Non-null here; non-BLANK is enforced at the service boundary, because
|
||||||
|
# a column that merely forbids NULL is satisfied by "" and a required
|
||||||
|
# field that accepts "" is a formality.
|
||||||
|
sa.Column("reason", sa.Text(), nullable=False),
|
||||||
|
)
|
||||||
|
# The only read this table has: one surface's history, newest first.
|
||||||
|
op.create_index(
|
||||||
|
"ix_retrieval_tuning_surface_created",
|
||||||
|
"retrieval_tuning_events",
|
||||||
|
["surface", sa.text("created_at DESC")],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index(
|
||||||
|
"ix_retrieval_tuning_surface_created", table_name="retrieval_tuning_events"
|
||||||
|
)
|
||||||
|
op.drop_table("retrieval_tuning_events")
|
||||||
@@ -97,6 +97,20 @@ const kbToolRuleThreshold = ref("0.68");
|
|||||||
// prose rather than anything a tool produced (#3852).
|
// prose rather than anything a tool produced (#3852).
|
||||||
const kbPromptRuleThreshold = ref("0.72");
|
const kbPromptRuleThreshold = ref("0.72");
|
||||||
const kbReportPrefThreshold = ref("0.72");
|
const kbReportPrefThreshold = ref("0.72");
|
||||||
|
// The BUDGETS, one per arm (#4102). Until this step only auto-inject had one
|
||||||
|
// and every other arm's ceiling was a module constant nobody could reach — so
|
||||||
|
// the only control an operator had over a noisy surface was to raise its bar,
|
||||||
|
// which discards the surface's best candidates along with its worst. A budget
|
||||||
|
// keeps the top of the ranking and drops the tail, which is what was wanted.
|
||||||
|
const kbWritePathTopK = ref("3");
|
||||||
|
const kbRuleHintTopK = ref("5");
|
||||||
|
const kbToolRuleTopK = ref("5");
|
||||||
|
const kbPromptRuleTopK = ref("3");
|
||||||
|
const kbReportPrefTopK = ref("3");
|
||||||
|
// What has been changed about retrieval, newest first — the review surface for
|
||||||
|
// changes the model made on the operator's behalf (#4102).
|
||||||
|
const tuningEvents = ref<TuningEvent[]>([]);
|
||||||
|
const loadingTuning = ref(false);
|
||||||
// Near-duplicate report floors, one per record kind (services/dedup.py).
|
// Near-duplicate report floors, one per record kind (services/dedup.py).
|
||||||
// Snippets are single-chunk, so their floor sits below the 0.90 write-time
|
// Snippets are single-chunk, so their floor sits below the 0.90 write-time
|
||||||
// gate and catches what it lets through. Notes/tasks are scored at chunk
|
// gate and catches what it lets through. Notes/tasks are scored at chunk
|
||||||
@@ -149,6 +163,37 @@ async function saveRetention() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One entry in the retrieval tuning trail (#4102). Mirrors
|
||||||
|
// RetrievalTuningEvent.to_dict() — `old_value` is null for the first change to
|
||||||
|
// a surface, which is a different event from moving off a value somebody chose
|
||||||
|
// and renders differently below.
|
||||||
|
interface TuningEvent {
|
||||||
|
id: number;
|
||||||
|
created_at: string | null;
|
||||||
|
surface: string;
|
||||||
|
dial: string;
|
||||||
|
old_value: number | null;
|
||||||
|
new_value: number;
|
||||||
|
actor: string;
|
||||||
|
reason: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadTuningHistory() {
|
||||||
|
loadingTuning.value = true;
|
||||||
|
try {
|
||||||
|
const res = await apiGet<{ events: TuningEvent[] }>(
|
||||||
|
"/api/retrieval/tuning-history?limit=25",
|
||||||
|
);
|
||||||
|
tuningEvents.value = res.events ?? [];
|
||||||
|
} catch {
|
||||||
|
// A history that cannot be read is not worth a toast on page load — the
|
||||||
|
// panel says so itself, and the settings above are still usable.
|
||||||
|
tuningEvents.value = [];
|
||||||
|
} finally {
|
||||||
|
loadingTuning.value = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function saveKbInject() {
|
async function saveKbInject() {
|
||||||
const t = Math.min(1, Math.max(0, Number(kbInjectThreshold.value) || 0));
|
const t = Math.min(1, Math.max(0, Number(kbInjectThreshold.value) || 0));
|
||||||
const k = Math.min(10, Math.max(1, Math.floor(Number(kbInjectTopK.value) || 1)));
|
const k = Math.min(10, Math.max(1, Math.floor(Number(kbInjectTopK.value) || 1)));
|
||||||
@@ -173,6 +218,22 @@ async function saveKbInject() {
|
|||||||
const trT = Math.min(1, Math.max(0, Number(kbToolRuleThreshold.value) || 0.68));
|
const trT = Math.min(1, Math.max(0, Number(kbToolRuleThreshold.value) || 0.68));
|
||||||
const prT = Math.min(1, Math.max(0, Number(kbPromptRuleThreshold.value) || 0.72));
|
const prT = Math.min(1, Math.max(0, Number(kbPromptRuleThreshold.value) || 0.72));
|
||||||
const rpT = Math.min(1, Math.max(0, Number(kbReportPrefThreshold.value) || 0.72));
|
const rpT = Math.min(1, Math.max(0, Number(kbReportPrefThreshold.value) || 0.72));
|
||||||
|
// The budgets, clamped the way the server clamps them: a whole number in
|
||||||
|
// [1, 10]. Never 0 — an arm turned off is turned off by its switch, and a
|
||||||
|
// budget of zero would run the search, log the retrieval and render nothing,
|
||||||
|
// which reads in the telemetry exactly like a bar nothing cleared.
|
||||||
|
const asK = (v: string, d: number) =>
|
||||||
|
Math.min(10, Math.max(1, Math.floor(Number(v) || d)));
|
||||||
|
const wpK = asK(kbWritePathTopK.value, 3);
|
||||||
|
const rhK = asK(kbRuleHintTopK.value, 5);
|
||||||
|
const trK = asK(kbToolRuleTopK.value, 5);
|
||||||
|
const prK = asK(kbPromptRuleTopK.value, 3);
|
||||||
|
const rpK = asK(kbReportPrefTopK.value, 3);
|
||||||
|
kbWritePathTopK.value = String(wpK);
|
||||||
|
kbRuleHintTopK.value = String(rhK);
|
||||||
|
kbToolRuleTopK.value = String(trK);
|
||||||
|
kbPromptRuleTopK.value = String(prK);
|
||||||
|
kbReportPrefTopK.value = String(rpK);
|
||||||
kbInjectThreshold.value = String(t);
|
kbInjectThreshold.value = String(t);
|
||||||
kbInjectTopK.value = String(k);
|
kbInjectTopK.value = String(k);
|
||||||
kbDupThresholdSnippet.value = String(dupSnip);
|
kbDupThresholdSnippet.value = String(dupSnip);
|
||||||
@@ -211,6 +272,15 @@ async function saveKbInject() {
|
|||||||
// and a constant that lands under the bar is a dead arm, not a quiet
|
// and a constant that lands under the bar is a dead arm, not a quiet
|
||||||
// one (#3860).
|
// one (#3860).
|
||||||
kb_reportpref_threshold: String(rpT),
|
kb_reportpref_threshold: String(rpT),
|
||||||
|
// The budgets. Every one of these keys is recognised by the server as a
|
||||||
|
// retrieval dial, so this save is recorded in the tuning history as a
|
||||||
|
// change the OPERATOR made — which is the one entry the model must not
|
||||||
|
// quietly tune back.
|
||||||
|
kb_writepath_top_k: String(wpK),
|
||||||
|
kb_rulehint_top_k: String(rhK),
|
||||||
|
kb_toolrule_top_k: String(trK),
|
||||||
|
kb_promptrule_top_k: String(prK),
|
||||||
|
kb_reportpref_top_k: String(rpK),
|
||||||
kb_duplicate_threshold_snippet: String(dupSnip),
|
kb_duplicate_threshold_snippet: String(dupSnip),
|
||||||
kb_duplicate_threshold_note: String(dupNote),
|
kb_duplicate_threshold_note: String(dupNote),
|
||||||
kb_duplicate_threshold_task: String(dupTask),
|
kb_duplicate_threshold_task: String(dupTask),
|
||||||
@@ -218,6 +288,10 @@ async function saveKbInject() {
|
|||||||
});
|
});
|
||||||
kbInjectSaved.value = true;
|
kbInjectSaved.value = true;
|
||||||
setTimeout(() => (kbInjectSaved.value = false), 2000);
|
setTimeout(() => (kbInjectSaved.value = false), 2000);
|
||||||
|
// The save just appended to the history it sits above, so re-read it —
|
||||||
|
// otherwise the panel shows a trail that is stale by exactly the change
|
||||||
|
// the operator is looking at it to confirm.
|
||||||
|
await loadTuningHistory();
|
||||||
} catch {
|
} catch {
|
||||||
toastStore.show('Failed to save auto-inject settings', 'error');
|
toastStore.show('Failed to save auto-inject settings', 'error');
|
||||||
} finally {
|
} finally {
|
||||||
@@ -672,6 +746,27 @@ onMounted(async () => {
|
|||||||
if (allSettings.kb_writepath_threshold !== undefined) {
|
if (allSettings.kb_writepath_threshold !== undefined) {
|
||||||
kbWritePathThreshold.value = allSettings.kb_writepath_threshold;
|
kbWritePathThreshold.value = allSettings.kb_writepath_threshold;
|
||||||
}
|
}
|
||||||
|
if (allSettings.kb_writepath_top_k !== undefined) {
|
||||||
|
kbWritePathTopK.value = allSettings.kb_writepath_top_k;
|
||||||
|
} else if (allSettings.kb_autoinject_top_k !== undefined) {
|
||||||
|
// The write path shared auto-inject's ceiling until it was given its own
|
||||||
|
// key, so an install that tuned the shared one must keep seeing that value
|
||||||
|
// here — the server falls back the same way.
|
||||||
|
kbWritePathTopK.value = allSettings.kb_autoinject_top_k;
|
||||||
|
}
|
||||||
|
if (allSettings.kb_rulehint_top_k !== undefined) {
|
||||||
|
kbRuleHintTopK.value = allSettings.kb_rulehint_top_k;
|
||||||
|
}
|
||||||
|
if (allSettings.kb_toolrule_top_k !== undefined) {
|
||||||
|
kbToolRuleTopK.value = allSettings.kb_toolrule_top_k;
|
||||||
|
}
|
||||||
|
if (allSettings.kb_promptrule_top_k !== undefined) {
|
||||||
|
kbPromptRuleTopK.value = allSettings.kb_promptrule_top_k;
|
||||||
|
}
|
||||||
|
if (allSettings.kb_reportpref_top_k !== undefined) {
|
||||||
|
kbReportPrefTopK.value = allSettings.kb_reportpref_top_k;
|
||||||
|
}
|
||||||
|
await loadTuningHistory();
|
||||||
if (allSettings.kb_duplicate_threshold_snippet !== undefined) {
|
if (allSettings.kb_duplicate_threshold_snippet !== undefined) {
|
||||||
kbDupThresholdSnippet.value = allSettings.kb_duplicate_threshold_snippet;
|
kbDupThresholdSnippet.value = allSettings.kb_duplicate_threshold_snippet;
|
||||||
}
|
}
|
||||||
@@ -1510,13 +1605,27 @@ async function deleteUser(userId: number) {
|
|||||||
Stricter than the prompt threshold above on purpose. Any two pieces of
|
Stricter than the prompt threshold above on purpose. Any two pieces of
|
||||||
code look somewhat alike — shared keywords, indentation, structure — so
|
code look somewhat alike — shared keywords, indentation, structure — so
|
||||||
resemblance scores start higher for code than for prose, and a bar tuned
|
resemblance scores start higher for code than for prose, and a bar tuned
|
||||||
for prompts flags unrelated code as prior art. Lower this if genuine
|
for prompts flags unrelated code as prior art. Claude keeps this
|
||||||
duplicates go unnoticed; raise it if you're being offered snippets that
|
one current from what the arm actually surfaced and refused; set it
|
||||||
have nothing to do with what's being written. Snippets recorded at the
|
yourself if you disagree with where it has landed. Snippets recorded at the
|
||||||
exact file are always shown regardless — those are prior art by
|
exact file are always shown regardless — those are prior art by
|
||||||
location, not by resemblance.
|
location, not by resemblance.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="field">
|
||||||
|
<label for="kb-writepath-topk">Max prior-art lines per edit</label>
|
||||||
|
<input
|
||||||
|
id="kb-writepath-topk"
|
||||||
|
v-model="kbWritePathTopK"
|
||||||
|
type="number"
|
||||||
|
min="1"
|
||||||
|
max="10"
|
||||||
|
step="1"
|
||||||
|
class="fs-input input"
|
||||||
|
style="max-width: 8rem"
|
||||||
|
/>
|
||||||
|
<p class="field-hint">How many snippets and issues one edit may be offered (1–10). Reach for this rather than the threshold when the hint feels long: lowering it keeps the best matches and drops the tail, while raising the bar above throws away good matches along with weak ones.</p>
|
||||||
|
</div>
|
||||||
<div class="field">
|
<div class="field">
|
||||||
<label for="kb-rulehint-threshold">Standing-rule confidence threshold (0–1)</label>
|
<label for="kb-rulehint-threshold">Standing-rule confidence threshold (0–1)</label>
|
||||||
<input
|
<input
|
||||||
@@ -1535,10 +1644,25 @@ async function deleteUser(userId: number) {
|
|||||||
preloaded any more, so this is the only way a rule reaches a write.
|
preloaded any more, so this is the only way a rule reaches a write.
|
||||||
Stricter than the threshold above, because there are far fewer rules
|
Stricter than the threshold above, because there are far fewer rules
|
||||||
than snippets: with a small set something always ranks first, so the
|
than snippets: with a small set something always ranks first, so the
|
||||||
bar has to carry more of the judgement. Raise it if rules keep
|
bar has to carry more of the judgement. If rules arrive unread, or one
|
||||||
arriving unread; lower it if a rule you needed never showed up.
|
you needed never showed up, that is Claude's to notice and correct
|
||||||
|
from the telemetry — and the budget below is usually the better lever.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="field">
|
||||||
|
<label for="kb-rulehint-topk">Max rules per edit</label>
|
||||||
|
<input
|
||||||
|
id="kb-rulehint-topk"
|
||||||
|
v-model="kbRuleHintTopK"
|
||||||
|
type="number"
|
||||||
|
min="1"
|
||||||
|
max="10"
|
||||||
|
step="1"
|
||||||
|
class="fs-input input"
|
||||||
|
style="max-width: 8rem"
|
||||||
|
/>
|
||||||
|
<p class="field-hint">How many standing rules one edit may be shown (1–10).</p>
|
||||||
|
</div>
|
||||||
<div class="field">
|
<div class="field">
|
||||||
<label for="kb-toolrule-threshold">Command confidence threshold (0–1)</label>
|
<label for="kb-toolrule-threshold">Command confidence threshold (0–1)</label>
|
||||||
<input
|
<input
|
||||||
@@ -1556,10 +1680,23 @@ async function deleteUser(userId: number) {
|
|||||||
query is the command text rather than code. Lower than the one above
|
query is the command text rather than code. Lower than the one above
|
||||||
on purpose: a shell command is short, so it scores lower for the same
|
on purpose: a shell command is short, so it scores lower for the same
|
||||||
relevance — at a shared bar this arm spoke on 2% of calls against the
|
relevance — at a shared bar this arm spoke on 2% of calls against the
|
||||||
write path's 37%. Raise it if commands attract rules that do not
|
write path's 37%.
|
||||||
apply; lower it if a <code>git push</code> arrives with nothing.
|
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="field">
|
||||||
|
<label for="kb-toolrule-topk">Max rules per command</label>
|
||||||
|
<input
|
||||||
|
id="kb-toolrule-topk"
|
||||||
|
v-model="kbToolRuleTopK"
|
||||||
|
type="number"
|
||||||
|
min="1"
|
||||||
|
max="10"
|
||||||
|
step="1"
|
||||||
|
class="fs-input input"
|
||||||
|
style="max-width: 8rem"
|
||||||
|
/>
|
||||||
|
<p class="field-hint">How many standing rules one command may be shown (1–10). This is the busiest arm there is — it fires before every command — so its budget is the one most worth keeping small.</p>
|
||||||
|
</div>
|
||||||
<div class="field">
|
<div class="field">
|
||||||
<label for="kb-promptrule-threshold">Prompt confidence threshold (0–1)</label>
|
<label for="kb-promptrule-threshold">Prompt confidence threshold (0–1)</label>
|
||||||
<input
|
<input
|
||||||
@@ -1580,6 +1717,20 @@ async function deleteUser(userId: number) {
|
|||||||
not a command or a file — which is why it carries its own number.
|
not a command or a file — which is why it carries its own number.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="field">
|
||||||
|
<label for="kb-promptrule-topk">Max rules per prompt</label>
|
||||||
|
<input
|
||||||
|
id="kb-promptrule-topk"
|
||||||
|
v-model="kbPromptRuleTopK"
|
||||||
|
type="number"
|
||||||
|
min="1"
|
||||||
|
max="10"
|
||||||
|
step="1"
|
||||||
|
class="fs-input input"
|
||||||
|
style="max-width: 8rem"
|
||||||
|
/>
|
||||||
|
<p class="field-hint">How many rules or preferences one message may be shown (1–10).</p>
|
||||||
|
</div>
|
||||||
<div class="field">
|
<div class="field">
|
||||||
<label for="kb-reportpref-threshold">Completion-report confidence threshold (0–1)</label>
|
<label for="kb-reportpref-threshold">Completion-report confidence threshold (0–1)</label>
|
||||||
<input
|
<input
|
||||||
@@ -1597,11 +1748,66 @@ async function deleteUser(userId: number) {
|
|||||||
written</em>, looked up when a task closes. Unlike every other bar
|
written</em>, looked up when a task closes. Unlike every other bar
|
||||||
here, the question this arm asks never changes — so its score is
|
here, the question this arm asks never changes — so its score is
|
||||||
fixed by your preferences alone, and it will either always find one
|
fixed by your preferences alone, and it will either always find one
|
||||||
or never find one. If you have written a preference for report shape
|
or never find one, and no run of calls will reveal a dead one on its
|
||||||
and it is not arriving, lower this; there is no run of calls that
|
own. That is why this arm is worth looking up in the panel below when
|
||||||
will reveal the problem on its own.
|
a report preference never seems to arrive.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="field">
|
||||||
|
<label for="kb-reportpref-topk">Max report preferences</label>
|
||||||
|
<input
|
||||||
|
id="kb-reportpref-topk"
|
||||||
|
v-model="kbReportPrefTopK"
|
||||||
|
type="number"
|
||||||
|
min="1"
|
||||||
|
max="10"
|
||||||
|
step="1"
|
||||||
|
class="fs-input input"
|
||||||
|
style="max-width: 8rem"
|
||||||
|
/>
|
||||||
|
<p class="field-hint">How many preferences a finished task may be shown (1–10).</p>
|
||||||
|
</div>
|
||||||
|
<!-- THE REVIEW SURFACE (#4102). These numbers are maintained by the
|
||||||
|
model that uses them: it reads which records each bar refused and
|
||||||
|
moves the dial with the argument attached. This panel is the other
|
||||||
|
half of that bargain — a change made on your behalf is one you can
|
||||||
|
read, disagree with, and set back by hand above. -->
|
||||||
|
<div class="tuning-history">
|
||||||
|
<h4 class="tuning-history-title">What has been tuned</h4>
|
||||||
|
<p class="field-hint">
|
||||||
|
Claude adjusts the bars and budgets above from what each arm actually
|
||||||
|
surfaced and refused, and has to state a reason to change one. Nothing
|
||||||
|
here needs your attention as a matter of course — it is here so that
|
||||||
|
when a surface behaves oddly, why it is set the way it is can be read
|
||||||
|
rather than guessed at. Anything you change yourself is recorded the
|
||||||
|
same way, and is the one entry Claude will not quietly move back.
|
||||||
|
</p>
|
||||||
|
<p v-if="loadingTuning" class="field-hint">Loading…</p>
|
||||||
|
<p v-else-if="!tuningEvents.length" class="field-hint">
|
||||||
|
Nothing has been changed yet — every surface is on the value Scribe
|
||||||
|
shipped. Those are starting points measured against one corpus with
|
||||||
|
one embedding model, not answers, so expect this to fill.
|
||||||
|
</p>
|
||||||
|
<ul v-else class="tuning-list">
|
||||||
|
<li v-for="ev in tuningEvents" :key="ev.id" class="tuning-item">
|
||||||
|
<div class="tuning-item-head">
|
||||||
|
<span class="tuning-surface">{{ ev.surface }}</span>
|
||||||
|
<span class="tuning-dial">{{ ev.dial }}</span>
|
||||||
|
<span class="tuning-move">
|
||||||
|
<template v-if="ev.old_value !== null">{{ ev.old_value }} →</template>
|
||||||
|
<template v-else>set to</template>
|
||||||
|
{{ ev.new_value }}
|
||||||
|
</span>
|
||||||
|
<span class="tuning-actor" :class="{ 'is-human': ev.actor === 'human' }">
|
||||||
|
{{ ev.actor === 'human' ? 'you' : 'Claude' }}
|
||||||
|
</span>
|
||||||
|
<span v-if="ev.created_at" class="tuning-when">{{ fmtDate(ev.created_at) }}</span>
|
||||||
|
</div>
|
||||||
|
<p class="tuning-reason">{{ ev.reason }}</p>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- A design system belongs to a PROJECT, and the picker for it lives on
|
<!-- A design system belongs to a PROJECT, and the picker for it lives on
|
||||||
the project. There was a setting here that designated the system
|
the project. There was a setting here that designated the system
|
||||||
this install's own interface was built from; it only ever described
|
this install's own interface was built from; it only ever described
|
||||||
@@ -3848,4 +4054,65 @@ async function deleteUser(userId: number) {
|
|||||||
.area-admin-form { display: flex; flex-direction: column; gap: 0.5rem; flex: 1; }
|
.area-admin-form { display: flex; flex-direction: column; gap: 0.5rem; flex: 1; }
|
||||||
.area-admin-create { margin-top: var(--fs-space-4); }
|
.area-admin-create { margin-top: var(--fs-space-4); }
|
||||||
.area-admin-actions { display: flex; gap: 0.4rem; }
|
.area-admin-actions { display: flex; gap: 0.4rem; }
|
||||||
|
|
||||||
|
/* The retrieval tuning trail (#4102). Reads as a record, not a control panel:
|
||||||
|
the operator is reviewing what was done, and the reason is the part worth
|
||||||
|
reading, so it gets the full-width line under a compact header row. */
|
||||||
|
.tuning-history {
|
||||||
|
margin-top: var(--fs-space-5);
|
||||||
|
padding-top: var(--fs-space-4);
|
||||||
|
border-top: 1px solid var(--fs-border-color);
|
||||||
|
}
|
||||||
|
.tuning-history-title {
|
||||||
|
margin: 0 0 var(--fs-space-2);
|
||||||
|
font-size: 0.95rem;
|
||||||
|
color: var(--fs-text-primary);
|
||||||
|
}
|
||||||
|
.tuning-list {
|
||||||
|
list-style: none;
|
||||||
|
margin: var(--fs-space-3) 0 0;
|
||||||
|
padding: 0;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: var(--fs-space-2);
|
||||||
|
}
|
||||||
|
.tuning-item {
|
||||||
|
padding: var(--fs-space-3);
|
||||||
|
background: var(--fs-surface-raised);
|
||||||
|
border: 1px solid var(--fs-border-color);
|
||||||
|
border-radius: var(--fs-radius-md);
|
||||||
|
}
|
||||||
|
.tuning-item-head {
|
||||||
|
display: flex;
|
||||||
|
align-items: baseline;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: var(--fs-space-2);
|
||||||
|
font-size: 0.85rem;
|
||||||
|
}
|
||||||
|
.tuning-surface {
|
||||||
|
font-family: var(--fs-font-mono);
|
||||||
|
color: var(--fs-text-primary);
|
||||||
|
}
|
||||||
|
.tuning-dial,
|
||||||
|
.tuning-move {
|
||||||
|
color: var(--fs-text-secondary);
|
||||||
|
}
|
||||||
|
.tuning-actor {
|
||||||
|
color: var(--fs-text-tertiary);
|
||||||
|
border: 1px solid var(--fs-border-color);
|
||||||
|
border-radius: var(--fs-radius-sm);
|
||||||
|
padding: 0 0.4rem;
|
||||||
|
}
|
||||||
|
/* The operator's own changes are marked, because "did I do this, or did the
|
||||||
|
session?" is the first question this list is asked. */
|
||||||
|
.tuning-actor.is-human {
|
||||||
|
color: var(--fs-accent);
|
||||||
|
border-color: var(--fs-accent);
|
||||||
|
}
|
||||||
|
.tuning-when { margin-left: auto; color: var(--fs-text-tertiary); }
|
||||||
|
.tuning-reason {
|
||||||
|
margin: var(--fs-space-2) 0 0;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: var(--fs-text-secondary);
|
||||||
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "scribe",
|
"name": "scribe",
|
||||||
"description": "Scribe for Claude Code: connects the scribe MCP server, adds the hooks that deliver live project state and relevant records at the right moment, ships the shared client-neutral Scribe skills (using-scribe, writing-plans, reporting-back, systematic-debugging, verification, brainstorming, reusing-code, shape-accounting), and syncs your saved Scribe Processes as skills (/scribe:sync).",
|
"description": "Scribe for Claude Code: connects the scribe MCP server, adds the hooks that deliver live project state and relevant records at the right moment, ships the shared client-neutral Scribe skills (using-scribe, writing-plans, reporting-back, systematic-debugging, verification, brainstorming, reusing-code, shape-accounting), and syncs your saved Scribe Processes as skills (/scribe:sync).",
|
||||||
"version": "2026.09.16.2102",
|
"version": "2026.09.17.0111",
|
||||||
"author": {
|
"author": {
|
||||||
"name": "Bryan Van Deusen"
|
"name": "Bryan Van Deusen"
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -308,6 +308,50 @@ scribe_held_query() {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Drop EVERY per-session ledger, matched by convention rather than listed (#4101).
|
||||||
|
#
|
||||||
|
# A LIST IS THE BUG. Until now the compact/clear branch named its files one at
|
||||||
|
# a time, and it named two of the five: `.rules.ids` and `.opened.ids` were
|
||||||
|
# cleared while `.ids` (notes), `.sync.ids` (shape signals) and `.derive.ids`
|
||||||
|
# survived. So milestone 386's defect — "a compaction destroys the context but
|
||||||
|
# not the ledger, so the most applicable records become permanently
|
||||||
|
# unreachable mid-session" — was fixed for rules and left standing on the note
|
||||||
|
# and snippet surfaces, which are the ones that fire most often.
|
||||||
|
#
|
||||||
|
# Nobody decided that. The list was written when rules were the only ledger
|
||||||
|
# that mattered and was never revisited when the others arrived, which is what
|
||||||
|
# a hand-maintained list of "things to remember to clean up" does. Adding three
|
||||||
|
# more `rm` lines would rebuild the same trap for the sixth ledger.
|
||||||
|
#
|
||||||
|
# So the rule is the NAME: a per-session ledger is `<sid>[.<kind>].ids`, and
|
||||||
|
# everything matching that goes. A new ledger following the convention is
|
||||||
|
# covered the day it is written, by nobody. One that does not follow it is a
|
||||||
|
# deliberate exception and has to say so.
|
||||||
|
#
|
||||||
|
# Scoped to `.ids` rather than `<sid>.*` so a marker that is REWRITTEN on
|
||||||
|
# compact rather than discarded can still live in these directories without
|
||||||
|
# being swept away by a glob that was never told about it. `<sid>.unreached`
|
||||||
|
# is the live example: it records that the instance could not be reached, not
|
||||||
|
# what the session holds, and #2932 needs it to outlive a compaction.
|
||||||
|
#
|
||||||
|
# TWO DIRECTORIES, WHICH IS ITS OWN LESSON. The first cut of this swept only
|
||||||
|
# `scribe-priorart` — every ledger named in the hooks was there, so the list
|
||||||
|
# looked complete. `scribe_autoinject.sh` keeps its note ledger in
|
||||||
|
# `scribe-autoinject`, so the arm that fires most (598 calls in five days) was
|
||||||
|
# the one the clear could not reach, and the fix read as finished. Hence the
|
||||||
|
# roster here rather than a path at the call site: one place to add to, and
|
||||||
|
# the tests read THIS string rather than a copy of it.
|
||||||
|
SCRIBE_LEDGER_DIRS="scribe-priorart scribe-autoinject"
|
||||||
|
|
||||||
|
scribe_clear_session_ledgers() {
|
||||||
|
local sid="$1" dir
|
||||||
|
[ -n "$sid" ] || return 0
|
||||||
|
for dir in $SCRIBE_LEDGER_DIRS; do
|
||||||
|
rm -f "${TMPDIR:-/tmp}/$dir/$sid"*.ids 2>/dev/null || true
|
||||||
|
done
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# WHICH PROJECT IS THIS DIRECTORY'S? (#4085)
|
# WHICH PROJECT IS THIS DIRECTORY'S? (#4085)
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -77,9 +77,11 @@ source=$(printf '%s' "$event" | jq -r '.source // empty' 2>/dev/null) || source=
|
|||||||
# this state are the ones that fire most often, which is to say the ones that
|
# this state are the ones that fire most often, which is to say the ones that
|
||||||
# apply most.
|
# apply most.
|
||||||
#
|
#
|
||||||
# The session id survives a compaction — the etag marker further down is
|
# The session id survives a compaction — the ledgers are keyed by it and are
|
||||||
# rewritten on `compact` and keyed by session_id, which is only meaningful if
|
# still found under the same name afterwards, so a stale ledger is genuinely
|
||||||
# the id is stable — so the stale ledger is genuinely found again, not orphaned.
|
# reached again rather than orphaned. (This used to point at an etag marker as
|
||||||
|
# the evidence for that; milestone 394 removed the preload the etag described,
|
||||||
|
# and `plugin_context.py` records its retirement.)
|
||||||
#
|
#
|
||||||
# CLEARED ON THE SOURCES THAT DESTROY CONTEXT, AND ONLY THOSE:
|
# CLEARED ON THE SOURCES THAT DESTROY CONTEXT, AND ONLY THOSE:
|
||||||
#
|
#
|
||||||
@@ -93,25 +95,42 @@ source=$(printf '%s' "$event" | jq -r '.source // empty' 2>/dev/null) || source=
|
|||||||
# fork carries the conversation, so if it inherits the id the ledger
|
# fork carries the conversation, so if it inherits the id the ledger
|
||||||
# is accurate, and if it gets a new one the file is empty anyway.
|
# is accurate, and if it gets a new one the file is empty anyway.
|
||||||
#
|
#
|
||||||
# ONLY the rules ledger. The same directory holds .ids / .sync.ids /
|
# EVERY LEDGER, AND THE NOTE ARMS NEEDED IT MOST (#4101).
|
||||||
# .derive.ids for the note arms. Whether a surfaced NOTE should return after a
|
#
|
||||||
# compaction is a different question with a different answer, and leaving those
|
# This used to clear the two rule ledgers by name and say, in a comment, that
|
||||||
# alone is a decision rather than an oversight.
|
# leaving `.ids` / `.sync.ids` / `.derive.ids` alone was "a decision rather than
|
||||||
|
# an oversight". Reading the note arms says otherwise, on two counts:
|
||||||
|
#
|
||||||
|
# - They are HARD exclusions. `exclude_ids` goes into `semantic_search_notes`
|
||||||
|
# itself, so a surfaced note is removed from the result set — it is not
|
||||||
|
# rendered as a reference the way #3750 made a repeated rule. There is no
|
||||||
|
# weaker form for it to fall back to.
|
||||||
|
# - They never AGE. #3751 gave the rules ledger a TTL precisely because
|
||||||
|
# salience decays without a context event; the note channels were left on a
|
||||||
|
# flat read.
|
||||||
|
#
|
||||||
|
# Hard plus permanent plus never cleared means a note surfaced in the first
|
||||||
|
# minute of a session is unreachable for the rest of it, through any number of
|
||||||
|
# compactions. That is milestone 386's original defect, alive on the arms that
|
||||||
|
# fire most often, and nothing about it was decided.
|
||||||
|
#
|
||||||
|
# THE LIST WAS THE BUG, so the fix is not a longer list. `scribe_clear_session_
|
||||||
|
# ledgers` matches on the naming convention — a per-session ledger is
|
||||||
|
# `<sid>[.<kind>].ids` — which covers all five and covers the sixth on the day
|
||||||
|
# it is written. Best-effort, like every other filesystem touch in these hooks:
|
||||||
|
# a ledger that cannot be removed costs a repeated exclusion, never a session.
|
||||||
|
#
|
||||||
|
# NOT swept: `<sid>.unreached`, which records that the instance was unreachable
|
||||||
|
# rather than what the session holds, and survives on purpose. The directories
|
||||||
|
# swept are `scribe_defs.sh`'s `SCRIBE_LEDGER_DIRS` — plural, because the
|
||||||
|
# auto-inject arm keeps its ledger somewhere else and a single-directory sweep
|
||||||
|
# missed exactly the arm that fires most.
|
||||||
case "$source" in
|
case "$source" in
|
||||||
compact|clear)
|
compact|clear)
|
||||||
sid=$(printf '%s' "$event" | jq -r '.session_id // empty' 2>/dev/null) || sid=""
|
sid=$(printf '%s' "$event" | jq -r '.session_id // empty' 2>/dev/null) || sid=""
|
||||||
if [ -n "$sid" ]; then
|
if [ -n "$sid" ]; then
|
||||||
safe_sid=$(printf '%s' "$sid" | tr -c 'A-Za-z0-9._-' '_')
|
safe_sid=$(printf '%s' "$sid" | tr -c 'A-Za-z0-9._-' '_')
|
||||||
# Best-effort, like every other filesystem touch in these hooks: a ledger
|
scribe_clear_session_ledgers "$safe_sid"
|
||||||
# that cannot be removed costs a repeated exclusion, never a session.
|
|
||||||
rm -f "${TMPDIR:-/tmp}/scribe-priorart/${safe_sid}.rules.ids" 2>/dev/null || true
|
|
||||||
# BOTH ledgers, for one reason (#4100). `.opened.ids` records what the
|
|
||||||
# session read; a compaction is exactly the event that takes it away
|
|
||||||
# again. Clearing the naming ledger while keeping this one would leave
|
|
||||||
# the surfacing arms telling a freshly-summarised session "you opened
|
|
||||||
# it earlier" about a rule that is no longer anywhere in its context —
|
|
||||||
# a more confident version of the claim this milestone removed.
|
|
||||||
rm -f "${TMPDIR:-/tmp}/scribe-priorart/${safe_sid}.opened.ids" 2>/dev/null || true
|
|
||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from scribe.routes.notes import notes_bp
|
|||||||
from scribe.routes.milestones import milestones_bp
|
from scribe.routes.milestones import milestones_bp
|
||||||
from scribe.routes.task_logs import task_logs_bp
|
from scribe.routes.task_logs import task_logs_bp
|
||||||
from scribe.routes.projects import projects_bp
|
from scribe.routes.projects import projects_bp
|
||||||
|
from scribe.routes.retrieval import retrieval_bp
|
||||||
from scribe.routes.settings import settings_bp
|
from scribe.routes.settings import settings_bp
|
||||||
from scribe.routes.tasks import tasks_bp
|
from scribe.routes.tasks import tasks_bp
|
||||||
from scribe.routes.groups import groups_bp
|
from scribe.routes.groups import groups_bp
|
||||||
@@ -79,6 +80,7 @@ def create_app() -> Quart:
|
|||||||
app.register_blueprint(milestones_bp)
|
app.register_blueprint(milestones_bp)
|
||||||
app.register_blueprint(notes_bp)
|
app.register_blueprint(notes_bp)
|
||||||
app.register_blueprint(projects_bp)
|
app.register_blueprint(projects_bp)
|
||||||
|
app.register_blueprint(retrieval_bp)
|
||||||
app.register_blueprint(settings_bp)
|
app.register_blueprint(settings_bp)
|
||||||
app.register_blueprint(task_logs_bp)
|
app.register_blueprint(task_logs_bp)
|
||||||
app.register_blueprint(tasks_bp)
|
app.register_blueprint(tasks_bp)
|
||||||
|
|||||||
@@ -131,6 +131,12 @@ _READ_ONLY_TOOLS = frozenset({
|
|||||||
# looked at names that had one (#3191). rule_history records a pull the way
|
# looked at names that had one (#3191). rule_history records a pull the way
|
||||||
# the getters above do.
|
# the getters above do.
|
||||||
"rules_due_for_verification", "rule_history",
|
"rules_due_for_verification", "rule_history",
|
||||||
|
# What each retrieval surface's floor and budget currently are, and what
|
||||||
|
# has been changed about them (#4102). Both pure reads; `tune_retrieval` is
|
||||||
|
# the write and is deliberately NOT here. Read access matters more than
|
||||||
|
# usual for these two: a session that cannot see the bar in force, or the
|
||||||
|
# reason it was last moved, is a session that will move it again blind.
|
||||||
|
"retrieval_surfaces", "retrieval_tuning_history",
|
||||||
})
|
})
|
||||||
|
|
||||||
# Every tool that WRITES, by name. Nothing reads this set at runtime — a tool
|
# Every tool that WRITES, by name. Nothing reads this set at runtime — a tool
|
||||||
@@ -166,6 +172,9 @@ _WRITE_TOOLS = frozenset({
|
|||||||
"create_rule", "create_project_rule", "update_rule", "move_rule", "delete_rule",
|
"create_rule", "create_project_rule", "update_rule", "move_rule", "delete_rule",
|
||||||
"create_preference", "update_preference",
|
"create_preference", "update_preference",
|
||||||
"relate_rules", "unrelate_rules", "mark_rule_verified",
|
"relate_rules", "unrelate_rules", "mark_rule_verified",
|
||||||
|
# retrieval tuning — a write in both senses: it moves the number the arm
|
||||||
|
# reads, and it appends the reason to the audit trail (#4102).
|
||||||
|
"tune_retrieval",
|
||||||
# trash
|
# trash
|
||||||
"restore", "purge_trash",
|
"restore", "purge_trash",
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -5,14 +5,15 @@ to a FastMCP instance. `register_all(mcp)` is the single entry point called
|
|||||||
from `mcp.server.build_mcp_server`.
|
from `mcp.server.build_mcp_server`.
|
||||||
"""
|
"""
|
||||||
from scribe.mcp.tools import (
|
from scribe.mcp.tools import (
|
||||||
design_systems, milestones, notes, processes, projects, recent, repos, rulebooks, search, shapes,
|
design_systems, milestones, notes, processes, projects, recent, repos, retrieval_tuning,
|
||||||
snippets, systems, tags, tasks, trash,
|
rulebooks, search, shapes, snippets, systems, tags, tasks, trash,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def register_all(mcp) -> None:
|
def register_all(mcp) -> None:
|
||||||
"""Register every tool module's tools on the given FastMCP instance."""
|
"""Register every tool module's tools on the given FastMCP instance."""
|
||||||
search.register(mcp)
|
search.register(mcp)
|
||||||
|
retrieval_tuning.register(mcp)
|
||||||
notes.register(mcp)
|
notes.register(mcp)
|
||||||
tasks.register(mcp)
|
tasks.register(mcp)
|
||||||
projects.register(mcp)
|
projects.register(mcp)
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
"""Reading and moving a retrieval surface's floor and budget (#4102).
|
||||||
|
|
||||||
|
The write half of the loop `retrieval_telemetry` opens. That tool says what a
|
||||||
|
bar did; these say what the bar IS, and let it be changed with the argument
|
||||||
|
attached.
|
||||||
|
"""
|
||||||
|
from scribe.mcp._context import current_user_id
|
||||||
|
from scribe.services import retrieval_tuning as tuning_svc
|
||||||
|
|
||||||
|
|
||||||
|
async def retrieval_surfaces() -> dict:
|
||||||
|
"""Every retrieval surface Scribe pushes on, with its floor and its budget.
|
||||||
|
|
||||||
|
Read this before changing either, and read it beside
|
||||||
|
`retrieval_telemetry(days=…, near_miss_samples=5)` — this tool says what is
|
||||||
|
in force, that one says what it did.
|
||||||
|
|
||||||
|
Each surface carries what it ASKS, what corpus it asks OVER, and how often
|
||||||
|
it FIRES, because a floor cannot be moved sensibly without all three. An arm
|
||||||
|
firing before every Bash call is spending attention on a scale an arm firing
|
||||||
|
once a turn is not, and the same number means different things to a query
|
||||||
|
that is a shell command, a code payload, or an operator's sentence.
|
||||||
|
|
||||||
|
`floor` is a COST floor: is this worth ranking at all. It is not a relevance
|
||||||
|
judgement — relevance is decided by the reader, which is the only
|
||||||
|
participant that can read a record's trigger against the actual situation,
|
||||||
|
and every injected line says so out loud ("read it before deciding it does
|
||||||
|
not apply").
|
||||||
|
|
||||||
|
`budget` is what actually binds. It is how many lines this surface may spend
|
||||||
|
on one injection, and it is the control to reach for when a surface feels
|
||||||
|
noisy — lowering a budget removes the weakest candidates, while raising a
|
||||||
|
floor removes whichever candidates happen to sit under a number.
|
||||||
|
|
||||||
|
`last_change` carries the reason last given for each dial, or is absent when
|
||||||
|
the surface is still on its shipped starting point. Those starting points
|
||||||
|
are starting points: they were measured against one corpus with one
|
||||||
|
embedding model and cannot be right for another install by construction.
|
||||||
|
That is why this tool exists rather than a better set of defaults.
|
||||||
|
"""
|
||||||
|
return {"surfaces": await tuning_svc.current_settings(current_user_id())}
|
||||||
|
|
||||||
|
|
||||||
|
async def tune_retrieval(
|
||||||
|
surface: str, dial: str, value: float, reason: str, actor: str = "model",
|
||||||
|
) -> dict:
|
||||||
|
"""Move one surface's floor or budget, recording why.
|
||||||
|
|
||||||
|
DO NOT CALL THIS FROM A PERCENTILE ALONE. `retrieval_telemetry` gives
|
||||||
|
`near_misses.p90` — the mass sitting just under the bar — and that number
|
||||||
|
says nothing about whether the mass is RELEVANT. The two have been measured
|
||||||
|
disagreeing: one surface logged 69 consecutive declines with the refused
|
||||||
|
record 0.0006 under its bar, and every percentile said "lower it". The
|
||||||
|
refused record turned out to be a rule about interpreting a REQUEST, matched
|
||||||
|
against a query about report layout — a false positive. Lowering would have
|
||||||
|
attached that rule to every completion report ever written. The statistic
|
||||||
|
and the correct action pointed in opposite directions, and only opening the
|
||||||
|
record could tell.
|
||||||
|
|
||||||
|
So the procedure is: `retrieval_telemetry(near_miss_samples=5)`, then
|
||||||
|
`get_rule` / `get_note` the `record_id`s it names, and decide whether those
|
||||||
|
records SHOULD have surfaced for those queries. Then move the dial, and say
|
||||||
|
in `reason` what you read and what it showed.
|
||||||
|
|
||||||
|
`reason` is required and must be substantive. It is the guardrail, not
|
||||||
|
bookkeeping: it is what lets the operator review a change they did not make,
|
||||||
|
disagree with it, and revert it. A number that moved with no stated basis is
|
||||||
|
one nobody can audit — including you, next week.
|
||||||
|
|
||||||
|
WHICH DIAL. Reach for `budget` when the complaint is volume, and `floor`
|
||||||
|
when the complaint is quality. Raising a floor to quieten a surface throws
|
||||||
|
away its best candidates along with its worst, because a floor cannot tell
|
||||||
|
rank from relevance; lowering a budget keeps the top of the ranking and
|
||||||
|
drops the tail, which is usually what was wanted.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
surface: the surface name from `retrieval_surfaces` — also its
|
||||||
|
`retrieval_telemetry` source, so the two always name the same arm.
|
||||||
|
dial: "floor" or "budget".
|
||||||
|
value: the new value. A floor is clamped to [0, 1], a budget to a whole
|
||||||
|
number in [1, 10]. The result says whether a clamp bit, which
|
||||||
|
matters: believing you set a value the registry corrected means
|
||||||
|
reading the next telemetry as evidence about a bar never in force.
|
||||||
|
reason: what you read and what it showed. Required.
|
||||||
|
actor: "model" (default) or "human" — pass "human" only when relaying a
|
||||||
|
value the operator chose themselves, so the audit trail can tell
|
||||||
|
a change they made from one made on their behalf.
|
||||||
|
"""
|
||||||
|
return await tuning_svc.set_dial(
|
||||||
|
current_user_id(), surface, dial, value, reason=reason, actor=actor,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def retrieval_tuning_history(surface: str = "", limit: int = 20) -> dict:
|
||||||
|
"""What has been changed about retrieval on this install, newest first.
|
||||||
|
|
||||||
|
Reach for it before moving a dial that has been moved before — the previous
|
||||||
|
reason is the argument the next change has to answer, and a surface that has
|
||||||
|
been walked up and down repeatedly is evidence the floor is not the problem.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
surface: restrict to one surface; omit for everything.
|
||||||
|
limit: how many events, default 20, capped at 200.
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"events": await tuning_svc.tuning_history(
|
||||||
|
current_user_id(), surface=surface or None, limit=limit,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def register(mcp) -> None:
|
||||||
|
mcp.tool(name="retrieval_surfaces")(retrieval_surfaces)
|
||||||
|
mcp.tool(name="tune_retrieval")(tune_retrieval)
|
||||||
|
mcp.tool(name="retrieval_tuning_history")(retrieval_tuning_history)
|
||||||
@@ -209,11 +209,13 @@ async def retrieval_telemetry(
|
|||||||
) -> dict:
|
) -> dict:
|
||||||
"""What the retrieval telemetry says about YOUR surfaces, over a window.
|
"""What the retrieval telemetry says about YOUR surfaces, over a window.
|
||||||
|
|
||||||
The read half of the loop the ranker's thresholds are meant to be tuned
|
The read half of the tuning loop, whose write half is `tune_retrieval`
|
||||||
from (#2975). Reach for it before changing a similarity threshold, a top-k,
|
(#2975, #4102). Reach for it before moving any floor or budget, and read
|
||||||
or deciding whether a reranker is worth building — the alternative is
|
the records it names rather than its percentiles alone: this readout has
|
||||||
hand-probing the live instance, which is how the last such decision had to
|
been measured pointing the WRONG WAY — 69 consecutive declines where every
|
||||||
be made.
|
percentile said "lower the bar" and the refused record was a false positive
|
||||||
|
— so `near_miss_samples=5` and opening the ids it returns is the step that
|
||||||
|
separates a real miss from a bar doing its job.
|
||||||
|
|
||||||
Three readouts, from the three tables built for them:
|
Three readouts, from the three tables built for them:
|
||||||
|
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ from scribe.models.password_reset import PasswordResetToken # noqa: E402, F401
|
|||||||
from scribe.models.invitation import InvitationToken # noqa: E402, F401
|
from scribe.models.invitation import InvitationToken # noqa: E402, F401
|
||||||
from scribe.models.embedding import MilestoneEmbedding, NoteEmbedding, RuleEmbedding # noqa: E402, F401
|
from scribe.models.embedding import MilestoneEmbedding, NoteEmbedding, RuleEmbedding # noqa: E402, F401
|
||||||
from scribe.models.retrieval_log import RetrievalLog # noqa: E402, F401
|
from scribe.models.retrieval_log import RetrievalLog # noqa: E402, F401
|
||||||
|
from scribe.models.retrieval_tuning import RetrievalTuningEvent # noqa: E402, F401
|
||||||
from scribe.models.note_usage import NoteUsageEvent # noqa: E402, F401
|
from scribe.models.note_usage import NoteUsageEvent # noqa: E402, F401
|
||||||
from scribe.models.rule_usage import RuleUsageEvent # noqa: E402, F401
|
from scribe.models.rule_usage import RuleUsageEvent # noqa: E402, F401
|
||||||
from scribe.models.project import Project # noqa: E402, F401
|
from scribe.models.project import Project # noqa: E402, F401
|
||||||
|
|||||||
@@ -0,0 +1,97 @@
|
|||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import DateTime, Float, Index, Integer, Text
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from scribe.models import Base
|
||||||
|
|
||||||
|
|
||||||
|
class RetrievalTuningEvent(Base):
|
||||||
|
"""One row per change to a retrieval surface's floor or budget (#4102).
|
||||||
|
|
||||||
|
WHY A TABLE AND NOT JUST THE SETTING
|
||||||
|
|
||||||
|
The setting holds the number. It cannot hold the argument, and from this
|
||||||
|
step onward the argument is the interesting part: the operator's decision is
|
||||||
|
that the model moves these values, *"9 times out of 10"*, without being
|
||||||
|
asked each time. Delegation like that is only safe if it leaves a trail the
|
||||||
|
operator can read afterwards — what changed, from what to what, and on what
|
||||||
|
evidence — and can disagree with.
|
||||||
|
|
||||||
|
So `reason` is not decoration and not optional. It is the guardrail: a model
|
||||||
|
that has to state why is a model that has to have looked, and a reason that
|
||||||
|
turns out to be wrong is a reason someone can point at. A bare number in a
|
||||||
|
settings row records that something moved and nothing about whether it
|
||||||
|
should have.
|
||||||
|
|
||||||
|
WHY `actor` IS NOT JUST `user_id`
|
||||||
|
|
||||||
|
Both a person and a model act as the same user — the MCP tools run under the
|
||||||
|
operator's own id — so the id cannot distinguish them. That distinction is
|
||||||
|
exactly what an operator scanning this table wants first: "did I do this, or
|
||||||
|
did the session?" A change the operator made themselves needs no review; one
|
||||||
|
made on their behalf might.
|
||||||
|
|
||||||
|
FK-FREE ON user_id, like `RetrievalLog` and `AppLog` above it: the history of
|
||||||
|
how a surface was tuned should outlive the account that tuned it, and a
|
||||||
|
deleted user must not cascade away the record of why a number is where it is.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "retrieval_tuning_events"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
# Declared here rather than via CreatedAtMixin because the index below
|
||||||
|
# orders on `created_at.desc()`, which needs the column object in this class
|
||||||
|
# body — a mixin's column is not in scope there.
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
|
||||||
|
)
|
||||||
|
user_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
|
||||||
|
# The surface's registry name, which is also its `retrieval_logs.source`.
|
||||||
|
# That identity is what lets a reader put a change next to what the change
|
||||||
|
# did; `tests/test_retrieval_surfaces.py` asserts it rather than trusting it.
|
||||||
|
surface: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
|
||||||
|
# "floor" or "budget". Text rather than an enum because rule 36 makes a new
|
||||||
|
# CHECK value a migration, and this is a young surface that may well grow a
|
||||||
|
# third tunable — a constraint bought nothing here except a future migration.
|
||||||
|
dial: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
|
||||||
|
# Both stored as float so one pair of columns serves both dials; a budget is
|
||||||
|
# a whole number and reads back as one. `old_value` is nullable because the
|
||||||
|
# first change to a surface has no stored predecessor — it moved off the
|
||||||
|
# shipped starting point, which is a different event from moving off a value
|
||||||
|
# somebody chose, and worth being able to tell apart.
|
||||||
|
old_value: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||||
|
new_value: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
|
||||||
|
# "model" | "human". See the class docstring: the user id cannot tell these
|
||||||
|
# apart, and the difference is the first thing a reviewer wants.
|
||||||
|
actor: Mapped[str] = mapped_column(Text, nullable=False, default="model")
|
||||||
|
|
||||||
|
# REQUIRED at the service boundary, not merely non-null here. A column that
|
||||||
|
# is technically non-null is satisfied by "", which is how a required field
|
||||||
|
# becomes a formality; the service refuses a blank one.
|
||||||
|
reason: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
# The only read this table has: one surface's history, newest first.
|
||||||
|
Index(
|
||||||
|
"ix_retrieval_tuning_surface_created",
|
||||||
|
"surface",
|
||||||
|
created_at.desc(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def to_dict(self) -> dict:
|
||||||
|
return {
|
||||||
|
"id": self.id,
|
||||||
|
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||||
|
"surface": self.surface,
|
||||||
|
"dial": self.dial,
|
||||||
|
"old_value": self.old_value,
|
||||||
|
"new_value": self.new_value,
|
||||||
|
"actor": self.actor,
|
||||||
|
"reason": self.reason,
|
||||||
|
}
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
"""The operator's view of retrieval tuning — what the dials are, and who moved them (#4102).
|
||||||
|
|
||||||
|
WHY THIS EXISTS SEPARATELY FROM /api/settings
|
||||||
|
|
||||||
|
The numbers themselves are ordinary settings rows and could be written through
|
||||||
|
the generic KV endpoint. They must not be, and that is the whole point of this
|
||||||
|
blueprint: a floor changed through `/api/settings` moves silently, leaving the
|
||||||
|
history saying nothing happened. From milestone 416 those dials are moved by
|
||||||
|
the model on the operator's behalf, so a trail with holes in it is worse than
|
||||||
|
no trail — it reads as complete.
|
||||||
|
|
||||||
|
So every change to a retrieval dial goes through `set_dial`, from the UI as
|
||||||
|
much as from the MCP tool, and the only difference is the `actor` recorded.
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from quart import Blueprint, jsonify, request
|
||||||
|
|
||||||
|
from scribe.auth import get_current_user_id, login_required
|
||||||
|
from scribe.services.retrieval_tuning import set_dial, current_settings, tuning_history
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
retrieval_bp = Blueprint("retrieval", __name__, url_prefix="/api/retrieval")
|
||||||
|
|
||||||
|
# What is recorded when the operator changes a dial in Settings and says
|
||||||
|
# nothing about why.
|
||||||
|
#
|
||||||
|
# The MCP tool REFUSES a blank reason, and this endpoint does not, which is a
|
||||||
|
# deliberate asymmetry rather than an oversight. The requirement exists to make
|
||||||
|
# the model look at the records before it moves a number on someone else's
|
||||||
|
# behalf. The operator IS that someone: they are the audience the trail is
|
||||||
|
# written for, they cannot be uninformed about their own decision, and a
|
||||||
|
# mandatory justification textarea on every control would be friction charged
|
||||||
|
# to the one participant who owes no explanation (rule 24).
|
||||||
|
#
|
||||||
|
# The event is still written, because "the operator set this by hand" is the
|
||||||
|
# single most useful thing the history can tell a later session — it is the one
|
||||||
|
# entry the model must not quietly tune back.
|
||||||
|
_OPERATOR_DEFAULT_REASON = "Set directly in Settings by the operator."
|
||||||
|
|
||||||
|
|
||||||
|
@retrieval_bp.route("/surfaces", methods=["GET"])
|
||||||
|
@login_required
|
||||||
|
async def get_surfaces_route():
|
||||||
|
"""Every tunable surface: its live floor and budget, what it asks and over
|
||||||
|
what, and the reason each dial was last moved."""
|
||||||
|
uid = get_current_user_id()
|
||||||
|
surfaces = await current_settings(uid)
|
||||||
|
return jsonify({"surfaces": surfaces, "total": len(surfaces)})
|
||||||
|
|
||||||
|
|
||||||
|
@retrieval_bp.route("/surfaces/<surface>", methods=["PUT"])
|
||||||
|
@login_required
|
||||||
|
async def tune_surface_route(surface: str):
|
||||||
|
"""Move one dial, recorded as a human change.
|
||||||
|
|
||||||
|
`actor` is fixed here rather than taken from the payload: this endpoint is
|
||||||
|
reached with a session cookie from the Settings form, so the actor is known
|
||||||
|
and accepting a claim about it would let the one field a reviewer relies on
|
||||||
|
be set to anything.
|
||||||
|
"""
|
||||||
|
uid = get_current_user_id()
|
||||||
|
data = await request.get_json()
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return jsonify({"error": "Expected a JSON object"}), 400
|
||||||
|
if "dial" not in data or "value" not in data:
|
||||||
|
return jsonify({"error": "dial and value are required"}), 400
|
||||||
|
|
||||||
|
try:
|
||||||
|
value = float(data["value"])
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return jsonify({"error": f"value must be a number, got {data['value']!r}"}), 400
|
||||||
|
|
||||||
|
reason = str(data.get("reason") or "").strip() or _OPERATOR_DEFAULT_REASON
|
||||||
|
try:
|
||||||
|
result = await set_dial(
|
||||||
|
uid, surface, str(data["dial"]), value, reason=reason, actor="human",
|
||||||
|
)
|
||||||
|
except ValueError as e:
|
||||||
|
# Unknown surface, unknown dial — the service names the alternatives,
|
||||||
|
# so the message is worth passing through rather than flattening.
|
||||||
|
return jsonify({"error": str(e)}), 400
|
||||||
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
|
@retrieval_bp.route("/tuning-history", methods=["GET"])
|
||||||
|
@login_required
|
||||||
|
async def tuning_history_route():
|
||||||
|
"""What has been changed about retrieval on this install, newest first.
|
||||||
|
|
||||||
|
The review surface. Unscoped by default because the operator's question
|
||||||
|
here is "what has been done on my behalf", not "why is this one arm set
|
||||||
|
like that" — the per-surface scoping is for the model.
|
||||||
|
"""
|
||||||
|
uid = get_current_user_id()
|
||||||
|
surface = request.args.get("surface") or None
|
||||||
|
try:
|
||||||
|
limit = int(request.args.get("limit", 50))
|
||||||
|
except ValueError:
|
||||||
|
return jsonify({"error": "limit must be a whole number"}), 400
|
||||||
|
try:
|
||||||
|
events = await tuning_history(uid, surface=surface, limit=limit)
|
||||||
|
except ValueError as e:
|
||||||
|
return jsonify({"error": str(e)}), 400
|
||||||
|
return jsonify({"events": events, "total": len(events)})
|
||||||
@@ -9,6 +9,8 @@ from quart import Blueprint, jsonify, request
|
|||||||
|
|
||||||
from scribe.auth import login_required, get_current_user_id
|
from scribe.auth import login_required, get_current_user_id
|
||||||
from scribe.config import Config
|
from scribe.config import Config
|
||||||
|
from scribe.services.retrieval_surfaces import dial_for_key, get_surface
|
||||||
|
from scribe.services.retrieval_tuning import set_dial
|
||||||
from scribe.services.settings import (
|
from scribe.services.settings import (
|
||||||
SECRET_MASK, delete_setting, get_all_settings, get_setting, set_settings_batch,
|
SECRET_MASK, delete_setting, get_all_settings, get_setting, set_settings_batch,
|
||||||
)
|
)
|
||||||
@@ -25,6 +27,13 @@ settings_bp = Blueprint("settings", __name__, url_prefix="/api/settings")
|
|||||||
# (forge_token left with 0078: forge credentials are keyring rows now, #2778.)
|
# (forge_token left with 0078: forge credentials are keyring rows now, #2778.)
|
||||||
_SECRET_KEYS = frozenset({"smtp_password", "forge_webhook_secret"})
|
_SECRET_KEYS = frozenset({"smtp_password", "forge_webhook_secret"})
|
||||||
|
|
||||||
|
# What the tuning history records for a dial changed through this form. The MCP
|
||||||
|
# tool refuses a blank reason; the operator is not asked for one, because the
|
||||||
|
# requirement exists to make the MODEL read the records before moving a number
|
||||||
|
# on someone else's behalf — and the operator is that someone. See
|
||||||
|
# routes/retrieval.py, which states the asymmetry in full.
|
||||||
|
_SETTINGS_FORM_REASON = "Changed in Settings by the operator."
|
||||||
|
|
||||||
|
|
||||||
def _masked(settings: dict) -> dict:
|
def _masked(settings: dict) -> dict:
|
||||||
return {
|
return {
|
||||||
@@ -52,6 +61,33 @@ async def update_settings_route():
|
|||||||
to_save = {}
|
to_save = {}
|
||||||
for k, v in data.items():
|
for k, v in data.items():
|
||||||
str_v = str(v)
|
str_v = str(v)
|
||||||
|
# A retrieval dial is never written as a plain settings row, wherever
|
||||||
|
# the request came from (#4102). The value would land correctly and the
|
||||||
|
# tuning history would say nothing happened — and since milestone 416
|
||||||
|
# those dials are moved by the model on the operator's behalf, a
|
||||||
|
# history with holes in it is worse than none: it reads as complete.
|
||||||
|
#
|
||||||
|
# Routed rather than refused on purpose. Refusing would work only for
|
||||||
|
# callers that had been updated; this way every caller that ever writes
|
||||||
|
# one of these keys — this form, a script, an old client — leaves the
|
||||||
|
# trail, and there is no version of "forgot to use the other endpoint".
|
||||||
|
dial = dial_for_key(k)
|
||||||
|
if dial:
|
||||||
|
surface, which = dial
|
||||||
|
s = get_surface(surface)
|
||||||
|
# A CLEARED control means "back to the shipped starting point", and
|
||||||
|
# that is a change like any other — it is the operator reverting
|
||||||
|
# something, which is the single most important move this history
|
||||||
|
# can record. So it is written as an explicit set to the default
|
||||||
|
# rather than deleted, which would leave the same value behind and
|
||||||
|
# no record of anyone having chosen it.
|
||||||
|
default = s.floor_default if which == "floor" else s.budget_default
|
||||||
|
try:
|
||||||
|
await set_dial(uid, surface, which, float(str_v or default),
|
||||||
|
reason=_SETTINGS_FORM_REASON, actor="human")
|
||||||
|
except (TypeError, ValueError) as e:
|
||||||
|
return jsonify({"error": f"{k}: {e}"}), 400
|
||||||
|
continue
|
||||||
# A masked secret round-tripping through a client is "unchanged", not
|
# A masked secret round-tripping through a client is "unchanged", not
|
||||||
# a request to store the mask over the real credential.
|
# a request to store the mask over the real credential.
|
||||||
if k in _SECRET_KEYS and str_v == SECRET_MASK:
|
if k in _SECRET_KEYS and str_v == SECRET_MASK:
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ from scribe.models.rule_version import RuleVersion
|
|||||||
from scribe.models.design_system import DesignSystem, DesignToken
|
from scribe.models.design_system import DesignSystem, DesignToken
|
||||||
from scribe.models.note_usage import NoteUsageEvent
|
from scribe.models.note_usage import NoteUsageEvent
|
||||||
from scribe.models.rule_usage import RuleUsageEvent
|
from scribe.models.rule_usage import RuleUsageEvent
|
||||||
|
from scribe.models.retrieval_tuning import RetrievalTuningEvent
|
||||||
from scribe.models.canonical_system import CanonicalSystem
|
from scribe.models.canonical_system import CanonicalSystem
|
||||||
from scribe.models.rulebook import RuleRelation, rule_systems as rule_systems_t
|
from scribe.models.rulebook import RuleRelation, rule_systems as rule_systems_t
|
||||||
from scribe.models.code_shape import CodeShape, CodeShapeEvent, CodeShapeUse
|
from scribe.models.code_shape import CodeShape, CodeShapeEvent, CodeShapeUse
|
||||||
@@ -67,8 +68,14 @@ logger = logging.getLogger(__name__)
|
|||||||
# topic_suppressions with their tables (milestone 414): a rule's scope is its
|
# topic_suppressions with their tables (milestone 414): a rule's scope is its
|
||||||
# home now. Older archives carrying those sections still restore — the keys are
|
# home now. Older archives carrying those sections still restore — the keys are
|
||||||
# simply not read — as do the subscribe_rulebooks inception choices they hold.
|
# simply not read — as do the subscribe_rulebooks inception choices they hold.
|
||||||
|
# v16 (2026-09) added retrieval_tuning_events (milestone 416): the REASON each
|
||||||
|
# retrieval floor and budget is where it is. `settings` already carried the
|
||||||
|
# numbers, so leaving this behind would restore six moved dials with the
|
||||||
|
# argument for them silently dropped — and from this step on those dials are
|
||||||
|
# moved by the model, which is exactly the case where the operator needs the
|
||||||
|
# argument to review.
|
||||||
# Bump when the serialized schema changes.
|
# Bump when the serialized schema changes.
|
||||||
BACKUP_VERSION = 15
|
BACKUP_VERSION = 16
|
||||||
|
|
||||||
# Every table this backup carries, by its REAL name. Paired with _NOT_INCLUDED
|
# Every table this backup carries, by its REAL name. Paired with _NOT_INCLUDED
|
||||||
# below, these two lists must together account for the entire schema — which is
|
# below, these two lists must together account for the entire schema — which is
|
||||||
@@ -100,6 +107,12 @@ _BACKED_UP = [
|
|||||||
# accumulated, so a restore that dropped it would silently reset the
|
# accumulated, so a restore that dropped it would silently reset the
|
||||||
# measurement to zero while everything still looked fine.
|
# measurement to zero while everything still looked fine.
|
||||||
"rule_usage_events",
|
"rule_usage_events",
|
||||||
|
# v16 (2026-09): why each retrieval floor and budget is where it is
|
||||||
|
# (milestone 416). The values live in `settings` and already travelled; the
|
||||||
|
# argument for them had nowhere to go. Now that the model moves these dials
|
||||||
|
# on the operator's behalf, a restore that kept the numbers and dropped the
|
||||||
|
# reasons would leave an install tuned by nobody it can name.
|
||||||
|
"retrieval_tuning_events",
|
||||||
]
|
]
|
||||||
|
|
||||||
# Tables intentionally NOT in the backup, surfaced in the payload so the gap is
|
# Tables intentionally NOT in the backup, surfaced in the payload so the gap is
|
||||||
@@ -189,6 +202,9 @@ _COLUMN_EXCLUSIONS: dict[str, set[str]] = {
|
|||||||
"note_usage_events": {"id"},
|
"note_usage_events": {"id"},
|
||||||
# Same as the note twin: the surrogate key is re-issued on insert.
|
# Same as the note twin: the surrogate key is re-issued on insert.
|
||||||
"rule_usage_events": {"id"},
|
"rule_usage_events": {"id"},
|
||||||
|
# Same again — and everything else travels, because each remaining column
|
||||||
|
# is part of the argument: what moved, from what, to what, by whom, why.
|
||||||
|
"retrieval_tuning_events": {"id"},
|
||||||
"design_systems": {"deleted_at", "deleted_batch_id", "created_at", "updated_at"},
|
"design_systems": {"deleted_at", "deleted_batch_id", "created_at", "updated_at"},
|
||||||
"design_tokens": {"deleted_at", "deleted_batch_id", "created_at", "updated_at"},
|
"design_tokens": {"deleted_at", "deleted_batch_id", "created_at", "updated_at"},
|
||||||
"repo_bindings": {"id", "created_at", "updated_at"},
|
"repo_bindings": {"id", "created_at", "updated_at"},
|
||||||
@@ -329,6 +345,25 @@ def _rule_usage_event_rows(rows) -> list[dict]:
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _retrieval_tuning_event_rows(rows) -> list[dict]:
|
||||||
|
"""The record of why a retrieval dial is where it is (#4102).
|
||||||
|
|
||||||
|
Not `to_dict()`: that method serves the MCP reader, which already knows
|
||||||
|
whose install it is asking about, so it omits `user_id`. A restore has to
|
||||||
|
remap it, and a row that arrived without it would have to be dropped or
|
||||||
|
guessed at.
|
||||||
|
"""
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"user_id": r.user_id, "surface": r.surface, "dial": r.dial,
|
||||||
|
"old_value": r.old_value, "new_value": r.new_value,
|
||||||
|
"actor": r.actor, "reason": r.reason,
|
||||||
|
"created_at": r.created_at.isoformat() if r.created_at else None,
|
||||||
|
}
|
||||||
|
for r in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def _code_shape_rows(rows) -> list[dict]:
|
def _code_shape_rows(rows) -> list[dict]:
|
||||||
return [r.to_dict() for r in rows]
|
return [r.to_dict() for r in rows]
|
||||||
|
|
||||||
@@ -617,6 +652,12 @@ async def export_full_backup() -> dict:
|
|||||||
rule_usage_events = (
|
rule_usage_events = (
|
||||||
await session.execute(select(RuleUsageEvent))
|
await session.execute(select(RuleUsageEvent))
|
||||||
).scalars().all()
|
).scalars().all()
|
||||||
|
# Oldest first, so a restored history reads in the order the dials
|
||||||
|
# actually moved — the sequence IS the argument when a surface has been
|
||||||
|
# walked up and down.
|
||||||
|
retrieval_tuning_events = (await session.execute(
|
||||||
|
select(RetrievalTuningEvent).order_by(RetrievalTuningEvent.id)
|
||||||
|
)).scalars().all()
|
||||||
repo_bindings = (await session.execute(select(RepoBinding))).scalars().all()
|
repo_bindings = (await session.execute(select(RepoBinding))).scalars().all()
|
||||||
code_shapes = (await session.execute(select(CodeShape))).scalars().all()
|
code_shapes = (await session.execute(select(CodeShape))).scalars().all()
|
||||||
code_shape_events = (await session.execute(
|
code_shape_events = (await session.execute(
|
||||||
@@ -661,6 +702,9 @@ async def export_full_backup() -> dict:
|
|||||||
"design_tokens": _design_token_rows(design_tokens),
|
"design_tokens": _design_token_rows(design_tokens),
|
||||||
"note_usage_events": _usage_event_rows(usage_events),
|
"note_usage_events": _usage_event_rows(usage_events),
|
||||||
"rule_usage_events": _rule_usage_event_rows(rule_usage_events),
|
"rule_usage_events": _rule_usage_event_rows(rule_usage_events),
|
||||||
|
"retrieval_tuning_events": _retrieval_tuning_event_rows(
|
||||||
|
retrieval_tuning_events
|
||||||
|
),
|
||||||
"repo_bindings": _repo_binding_rows(repo_bindings),
|
"repo_bindings": _repo_binding_rows(repo_bindings),
|
||||||
"note_supersessions": _note_supersession_rows(supersessions),
|
"note_supersessions": _note_supersession_rows(supersessions),
|
||||||
"code_shapes": _code_shape_rows(code_shapes),
|
"code_shapes": _code_shape_rows(code_shapes),
|
||||||
@@ -795,6 +839,15 @@ async def export_user_backup(user_id: int) -> dict:
|
|||||||
rule_usage_events = (await session.execute(
|
rule_usage_events = (await session.execute(
|
||||||
select(RuleUsageEvent).where(RuleUsageEvent.rule_id.in_(_rule_ids))
|
select(RuleUsageEvent).where(RuleUsageEvent.rule_id.in_(_rule_ids))
|
||||||
)).scalars().all() if _rule_ids else []
|
)).scalars().all() if _rule_ids else []
|
||||||
|
# Scoped on user_id, and here that IS the right column — unlike the
|
||||||
|
# rule usage events directly above. These record changes to this user's
|
||||||
|
# OWN retrieval settings, which is what `user_id` means on this table;
|
||||||
|
# there is no second owner to route around.
|
||||||
|
retrieval_tuning_events = (await session.execute(
|
||||||
|
select(RetrievalTuningEvent)
|
||||||
|
.where(RetrievalTuningEvent.user_id == user_id)
|
||||||
|
.order_by(RetrievalTuningEvent.id)
|
||||||
|
)).scalars().all()
|
||||||
rule_relations = (await session.execute(
|
rule_relations = (await session.execute(
|
||||||
select(RuleRelation).where(
|
select(RuleRelation).where(
|
||||||
RuleRelation.from_rule_id.in_(_rule_ids),
|
RuleRelation.from_rule_id.in_(_rule_ids),
|
||||||
@@ -836,6 +889,9 @@ async def export_user_backup(user_id: int) -> dict:
|
|||||||
"design_tokens": _design_token_rows(design_tokens),
|
"design_tokens": _design_token_rows(design_tokens),
|
||||||
"note_usage_events": _usage_event_rows(usage_events),
|
"note_usage_events": _usage_event_rows(usage_events),
|
||||||
"rule_usage_events": _rule_usage_event_rows(rule_usage_events),
|
"rule_usage_events": _rule_usage_event_rows(rule_usage_events),
|
||||||
|
"retrieval_tuning_events": _retrieval_tuning_event_rows(
|
||||||
|
retrieval_tuning_events
|
||||||
|
),
|
||||||
"repo_bindings": _repo_binding_rows(repo_bindings),
|
"repo_bindings": _repo_binding_rows(repo_bindings),
|
||||||
"note_supersessions": _note_supersession_rows(supersessions),
|
"note_supersessions": _note_supersession_rows(supersessions),
|
||||||
"code_shapes": _code_shape_rows(code_shapes),
|
"code_shapes": _code_shape_rows(code_shapes),
|
||||||
@@ -975,6 +1031,7 @@ async def _restore_v2(data: dict) -> dict:
|
|||||||
"note_supersessions": 0, "code_shapes": 0, "code_shape_events": 0,
|
"note_supersessions": 0, "code_shapes": 0, "code_shape_events": 0,
|
||||||
"code_shape_uses": 0, "canonical_systems": 0,
|
"code_shape_uses": 0, "canonical_systems": 0,
|
||||||
"rule_systems": 0, "rule_relations": 0, "rule_versions": 0,
|
"rule_systems": 0, "rule_relations": 0, "rule_versions": 0,
|
||||||
|
"retrieval_tuning_events": 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
async with async_session() as session:
|
async with async_session() as session:
|
||||||
@@ -1169,6 +1226,31 @@ async def _restore_v2(data: dict) -> dict:
|
|||||||
session.add(Setting(user_id=mapped_uid, key=s_data["key"], value=s_data.get("value", "")))
|
session.add(Setting(user_id=mapped_uid, key=s_data["key"], value=s_data.get("value", "")))
|
||||||
stats["settings"] += 1
|
stats["settings"] += 1
|
||||||
|
|
||||||
|
# 8b. Retrieval tuning history (v16) — restored beside the settings it
|
||||||
|
# explains, and for the same reason: the numbers above are the state,
|
||||||
|
# these rows are the argument for it. From milestone 416 those dials
|
||||||
|
# move on the operator's behalf, so an install restored with the values
|
||||||
|
# and without the reasons is one tuned by nobody it can name.
|
||||||
|
#
|
||||||
|
# No id remapping beyond the user: `surface` is a registry NAME, not a
|
||||||
|
# foreign key, which is what lets this history survive a restore into
|
||||||
|
# an install whose row ids all differ.
|
||||||
|
for t_data in data.get("retrieval_tuning_events", []):
|
||||||
|
mapped_uid = user_id_map.get(t_data.get("user_id") or 0)
|
||||||
|
if mapped_uid is None:
|
||||||
|
continue
|
||||||
|
session.add(RetrievalTuningEvent(
|
||||||
|
user_id=mapped_uid,
|
||||||
|
surface=t_data.get("surface", ""),
|
||||||
|
dial=t_data.get("dial", ""),
|
||||||
|
old_value=t_data.get("old_value"),
|
||||||
|
new_value=t_data.get("new_value"),
|
||||||
|
actor=t_data.get("actor") or "model",
|
||||||
|
reason=t_data.get("reason", ""),
|
||||||
|
created_at=_dt(t_data.get("created_at")),
|
||||||
|
))
|
||||||
|
stats["retrieval_tuning_events"] += 1
|
||||||
|
|
||||||
# 9. Rulebooks (v3)
|
# 9. Rulebooks (v3)
|
||||||
for rb_data in data.get("rulebooks", []):
|
for rb_data in data.get("rulebooks", []):
|
||||||
mapped_uid = user_id_map.get(rb_data.get("owner_user_id", 0))
|
mapped_uid = user_id_map.get(rb_data.get("owner_user_id", 0))
|
||||||
|
|||||||
@@ -31,6 +31,12 @@ from scribe.services.embeddings import semantic_search_notes, semantic_search_ru
|
|||||||
from scribe.services.note_usage import record_surfaced
|
from scribe.services.note_usage import record_surfaced
|
||||||
from scribe.services.rule_usage import record_rule_surfaced
|
from scribe.services.rule_usage import record_rule_surfaced
|
||||||
from scribe.services.supersession import superseded_ids
|
from scribe.services.supersession import superseded_ids
|
||||||
|
from scribe.services.retrieval_surfaces import (
|
||||||
|
MAX_BUDGET,
|
||||||
|
SURFACES,
|
||||||
|
budget_for,
|
||||||
|
floor_for,
|
||||||
|
)
|
||||||
from scribe.services.retrieval_telemetry import record_retrieval
|
from scribe.services.retrieval_telemetry import record_retrieval
|
||||||
from scribe.services.settings import get_setting
|
from scribe.services.settings import get_setting
|
||||||
|
|
||||||
@@ -50,16 +56,25 @@ _GOAL_CHARS = 200
|
|||||||
# Per-user settings (keys live in the generic settings table). The threshold is
|
# Per-user settings (keys live in the generic settings table). The threshold is
|
||||||
# deliberately STRICTER than the pull-search default (embeddings
|
# deliberately STRICTER than the pull-search default (embeddings
|
||||||
# DEFAULT_SIMILARITY_THRESHOLD = 0.45): an unsolicited per-turn inject must clear
|
# DEFAULT_SIMILARITY_THRESHOLD = 0.45): an unsolicited per-turn inject must clear
|
||||||
# a higher bar than a search the agent chose to run. Defaults start conservative
|
# a higher bar than a search the agent chose to run.
|
||||||
# and are meant to be tuned from retrieval_logs (source='auto_inject') once data
|
#
|
||||||
# accrues — they're exposed in the Settings UI, no restart needed.
|
# The defaults below are STARTING POINTS, and correcting them is the model's
|
||||||
|
# job, not the operator's (#4102): `retrieval_surfaces` says what is in force,
|
||||||
|
# `retrieval_telemetry(near_miss_samples=N)` says what it refused, and
|
||||||
|
# `tune_retrieval` moves it with the reason attached. The operator can set any
|
||||||
|
# of them in Settings and their change is recorded the same way — but nobody
|
||||||
|
# has to read a log to get correct behaviour out of this.
|
||||||
AUTOINJECT_ENABLED_KEY = "kb_autoinject_enabled"
|
AUTOINJECT_ENABLED_KEY = "kb_autoinject_enabled"
|
||||||
AUTOINJECT_THRESHOLD_KEY = "kb_autoinject_threshold"
|
# The key and the value both live in the registry now (#4102); these names
|
||||||
AUTOINJECT_TOP_K_KEY = "kb_autoinject_top_k"
|
# survive because the comments above them are where each number's measurement
|
||||||
|
# is recorded, and because the Settings UI agreement test pairs them with the
|
||||||
|
# Vue refs by name. One definition, two readable places.
|
||||||
|
AUTOINJECT_THRESHOLD_KEY = SURFACES["auto_inject"].floor_key
|
||||||
|
AUTOINJECT_TOP_K_KEY = SURFACES["auto_inject"].budget_key
|
||||||
|
|
||||||
AUTOINJECT_DEFAULT_ENABLED = True
|
AUTOINJECT_DEFAULT_ENABLED = True
|
||||||
AUTOINJECT_DEFAULT_THRESHOLD = 0.55
|
AUTOINJECT_DEFAULT_THRESHOLD = SURFACES["auto_inject"].floor_default
|
||||||
AUTOINJECT_DEFAULT_TOP_K = 3
|
AUTOINJECT_DEFAULT_TOP_K = SURFACES["auto_inject"].budget_default
|
||||||
|
|
||||||
# The write-path trigger (#2082) gets its own on/off switch, its own threshold,
|
# The write-path trigger (#2082) gets its own on/off switch, its own threshold,
|
||||||
# and shares only top-k. It originally shared the threshold too, on the argument
|
# and shares only top-k. It originally shared the threshold too, on the argument
|
||||||
@@ -83,9 +98,9 @@ AUTOINJECT_DEFAULT_TOP_K = 3
|
|||||||
# pull-through (#2085) once a real corpus accrues; a cross-encoder rerank
|
# pull-through (#2085) once a real corpus accrues; a cross-encoder rerank
|
||||||
# (#1038) would subsume this bump.
|
# (#1038) would subsume this bump.
|
||||||
WRITEPATH_ENABLED_KEY = "kb_writepath_enabled"
|
WRITEPATH_ENABLED_KEY = "kb_writepath_enabled"
|
||||||
WRITEPATH_THRESHOLD_KEY = "kb_writepath_threshold"
|
WRITEPATH_THRESHOLD_KEY = SURFACES["write_path"].floor_key
|
||||||
WRITEPATH_DEFAULT_ENABLED = True
|
WRITEPATH_DEFAULT_ENABLED = True
|
||||||
WRITEPATH_DEFAULT_THRESHOLD = 0.68
|
WRITEPATH_DEFAULT_THRESHOLD = SURFACES["write_path"].floor_default
|
||||||
|
|
||||||
# The standing-rule arm (milestone 307) gets its own bar — the split #2223 made
|
# The standing-rule arm (milestone 307) gets its own bar — the split #2223 made
|
||||||
# one surface down, now made for the THIRD corpus. It inherited 0.68 above, and
|
# one surface down, now made for the THIRD corpus. It inherited 0.68 above, and
|
||||||
@@ -133,8 +148,8 @@ WRITEPATH_DEFAULT_THRESHOLD = 0.68
|
|||||||
# case 0.72 was calibrated on, and the telemetry says it is working — the
|
# case 0.72 was calibrated on, and the telemetry says it is working — the
|
||||||
# write-path rule arm speaks on 37% of its calls and its refused mass sits at
|
# write-path rule arm speaks on 37% of its calls and its refused mass sits at
|
||||||
# p50 0.6989, comfortably under the bar rather than piled against it.
|
# p50 0.6989, comfortably under the bar rather than piled against it.
|
||||||
RULEHINT_THRESHOLD_KEY = "kb_rulehint_threshold"
|
RULEHINT_THRESHOLD_KEY = SURFACES["write_path_rule"].floor_key
|
||||||
RULEHINT_DEFAULT_THRESHOLD = 0.72
|
RULEHINT_DEFAULT_THRESHOLD = SURFACES["write_path_rule"].floor_default
|
||||||
|
|
||||||
# THE COMMAND ARM'S OWN BAR, AND WHY IT IS NOT THE WRITE PATH'S (#3853).
|
# THE COMMAND ARM'S OWN BAR, AND WHY IT IS NOT THE WRITE PATH'S (#3853).
|
||||||
#
|
#
|
||||||
@@ -193,8 +208,8 @@ RULEHINT_DEFAULT_THRESHOLD = 0.72
|
|||||||
# eligible. Retrieval is ownership-scoped, not project-scoped. Scoping it
|
# eligible. Retrieval is ownership-scoped, not project-scoped. Scoping it
|
||||||
# would drop that ceiling and widen the 0.0115, which is the larger fix and
|
# would drop that ceiling and widen the 0.0115, which is the larger fix and
|
||||||
# the reason to settle project scoping before tuning this number twice.
|
# the reason to settle project scoping before tuning this number twice.
|
||||||
TOOLRULE_THRESHOLD_KEY = "kb_toolrule_threshold"
|
TOOLRULE_THRESHOLD_KEY = SURFACES["pre_tool_rule"].floor_key
|
||||||
TOOLRULE_DEFAULT_THRESHOLD = 0.68
|
TOOLRULE_DEFAULT_THRESHOLD = SURFACES["pre_tool_rule"].floor_default
|
||||||
|
|
||||||
# A SET OF RULES PER ACT, NOT THE SINGLE BEST ONE (#3851).
|
# A SET OF RULES PER ACT, NOT THE SINGLE BEST ONE (#3851).
|
||||||
#
|
#
|
||||||
@@ -223,7 +238,10 @@ TOOLRULE_DEFAULT_THRESHOLD = 0.68
|
|||||||
# and nothing else, so a moment with one clearly-relevant rule still shows
|
# and nothing else, so a moment with one clearly-relevant rule still shows
|
||||||
# one, and a moment with four shows four. The corpus decides, not a constant.
|
# one, and a moment with four shows four. The corpus decides, not a constant.
|
||||||
# The cap survives as a ceiling on the worst case, not as the usual answer.
|
# The cap survives as a ceiling on the worst case, not as the usual answer.
|
||||||
RULEHINT_LIMIT = 5
|
# NOW A DEFAULT RATHER THAN A CAP (#4102): both rule act-arms read their own
|
||||||
|
# budget from the registry, so this is the value an install starts at and not
|
||||||
|
# the value it is stuck with. The reasoning below is why 5 is where it starts.
|
||||||
|
RULEHINT_LIMIT = SURFACES["write_path_rule"].budget_default
|
||||||
|
|
||||||
# MEASURED, NOT REASONED — and the reasoning it replaced was wrong (#3851).
|
# MEASURED, NOT REASONED — and the reasoning it replaced was wrong (#3851).
|
||||||
#
|
#
|
||||||
@@ -433,7 +451,11 @@ _CONCEPT_MIN_CHARS = 16
|
|||||||
_AUTOINJECT_BAND = 0.10
|
_AUTOINJECT_BAND = 0.10
|
||||||
# Hard ceiling on top-k regardless of the user's setting — this is an
|
# Hard ceiling on top-k regardless of the user's setting — this is an
|
||||||
# awareness menu (titles only), never a content dump.
|
# awareness menu (titles only), never a content dump.
|
||||||
_AUTOINJECT_MAX_TOP_K = 10
|
# The budget ceiling, now shared by every surface rather than owned by this
|
||||||
|
# one (#4102). It bounds the same thing everywhere — how many lines a single
|
||||||
|
# unsolicited injection may occupy — so one surface having a private ceiling
|
||||||
|
# was an accident of which arm got a configurable budget first.
|
||||||
|
_AUTOINJECT_MAX_TOP_K = MAX_BUDGET
|
||||||
|
|
||||||
# --- the prompt-boundary rule arm (#3852) ------------------------------------
|
# --- the prompt-boundary rule arm (#3852) ------------------------------------
|
||||||
#
|
#
|
||||||
@@ -446,7 +468,7 @@ _AUTOINJECT_MAX_TOP_K = 10
|
|||||||
# The operator's message is the only query that exists before one is composed,
|
# The operator's message is the only query that exists before one is composed,
|
||||||
# and this arm is what runs against it. Until now that hook searched notes
|
# and this arm is what runs against it. Until now that hook searched notes
|
||||||
# alone, so no rule had ever been retrieved against a thing the operator said.
|
# alone, so no rule had ever been retrieved against a thing the operator said.
|
||||||
PROMPTRULE_THRESHOLD_KEY = "kb_promptrule_threshold"
|
PROMPTRULE_THRESHOLD_KEY = SURFACES["prompt_rule"].floor_key
|
||||||
# INHERITED FROM THE ACT ARMS, AND NOT YET EARNED HERE. 0.72 was tuned against
|
# INHERITED FROM THE ACT ARMS, AND NOT YET EARNED HERE. 0.72 was tuned against
|
||||||
# code and shell commands. An operator's prose is a different query shape
|
# code and shell commands. An operator's prose is a different query shape
|
||||||
# against the same documents, and nothing yet says the two distributions line
|
# against the same documents, and nothing yet says the two distributions line
|
||||||
@@ -458,7 +480,7 @@ PROMPTRULE_THRESHOLD_KEY = "kb_promptrule_threshold"
|
|||||||
# front of a corpus that binds. Every call is logged under `prompt_rule` from
|
# front of a corpus that binds. Every call is logged under `prompt_rule` from
|
||||||
# the first deploy, so a few days of real traffic settles it — read
|
# the first deploy, so a few days of real traffic settles it — read
|
||||||
# `near_miss_samples` (#3807) before moving this, not the percentile alone.
|
# `near_miss_samples` (#3807) before moving this, not the percentile alone.
|
||||||
PROMPTRULE_DEFAULT_THRESHOLD = 0.72
|
PROMPTRULE_DEFAULT_THRESHOLD = SURFACES["prompt_rule"].floor_default
|
||||||
|
|
||||||
# MORE THAN THE ACT ARMS' SINGLE SLOT, anchored on this hook's budget rather
|
# MORE THAN THE ACT ARMS' SINGLE SLOT, anchored on this hook's budget rather
|
||||||
# than theirs. RULEHINT_LIMIT is 1 because that arm fires before EVERY Bash
|
# than theirs. RULEHINT_LIMIT is 1 because that arm fires before EVERY Bash
|
||||||
@@ -469,7 +491,7 @@ PROMPTRULE_DEFAULT_THRESHOLD = 0.72
|
|||||||
# And a prompt genuinely contains more than one act. "Merge to main and then
|
# And a prompt genuinely contains more than one act. "Merge to main and then
|
||||||
# start on X" is two, governed by different rules; k=1 cannot serve that case
|
# start on X" is two, governed by different rules; k=1 cannot serve that case
|
||||||
# at all, where the act arms never face it because a command is one thing.
|
# at all, where the act arms never face it because a command is one thing.
|
||||||
PROMPTRULE_LIMIT = 3
|
PROMPTRULE_LIMIT = SURFACES["prompt_rule"].budget_default
|
||||||
|
|
||||||
# THE COMPLETION-REPORT ARM'S OWN BAR (services/reply_preferences.py).
|
# THE COMPLETION-REPORT ARM'S OWN BAR (services/reply_preferences.py).
|
||||||
#
|
#
|
||||||
@@ -487,8 +509,8 @@ PROMPTRULE_LIMIT = 3
|
|||||||
# Kept at the prose arm's starting value rather than tuned: the split is what
|
# Kept at the prose arm's starting value rather than tuned: the split is what
|
||||||
# makes the two independently movable, and a default is a product decision
|
# makes the two independently movable, and a default is a product decision
|
||||||
# that this install's corpus cannot settle (rule 115).
|
# that this install's corpus cannot settle (rule 115).
|
||||||
REPORTPREF_THRESHOLD_KEY = "kb_reportpref_threshold"
|
REPORTPREF_THRESHOLD_KEY = SURFACES["report_preference"].floor_key
|
||||||
REPORTPREF_DEFAULT_THRESHOLD = 0.72
|
REPORTPREF_DEFAULT_THRESHOLD = SURFACES["report_preference"].floor_default
|
||||||
|
|
||||||
|
|
||||||
def _slugify(text: str) -> str:
|
def _slugify(text: str) -> str:
|
||||||
@@ -592,8 +614,17 @@ async def build_process_manifest(user_id: int) -> dict:
|
|||||||
async def get_autoinject_config(user_id: int) -> dict:
|
async def get_autoinject_config(user_id: int) -> dict:
|
||||||
"""Resolve a user's auto-inject settings, falling back to the defaults.
|
"""Resolve a user's auto-inject settings, falling back to the defaults.
|
||||||
|
|
||||||
Returns {"enabled": bool, "threshold": float, "top_k": int}, clamped to
|
Returns {"enabled": bool, "threshold": float, "top_k": int}.
|
||||||
sane ranges (threshold to [0,1]; top_k to [1, _AUTOINJECT_MAX_TOP_K]).
|
|
||||||
|
THE TWO NUMBERS COME FROM THE REGISTRY NOW (#4102). They used to be read and
|
||||||
|
clamped here, and identically again in `get_writepath_config`, and again in
|
||||||
|
three rule arms, and once more in `reply_preferences`. That was
|
||||||
|
tolerable while the values were shipped constants. It stops being tolerable
|
||||||
|
once a tool is expected to MOVE them, because a tuning surface cannot be
|
||||||
|
consistent across arms that each spell their configuration differently.
|
||||||
|
|
||||||
|
`enabled` stays here: it is this surface's own switch, not a tunable number,
|
||||||
|
and the registry deliberately holds only the pair a floor-tuner touches.
|
||||||
"""
|
"""
|
||||||
enabled_raw = await get_setting(
|
enabled_raw = await get_setting(
|
||||||
user_id, AUTOINJECT_ENABLED_KEY,
|
user_id, AUTOINJECT_ENABLED_KEY,
|
||||||
@@ -601,21 +632,11 @@ async def get_autoinject_config(user_id: int) -> dict:
|
|||||||
)
|
)
|
||||||
enabled = enabled_raw.strip().lower() in ("true", "1", "yes", "on")
|
enabled = enabled_raw.strip().lower() in ("true", "1", "yes", "on")
|
||||||
|
|
||||||
try:
|
return {
|
||||||
threshold = float(await get_setting(
|
"enabled": enabled,
|
||||||
user_id, AUTOINJECT_THRESHOLD_KEY, str(AUTOINJECT_DEFAULT_THRESHOLD)))
|
"threshold": await floor_for(user_id, "auto_inject"),
|
||||||
except (TypeError, ValueError):
|
"top_k": await budget_for(user_id, "auto_inject"),
|
||||||
threshold = AUTOINJECT_DEFAULT_THRESHOLD
|
}
|
||||||
threshold = min(1.0, max(0.0, threshold))
|
|
||||||
|
|
||||||
try:
|
|
||||||
top_k = int(float(await get_setting(
|
|
||||||
user_id, AUTOINJECT_TOP_K_KEY, str(AUTOINJECT_DEFAULT_TOP_K))))
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
top_k = AUTOINJECT_DEFAULT_TOP_K
|
|
||||||
top_k = min(_AUTOINJECT_MAX_TOP_K, max(1, top_k))
|
|
||||||
|
|
||||||
return {"enabled": enabled, "threshold": threshold, "top_k": top_k}
|
|
||||||
|
|
||||||
|
|
||||||
def _record_kind(note) -> str:
|
def _record_kind(note) -> str:
|
||||||
@@ -644,7 +665,7 @@ async def _reserve_slot_for_reuse(
|
|||||||
cfg: dict,
|
cfg: dict,
|
||||||
*,
|
*,
|
||||||
project_id: int | None,
|
project_id: int | None,
|
||||||
exclude_ids: set[int],
|
already: set[int],
|
||||||
) -> list:
|
) -> list:
|
||||||
"""Guarantee the reuse-shaped kinds one slot, if one clears threshold (#2246).
|
"""Guarantee the reuse-shaped kinds one slot, if one clears threshold (#2246).
|
||||||
|
|
||||||
@@ -671,16 +692,23 @@ async def _reserve_slot_for_reuse(
|
|||||||
top_k = cfg["top_k"]
|
top_k = cfg["top_k"]
|
||||||
_t0 = time.perf_counter()
|
_t0 = time.perf_counter()
|
||||||
_rep: dict = {}
|
_rep: dict = {}
|
||||||
|
# THE LEDGER IS NOT AN EXCLUSION HERE EITHER (#4101) — but what is already
|
||||||
|
# in `kept` still is, and the two are different claims. A record sitting in
|
||||||
|
# this call's own menu must not be shown twice in it; a record shown in an
|
||||||
|
# EARLIER call is exactly what the slot should be allowed to spend itself
|
||||||
|
# on, because a snippet that was relevant then and is relevant now is the
|
||||||
|
# reuse case rather than a duplicate of it.
|
||||||
reuse = await semantic_search_notes(
|
reuse = await semantic_search_notes(
|
||||||
user_id, query,
|
user_id, query,
|
||||||
limit=1,
|
limit=1,
|
||||||
threshold=cfg["threshold"],
|
threshold=cfg["threshold"],
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
exclude_ids=exclude_ids | {int(n.id) for _s, n in kept},
|
exclude_ids={int(n.id) for _s, n in kept},
|
||||||
note_type=_REUSE_KINDS,
|
note_type=_REUSE_KINDS,
|
||||||
scope="browse",
|
scope="browse",
|
||||||
report=_rep,
|
report=_rep,
|
||||||
)
|
)
|
||||||
|
fresh_reuse = [(s, n) for s, n in reuse if int(n.id) not in already]
|
||||||
# A real semantic query competing for a menu slot — logged like the scored
|
# A real semantic query competing for a menu slot — logged like the scored
|
||||||
# arm it displaces. Before this, the hit it PUSHED OUT was in
|
# arm it displaces. Before this, the hit it PUSHED OUT was in
|
||||||
# retrieval_logs and the query that pushed it out was not, so the slot
|
# retrieval_logs and the query that pushed it out was not, so the slot
|
||||||
@@ -689,16 +717,17 @@ async def _reserve_slot_for_reuse(
|
|||||||
record_retrieval(
|
record_retrieval(
|
||||||
user_id=user_id, source="reuse_slot", query=query,
|
user_id=user_id, source="reuse_slot", query=query,
|
||||||
threshold=cfg["threshold"], limit=1, project_id=project_id,
|
threshold=cfg["threshold"], limit=1, project_id=project_id,
|
||||||
is_task=None, results=reuse,
|
is_task=None, results=fresh_reuse,
|
||||||
best_available=_rep.get("best_available_score"),
|
best_available=_rep.get("best_available_score"),
|
||||||
best_available_id=_rep.get("best_available_id"),
|
best_available_id=_rep.get("best_available_id"),
|
||||||
searched=bool(_rep.get("searched", True)),
|
searched=bool(_rep.get("searched", True)),
|
||||||
|
suppressed=len(reuse) - len(fresh_reuse),
|
||||||
duration_ms=(time.perf_counter() - _t0) * 1000.0,
|
duration_ms=(time.perf_counter() - _t0) * 1000.0,
|
||||||
)
|
)
|
||||||
# Verify the kind rather than trusting the query that asked for it, and
|
# Verify the kind rather than trusting the query that asked for it, and
|
||||||
# dedup on top of exclude_ids. This slot exists FOR reuse kinds — a slot
|
# dedup against this call's own menu. This slot exists FOR reuse kinds — a
|
||||||
# silently spent on something else is worse than no slot, because the line
|
# slot silently spent on something else is worse than no slot, because the
|
||||||
# is indistinguishable from one that earned its place on score.
|
# line is indistinguishable from one that earned its place on score.
|
||||||
kept_ids = {int(n.id) for _s, n in kept}
|
kept_ids = {int(n.id) for _s, n in kept}
|
||||||
fresh = [
|
fresh = [
|
||||||
(s, n) for s, n in reuse
|
(s, n) for s, n in reuse
|
||||||
@@ -726,7 +755,8 @@ async def build_autoinject_hint(
|
|||||||
The four anti-bloat gates (see the module + milestone-93 design):
|
The four anti-bloat gates (see the module + milestone-93 design):
|
||||||
1. high-confidence threshold (stricter than pull) — set per-user;
|
1. high-confidence threshold (stricter than pull) — set per-user;
|
||||||
2. margin gate — keep only hits within _AUTOINJECT_BAND of the top score;
|
2. margin gate — keep only hits within _AUTOINJECT_BAND of the top score;
|
||||||
3. session dedup — caller passes already-injected ids as `exclude_ids`;
|
3. session marking — caller passes already-injected ids as `exclude_ids`
|
||||||
|
and they are rendered again with `[seen]`, never withheld (#4101);
|
||||||
4. title-first payload — id + kind + title + score only, never bodies.
|
4. title-first payload — id + kind + title + score only, never bodies.
|
||||||
Disabled, blank-query, or nothing-clears-the-gates all return empty context,
|
Disabled, blank-query, or nothing-clears-the-gates all return empty context,
|
||||||
so most turns inject nothing.
|
so most turns inject nothing.
|
||||||
@@ -741,6 +771,24 @@ async def build_autoinject_hint(
|
|||||||
if not cfg["enabled"] or not q:
|
if not cfg["enabled"] or not q:
|
||||||
return empty
|
return empty
|
||||||
|
|
||||||
|
# THE LEDGER LEAVES THE SEARCH (#4101). `exclude_ids` used to go into
|
||||||
|
# `semantic_search_notes` itself, so a record this session had already been
|
||||||
|
# shown was removed from the candidate set — which is #3750's defect, on the
|
||||||
|
# arm that fires most. Three things followed from it, none of them intended:
|
||||||
|
#
|
||||||
|
# - the second time a note was the best answer, the session got SILENCE,
|
||||||
|
# indistinguishable from "nothing matched";
|
||||||
|
# - a compaction made that permanent, because the ledger outlived the
|
||||||
|
# context it described (fixed one layer down in this same step);
|
||||||
|
# - and the score the search reported was measured against a candidate set
|
||||||
|
# the caller had already edited, so `best_available` could name a bar
|
||||||
|
# that turned nothing away (#3739).
|
||||||
|
#
|
||||||
|
# Now the ledger is a RENDERING fact, not a retrieval one: every repeat is
|
||||||
|
# still ranked, still shown, and carries a marker saying it was surfaced
|
||||||
|
# before. A note line is a title and a score — the repeat costs about as
|
||||||
|
# much as the comma in this sentence — so there is nothing here to save.
|
||||||
|
already = {int(i) for i in (exclude_ids or [])}
|
||||||
t0 = time.perf_counter()
|
t0 = time.perf_counter()
|
||||||
_rep_ai: dict = {}
|
_rep_ai: dict = {}
|
||||||
hits = await semantic_search_notes(
|
hits = await semantic_search_notes(
|
||||||
@@ -748,7 +796,6 @@ async def build_autoinject_hint(
|
|||||||
limit=cfg["top_k"],
|
limit=cfg["top_k"],
|
||||||
threshold=cfg["threshold"],
|
threshold=cfg["threshold"],
|
||||||
project_id=(project_id or None),
|
project_id=(project_id or None),
|
||||||
exclude_ids=set(exclude_ids or []),
|
|
||||||
# Injection is the one retrieval nobody asked for, so it takes the BROWSE
|
# Injection is the one retrieval nobody asked for, so it takes the BROWSE
|
||||||
# scope: never a record shared one-to-one with the operator. What can
|
# scope: never a record shared one-to-one with the operator. What can
|
||||||
# still appear is a collaborator's note inside a shared project — legible
|
# still appear is a collaborator's note inside a shared project — legible
|
||||||
@@ -756,24 +803,37 @@ async def build_autoinject_hint(
|
|||||||
scope="browse",
|
scope="browse",
|
||||||
report=_rep_ai,
|
report=_rep_ai,
|
||||||
)
|
)
|
||||||
|
# `results=fresh` and `suppressed` together, on the rule arms' contract
|
||||||
|
# (#3752): a rendered repeat is not a new surfacing, so it stays out of the
|
||||||
|
# row's result set and is COUNTED instead. That keeps this source's surfaced
|
||||||
|
# set identical to its own log row (#3668) while making a zero-result call
|
||||||
|
# readable — `result_count == 0` with `suppressed_count > 0` is "everything
|
||||||
|
# that matched, this session has already seen", which is a different fact
|
||||||
|
# about the bar from "nothing cleared it" and used to be unreportable here.
|
||||||
|
fresh = [(s, n) for s, n in hits if int(n.id) not in already]
|
||||||
record_retrieval(
|
record_retrieval(
|
||||||
user_id=user_id, source="auto_inject", query=q,
|
user_id=user_id, source="auto_inject", query=q,
|
||||||
threshold=cfg["threshold"], limit=cfg["top_k"],
|
threshold=cfg["threshold"], limit=cfg["top_k"],
|
||||||
project_id=(project_id or None), is_task=None, results=hits,
|
project_id=(project_id or None), is_task=None, results=fresh,
|
||||||
best_available=_rep_ai.get("best_available_score"),
|
best_available=_rep_ai.get("best_available_score"),
|
||||||
best_available_id=_rep_ai.get("best_available_id"),
|
best_available_id=_rep_ai.get("best_available_id"),
|
||||||
searched=bool(_rep_ai.get("searched", True)),
|
searched=bool(_rep_ai.get("searched", True)),
|
||||||
|
suppressed=len(hits) - len(fresh),
|
||||||
duration_ms=(time.perf_counter() - t0) * 1000.0,
|
duration_ms=(time.perf_counter() - t0) * 1000.0,
|
||||||
)
|
)
|
||||||
if not hits:
|
if not hits:
|
||||||
return empty
|
return empty
|
||||||
|
|
||||||
# Margin gate: keep only hits close to the strongest one.
|
# Margin gate: keep only hits close to the strongest one. Computed over ALL
|
||||||
|
# hits, repeats included — the band measures distance from the top SCORE,
|
||||||
|
# and letting the ledger move that cutoff would make "you were shown this"
|
||||||
|
# change what counts as relevant, which is the axis independence the rule
|
||||||
|
# band keeps for the same reason.
|
||||||
top_score = hits[0][0]
|
top_score = hits[0][0]
|
||||||
kept = [(s, n) for s, n in hits if s >= top_score - _AUTOINJECT_BAND]
|
kept = [(s, n) for s, n in hits if s >= top_score - _AUTOINJECT_BAND]
|
||||||
kept = await _reserve_slot_for_reuse(
|
kept = await _reserve_slot_for_reuse(
|
||||||
user_id, q, kept, cfg, project_id=(project_id or None),
|
user_id, q, kept, cfg, project_id=(project_id or None),
|
||||||
exclude_ids=set(exclude_ids or []),
|
already=already,
|
||||||
)
|
)
|
||||||
|
|
||||||
# A collaborator's note can reach this menu via a shared project, and the
|
# A collaborator's note can reach this menu via a shared project, and the
|
||||||
@@ -786,10 +846,15 @@ async def build_autoinject_hint(
|
|||||||
# "records", not "notes" — the menu can hold snippets, processes and tasks
|
# "records", not "notes" — the menu can hold snippets, processes and tasks
|
||||||
# too, and the kind marker on each line is only legible if the header doesn't
|
# too, and the kind marker on each line is only legible if the header doesn't
|
||||||
# already claim they're all one thing.
|
# already claim they're all one thing.
|
||||||
|
# "injected once per session" was true and is not any more (#4101): a repeat
|
||||||
|
# is shown again with a marker rather than withheld, so the header must stop
|
||||||
|
# promising the old contract. It now says what the marker means instead,
|
||||||
|
# once, rather than each repeated line having to explain itself.
|
||||||
lines = [
|
lines = [
|
||||||
"> Possibly relevant from your Scribe records — open any in full with "
|
"> Possibly relevant from your Scribe records — open any in full with "
|
||||||
"`get_note(id)`, or `get_snippet` / `get_process` for those kinds "
|
"`get_note(id)`, or `get_snippet` / `get_process` for those kinds "
|
||||||
"(titles only; injected once per session):",
|
"(titles only; a line marked `seen` was surfaced earlier this session "
|
||||||
|
"and may no longer be in context):",
|
||||||
]
|
]
|
||||||
# A superseded record is DEMOTED, not removed (#278) — so one can still reach
|
# A superseded record is DEMOTED, not removed (#278) — so one can still reach
|
||||||
# this menu, and when it does the reader has to be told. An agent handed
|
# this menu, and when it does the reader has to be told. An agent handed
|
||||||
@@ -802,6 +867,13 @@ async def build_autoinject_hint(
|
|||||||
note_ids.append(int(note.id))
|
note_ids.append(int(note.id))
|
||||||
title = (note.title or "(untitled)").replace("\n", " ").strip()
|
title = (note.title or "(untitled)").replace("\n", " ").strip()
|
||||||
line = f"> - #{note.id} [{_record_kind(note)}] \"{title}\" ({score:.2f})"
|
line = f"> - #{note.id} [{_record_kind(note)}] \"{title}\" ({score:.2f})"
|
||||||
|
# ONE WORD, NOT A SENTENCE, and deliberately not the rule arms' phrasing.
|
||||||
|
# A rule line says "before deciding it does not apply", which is the
|
||||||
|
# voice of a record that BINDS; a note binds nothing, and borrowing that
|
||||||
|
# tone would tell the reader a dev-log has authority it does not have.
|
||||||
|
# The header carries the meaning, so the line carries only the flag.
|
||||||
|
if int(note.id) in already:
|
||||||
|
line += " [seen]"
|
||||||
if int(note.id) in stale:
|
if int(note.id) in stale:
|
||||||
line += " — SUPERSEDED, a later record covers this; check that first"
|
line += " — SUPERSEDED, a later record covers this; check that first"
|
||||||
if note.user_id != user_id:
|
if note.user_id != user_id:
|
||||||
@@ -813,7 +885,17 @@ async def build_autoinject_hint(
|
|||||||
# menu the agent actually saw. retrieval_logs already holds the full
|
# menu the agent actually saw. retrieval_logs already holds the full
|
||||||
# candidate set for threshold tuning; conflating the two would make
|
# candidate set for threshold tuning; conflating the two would make
|
||||||
# "surfaced" mean two different things depending on the surface (#2085).
|
# "surfaced" mean two different things depending on the surface (#2085).
|
||||||
record_surfaced(user_id=user_id, note_ids=note_ids, source="auto_inject")
|
#
|
||||||
|
# FRESH ONLY, which is the same cut the log row above takes (#4101). A
|
||||||
|
# repeat is rendered but is not a new surfacing, and counting it again would
|
||||||
|
# make this table disagree with `retrieval_logs` about the same call —
|
||||||
|
# #3668's identity, which is the cheapest true statement available about
|
||||||
|
# this pair of tables and is not worth a marker's convenience.
|
||||||
|
record_surfaced(
|
||||||
|
user_id=user_id,
|
||||||
|
note_ids=[i for i in note_ids if i not in already],
|
||||||
|
source="auto_inject",
|
||||||
|
)
|
||||||
|
|
||||||
return {"context": "\n".join(lines), "note_ids": note_ids, "config": cfg}
|
return {"context": "\n".join(lines), "note_ids": note_ids, "config": cfg}
|
||||||
|
|
||||||
@@ -963,13 +1045,8 @@ async def build_prompt_rule_hint(
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
try:
|
try:
|
||||||
try:
|
threshold = await floor_for(user_id, "prompt_rule")
|
||||||
threshold = float(await get_setting(
|
limit = await budget_for(user_id, "prompt_rule")
|
||||||
user_id, PROMPTRULE_THRESHOLD_KEY,
|
|
||||||
str(PROMPTRULE_DEFAULT_THRESHOLD)))
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
threshold = PROMPTRULE_DEFAULT_THRESHOLD
|
|
||||||
threshold = min(1.0, max(0.0, threshold))
|
|
||||||
|
|
||||||
t0 = time.perf_counter()
|
t0 = time.perf_counter()
|
||||||
_rep: dict = {}
|
_rep: dict = {}
|
||||||
@@ -980,7 +1057,7 @@ async def build_prompt_rule_hint(
|
|||||||
# sessions — this surface speaks unasked, and a whole-rulebook answer
|
# sessions — this surface speaks unasked, and a whole-rulebook answer
|
||||||
# is only right for someone who asked the whole rulebook.
|
# is only right for someone who asked the whole rulebook.
|
||||||
hits = await semantic_search_rules(
|
hits = await semantic_search_rules(
|
||||||
user_id, q, limit=PROMPTRULE_LIMIT, threshold=threshold,
|
user_id, q, limit=limit, threshold=threshold,
|
||||||
report=_rep, project_id=project_id or None,
|
report=_rep, project_id=project_id or None,
|
||||||
)
|
)
|
||||||
duration_ms = (time.perf_counter() - t0) * 1000.0
|
duration_ms = (time.perf_counter() - t0) * 1000.0
|
||||||
@@ -996,7 +1073,7 @@ async def build_prompt_rule_hint(
|
|||||||
# and unverified for this corpus, so the zero rows are the point.
|
# and unverified for this corpus, so the zero rows are the point.
|
||||||
record_retrieval(
|
record_retrieval(
|
||||||
user_id=user_id, source="prompt_rule", query=q,
|
user_id=user_id, source="prompt_rule", query=q,
|
||||||
threshold=threshold, limit=PROMPTRULE_LIMIT,
|
threshold=threshold, limit=limit,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
is_task=None, results=fresh, duration_ms=duration_ms,
|
is_task=None, results=fresh, duration_ms=duration_ms,
|
||||||
best_available=_rep.get("best_available_score"),
|
best_available=_rep.get("best_available_score"),
|
||||||
@@ -1233,52 +1310,37 @@ def concept_query(code: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
async def get_writepath_config(user_id: int) -> dict:
|
async def get_writepath_config(user_id: int) -> dict:
|
||||||
"""Write-path trigger settings: its own `enabled` and `threshold`, auto-inject's top_k.
|
"""Write-path trigger settings — and the two rule arms' numbers alongside.
|
||||||
|
|
||||||
The threshold OVERRIDES the inherited auto-inject value — code embeddings
|
Three surfaces' worth of configuration arrives in one call because one hook
|
||||||
have a much higher similarity floor than prose, so the two surfaces need
|
request drives all three arms. They are still three SURFACES with three
|
||||||
different bars. See WRITEPATH_DEFAULT_THRESHOLD for the measurements (#2223).
|
independent pairs, resolved from the registry (#4102).
|
||||||
top_k is still shared: "how many titles at once" means the same thing on
|
|
||||||
both surfaces, and nothing suggests they want different ceilings.
|
The floors were split apart one at a time, each on its own measurement, and
|
||||||
|
those measurements are recorded where the defaults are: WRITEPATH (#2223 —
|
||||||
|
code embeddings sit on a much higher similarity floor than prose), RULEHINT
|
||||||
|
(a third corpus again), TOOLRULE (#3853 — a shell command is a different
|
||||||
|
query shape from a code payload and scores lower for the same relevance).
|
||||||
|
|
||||||
|
THE BUDGET IS NOW SPLIT TOO. `top_k` used to be auto-inject's outright, on
|
||||||
|
the argument that "how many titles at once" means the same thing on both
|
||||||
|
surfaces. It does not: this arm fires before every Write and Edit while
|
||||||
|
auto-inject fires once a turn, so the same number buys wildly different
|
||||||
|
amounts of attention. `write_path` inherits auto-inject's value when it has
|
||||||
|
none of its own, so no install that tuned the shared knob loses it.
|
||||||
"""
|
"""
|
||||||
cfg = await get_autoinject_config(user_id)
|
|
||||||
enabled_raw = await get_setting(
|
enabled_raw = await get_setting(
|
||||||
user_id, WRITEPATH_ENABLED_KEY,
|
user_id, WRITEPATH_ENABLED_KEY,
|
||||||
"true" if WRITEPATH_DEFAULT_ENABLED else "false",
|
"true" if WRITEPATH_DEFAULT_ENABLED else "false",
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
|
||||||
threshold = float(await get_setting(
|
|
||||||
user_id, WRITEPATH_THRESHOLD_KEY, str(WRITEPATH_DEFAULT_THRESHOLD)))
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
threshold = WRITEPATH_DEFAULT_THRESHOLD
|
|
||||||
threshold = min(1.0, max(0.0, threshold))
|
|
||||||
|
|
||||||
try:
|
|
||||||
rule_threshold = float(await get_setting(
|
|
||||||
user_id, RULEHINT_THRESHOLD_KEY, str(RULEHINT_DEFAULT_THRESHOLD)))
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
rule_threshold = RULEHINT_DEFAULT_THRESHOLD
|
|
||||||
rule_threshold = min(1.0, max(0.0, rule_threshold))
|
|
||||||
|
|
||||||
try:
|
|
||||||
tool_rule_threshold = float(await get_setting(
|
|
||||||
user_id, TOOLRULE_THRESHOLD_KEY, str(TOOLRULE_DEFAULT_THRESHOLD)))
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
tool_rule_threshold = TOOLRULE_DEFAULT_THRESHOLD
|
|
||||||
tool_rule_threshold = min(1.0, max(0.0, tool_rule_threshold))
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
**cfg,
|
|
||||||
"enabled": enabled_raw.strip().lower() in ("true", "1", "yes", "on"),
|
"enabled": enabled_raw.strip().lower() in ("true", "1", "yes", "on"),
|
||||||
"threshold": threshold,
|
"threshold": await floor_for(user_id, "write_path"),
|
||||||
# Its own bar, for a third corpus — see RULEHINT_DEFAULT_THRESHOLD.
|
"top_k": await budget_for(user_id, "write_path"),
|
||||||
"rule_threshold": rule_threshold,
|
"rule_threshold": await floor_for(user_id, "write_path_rule"),
|
||||||
# And the COMMAND arm's own bar again, for the same reason one level
|
"rule_top_k": await budget_for(user_id, "write_path_rule"),
|
||||||
# down: a shell command is a different query shape from a code payload
|
"tool_rule_threshold": await floor_for(user_id, "pre_tool_rule"),
|
||||||
# and scores lower for the same relevance (#3853). Separate keys, so an
|
"tool_rule_top_k": await budget_for(user_id, "pre_tool_rule"),
|
||||||
# install can move one without the other — which is the whole finding.
|
|
||||||
"tool_rule_threshold": tool_rule_threshold,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def _rule_band(hits: list) -> list:
|
def _rule_band(hits: list) -> list:
|
||||||
@@ -1468,12 +1530,16 @@ async def build_write_path_hint(
|
|||||||
#2707): the record gets corrected in the session that has the context,
|
#2707): the record gets corrected in the session that has the context,
|
||||||
at the moment of change. Nearby and semantic hits stay the REUSE menu.
|
at the moment of change. Nearby and semantic hits stay the REUSE menu.
|
||||||
|
|
||||||
The two classes dedup on SEPARATE channels — `exclude_ids` (reuse) and
|
The two classes track the session on SEPARATE channels — `exclude_ids`
|
||||||
`exclude_sync_ids` (sync) — because they answer different questions: a
|
(reuse) and `exclude_sync_ids` (sync) — because they answer different
|
||||||
title shown as "consider reusing this" twenty turns ago must not silence
|
questions: a title shown as "consider reusing this" twenty turns ago must
|
||||||
"you are editing the recorded file right now" (#2708).
|
not silence "you are editing the recorded file right now" (#2708). The two
|
||||||
|
channels also now ACT differently, which is #4101: a reuse repeat is
|
||||||
|
rendered again with a `seen` marker, while the sync class still shows
|
||||||
|
once, because its claim is about an edit in progress rather than about a
|
||||||
|
record's continuing relevance and repeating it would be nagging.
|
||||||
|
|
||||||
Carries auto-inject's anti-bloat gates (margin, session dedup,
|
Carries auto-inject's anti-bloat gates (margin, session marking,
|
||||||
titles-never-bodies) plus the shared top-k cap across ALL arms — so a file
|
titles-never-bodies) plus the shared top-k cap across ALL arms — so a file
|
||||||
with a lot of recorded history can't turn one edit into a wall of text. Two
|
with a lot of recorded history can't turn one edit into a wall of text. Two
|
||||||
gates are its OWN, because code is not prose: a stricter similarity
|
gates are its OWN, because code is not prose: a stricter similarity
|
||||||
@@ -1546,7 +1612,13 @@ async def build_write_path_hint(
|
|||||||
# stay eligible here, which is the whole point of the split. Either way
|
# stay eligible here, which is the whole point of the split. Either way
|
||||||
# they join `seen`, so the reuse arms (where the directory query would
|
# they join `seen`, so the reuse arms (where the directory query would
|
||||||
# surface them again) never re-list a record the sync block owns.
|
# surface them again) never re-list a record the sync block owns.
|
||||||
seen: set[int] = set(excluded)
|
#
|
||||||
|
# `seen` IS THIS CALL'S OWN MENU, and nothing else (#4101). It used to start
|
||||||
|
# from `excluded` — the session ledger — which folded two different claims
|
||||||
|
# into one variable: "already listed a few lines above" and "shown at some
|
||||||
|
# earlier point in the session". Only the first is a reason to stay quiet.
|
||||||
|
# The ledger is now a marker instead, on both reuse arms.
|
||||||
|
seen: set[int] = set()
|
||||||
synced: list[dict] = []
|
synced: list[dict] = []
|
||||||
for item in here:
|
for item in here:
|
||||||
nid = int(item["id"])
|
nid = int(item["id"])
|
||||||
@@ -1560,7 +1632,12 @@ async def build_write_path_hint(
|
|||||||
if nid in seen:
|
if nid in seen:
|
||||||
continue
|
continue
|
||||||
seen.add(nid)
|
seen.add(nid)
|
||||||
placed.append(("nearby", item))
|
# MARKED HERE TOO, not just on the semantic arm, because these are one
|
||||||
|
# menu. A hint where some repeats carry `seen` and others are silently
|
||||||
|
# dropped — decided by which arm happened to find them — is worse than
|
||||||
|
# either rule applied consistently: the marker would read as a complete
|
||||||
|
# account of what the session has met before, and it would not be one.
|
||||||
|
placed.append(("nearby · seen" if nid in excluded else "nearby", item))
|
||||||
|
|
||||||
# The stamping feed's "actually pulled it" half (#2791). Read once, before
|
# The stamping feed's "actually pulled it" half (#2791). Read once, before
|
||||||
# the semantic arm, because the arm's query doubles as the resemblance
|
# the semantic arm, because the arm's query doubles as the resemblance
|
||||||
@@ -1592,16 +1669,19 @@ async def build_write_path_hint(
|
|||||||
query = concept_query(query) or query
|
query = concept_query(query) or query
|
||||||
if remaining > 0 and query:
|
if remaining > 0 and query:
|
||||||
t0 = time.perf_counter()
|
t0 = time.perf_counter()
|
||||||
# Pulled-and-seen ids stay in the query (as evidence) but never in
|
# Pulled-and-already-listed ids stay in the query (as evidence for
|
||||||
# the menu — the dedup contract holds, the resemblance still lands.
|
# `resembles`) but never in the menu, so the limit has to cover them.
|
||||||
pulled_seen = seen & set(pulled)
|
# `seen` is this call's own menu now, which is the only thing left that
|
||||||
|
# is a reason to withhold (#4101).
|
||||||
|
in_menu = seen
|
||||||
|
pulled_in_menu = in_menu & set(pulled)
|
||||||
_rep_wp: dict = {}
|
_rep_wp: dict = {}
|
||||||
hits = await semantic_search_notes(
|
hits = await semantic_search_notes(
|
||||||
user_id, query,
|
user_id, query,
|
||||||
limit=remaining + len(pulled_seen),
|
limit=remaining + len(pulled_in_menu),
|
||||||
threshold=cfg["threshold"],
|
threshold=cfg["threshold"],
|
||||||
project_id=scope_project,
|
project_id=scope_project,
|
||||||
exclude_ids=seen - pulled_seen,
|
exclude_ids=in_menu - set(pulled),
|
||||||
# Snippets AND recorded experience (#2246). This arm was
|
# Snippets AND recorded experience (#2246). This arm was
|
||||||
# snippets-only, which is auto-inject's mistake inverted: an issue
|
# snippets-only, which is auto-inject's mistake inverted: an issue
|
||||||
# saying "we tried this and it deadlocked", or a dev-log recording
|
# saying "we tried this and it deadlocked", or a dev-log recording
|
||||||
@@ -1625,41 +1705,50 @@ async def build_write_path_hint(
|
|||||||
int(note.id): float(score) for score, note in hits
|
int(note.id): float(score) for score, note in hits
|
||||||
if int(note.id) in pulled
|
if int(note.id) in pulled
|
||||||
}
|
}
|
||||||
shown = [(s, n) for s, n in hits if int(n.id) not in seen]
|
shown = [(s, n) for s, n in hits if int(n.id) not in in_menu]
|
||||||
# WHAT THIS ARM WITHHELD AFTER THE SEARCH ANSWERED, and the reason
|
# WHAT THIS ARM WITHHELD AFTER THE SEARCH ANSWERED, and the reason
|
||||||
# `best_available_score` cannot always be reported here (#3739 again,
|
# `best_available_score` cannot always be reported here (#3739 again,
|
||||||
# from the side its fix did not reach).
|
# from the side its fix did not reach).
|
||||||
#
|
#
|
||||||
# This arm is the one note arm that filters TWICE. `exclude_ids` takes
|
# This arm is the one note arm that filters TWICE. `exclude_ids` takes
|
||||||
# `seen - pulled_seen` into the search, but the pulled-and-seen ids stay
|
# the already-listed ids into the search, but the PULLED ones among them
|
||||||
# in the query deliberately — `resembles` above needs them — and are
|
# stay in the query deliberately — `resembles` above needs them — and
|
||||||
# dropped in the line above instead. So the score the search reported is
|
# are dropped in the line above instead. So the score the search
|
||||||
# PRE that drop while the row's `result_count` is POST it, and a record
|
# reported is PRE that drop while the row's `result_count` is POST it,
|
||||||
# the session had already been shown could be logged as something the
|
# and a record already listed in this same menu could be logged as
|
||||||
# BAR turned away. Live proof on the first read after #3739 shipped:
|
# something the BAR turned away. Live proof on the first read after
|
||||||
# write_path's near-miss max was 0.822 while the lowest score it ever
|
# #3739 shipped: write_path's near-miss max was 0.822 while the lowest
|
||||||
# RETURNED was 0.6857 — a "rejection" that beat every acceptance.
|
# score it ever RETURNED was 0.6857 — a "rejection" that beat every
|
||||||
#
|
# acceptance.
|
||||||
# The suppression column cannot rescue it the way it does for the rule
|
|
||||||
# arms: this arm's count would be PARTIAL, covering only the drops made
|
|
||||||
# here and not the ones `exclude_ids` made inside the search, and a
|
|
||||||
# partial number under a name that reads as complete is the substitution
|
|
||||||
# this whole milestone exists to stop.
|
|
||||||
#
|
#
|
||||||
# So the honest answer is null — "not measured on this call" — whenever
|
# So the honest answer is null — "not measured on this call" — whenever
|
||||||
# this filter removed anything, because then the bar is not the only
|
# this filter removed anything, because then the bar is not the only
|
||||||
# thing that turned something away and the reported score may belong to
|
# thing that turned something away and the reported score may belong to
|
||||||
# a record we withheld ourselves. Calls where nothing was dropped keep
|
# a record we withheld ourselves. Calls where nothing was dropped keep
|
||||||
# reporting it, which is most of them.
|
# reporting it, which is most of them.
|
||||||
|
#
|
||||||
|
# THE LEDGER IS NO LONGER PART OF THIS (#4101), and that is why the
|
||||||
|
# suppression column can now be filled in where it could not before.
|
||||||
|
# The old objection was that a count here would be PARTIAL — covering
|
||||||
|
# the drops made in this function but not the ones `exclude_ids` made
|
||||||
|
# inside the search — and a partial number under a name that reads as
|
||||||
|
# complete is the substitution this milestone exists to stop. That was
|
||||||
|
# right while the ledger was one of the things `exclude_ids` carried.
|
||||||
|
# It no longer is: every ledger repeat comes back from the search and is
|
||||||
|
# rendered, so `suppressed` counts all of them and none are hidden
|
||||||
|
# inside the query. What `exclude_ids` still removes is this call's own
|
||||||
|
# menu, which is not suppression at all — those records ARE being shown,
|
||||||
|
# one block further up.
|
||||||
withheld_here = len(hits) - len(shown)
|
withheld_here = len(hits) - len(shown)
|
||||||
hits = shown[:remaining]
|
hits = shown[:remaining]
|
||||||
|
fresh = [(s, n) for s, n in hits if int(n.id) not in excluded]
|
||||||
record_retrieval(
|
record_retrieval(
|
||||||
user_id=user_id, source="write_path", query=query,
|
user_id=user_id, source="write_path", query=query,
|
||||||
threshold=cfg["threshold"], limit=remaining,
|
threshold=cfg["threshold"], limit=remaining,
|
||||||
# is_task is None, not False: this arm now returns issues too, and
|
# is_task is None, not False: this arm now returns issues too, and
|
||||||
# recording it as a notes-only retrieval would misdescribe the
|
# recording it as a notes-only retrieval would misdescribe the
|
||||||
# candidate set the threshold is being tuned against.
|
# candidate set the threshold is being tuned against.
|
||||||
project_id=scope_project, is_task=None, results=hits,
|
project_id=scope_project, is_task=None, results=fresh,
|
||||||
best_available=(
|
best_available=(
|
||||||
None if withheld_here else _rep_wp.get("best_available_score")
|
None if withheld_here else _rep_wp.get("best_available_score")
|
||||||
),
|
),
|
||||||
@@ -1670,6 +1759,7 @@ async def build_write_path_hint(
|
|||||||
None if withheld_here else _rep_wp.get("best_available_id")
|
None if withheld_here else _rep_wp.get("best_available_id")
|
||||||
),
|
),
|
||||||
searched=bool(_rep_wp.get("searched", True)),
|
searched=bool(_rep_wp.get("searched", True)),
|
||||||
|
suppressed=len(hits) - len(fresh),
|
||||||
duration_ms=(time.perf_counter() - t0) * 1000.0,
|
duration_ms=(time.perf_counter() - t0) * 1000.0,
|
||||||
)
|
)
|
||||||
if hits:
|
if hits:
|
||||||
@@ -1683,9 +1773,19 @@ async def build_write_path_hint(
|
|||||||
# and an unlabelled line would be read as "here is code to
|
# and an unlabelled line would be read as "here is code to
|
||||||
# reuse", which is the opposite of what it says.
|
# reuse", which is the opposite of what it says.
|
||||||
kind = _record_kind(note)
|
kind = _record_kind(note)
|
||||||
scored.append((
|
marker = (
|
||||||
f"similar {score:.2f}" if kind == "snippet"
|
f"similar {score:.2f}" if kind == "snippet"
|
||||||
else f"similar {score:.2f} · {kind}",
|
else f"similar {score:.2f} · {kind}"
|
||||||
|
)
|
||||||
|
# The repeat marker rides the same dotted list as the kind, so a
|
||||||
|
# reference costs four characters and needs no second line
|
||||||
|
# (#4101). Same word as the auto-inject menu deliberately: a
|
||||||
|
# reader meeting `seen` on two different surfaces should not
|
||||||
|
# have to work out whether they mean the same thing.
|
||||||
|
if int(note.id) in excluded:
|
||||||
|
marker += " · seen"
|
||||||
|
scored.append((
|
||||||
|
marker,
|
||||||
{
|
{
|
||||||
"id": int(note.id), "title": note.title, "user_id": note.user_id,
|
"id": int(note.id), "title": note.title, "user_id": note.user_id,
|
||||||
# Carried so the line can disclose a cross-language hit
|
# Carried so the line can disclose a cross-language hit
|
||||||
@@ -1811,7 +1911,8 @@ async def build_write_path_hint(
|
|||||||
"`get_snippet(id)` for a snippet, `get_task(id)` for an issue, "
|
"`get_snippet(id)` for a snippet, `get_task(id)` for an issue, "
|
||||||
"`get_note(id)` otherwise. Reuse a snippet rather than writing a fresh "
|
"`get_note(id)` otherwise. Reuse a snippet rather than writing a fresh "
|
||||||
"one-off; read an issue before repeating what it records "
|
"one-off; read an issue before repeating what it records "
|
||||||
"(titles only; shown once per session):"
|
"(titles only; a line marked `seen` was surfaced earlier this "
|
||||||
|
"session and may no longer be in context):"
|
||||||
)
|
)
|
||||||
# Say what a language tag MEANS, and only when one is actually on the menu.
|
# Say what a language tag MEANS, and only when one is actually on the menu.
|
||||||
# Without this the reader has to infer why "· python" is attached to a hit on
|
# Without this the reader has to infer why "· python" is attached to a hit on
|
||||||
@@ -1848,7 +1949,13 @@ async def build_write_path_hint(
|
|||||||
if sync_note_ids:
|
if sync_note_ids:
|
||||||
by_arm["write_path_sync"] = list(sync_note_ids)
|
by_arm["write_path_sync"] = list(sync_note_ids)
|
||||||
for marker, item in menu:
|
for marker, item in menu:
|
||||||
arm = "write_path_place" if marker == "nearby" else "write_path_semantic"
|
# A rendered repeat is not a new surfacing (#4101) — same cut the log
|
||||||
|
# row takes, so this table and `retrieval_logs` keep agreeing about the
|
||||||
|
# same call (#3668).
|
||||||
|
if int(item["id"]) in excluded:
|
||||||
|
continue
|
||||||
|
arm = ("write_path_place" if marker.startswith("nearby")
|
||||||
|
else "write_path_semantic")
|
||||||
by_arm.setdefault(arm, []).append(int(item["id"]))
|
by_arm.setdefault(arm, []).append(int(item["id"]))
|
||||||
for arm, ids in by_arm.items():
|
for arm, ids in by_arm.items():
|
||||||
record_surfaced(user_id=user_id, note_ids=ids, source=arm)
|
record_surfaced(user_id=user_id, note_ids=ids, source=arm)
|
||||||
@@ -1882,7 +1989,7 @@ async def build_write_path_hint(
|
|||||||
rule_t0 = time.perf_counter()
|
rule_t0 = time.perf_counter()
|
||||||
_rep_wpr: dict = {}
|
_rep_wpr: dict = {}
|
||||||
hits = await semantic_search_rules(
|
hits = await semantic_search_rules(
|
||||||
user_id, code or path, limit=RULEHINT_LIMIT,
|
user_id, code or path, limit=cfg["rule_top_k"],
|
||||||
threshold=cfg["rule_threshold"],
|
threshold=cfg["rule_threshold"],
|
||||||
report=_rep_wpr, project_id=project_id or None,
|
report=_rep_wpr, project_id=project_id or None,
|
||||||
)
|
)
|
||||||
@@ -1943,7 +2050,7 @@ async def build_write_path_hint(
|
|||||||
# the same readout.
|
# the same readout.
|
||||||
record_retrieval(
|
record_retrieval(
|
||||||
user_id=user_id, source="write_path_rule", query=code or path,
|
user_id=user_id, source="write_path_rule", query=code or path,
|
||||||
threshold=cfg["rule_threshold"], limit=RULEHINT_LIMIT,
|
threshold=cfg["rule_threshold"], limit=cfg["rule_top_k"],
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
is_task=None, results=fresh, duration_ms=rule_ms,
|
is_task=None, results=fresh, duration_ms=rule_ms,
|
||||||
best_available=_rep_wpr.get("best_available_score"),
|
best_available=_rep_wpr.get("best_available_score"),
|
||||||
@@ -2039,7 +2146,7 @@ async def build_tool_rule_hint(
|
|||||||
t0 = time.perf_counter()
|
t0 = time.perf_counter()
|
||||||
_rep_ptr: dict = {}
|
_rep_ptr: dict = {}
|
||||||
hits = await semantic_search_rules(
|
hits = await semantic_search_rules(
|
||||||
user_id, query, limit=RULEHINT_LIMIT,
|
user_id, query, limit=cfg["tool_rule_top_k"],
|
||||||
threshold=cfg["tool_rule_threshold"],
|
threshold=cfg["tool_rule_threshold"],
|
||||||
report=_rep_ptr, project_id=project_id or None,
|
report=_rep_ptr, project_id=project_id or None,
|
||||||
)
|
)
|
||||||
@@ -2062,7 +2169,7 @@ async def build_tool_rule_hint(
|
|||||||
# failure the arm was built to stop.
|
# failure the arm was built to stop.
|
||||||
record_retrieval(
|
record_retrieval(
|
||||||
user_id=user_id, source="pre_tool_rule", query=query,
|
user_id=user_id, source="pre_tool_rule", query=query,
|
||||||
threshold=cfg["tool_rule_threshold"], limit=RULEHINT_LIMIT,
|
threshold=cfg["tool_rule_threshold"], limit=cfg["tool_rule_top_k"],
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
is_task=None, results=fresh, duration_ms=duration_ms,
|
is_task=None, results=fresh, duration_ms=duration_ms,
|
||||||
best_available=_rep_ptr.get("best_available_score"),
|
best_available=_rep_ptr.get("best_available_score"),
|
||||||
|
|||||||
@@ -61,13 +61,9 @@ import logging
|
|||||||
import time
|
import time
|
||||||
|
|
||||||
from scribe.services.embeddings import semantic_search_rules
|
from scribe.services.embeddings import semantic_search_rules
|
||||||
from scribe.services.plugin_context import (
|
from scribe.services.retrieval_surfaces import SURFACES, budget_for, floor_for
|
||||||
REPORTPREF_DEFAULT_THRESHOLD,
|
|
||||||
REPORTPREF_THRESHOLD_KEY,
|
|
||||||
)
|
|
||||||
from scribe.services.retrieval_telemetry import record_retrieval
|
from scribe.services.retrieval_telemetry import record_retrieval
|
||||||
from scribe.services.rule_usage import record_rule_surfaced
|
from scribe.services.rule_usage import record_rule_surfaced
|
||||||
from scribe.services.settings import get_setting
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -85,16 +81,20 @@ COMPLETION_QUERY = (
|
|||||||
# A handful, not a menu. More than a few shape preferences for ONE kind of
|
# A handful, not a menu. More than a few shape preferences for ONE kind of
|
||||||
# reply would contradict each other before they helped; the limit is here to
|
# reply would contradict each other before they helped; the limit is here to
|
||||||
# keep one noisy corpus from turning a status change into a wall of text.
|
# keep one noisy corpus from turning a status change into a wall of text.
|
||||||
LIMIT = 3
|
# The STARTING budget, not the budget (#4102). Both numbers this arm runs on
|
||||||
|
# now come from the surface registry, so the model that reads this arm's
|
||||||
|
# telemetry can move either — which matters more here than anywhere else,
|
||||||
|
# because a fixed query makes this arm's score a constant and a floor a hair
|
||||||
|
# above it produces a dead arm no amount of traffic will ever reveal.
|
||||||
|
LIMIT = SURFACES["report_preference"].budget_default
|
||||||
|
|
||||||
|
|
||||||
async def _threshold(user_id: int) -> float:
|
async def _threshold(user_id: int) -> float:
|
||||||
try:
|
return await floor_for(user_id, SOURCE)
|
||||||
value = float(await get_setting(
|
|
||||||
user_id, REPORTPREF_THRESHOLD_KEY, str(REPORTPREF_DEFAULT_THRESHOLD)))
|
|
||||||
except (TypeError, ValueError):
|
async def _limit(user_id: int) -> int:
|
||||||
value = REPORTPREF_DEFAULT_THRESHOLD
|
return await budget_for(user_id, SOURCE)
|
||||||
return min(1.0, max(0.0, value))
|
|
||||||
|
|
||||||
|
|
||||||
async def completion_preferences(user_id: int, *, project_id: int | None = None) -> list[dict]:
|
async def completion_preferences(user_id: int, *, project_id: int | None = None) -> list[dict]:
|
||||||
@@ -115,16 +115,17 @@ async def completion_preferences(user_id: int, *, project_id: int | None = None)
|
|||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
threshold = await _threshold(user_id)
|
threshold = await _threshold(user_id)
|
||||||
|
limit = await _limit(user_id)
|
||||||
report: dict = {}
|
report: dict = {}
|
||||||
t0 = time.perf_counter()
|
t0 = time.perf_counter()
|
||||||
hits = await semantic_search_rules(
|
hits = await semantic_search_rules(
|
||||||
user_id, COMPLETION_QUERY, limit=LIMIT, threshold=threshold,
|
user_id, COMPLETION_QUERY, limit=limit, threshold=threshold,
|
||||||
kind="preference", report=report, project_id=project_id,
|
kind="preference", report=report, project_id=project_id,
|
||||||
)
|
)
|
||||||
hits = [(score, rule) for score, rule in hits if rule.kind == "preference"]
|
hits = [(score, rule) for score, rule in hits if rule.kind == "preference"]
|
||||||
record_retrieval(
|
record_retrieval(
|
||||||
user_id=user_id, source=SOURCE, query=COMPLETION_QUERY,
|
user_id=user_id, source=SOURCE, query=COMPLETION_QUERY,
|
||||||
threshold=threshold, limit=LIMIT, project_id=project_id,
|
threshold=threshold, limit=limit, project_id=project_id,
|
||||||
is_task=None, results=hits,
|
is_task=None, results=hits,
|
||||||
best_available=report.get("best_available_score"),
|
best_available=report.get("best_available_score"),
|
||||||
best_available_id=report.get("best_available_id"),
|
best_available_id=report.get("best_available_id"),
|
||||||
|
|||||||
@@ -0,0 +1,264 @@
|
|||||||
|
"""One registry of the retrieval surfaces, and the two numbers each one has (#4102).
|
||||||
|
|
||||||
|
WHY THIS EXISTS
|
||||||
|
|
||||||
|
Six push arms each carried their own loose copy of the same shape: a settings
|
||||||
|
key, a default, and a limit that was usually a module constant nobody could
|
||||||
|
change. The read-and-clamp was written out separately in `plugin_context` (twice
|
||||||
|
over, for auto-inject and the write path), in three rule arms, and again as
|
||||||
|
`reply_preferences._threshold`. That was survivable while the numbers were
|
||||||
|
shipped constants an operator occasionally edited.
|
||||||
|
|
||||||
|
It stops being survivable once the numbers are meant to MOVE. The operator's
|
||||||
|
decision for this step:
|
||||||
|
|
||||||
|
"the floor should be chosen and adjusted by the model using it. we've come
|
||||||
|
back to something either fails or has to be looked at by the user we need a
|
||||||
|
model consistent surface for the adjustment of these floor values. the user
|
||||||
|
should be able to touch it but the model should be the thing handling it 9
|
||||||
|
times out of 10."
|
||||||
|
|
||||||
|
A tuning surface cannot be "model consistent" if every arm spells its own
|
||||||
|
configuration differently. So the arms stop owning their numbers and read them
|
||||||
|
from here instead, and the tuning tool, the routes and the Settings UI all
|
||||||
|
enumerate THIS table rather than hard-coding six special cases.
|
||||||
|
|
||||||
|
WHAT A FLOOR IS NOW, AND WHAT IT IS NOT
|
||||||
|
|
||||||
|
It is not a relevance judgement. Relevance is decided by the reader, which is
|
||||||
|
the only participant that can read a trigger against a situation — that is the
|
||||||
|
milestone's whole argument, and the injected line has always said so out loud
|
||||||
|
("read it before deciding it does not apply").
|
||||||
|
|
||||||
|
A floor answers the cheaper question: *is this worth ranking at all*. `k` is
|
||||||
|
what binds, and `k` is a BUDGET — how much of this surface's attention a
|
||||||
|
candidate list may spend. An arm that fires before every Bash call cannot
|
||||||
|
afford what an arm that fires once a turn can.
|
||||||
|
|
||||||
|
WHY THE DEFAULTS BELOW ARE STARTING POINTS AND NOT ANSWERS
|
||||||
|
|
||||||
|
A cosine score is a distance in BAAI/bge-small-en-v1.5's vector space, measured
|
||||||
|
against THIS corpus. It cannot transfer to an install with different records,
|
||||||
|
and rule 115 forbids defending a shipped default from this instance's
|
||||||
|
telemetry — which is what made the old design unbuildable: every number was a
|
||||||
|
guess everywhere except here, and there was no mechanism that could ever
|
||||||
|
improve it.
|
||||||
|
|
||||||
|
The mechanism is the fix. Scribe ships a starting point and the means to
|
||||||
|
correct it, so the values below carry the measurement that motivated them (see
|
||||||
|
the long comments in `plugin_context.py`, which are kept where they are because
|
||||||
|
they record how each number was first arrived at) without claiming to be right
|
||||||
|
for anybody else.
|
||||||
|
|
||||||
|
HOW A FLOOR SHOULD ACTUALLY BE MOVED
|
||||||
|
|
||||||
|
By reading the records the floor refused — `retrieval_telemetry(
|
||||||
|
near_miss_samples=N)` returns them by id — and never by the percentile alone.
|
||||||
|
That is not a style preference; it is the one case where the two disagreed and
|
||||||
|
was checked. `report_preference` logged 69 consecutive declines with the
|
||||||
|
refused score 0.0006 under the bar, and every percentile said "lower it".
|
||||||
|
Reading the refused record showed it was rule 77 "Extract intent from loose
|
||||||
|
phrasing", a false positive, so lowering the bar would have delivered that rule
|
||||||
|
on every completion report ever written. The statistic and the correct action
|
||||||
|
pointed in opposite directions, and only opening the record could tell.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from scribe.services.settings import get_setting
|
||||||
|
|
||||||
|
# A budget nobody should be able to set past. Not a tuning value — a guard on
|
||||||
|
# the worst case, so a mistyped setting cannot turn a menu into a wall of text.
|
||||||
|
# Shared by every surface because it bounds the same thing everywhere: how many
|
||||||
|
# lines a single unsolicited injection may occupy.
|
||||||
|
MAX_BUDGET = 10
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Surface:
|
||||||
|
"""One push arm's tunable pair, plus enough prose to tune it responsibly.
|
||||||
|
|
||||||
|
`asks` / `over` / `fires` are not documentation for this file — they are
|
||||||
|
rendered by the tuning tool and the Settings UI. A floor cannot be moved
|
||||||
|
sensibly by anyone, model or human, who does not know what the query is, what
|
||||||
|
corpus it runs against, or how often it costs something. Those three facts
|
||||||
|
are exactly what separates these arms from each other, and they were
|
||||||
|
previously recoverable only by reading `plugin_context.py`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
name: str
|
||||||
|
"""The telemetry `source` value, and the join key.
|
||||||
|
|
||||||
|
MUST equal the string this arm passes to `record_retrieval`. Everything
|
||||||
|
useful about tuning depends on that identity: the tool that moves a floor
|
||||||
|
and the table that says what the floor did have to be talking about the same
|
||||||
|
arm. A test asserts it rather than a comment asking nicely.
|
||||||
|
"""
|
||||||
|
|
||||||
|
floor_key: str
|
||||||
|
floor_default: float
|
||||||
|
budget_key: str
|
||||||
|
budget_default: int
|
||||||
|
asks: str
|
||||||
|
over: str
|
||||||
|
fires: str
|
||||||
|
budget_falls_back_to: str = ""
|
||||||
|
"""A budget key to inherit when this surface has none of its own set.
|
||||||
|
|
||||||
|
Only `write_path` uses it, and only because it USED to share auto-inject's
|
||||||
|
`top_k` outright. Giving it a key without this would silently reset the
|
||||||
|
budget of every install that had tuned the shared one — a behaviour change
|
||||||
|
delivered as a default, which is the shape of regression nobody reports
|
||||||
|
because nothing looks broken.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
SURFACES: dict[str, Surface] = {
|
||||||
|
"auto_inject": Surface(
|
||||||
|
name="auto_inject",
|
||||||
|
floor_key="kb_autoinject_threshold",
|
||||||
|
floor_default=0.55,
|
||||||
|
budget_key="kb_autoinject_top_k",
|
||||||
|
budget_default=3,
|
||||||
|
asks="the operator's message, as they typed it",
|
||||||
|
over="notes, snippets, processes and issues",
|
||||||
|
fires="once per operator turn",
|
||||||
|
),
|
||||||
|
"write_path": Surface(
|
||||||
|
name="write_path",
|
||||||
|
floor_key="kb_writepath_threshold",
|
||||||
|
floor_default=0.68,
|
||||||
|
budget_key="kb_writepath_top_k",
|
||||||
|
budget_default=3,
|
||||||
|
budget_falls_back_to="kb_autoinject_top_k",
|
||||||
|
asks="the code being written, rewritten as a concept query",
|
||||||
|
over="snippets and recorded issues",
|
||||||
|
fires="before every Write and Edit",
|
||||||
|
),
|
||||||
|
"write_path_rule": Surface(
|
||||||
|
name="write_path_rule",
|
||||||
|
floor_key="kb_rulehint_threshold",
|
||||||
|
floor_default=0.72,
|
||||||
|
budget_key="kb_rulehint_top_k",
|
||||||
|
budget_default=5,
|
||||||
|
asks="the code being written, against rule triggers",
|
||||||
|
over="global rules plus the bound project's own",
|
||||||
|
fires="before every Write and Edit",
|
||||||
|
),
|
||||||
|
"pre_tool_rule": Surface(
|
||||||
|
name="pre_tool_rule",
|
||||||
|
floor_key="kb_toolrule_threshold",
|
||||||
|
floor_default=0.68,
|
||||||
|
budget_key="kb_toolrule_top_k",
|
||||||
|
budget_default=5,
|
||||||
|
asks="the command about to run, against rule triggers",
|
||||||
|
over="global rules plus the bound project's own",
|
||||||
|
fires="before every Bash call — the busiest arm there is",
|
||||||
|
),
|
||||||
|
"prompt_rule": Surface(
|
||||||
|
name="prompt_rule",
|
||||||
|
floor_key="kb_promptrule_threshold",
|
||||||
|
floor_default=0.72,
|
||||||
|
budget_key="kb_promptrule_top_k",
|
||||||
|
budget_default=3,
|
||||||
|
asks="the operator's message, against rule triggers",
|
||||||
|
over="global rules plus the bound project's own",
|
||||||
|
fires="once per operator turn",
|
||||||
|
),
|
||||||
|
"report_preference": Surface(
|
||||||
|
name="report_preference",
|
||||||
|
floor_key="kb_reportpref_threshold",
|
||||||
|
floor_default=0.72,
|
||||||
|
budget_key="kb_reportpref_top_k",
|
||||||
|
budget_default=3,
|
||||||
|
# THE ONE FIXED QUERY, and the reason this arm behaves unlike the rest.
|
||||||
|
# The others score something that varies per call; this one scores a
|
||||||
|
# constant string, so its top score for a given corpus is also a
|
||||||
|
# constant. A floor a hair above that constant is not a quiet arm, it is
|
||||||
|
# a dead one, and no amount of traffic will ever reveal it — which is
|
||||||
|
# precisely how this arm spent 69 calls declining the same record.
|
||||||
|
asks="a fixed question about how to lay out a completion report",
|
||||||
|
over="preferences",
|
||||||
|
fires="when a task finishes",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
# Reserved slots are deliberately absent. `preference_slot` and `reuse_slot`
|
||||||
|
# borrow their parent arm's floor and are hard-limited to one hit each, because
|
||||||
|
# their entire purpose is to guarantee a single line to a kind of record that
|
||||||
|
# keeps losing a general score contest (#2246, #3894). A budget of "1" is the
|
||||||
|
# feature; exposing it as tunable would invite setting it to 0 and silently
|
||||||
|
# removing the guarantee.
|
||||||
|
|
||||||
|
|
||||||
|
def surface_names() -> list[str]:
|
||||||
|
"""Every tunable surface, in a stable order for menus and listings."""
|
||||||
|
return list(SURFACES)
|
||||||
|
|
||||||
|
|
||||||
|
def get_surface(name: str) -> Surface:
|
||||||
|
"""Look one up, refusing an unknown name loudly.
|
||||||
|
|
||||||
|
A typo'd surface must not be writable. Settings keys are free-form strings
|
||||||
|
in a generic table, so a tuning call naming `pretool_rule` would otherwise
|
||||||
|
write a key nothing ever reads — a change that appears to succeed, reports a
|
||||||
|
new value, and alters nothing.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return SURFACES[name]
|
||||||
|
except KeyError:
|
||||||
|
raise ValueError(
|
||||||
|
f"unknown retrieval surface {name!r}. Tunable surfaces are: "
|
||||||
|
+ ", ".join(surface_names())
|
||||||
|
) from None
|
||||||
|
|
||||||
|
|
||||||
|
def dial_for_key(key: str) -> tuple[str, str] | None:
|
||||||
|
"""Which `(surface, dial)` a settings key belongs to, or None.
|
||||||
|
|
||||||
|
The registry read backwards, and it exists for one caller: the generic
|
||||||
|
`/api/settings` endpoint, which accepts any key at all. Without this, a
|
||||||
|
floor written through that endpoint moves with no event recorded, and the
|
||||||
|
tuning history says nothing happened — a trail with holes in it, which is
|
||||||
|
worse than no trail because it reads as complete.
|
||||||
|
|
||||||
|
Derived rather than listed so a seventh surface is covered the moment it is
|
||||||
|
added here, which is the only way this stays true.
|
||||||
|
"""
|
||||||
|
for surface in SURFACES.values():
|
||||||
|
if key == surface.floor_key:
|
||||||
|
return (surface.name, "floor")
|
||||||
|
if key == surface.budget_key:
|
||||||
|
return (surface.name, "budget")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def floor_for(user_id: int, name: str) -> float:
|
||||||
|
"""This install's current floor for a surface, clamped to [0, 1]."""
|
||||||
|
s = get_surface(name)
|
||||||
|
try:
|
||||||
|
value = float(await get_setting(user_id, s.floor_key, str(s.floor_default)))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
value = s.floor_default
|
||||||
|
return min(1.0, max(0.0, value))
|
||||||
|
|
||||||
|
|
||||||
|
async def budget_for(user_id: int, name: str) -> int:
|
||||||
|
"""This install's current budget for a surface, clamped to [1, MAX_BUDGET].
|
||||||
|
|
||||||
|
The lower clamp is 1, never 0: a surface turned off is turned off by its
|
||||||
|
`enabled` switch, which says so. A budget of zero would be an arm that runs
|
||||||
|
a search, logs a retrieval, and renders nothing — indistinguishable in the
|
||||||
|
telemetry from a bar nothing cleared, which is the exact confusion this
|
||||||
|
milestone exists to remove.
|
||||||
|
"""
|
||||||
|
s = get_surface(name)
|
||||||
|
raw = await get_setting(user_id, s.budget_key, "")
|
||||||
|
if not raw and s.budget_falls_back_to:
|
||||||
|
raw = await get_setting(user_id, s.budget_falls_back_to, "")
|
||||||
|
try:
|
||||||
|
value = int(float(raw)) if raw else s.budget_default
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
value = s.budget_default
|
||||||
|
return min(MAX_BUDGET, max(1, value))
|
||||||
@@ -0,0 +1,226 @@
|
|||||||
|
"""Moving a retrieval surface's floor or budget, with the argument attached (#4102).
|
||||||
|
|
||||||
|
WHY THIS EXISTS
|
||||||
|
|
||||||
|
The operator's decision for milestone 416 step 4:
|
||||||
|
|
||||||
|
"the floor should be chosen and adjusted by the model using it. we've come
|
||||||
|
back to something either fails or has to be looked at by the user we need a
|
||||||
|
model consistent surface for the adjustment of these floor values. the user
|
||||||
|
should be able to touch it but the model should be the thing handling it 9
|
||||||
|
times out of 10."
|
||||||
|
|
||||||
|
`retrieval_surfaces` made the six arms describe their numbers the same way.
|
||||||
|
This module is the write half: one call that moves one dial on one surface,
|
||||||
|
records what it was, what it became, who moved it and why, and refuses to do
|
||||||
|
any of that without a reason.
|
||||||
|
|
||||||
|
WHY A REASON IS REQUIRED
|
||||||
|
|
||||||
|
Because the alternative was tried and measured. The milestone originally listed
|
||||||
|
self-tuning as a non-goal on the strength of one case: `report_preference`
|
||||||
|
logged 69 consecutive declines with the refused record 0.0006 under the bar, and
|
||||||
|
every percentile in the readout said "lower it". Reading the refused record
|
||||||
|
showed it was rule 77 "Extract intent from loose phrasing" — a false positive —
|
||||||
|
so lowering the bar would have attached that rule to every completion report
|
||||||
|
ever written. The statistic and the correct action pointed in opposite
|
||||||
|
directions.
|
||||||
|
|
||||||
|
What separated them was opening the record. A required `reason` is the cheapest
|
||||||
|
mechanism that makes that step happen: a caller who must write down why has to
|
||||||
|
have looked, and a caller who writes down something wrong has left the operator
|
||||||
|
a sentence to disagree with. A number moved silently leaves nothing.
|
||||||
|
|
||||||
|
WHAT THIS DELIBERATELY DOES NOT DO
|
||||||
|
|
||||||
|
It does not decide anything itself. There is no rule here that reads a
|
||||||
|
percentile and picks a value, and that absence is the design — the non-goal that
|
||||||
|
survived is *statistical* auto-tuning, precisely because the statistic was the
|
||||||
|
thing that was wrong. The judgement stays with the reader; this module only
|
||||||
|
makes the judgement recordable and reversible.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from scribe.models import async_session
|
||||||
|
from scribe.models.retrieval_tuning import RetrievalTuningEvent
|
||||||
|
from scribe.services.retrieval_surfaces import (
|
||||||
|
MAX_BUDGET,
|
||||||
|
budget_for,
|
||||||
|
floor_for,
|
||||||
|
get_surface,
|
||||||
|
surface_names,
|
||||||
|
)
|
||||||
|
from scribe.services.settings import set_setting
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DIALS = ("floor", "budget")
|
||||||
|
|
||||||
|
# Long enough to say what was read and what it showed; short enough that nobody
|
||||||
|
# pastes a telemetry dump in. The number is not a measurement — it is the point
|
||||||
|
# at which "0.66" stops being an acceptable answer to "why".
|
||||||
|
_MIN_REASON_CHARS = 20
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_reason(reason: str) -> str:
|
||||||
|
"""The guardrail, enforced here rather than by the column.
|
||||||
|
|
||||||
|
`reason` is NOT NULL in the schema, which "" satisfies. A required field
|
||||||
|
that accepts an empty string is a formality, and this one is the whole
|
||||||
|
mechanism — see the module docstring.
|
||||||
|
"""
|
||||||
|
text = (reason or "").strip()
|
||||||
|
if len(text) < _MIN_REASON_CHARS:
|
||||||
|
raise ValueError(
|
||||||
|
"reason is required, and has to say what you read. A floor moved "
|
||||||
|
"without a stated basis is a number nobody can review or revert. "
|
||||||
|
"Name the evidence: which surface's telemetry, and what the refused "
|
||||||
|
"records actually were — `retrieval_telemetry(near_miss_samples=N)` "
|
||||||
|
"returns them by id, and reading them is the step that separates a "
|
||||||
|
"real miss from a bar doing its job."
|
||||||
|
)
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
async def current_settings(user_id: int) -> list[dict]:
|
||||||
|
"""Every tunable surface with its live pair and its last stated reason.
|
||||||
|
|
||||||
|
The read side of the tuning surface, and shaped for a reader who is about to
|
||||||
|
change something: the value, what the arm asks and over what corpus and how
|
||||||
|
often — because a floor cannot be moved sensibly without those three — and
|
||||||
|
the reason last given, so the next change argues with the last one instead
|
||||||
|
of overwriting it blind.
|
||||||
|
"""
|
||||||
|
out: list[dict] = []
|
||||||
|
async with async_session() as session:
|
||||||
|
for name in surface_names():
|
||||||
|
s = get_surface(name)
|
||||||
|
rows = (
|
||||||
|
await session.execute(
|
||||||
|
select(RetrievalTuningEvent)
|
||||||
|
.where(
|
||||||
|
RetrievalTuningEvent.surface == name,
|
||||||
|
RetrievalTuningEvent.user_id == user_id,
|
||||||
|
)
|
||||||
|
.order_by(RetrievalTuningEvent.created_at.desc())
|
||||||
|
.limit(len(DIALS))
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
last = {r.dial: r for r in rows}
|
||||||
|
out.append({
|
||||||
|
"surface": name,
|
||||||
|
"floor": await floor_for(user_id, name),
|
||||||
|
"budget": await budget_for(user_id, name),
|
||||||
|
"floor_default": s.floor_default,
|
||||||
|
"budget_default": s.budget_default,
|
||||||
|
"asks": s.asks,
|
||||||
|
"over": s.over,
|
||||||
|
"fires": s.fires,
|
||||||
|
# Absent rather than empty when a surface has never been moved:
|
||||||
|
# "still on the shipped starting point" is a real state and
|
||||||
|
# should not render as a blank reason somebody wrote.
|
||||||
|
"last_change": {
|
||||||
|
dial: last[dial].to_dict() for dial in DIALS if dial in last
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
async def set_dial(
|
||||||
|
user_id: int,
|
||||||
|
surface: str,
|
||||||
|
dial: str,
|
||||||
|
value: float,
|
||||||
|
*,
|
||||||
|
reason: str,
|
||||||
|
actor: str = "model",
|
||||||
|
) -> dict:
|
||||||
|
"""Move one dial on one surface, recording the change and its argument.
|
||||||
|
|
||||||
|
Returns the applied value alongside the previous one, so a caller can see
|
||||||
|
that a clamp bit rather than assuming the number it sent is the number in
|
||||||
|
force — the failure being avoided is a tool reporting success for a value
|
||||||
|
the registry silently corrected.
|
||||||
|
"""
|
||||||
|
s = get_surface(surface) # refuses an unknown name
|
||||||
|
if dial not in DIALS:
|
||||||
|
raise ValueError(f"dial must be one of {DIALS}, got {dial!r}")
|
||||||
|
if actor not in ("model", "human"):
|
||||||
|
raise ValueError(f"actor must be 'model' or 'human', got {actor!r}")
|
||||||
|
text = _clean_reason(reason)
|
||||||
|
|
||||||
|
if dial == "floor":
|
||||||
|
old = await floor_for(user_id, surface)
|
||||||
|
applied = min(1.0, max(0.0, float(value)))
|
||||||
|
key, stored = s.floor_key, str(applied)
|
||||||
|
else:
|
||||||
|
old = float(await budget_for(user_id, surface))
|
||||||
|
applied = float(min(MAX_BUDGET, max(1, int(float(value)))))
|
||||||
|
key, stored = s.budget_key, str(int(applied))
|
||||||
|
|
||||||
|
# A change to the value it already has is not a change, and must not be
|
||||||
|
# written. The Settings form re-sends every field on every save, so without
|
||||||
|
# this the history fills with rows saying the operator set six dials to the
|
||||||
|
# numbers they were already on — and a history nobody can skim is one
|
||||||
|
# nobody reads, which costs the surface its entire purpose.
|
||||||
|
#
|
||||||
|
# Reported rather than silently skipped, so a caller that expected to move
|
||||||
|
# something learns that it did not.
|
||||||
|
if abs(applied - old) < 1e-9:
|
||||||
|
return {
|
||||||
|
"surface": surface, "dial": dial, "previous": old,
|
||||||
|
"applied": applied, "clamped": abs(applied - float(value)) > 1e-9,
|
||||||
|
"reason": text, "actor": actor, "unchanged": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
await set_setting(user_id, key, stored)
|
||||||
|
async with async_session() as session:
|
||||||
|
session.add(RetrievalTuningEvent(
|
||||||
|
user_id=user_id, surface=surface, dial=dial,
|
||||||
|
old_value=old, new_value=applied, actor=actor, reason=text,
|
||||||
|
))
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
return {
|
||||||
|
"surface": surface,
|
||||||
|
"dial": dial,
|
||||||
|
"previous": old,
|
||||||
|
"applied": applied,
|
||||||
|
# True when the registry corrected what was asked for. Said out loud
|
||||||
|
# because a caller that believes it set 1.4 will read the next
|
||||||
|
# telemetry as evidence about a bar that was never in force.
|
||||||
|
"clamped": abs(applied - float(value)) > 1e-9,
|
||||||
|
"reason": text,
|
||||||
|
"actor": actor,
|
||||||
|
# Always present, both ways round: a caller that has to test for the
|
||||||
|
# key's absence to learn the answer will eventually forget to.
|
||||||
|
"unchanged": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def tuning_history(
|
||||||
|
user_id: int, *, surface: str | None = None, limit: int = 20
|
||||||
|
) -> list[dict]:
|
||||||
|
"""What has been moved, newest first — the operator's review surface.
|
||||||
|
|
||||||
|
Scoped to one surface when asked, because the question is almost always
|
||||||
|
"why is THIS arm set like this", and an unscoped list buries one surface's
|
||||||
|
two changes under another's twenty.
|
||||||
|
"""
|
||||||
|
if surface is not None:
|
||||||
|
get_surface(surface) # refuse a typo on the read too
|
||||||
|
async with async_session() as session:
|
||||||
|
q = (
|
||||||
|
select(RetrievalTuningEvent)
|
||||||
|
.where(RetrievalTuningEvent.user_id == user_id)
|
||||||
|
.order_by(RetrievalTuningEvent.created_at.desc())
|
||||||
|
.limit(max(1, min(int(limit), 200)))
|
||||||
|
)
|
||||||
|
if surface is not None:
|
||||||
|
q = q.where(RetrievalTuningEvent.surface == surface)
|
||||||
|
rows = (await session.execute(q)).scalars().all()
|
||||||
|
return [r.to_dict() for r in rows]
|
||||||
@@ -322,3 +322,32 @@ def http_sink(reply: bytes = b'{"context":"","note_ids":[]}'):
|
|||||||
finally:
|
finally:
|
||||||
server.shutdown()
|
server.shutdown()
|
||||||
server.server_close()
|
server.server_close()
|
||||||
|
|
||||||
|
|
||||||
|
def writepath_cfg(**over):
|
||||||
|
"""A complete `get_writepath_config` stand-in, built from the registry (#4102).
|
||||||
|
|
||||||
|
DERIVED, NOT LITERAL, and the reason is a failure mode this file already
|
||||||
|
warned about in prose without being able to prevent: the write-path hint
|
||||||
|
drives three arms, each of which reads its numbers out of the config dict
|
||||||
|
inside a fail-open `except`. A dict missing one key does not raise where a
|
||||||
|
reader would see it — the arm silently becomes a no-op, which is
|
||||||
|
indistinguishable from the arm working and finding nothing.
|
||||||
|
|
||||||
|
So the keys come from `retrieval_surfaces.SURFACES`. A seventh surface, or a
|
||||||
|
rename, changes this helper for free and cannot quietly disable an arm in
|
||||||
|
ten hand-written dicts that each looked complete on the day they were typed.
|
||||||
|
"""
|
||||||
|
from scribe.services.retrieval_surfaces import SURFACES
|
||||||
|
|
||||||
|
cfg = {
|
||||||
|
"enabled": True,
|
||||||
|
"threshold": SURFACES["write_path"].floor_default,
|
||||||
|
"top_k": SURFACES["write_path"].budget_default,
|
||||||
|
"rule_threshold": SURFACES["write_path_rule"].floor_default,
|
||||||
|
"rule_top_k": SURFACES["write_path_rule"].budget_default,
|
||||||
|
"tool_rule_threshold": SURFACES["pre_tool_rule"].floor_default,
|
||||||
|
"tool_rule_top_k": SURFACES["pre_tool_rule"].budget_default,
|
||||||
|
}
|
||||||
|
cfg.update(over)
|
||||||
|
return cfg
|
||||||
|
|||||||
@@ -0,0 +1,241 @@
|
|||||||
|
"""A repeat on a note arm is referenced, not withheld (#4101).
|
||||||
|
|
||||||
|
WHY THIS EXISTS
|
||||||
|
|
||||||
|
#3750 settled the argument for rules: a record the session was told about an
|
||||||
|
hour ago is not a record in front of the reader now, so the second time it is
|
||||||
|
the best answer the session gets the line again with a tail saying so — never
|
||||||
|
silence, which is indistinguishable from "nothing matched".
|
||||||
|
|
||||||
|
The note and snippet arms never got that fix. Worse, their ledger went into
|
||||||
|
`semantic_search_notes` as `exclude_ids`, so the repeat left the candidate set
|
||||||
|
entirely, which had three consequences:
|
||||||
|
|
||||||
|
- the session got silence the second time, on the arms that fire most;
|
||||||
|
- a compaction made it permanent, because the ledger outlived the context it
|
||||||
|
described (the same step fixes that one layer down);
|
||||||
|
- and `best_available` was measured against a candidate set the caller had
|
||||||
|
already edited, so the bar could be blamed for a record the caller withheld
|
||||||
|
(#3739, from the side its fix never reached).
|
||||||
|
|
||||||
|
WHAT THIS PINS
|
||||||
|
|
||||||
|
1. The ledger does not reach the search. Asserted on the call's kwargs,
|
||||||
|
because this is the difference between a reference and silence and every
|
||||||
|
behavioural assertion below rests on it.
|
||||||
|
2. The repeat is rendered, and marked. The wording is NOT the rule arms' —
|
||||||
|
"before deciding it does not apply" is the voice of a record that binds,
|
||||||
|
and a dev-log borrowing it would claim authority it does not have — so
|
||||||
|
what is pinned is that the line appears and is distinguishable, not its
|
||||||
|
prose.
|
||||||
|
3. The telemetry splits the two (#3752 / #3668): the row's result set and the
|
||||||
|
surfaced table both take FRESH only, and the repeat is COUNTED in
|
||||||
|
`suppressed` instead. This is what makes a zero-result call readable —
|
||||||
|
`result_count == 0` with `suppressed_count > 0` says "everything that
|
||||||
|
matched, the session has already seen", which was unreportable here.
|
||||||
|
4. The deliberate exception: the write-path SYNC class still shows once. Its
|
||||||
|
claim is about an edit in progress, not about a record's continuing
|
||||||
|
relevance, and repeating it would be nagging rather than reminding.
|
||||||
|
"""
|
||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tests.helpers import fake_note, writepath_cfg
|
||||||
|
|
||||||
|
REAL_CODE = '''def debounce(fn, wait=0.25):
|
||||||
|
"""Rate-limit a callback so it fires once after the last call."""
|
||||||
|
timer = None
|
||||||
|
|
||||||
|
def wrapped(*a, **kw):
|
||||||
|
nonlocal timer
|
||||||
|
if timer:
|
||||||
|
timer.cancel()
|
||||||
|
timer = threading.Timer(wait, fn, a, kw)
|
||||||
|
timer.start()
|
||||||
|
|
||||||
|
return wrapped
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
def _wp_cfg(**over):
|
||||||
|
return writepath_cfg(**over)
|
||||||
|
|
||||||
|
|
||||||
|
def _snippet_item(nid, title, user_id=1):
|
||||||
|
return {"id": nid, "title": title, "user_id": user_id, "note_type": "snippet"}
|
||||||
|
|
||||||
|
|
||||||
|
# ── auto-inject ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async def _autoinject(hits, exclude_ids, *, rec=None, surf=None):
|
||||||
|
from scribe.services import plugin_context as pc
|
||||||
|
search = AsyncMock(return_value=hits)
|
||||||
|
with patch.object(pc, "get_autoinject_config", AsyncMock(
|
||||||
|
return_value={"enabled": True, "threshold": 0.55, "top_k": 3})), \
|
||||||
|
patch.object(pc, "semantic_search_notes", search), \
|
||||||
|
patch.object(pc, "record_retrieval", rec or MagicMock()), \
|
||||||
|
patch.object(pc, "record_surfaced", surf or MagicMock()), \
|
||||||
|
patch.object(pc, "owner_names_for", AsyncMock(return_value={})), \
|
||||||
|
patch.object(pc, "superseded_ids", AsyncMock(return_value=set())):
|
||||||
|
out = await pc.build_autoinject_hint(
|
||||||
|
1, "postgres pool", project_id=2, exclude_ids=exclude_ids)
|
||||||
|
return out, search
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_the_ledger_never_reaches_the_search():
|
||||||
|
"""The load-bearing one. A ledger inside the query IS the silence."""
|
||||||
|
hits = [(0.80, fake_note(id=11, title="Pool sizing decision", user_id=1))]
|
||||||
|
_out, search = await _autoinject(hits, [11])
|
||||||
|
# The menu arm runs first; the reuse slot's own call may legitimately
|
||||||
|
# exclude this call's menu, which is a different claim (see below).
|
||||||
|
menu_call = search.call_args_list[0]
|
||||||
|
assert not menu_call.kwargs.get("exclude_ids"), (
|
||||||
|
"the session ledger was passed into the search, so a repeat is "
|
||||||
|
"withheld rather than referenced"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_repeat_is_shown_again_and_marked():
|
||||||
|
hits = [(0.80, fake_note(id=11, title="Pool sizing decision", user_id=1)),
|
||||||
|
(0.78, fake_note(id=22, title="run_maintenance thresholds", user_id=1))]
|
||||||
|
out, _ = await _autoinject(hits, [11])
|
||||||
|
assert "#11" in out["context"] and "#22" in out["context"]
|
||||||
|
# Distinguishable, without pinning the word's neighbours in the sentence.
|
||||||
|
line_11 = next(ln for ln in out["context"].splitlines() if "#11" in ln)
|
||||||
|
line_22 = next(ln for ln in out["context"].splitlines() if "#22" in ln)
|
||||||
|
assert "seen" in line_11 and "seen" not in line_22
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_the_repeat_is_counted_not_reported_as_a_result():
|
||||||
|
rec = MagicMock()
|
||||||
|
hits = [(0.80, fake_note(id=11, title="Pool sizing decision", user_id=1)),
|
||||||
|
(0.78, fake_note(id=22, title="run_maintenance thresholds", user_id=1))]
|
||||||
|
await _autoinject(hits, [11], rec=rec)
|
||||||
|
row = next(c.kwargs for c in rec.call_args_list
|
||||||
|
if c.kwargs["source"] == "auto_inject")
|
||||||
|
assert [int(n.id) for _s, n in row["results"]] == [22]
|
||||||
|
assert row["suppressed"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_call_where_everything_was_already_seen_is_readable():
|
||||||
|
"""The fact that could not be expressed before.
|
||||||
|
|
||||||
|
Previously this call logged zero results with no suppression count, so it
|
||||||
|
was indistinguishable from a bar nothing cleared — and the operator tuning
|
||||||
|
that bar would have been reading the wrong number.
|
||||||
|
"""
|
||||||
|
rec = MagicMock()
|
||||||
|
hits = [(0.80, fake_note(id=11, title="Pool sizing decision", user_id=1))]
|
||||||
|
out, _ = await _autoinject(hits, [11], rec=rec)
|
||||||
|
row = next(c.kwargs for c in rec.call_args_list
|
||||||
|
if c.kwargs["source"] == "auto_inject")
|
||||||
|
assert row["results"] == [] and row["suppressed"] == 1
|
||||||
|
# And the session still gets the line, which is the whole point.
|
||||||
|
assert "#11" in out["context"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_repeat_is_not_recorded_as_a_fresh_surfacing():
|
||||||
|
"""#3668's identity: this table and the log row describe the same call."""
|
||||||
|
surf = MagicMock()
|
||||||
|
hits = [(0.80, fake_note(id=11, title="Pool sizing decision", user_id=1)),
|
||||||
|
(0.78, fake_note(id=22, title="run_maintenance thresholds", user_id=1))]
|
||||||
|
await _autoinject(hits, [11], surf=surf)
|
||||||
|
rows = [c.kwargs for c in surf.call_args_list
|
||||||
|
if c.kwargs.get("source") == "auto_inject"]
|
||||||
|
assert rows and rows[0]["note_ids"] == [22]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_the_header_no_longer_promises_once_per_session():
|
||||||
|
"""A contract stated in the prose is a contract, and this one changed.
|
||||||
|
|
||||||
|
Cheap to forget and invisible when wrong: the menu would carry a marker
|
||||||
|
the header had never explained, and a reader would have to guess.
|
||||||
|
"""
|
||||||
|
hits = [(0.80, fake_note(id=11, title="Pool sizing decision", user_id=1))]
|
||||||
|
out, _ = await _autoinject(hits, [])
|
||||||
|
header = out["context"].splitlines()[0]
|
||||||
|
assert "once per session" not in header
|
||||||
|
assert "seen" in header
|
||||||
|
|
||||||
|
|
||||||
|
# ── the write path ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async def _write_path(hits, exclude_ids, *, here=(), rec=None, sync_exclude=()):
|
||||||
|
from scribe.services import plugin_context as pc
|
||||||
|
search = AsyncMock(return_value=hits)
|
||||||
|
listing = AsyncMock(side_effect=[(list(here), len(here)), ([], 0)])
|
||||||
|
with patch.object(pc, "get_writepath_config", AsyncMock(return_value=_wp_cfg())), \
|
||||||
|
patch.object(pc.snippets_svc, "list_snippets", listing), \
|
||||||
|
patch.object(pc, "semantic_search_notes", search), \
|
||||||
|
patch.object(pc, "record_retrieval", rec or MagicMock()), \
|
||||||
|
patch.object(pc, "record_surfaced", MagicMock()), \
|
||||||
|
patch.object(pc, "owner_names_for", AsyncMock(return_value={})):
|
||||||
|
out = await pc.build_write_path_hint(
|
||||||
|
1, "src/x.py", code=REAL_CODE,
|
||||||
|
exclude_ids=list(exclude_ids),
|
||||||
|
exclude_sync_ids=list(sync_exclude),
|
||||||
|
)
|
||||||
|
return out, search
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_the_write_path_ledger_does_not_reach_its_search_either():
|
||||||
|
hits = [(0.83, fake_note(id=11, title="debounce helper", user_id=1))]
|
||||||
|
_out, search = await _write_path(hits, [11])
|
||||||
|
assert 11 not in (search.call_args.kwargs.get("exclude_ids") or set())
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_write_path_repeat_is_rendered_with_a_marker():
|
||||||
|
hits = [(0.83, fake_note(id=11, title="debounce helper", user_id=1))]
|
||||||
|
out, _ = await _write_path(hits, [11])
|
||||||
|
assert "#11" in out["context"]
|
||||||
|
assert "seen" in next(ln for ln in out["context"].splitlines() if "#11" in ln)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_the_write_path_counts_its_repeat():
|
||||||
|
rec = MagicMock()
|
||||||
|
hits = [(0.83, fake_note(id=11, title="debounce helper", user_id=1)),
|
||||||
|
(0.80, fake_note(id=22, title="throttle helper", user_id=1))]
|
||||||
|
await _write_path(hits, [11], rec=rec)
|
||||||
|
row = next(c.kwargs for c in rec.call_args_list
|
||||||
|
if c.kwargs["source"] == "write_path")
|
||||||
|
assert [int(n.id) for _s, n in row["results"]] == [22]
|
||||||
|
assert row["suppressed"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_this_calls_own_menu_is_still_excluded_from_its_search():
|
||||||
|
"""The claim that stayed an exclusion, and must not be lost with the other.
|
||||||
|
|
||||||
|
A snippet already listed by PLACE in this same hint has nothing to gain
|
||||||
|
from a second line in it. That is same-call duplication, not a repeat
|
||||||
|
across calls, and the two were the same variable until this step.
|
||||||
|
"""
|
||||||
|
hits = []
|
||||||
|
_out, search = await _write_path(
|
||||||
|
hits, [], here=[_snippet_item(7, "records this file")])
|
||||||
|
assert 7 in (search.call_args.kwargs.get("exclude_ids") or set())
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_the_sync_class_still_shows_only_once():
|
||||||
|
"""The deliberate exception, pinned so it reads as a decision.
|
||||||
|
|
||||||
|
The sync nudge says "you are editing the file this record describes, so
|
||||||
|
updating it is part of the edit". Repeated every write to the same file it
|
||||||
|
is nagging, and unlike a reuse suggestion it is not a claim whose relevance
|
||||||
|
can return — it either got acted on or it did not.
|
||||||
|
"""
|
||||||
|
out, _ = await _write_path(
|
||||||
|
[], [], here=[_snippet_item(7, "records this file")], sync_exclude=[7])
|
||||||
|
assert out["sync_note_ids"] == []
|
||||||
|
assert "#7" not in out["context"]
|
||||||
@@ -20,7 +20,7 @@ from scribe.services.note_usage import (
|
|||||||
record_surfaced,
|
record_surfaced,
|
||||||
usage_for_notes,
|
usage_for_notes,
|
||||||
)
|
)
|
||||||
from tests.helpers import fake_note
|
from tests.helpers import fake_note, writepath_cfg
|
||||||
|
|
||||||
|
|
||||||
# --- recording ------------------------------------------------------------
|
# --- recording ------------------------------------------------------------
|
||||||
@@ -154,8 +154,7 @@ async def test_unscored_location_arms_are_recorded(lookups, expected_source):
|
|||||||
patch.object(
|
patch.object(
|
||||||
plugin_context,
|
plugin_context,
|
||||||
"get_writepath_config",
|
"get_writepath_config",
|
||||||
AsyncMock(return_value={"enabled": True, "threshold": 0.55, "top_k": 3,
|
AsyncMock(return_value=writepath_cfg(threshold=0.55)),
|
||||||
"rule_threshold": 0.72}),
|
|
||||||
),
|
),
|
||||||
patch.object(
|
patch.object(
|
||||||
plugin_context.snippets_svc,
|
plugin_context.snippets_svc,
|
||||||
|
|||||||
@@ -0,0 +1,187 @@
|
|||||||
|
"""The surface registry, and the identity the whole tuning story rests on (#4102).
|
||||||
|
|
||||||
|
WHY THIS EXISTS
|
||||||
|
|
||||||
|
Six arms used to own their own numbers: a settings key, a default, and a limit
|
||||||
|
that was usually a module constant nobody could change. That survived while the
|
||||||
|
values were shipped constants. It stops surviving once the operator's decision
|
||||||
|
is that the numbers MOVE:
|
||||||
|
|
||||||
|
"the floor should be chosen and adjusted by the model using it… the user
|
||||||
|
should be able to touch it but the model should be the thing handling it 9
|
||||||
|
times out of 10."
|
||||||
|
|
||||||
|
A tuning surface cannot be model-consistent across arms that each spell their
|
||||||
|
configuration differently, so the arms now read `{floor, budget}` from one
|
||||||
|
registry.
|
||||||
|
|
||||||
|
WHAT THIS PINS
|
||||||
|
|
||||||
|
1. **A surface's name IS its telemetry source.** This is the load-bearing
|
||||||
|
one. The tool that moves a floor and the table that reports what the floor
|
||||||
|
did have to be naming the same arm, and nothing in the type system says so
|
||||||
|
— `record_retrieval(source="pre_tool_rule")` is a string literal in a
|
||||||
|
different file. Renaming one without the other produces a surface that can
|
||||||
|
be tuned and cannot be measured, or measured and not tuned, and both fail
|
||||||
|
silently.
|
||||||
|
2. **Keys are unique.** Two surfaces sharing a settings key is how
|
||||||
|
`report_preference` spent its first release moving whenever the prompt arm
|
||||||
|
was tuned (#3860) — one dial wearing two labels.
|
||||||
|
3. **An unknown surface is refused.** Settings keys are free-form strings in
|
||||||
|
a generic table, so a typo'd name would write a key nothing reads: a
|
||||||
|
change that appears to succeed, reports a new value, and alters nothing.
|
||||||
|
4. **The budget clamps at 1, never 0.** A zero-budget arm searches, logs a
|
||||||
|
retrieval and renders nothing — indistinguishable in the telemetry from a
|
||||||
|
bar nothing cleared, which is the exact confusion this milestone exists to
|
||||||
|
remove. "Off" is what the `enabled` switch is for.
|
||||||
|
5. **`write_path` inherits auto-inject's budget when it has none.** It used
|
||||||
|
to share that key outright; giving it its own without a fallback would
|
||||||
|
silently reset the budget on every install that had tuned the shared one.
|
||||||
|
"""
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from scribe.services import retrieval_surfaces as rs
|
||||||
|
|
||||||
|
SERVICES = ("plugin_context", "reply_preferences")
|
||||||
|
|
||||||
|
|
||||||
|
def _service_source(name: str) -> str:
|
||||||
|
from pathlib import Path
|
||||||
|
root = Path(__file__).resolve().parents[1] / "src" / "scribe" / "services"
|
||||||
|
return (root / f"{name}.py").read_text()
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_surface_name_is_a_real_telemetry_source():
|
||||||
|
"""The join key, checked against the arms that emit it.
|
||||||
|
|
||||||
|
Asserted on the source text rather than by calling the arms, because what
|
||||||
|
can rot here is the literal: a surface renamed in the registry and not in
|
||||||
|
the `record_retrieval(source=…)` call still runs, still logs, and still
|
||||||
|
tunes — just against two different names, so the readout an operator uses
|
||||||
|
to justify a change describes a different arm from the one the change hits.
|
||||||
|
"""
|
||||||
|
blob = "\n".join(_service_source(n) for n in SERVICES)
|
||||||
|
# `report_preference` passes its name through a module constant rather than
|
||||||
|
# a literal, so that one name is satisfied by the constant holding it.
|
||||||
|
from scribe.services.reply_preferences import SOURCE
|
||||||
|
|
||||||
|
missing = [
|
||||||
|
s.name for s in rs.SURFACES.values()
|
||||||
|
if f'source="{s.name}"' not in blob and s.name != SOURCE
|
||||||
|
]
|
||||||
|
assert not missing, (
|
||||||
|
f"these surfaces can be tuned but never measured: {missing}. The "
|
||||||
|
"registry name must match the string the arm passes to record_retrieval."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_two_surfaces_share_a_settings_key():
|
||||||
|
"""One dial, one label. #3860 is what the other way costs."""
|
||||||
|
floors = [s.floor_key for s in rs.SURFACES.values()]
|
||||||
|
budgets = [s.budget_key for s in rs.SURFACES.values()]
|
||||||
|
assert len(set(floors)) == len(floors), f"duplicate floor key in {floors}"
|
||||||
|
assert len(set(budgets)) == len(budgets), f"duplicate budget key in {budgets}"
|
||||||
|
assert not (set(floors) & set(budgets)), "a floor key doubles as a budget key"
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_surface_says_what_it_asks_over_what_and_how_often():
|
||||||
|
"""The prose is rendered, not decorative.
|
||||||
|
|
||||||
|
A floor cannot be moved responsibly by anyone — model or human — who does
|
||||||
|
not know the query shape, the corpus, or how often the arm costs something.
|
||||||
|
An empty string here reaches the tuning tool and the Settings UI as a blank.
|
||||||
|
"""
|
||||||
|
for s in rs.SURFACES.values():
|
||||||
|
for field in ("asks", "over", "fires"):
|
||||||
|
assert getattr(s, field).strip(), f"{s.name}.{field} is empty"
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_unknown_surface_is_refused_rather_than_written():
|
||||||
|
with pytest.raises(ValueError) as e:
|
||||||
|
rs.get_surface("pretool_rule") # a real typo for pre_tool_rule
|
||||||
|
# The message has to name the alternatives, or the caller's next move is a
|
||||||
|
# second guess.
|
||||||
|
assert "pre_tool_rule" in str(e.value)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("stored, expected", [
|
||||||
|
("0.8", 0.8),
|
||||||
|
("5", 1.0), # clamped, not believed
|
||||||
|
("-3", 0.0),
|
||||||
|
("banana", 0.55), # unparseable falls back to the surface's default
|
||||||
|
("", 0.55),
|
||||||
|
])
|
||||||
|
async def test_a_floor_is_clamped_to_the_unit_interval(stored, expected):
|
||||||
|
with patch.object(rs, "get_setting", AsyncMock(return_value=stored)):
|
||||||
|
assert await rs.floor_for(1, "auto_inject") == expected
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("stored, expected", [
|
||||||
|
("4", 4),
|
||||||
|
("999", rs.MAX_BUDGET),
|
||||||
|
("0", 1),
|
||||||
|
("-2", 1),
|
||||||
|
("banana", 3),
|
||||||
|
])
|
||||||
|
async def test_a_budget_is_clamped_with_a_floor_of_one(stored, expected):
|
||||||
|
"""Zero is the value that must not get through.
|
||||||
|
|
||||||
|
An arm with a budget of 0 runs its search, writes a `retrieval_logs` row
|
||||||
|
with `result_count == 0`, and renders nothing — which reads in the telemetry
|
||||||
|
exactly like a bar nothing cleared. Turning a surface off is the `enabled`
|
||||||
|
switch's job, and that one says so.
|
||||||
|
"""
|
||||||
|
with patch.object(rs, "get_setting", AsyncMock(return_value=stored)):
|
||||||
|
assert await rs.budget_for(1, "auto_inject") == expected
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_the_write_path_budget_falls_back_to_auto_injects():
|
||||||
|
"""The migration this fallback exists for.
|
||||||
|
|
||||||
|
`write_path` had no budget key of its own — it read auto-inject's. An
|
||||||
|
install that had tuned that shared knob to 6 must not silently drop to the
|
||||||
|
new key's default the day this ships; nothing would look broken and the
|
||||||
|
operator would never know to look.
|
||||||
|
"""
|
||||||
|
async def _setting(_uid, key, default=""):
|
||||||
|
return {"kb_writepath_top_k": "", "kb_autoinject_top_k": "6"}.get(key, default)
|
||||||
|
|
||||||
|
with patch.object(rs, "get_setting", AsyncMock(side_effect=_setting)):
|
||||||
|
assert await rs.budget_for(1, "write_path") == 6
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_its_own_budget_wins_once_it_is_set():
|
||||||
|
"""And the fallback must not become a permanent override."""
|
||||||
|
async def _setting(_uid, key, default=""):
|
||||||
|
return {"kb_writepath_top_k": "2", "kb_autoinject_top_k": "6"}.get(key, default)
|
||||||
|
|
||||||
|
with patch.object(rs, "get_setting", AsyncMock(side_effect=_setting)):
|
||||||
|
assert await rs.budget_for(1, "write_path") == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_the_write_path_config_carries_every_arm_it_drives():
|
||||||
|
"""One hook request drives three arms, and each reads its pair out of this.
|
||||||
|
|
||||||
|
A missing key does not raise where a reader would see it — the rule arms
|
||||||
|
fail open, so a KeyError becomes an empty hint, and the arm reads as "fired
|
||||||
|
and found nothing". That is the failure this asserts against, and it is the
|
||||||
|
reason tests build the dict from the registry rather than by hand.
|
||||||
|
"""
|
||||||
|
from scribe.services import plugin_context as pc
|
||||||
|
|
||||||
|
# Both modules read settings: `pc` for the enabled switch, `rs` for the
|
||||||
|
# pairs. Patching one and not the other reaches a real database.
|
||||||
|
with patch.object(pc, "get_setting", AsyncMock(return_value="")), \
|
||||||
|
patch.object(rs, "get_setting", AsyncMock(return_value="")):
|
||||||
|
cfg = await pc.get_writepath_config(1)
|
||||||
|
|
||||||
|
for key in ("threshold", "top_k", "rule_threshold", "rule_top_k",
|
||||||
|
"tool_rule_threshold", "tool_rule_top_k"):
|
||||||
|
assert key in cfg, f"{key} missing — its arm will silently no-op"
|
||||||
@@ -0,0 +1,199 @@
|
|||||||
|
"""Moving a floor, and the reason that has to come with it (#4102).
|
||||||
|
|
||||||
|
WHY THIS EXISTS
|
||||||
|
|
||||||
|
The operator handed the dial to the model: *"the user should be able to touch it
|
||||||
|
but the model should be the thing handling it 9 times out of 10."* Everything
|
||||||
|
here guards the half of that sentence people skip — the operator still has to be
|
||||||
|
able to see what was done on their behalf and disagree with it.
|
||||||
|
|
||||||
|
WHAT THIS PINS
|
||||||
|
|
||||||
|
1. **A reason is required, and "" does not count.** The column is NOT NULL,
|
||||||
|
which an empty string satisfies; the service is where the requirement is
|
||||||
|
real. This is the whole guardrail: a caller who must write down why has to
|
||||||
|
have looked, and a caller who writes down something wrong leaves the
|
||||||
|
operator a sentence to argue with. A number that moved silently leaves
|
||||||
|
nothing.
|
||||||
|
2. **The change is recorded, with what it was before.** Without `old_value` a
|
||||||
|
history cannot answer "was this always like that", which is the first
|
||||||
|
question anyone asks of a surface behaving oddly.
|
||||||
|
3. **A clamp is reported, never swallowed.** A caller that believes it set 1.4
|
||||||
|
will read the next telemetry as evidence about a bar that was never in
|
||||||
|
force — the same class of error as #3739, one layer up.
|
||||||
|
4. **An unknown surface is refused before anything is written.** Settings keys
|
||||||
|
are free-form strings, so a typo'd surface would write a key nothing reads:
|
||||||
|
a change that reports success and alters nothing.
|
||||||
|
5. **The tool teaches the procedure that works**, not the one the numbers
|
||||||
|
suggest. Asserted on the docstring because the docstring IS the contract an
|
||||||
|
agent reads, and the failure it prevents is a model tuning from a
|
||||||
|
percentile — which has been measured pointing the wrong way.
|
||||||
|
"""
|
||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from scribe.services import retrieval_tuning as rt
|
||||||
|
from tests.helpers import make_mock_session
|
||||||
|
|
||||||
|
|
||||||
|
def _patches(floor=0.72, budget=3):
|
||||||
|
"""Patch everything `set_dial` touches except the thing under test."""
|
||||||
|
session = make_mock_session()
|
||||||
|
return session, (
|
||||||
|
patch.object(rt, "async_session", MagicMock(return_value=session)),
|
||||||
|
patch.object(rt, "set_setting", AsyncMock()),
|
||||||
|
patch.object(rt, "floor_for", AsyncMock(return_value=floor)),
|
||||||
|
patch.object(rt, "budget_for", AsyncMock(return_value=budget)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("reason", ["", " ", "noisy", "too high"])
|
||||||
|
async def test_a_change_without_a_real_reason_is_refused(reason):
|
||||||
|
"""The guardrail. "too high" is a restatement of the change, not a basis."""
|
||||||
|
session, ctx = _patches()
|
||||||
|
with ctx[0], ctx[1], ctx[2], ctx[3], pytest.raises(ValueError) as e:
|
||||||
|
await rt.set_dial(1, "prompt_rule", "floor", 0.66, reason=reason)
|
||||||
|
# The message has to name the tool that produces a real basis, or the
|
||||||
|
# caller's next move is a longer sentence rather than a look at the records.
|
||||||
|
assert "near_miss_samples" in str(e.value)
|
||||||
|
session.add.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_nothing_is_written_when_the_reason_is_refused():
|
||||||
|
"""Refused BEFORE the setting is touched, not after.
|
||||||
|
|
||||||
|
Writing the value and then raising would leave the number moved and the
|
||||||
|
history empty — the exact state this table exists to make impossible.
|
||||||
|
"""
|
||||||
|
session, ctx = _patches()
|
||||||
|
with ctx[0], patch.object(rt, "set_setting", AsyncMock()) as setter, \
|
||||||
|
ctx[2], ctx[3]:
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
await rt.set_dial(1, "prompt_rule", "floor", 0.66, reason="x")
|
||||||
|
setter.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_good_change_writes_the_setting_and_the_event():
|
||||||
|
session, ctx = _patches(floor=0.72)
|
||||||
|
reason = ("read prompt_rule's 5 highest declines: 3 were project rules for "
|
||||||
|
"another repo, so the bar is doing its job here")
|
||||||
|
with ctx[0], patch.object(rt, "set_setting", AsyncMock()) as setter, \
|
||||||
|
ctx[2], ctx[3]:
|
||||||
|
out = await rt.set_dial(1, "prompt_rule", "floor", 0.66, reason=reason)
|
||||||
|
|
||||||
|
setter.assert_awaited_once()
|
||||||
|
_uid, key, stored = setter.await_args.args
|
||||||
|
assert key == "kb_promptrule_threshold" and stored == "0.66"
|
||||||
|
|
||||||
|
event = session.add.call_args.args[0]
|
||||||
|
assert event.surface == "prompt_rule" and event.dial == "floor"
|
||||||
|
# The before-value is what makes the history answerable.
|
||||||
|
assert event.old_value == 0.72 and event.new_value == 0.66
|
||||||
|
assert event.reason == reason and event.actor == "model"
|
||||||
|
assert out["previous"] == 0.72 and out["applied"] == 0.66
|
||||||
|
assert out["clamped"] is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("dial, sent, applied", [
|
||||||
|
("floor", 1.4, 1.0),
|
||||||
|
("floor", -0.2, 0.0),
|
||||||
|
("budget", 99, 10),
|
||||||
|
("budget", 0, 1),
|
||||||
|
])
|
||||||
|
async def test_a_clamp_is_reported_rather_than_swallowed(dial, sent, applied):
|
||||||
|
"""Said out loud, because silence here poisons the next reading.
|
||||||
|
|
||||||
|
A caller that believes it set 1.4 treats the following week's telemetry as
|
||||||
|
evidence about a bar that never existed, and then moves the dial again to
|
||||||
|
fix a problem it invented.
|
||||||
|
"""
|
||||||
|
session, ctx = _patches()
|
||||||
|
reason = "checked the refused records for this surface and they were fine"
|
||||||
|
with ctx[0], ctx[1], ctx[2], ctx[3]:
|
||||||
|
out = await rt.set_dial(1, "auto_inject", dial, sent, reason=reason)
|
||||||
|
assert out["applied"] == applied
|
||||||
|
assert out["clamped"] is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_an_unknown_surface_is_refused_before_anything_is_written():
|
||||||
|
session, ctx = _patches()
|
||||||
|
with ctx[0], patch.object(rt, "set_setting", AsyncMock()) as setter, \
|
||||||
|
ctx[2], ctx[3]:
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
await rt.set_dial(1, "pretool_rule", "floor", 0.6,
|
||||||
|
reason="a perfectly good reason that is long enough")
|
||||||
|
setter.assert_not_called()
|
||||||
|
session.add.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("dial", ["threshold", "limit", "", "FLOOR"])
|
||||||
|
async def test_an_unknown_dial_is_refused(dial):
|
||||||
|
"""The near-misses are the old vocabulary — `threshold` and `limit` are what
|
||||||
|
these were called before this step, so they are exactly what a stale caller
|
||||||
|
will send, and a silent no-op there would be indistinguishable from a
|
||||||
|
change that did not take."""
|
||||||
|
session, ctx = _patches()
|
||||||
|
with ctx[0], ctx[1], ctx[2], ctx[3], pytest.raises(ValueError):
|
||||||
|
await rt.set_dial(1, "auto_inject", dial, 0.6,
|
||||||
|
reason="a perfectly good reason that is long enough")
|
||||||
|
session.add.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_human_change_is_distinguishable_from_the_models():
|
||||||
|
"""Both act as the same user, so the id cannot tell them apart — and "did I
|
||||||
|
do this, or did the session?" is the first question the history is asked."""
|
||||||
|
session, ctx = _patches()
|
||||||
|
with ctx[0], ctx[1], ctx[2], ctx[3]:
|
||||||
|
await rt.set_dial(1, "auto_inject", "floor", 0.6, actor="human",
|
||||||
|
reason="operator set this themselves in Settings")
|
||||||
|
assert session.add.call_args.args[0].actor == "human"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_an_unknown_actor_is_refused():
|
||||||
|
"""Free text here would make the column unreadable within a month."""
|
||||||
|
session, ctx = _patches()
|
||||||
|
with ctx[0], ctx[1], ctx[2], ctx[3], pytest.raises(ValueError):
|
||||||
|
await rt.set_dial(1, "auto_inject", "floor", 0.6, actor="agent",
|
||||||
|
reason="a perfectly good reason that is long enough")
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_tool_teaches_reading_the_records_not_the_percentile():
|
||||||
|
"""The contract an agent actually reads (rule 167).
|
||||||
|
|
||||||
|
The failure this prevents is a model moving a bar because `near_misses.p90`
|
||||||
|
sat close to it. That has been measured pointing the wrong way — 69 declines
|
||||||
|
where every percentile said "lower it" and the refused record was a false
|
||||||
|
positive — so the docstring has to carry the method, not just the warning.
|
||||||
|
"""
|
||||||
|
from scribe.mcp.tools import retrieval_tuning as tool
|
||||||
|
|
||||||
|
doc = tool.tune_retrieval.__doc__
|
||||||
|
assert "near_miss_samples" in doc, "the tool does not name how to get records"
|
||||||
|
assert "PERCENTILE ALONE" in doc.upper()
|
||||||
|
# And the worked example, because an abstract warning loses to a number.
|
||||||
|
assert "69" in doc
|
||||||
|
|
||||||
|
|
||||||
|
def test_all_three_tools_are_registered():
|
||||||
|
from scribe.mcp.tools import retrieval_tuning as tool
|
||||||
|
|
||||||
|
names = []
|
||||||
|
|
||||||
|
class _MCP:
|
||||||
|
def tool(self, name):
|
||||||
|
names.append(name)
|
||||||
|
return lambda fn: fn
|
||||||
|
|
||||||
|
tool.register(_MCP())
|
||||||
|
assert names == [
|
||||||
|
"retrieval_surfaces", "tune_retrieval", "retrieval_tuning_history",
|
||||||
|
]
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
"""The operator's half of the tuning surface (#4102).
|
||||||
|
|
||||||
|
WHY THIS EXISTS
|
||||||
|
|
||||||
|
The model moves these dials — that is the operator's decision for milestone
|
||||||
|
416 — so the browser's job is no longer "set the number". It is to show what
|
||||||
|
was set, by whom, and on what argument, and to let the operator disagree.
|
||||||
|
|
||||||
|
The load-bearing guard here is the last one. `/api/settings` is a generic
|
||||||
|
key-value endpoint that accepts any key at all, and every retrieval floor IS an
|
||||||
|
ordinary settings key. A floor written straight through it would land correctly
|
||||||
|
and record nothing — a tuning history with holes in it, which is worse than no
|
||||||
|
history because it reads as complete. So the generic endpoint routes those keys
|
||||||
|
through `set_dial`, and that routing is asserted rather than remembered.
|
||||||
|
"""
|
||||||
|
import inspect
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
def test_retrieval_blueprint_registered():
|
||||||
|
from scribe.routes.retrieval import retrieval_bp
|
||||||
|
assert retrieval_bp.name == "retrieval"
|
||||||
|
assert retrieval_bp.url_prefix == "/api/retrieval"
|
||||||
|
|
||||||
|
|
||||||
|
def test_retrieval_blueprint_registered_in_app():
|
||||||
|
from scribe.app import create_app
|
||||||
|
app = create_app()
|
||||||
|
assert "retrieval" in app.blueprints
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_endpoint_is_reachable_on_the_app():
|
||||||
|
"""Handlers existing is not the same as them being routed."""
|
||||||
|
from scribe.app import create_app
|
||||||
|
app = create_app()
|
||||||
|
rules = {
|
||||||
|
str(r.rule) for r in app.url_map.iter_rules()
|
||||||
|
if r.endpoint.startswith("retrieval.")
|
||||||
|
}
|
||||||
|
assert rules == {
|
||||||
|
"/api/retrieval/surfaces",
|
||||||
|
"/api/retrieval/surfaces/<surface>",
|
||||||
|
"/api/retrieval/tuning-history",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_browser_and_the_agent_call_the_same_service():
|
||||||
|
"""Rule 33 parity. Two callers, one service — or the two surfaces drift and
|
||||||
|
the guard that a reason is required exists on only one of them."""
|
||||||
|
from scribe.mcp.tools import retrieval_tuning as tool
|
||||||
|
from scribe.routes import retrieval as routes
|
||||||
|
from scribe.services import retrieval_tuning as svc
|
||||||
|
|
||||||
|
assert routes.set_dial is svc.set_dial
|
||||||
|
assert tool.tuning_svc is svc
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_route_does_not_take_the_actor_from_the_caller():
|
||||||
|
"""`actor` is the one field a reviewer leans on to answer "did I do this,
|
||||||
|
or did the session?". A payload-supplied actor would let a model claim to
|
||||||
|
be the operator, which turns the column into decoration."""
|
||||||
|
from scribe.routes import retrieval as routes
|
||||||
|
|
||||||
|
src = inspect.getsource(routes.tune_surface_route)
|
||||||
|
assert 'actor="human"' in src
|
||||||
|
assert 'data.get("actor"' not in src and 'data["actor"]' not in src
|
||||||
|
|
||||||
|
|
||||||
|
# ── the hole the generic settings endpoint would otherwise leave ────────────
|
||||||
|
|
||||||
|
def test_every_registry_key_is_recognised_as_a_dial():
|
||||||
|
"""Derived from the registry, both directions. A seventh surface added
|
||||||
|
without a row here is a floor that can be written silently again."""
|
||||||
|
from scribe.services.retrieval_surfaces import SURFACES, dial_for_key
|
||||||
|
|
||||||
|
for name, surface in SURFACES.items():
|
||||||
|
assert dial_for_key(surface.floor_key) == (name, "floor")
|
||||||
|
assert dial_for_key(surface.budget_key) == (name, "budget")
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_ordinary_setting_is_not_mistaken_for_a_dial():
|
||||||
|
"""The interception must be narrow. A false positive here would send an
|
||||||
|
unrelated setting through a service that parses it as a float and rejects
|
||||||
|
the save."""
|
||||||
|
from scribe.services.retrieval_surfaces import dial_for_key
|
||||||
|
|
||||||
|
for key in ("smtp_password", "kb_autoinject_enabled", "theme",
|
||||||
|
"kb_planmatch_threshold", ""):
|
||||||
|
assert dial_for_key(key) is None, key
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_settings_endpoint_routes_a_dial_through_the_recorder():
|
||||||
|
"""THE GUARD. Asserted on the source because the alternative is a full
|
||||||
|
request-context round trip for a branch whose whole content is "which
|
||||||
|
function gets called" — and because what must not regress is precisely
|
||||||
|
that this module reaches for `set_dial` at all."""
|
||||||
|
from scribe.routes import settings as routes
|
||||||
|
|
||||||
|
src = inspect.getsource(routes.update_settings_route)
|
||||||
|
assert "dial_for_key" in src, (
|
||||||
|
"the generic settings endpoint no longer recognises retrieval dials — "
|
||||||
|
"a floor written through it now moves with no event recorded, and the "
|
||||||
|
"tuning history will say nothing happened"
|
||||||
|
)
|
||||||
|
assert "set_dial" in src
|
||||||
|
assert 'actor="human"' in src
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_setting_a_dial_to_the_value_it_already_has_records_nothing():
|
||||||
|
"""The Settings form re-sends every field on every save. Without this, one
|
||||||
|
press of Save writes six rows saying the operator set six dials to the
|
||||||
|
numbers they were already on — and a history nobody can skim is one nobody
|
||||||
|
reads."""
|
||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
|
from scribe.services import retrieval_tuning as rt
|
||||||
|
from tests.helpers import make_mock_session
|
||||||
|
|
||||||
|
session = make_mock_session()
|
||||||
|
with patch.object(rt, "async_session", MagicMock(return_value=session)), \
|
||||||
|
patch.object(rt, "set_setting", AsyncMock()) as setter, \
|
||||||
|
patch.object(rt, "floor_for", AsyncMock(return_value=0.72)), \
|
||||||
|
patch.object(rt, "budget_for", AsyncMock(return_value=3)):
|
||||||
|
out = await rt.set_dial(1, "prompt_rule", "floor", 0.72,
|
||||||
|
reason="re-saved the settings form untouched")
|
||||||
|
|
||||||
|
assert out["unchanged"] is True
|
||||||
|
session.add.assert_not_called()
|
||||||
|
# And the setting is left alone too: rewriting the same value would bump
|
||||||
|
# whatever timestamp the row carries for no reason.
|
||||||
|
setter.assert_not_called()
|
||||||
@@ -135,21 +135,9 @@ def test_an_event_with_nothing_usable_records_nothing_and_still_exits_zero(event
|
|||||||
assert not led.exists()
|
assert not led.exists()
|
||||||
|
|
||||||
|
|
||||||
# ── the two ledgers stay in step ───────────────────────────────────────────
|
# What a compaction clears — including `.opened.ids`, whose claim is the one
|
||||||
|
# that would be worst to get wrong — is asserted against the running hook in
|
||||||
def test_a_compaction_clears_both_ledgers():
|
# tests/test_session_ledger_clear.py, so there is one home for it.
|
||||||
"""The one that would be worst to get wrong. `.opened.ids` describes a
|
|
||||||
context the compaction just destroyed, so keeping it while clearing the
|
|
||||||
naming ledger would have the arms telling a freshly-summarised session
|
|
||||||
"you opened it earlier" about a rule now nowhere in its context.
|
|
||||||
"""
|
|
||||||
sh = (HOOKS / "scribe_session_context.sh").read_text()
|
|
||||||
block = sh.split("case \"$source\" in")[1].split("esac")[0]
|
|
||||||
assert "compact|clear)" in block
|
|
||||||
for led in (".rules.ids", ".opened.ids"):
|
|
||||||
assert re.search(rf"rm -f .*{re.escape(led)}", block), (
|
|
||||||
f"{led} survives a compaction that destroyed what it describes"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_recorder_is_registered_on_the_get_rule_tool():
|
def test_the_recorder_is_registered_on_the_get_rule_tool():
|
||||||
|
|||||||
@@ -17,7 +17,8 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from tests.helpers import fake_note, fake_rule
|
from scribe.services import retrieval_surfaces as rs
|
||||||
|
from tests.helpers import fake_note, fake_rule, writepath_cfg
|
||||||
|
|
||||||
# The MCP tool layer reads its caller from a ContextVar the HTTP transport sets
|
# The MCP tool layer reads its caller from a ContextVar the HTTP transport sets
|
||||||
# per request; a unit test has no request, so it binds the caller itself. The
|
# per request; a unit test has no request, so it binds the caller itself. The
|
||||||
@@ -59,16 +60,15 @@ def _arm_patches(pc, hits, recorder, prior_art=None, cfg=None, rule_search=None,
|
|||||||
"""
|
"""
|
||||||
return (
|
return (
|
||||||
patch.object(pc, "get_writepath_config",
|
patch.object(pc, "get_writepath_config",
|
||||||
AsyncMock(return_value=cfg or {
|
# Every key, derived from the surface registry (#4102).
|
||||||
"enabled": True, "threshold": 0.6,
|
# A stub missing one does not fail where a reader would
|
||||||
"top_k": 3, "rule_threshold": 0.6,
|
# see it: the arm fails open, so the KeyError becomes an
|
||||||
# The command arm reads its OWN bar since #3853, and
|
# empty hint and every case in _ARMS reports the arm went
|
||||||
# a stub missing this key does not fail where a
|
# silent instead.
|
||||||
# reader would see it: the arm fails open, so the
|
AsyncMock(return_value=cfg or writepath_cfg(
|
||||||
# KeyError becomes an empty hint and every case in
|
threshold=0.6, rule_threshold=0.6,
|
||||||
# _ARMS reports the arm went silent instead.
|
tool_rule_threshold=0.6,
|
||||||
"tool_rule_threshold": 0.6,
|
))),
|
||||||
})),
|
|
||||||
patch.object(pc.snippets_svc, "list_snippets", AsyncMock(return_value=([], 0))),
|
patch.object(pc.snippets_svc, "list_snippets", AsyncMock(return_value=([], 0))),
|
||||||
patch.object(pc, "semantic_search_notes",
|
patch.object(pc, "semantic_search_notes",
|
||||||
AsyncMock(return_value=_PRIOR_ART if prior_art is None
|
AsyncMock(return_value=_PRIOR_ART if prior_art is None
|
||||||
@@ -156,8 +156,7 @@ async def test_the_arm_searches_on_its_OWN_bar_not_the_code_one():
|
|||||||
with ExitStack() as stack:
|
with ExitStack() as stack:
|
||||||
for ctx in _arm_patches(
|
for ctx in _arm_patches(
|
||||||
pc, [], MagicMock(), rule_search=search,
|
pc, [], MagicMock(), rule_search=search,
|
||||||
cfg={"enabled": True, "threshold": 0.60,
|
cfg=writepath_cfg(threshold=0.60, top_k=3, rule_threshold=0.81),
|
||||||
"top_k": 3, "rule_threshold": 0.81},
|
|
||||||
):
|
):
|
||||||
stack.enter_context(ctx)
|
stack.enter_context(ctx)
|
||||||
await pc.build_write_path_hint(
|
await pc.build_write_path_hint(
|
||||||
@@ -392,16 +391,15 @@ def test_every_rules_payload_caller_names_itself():
|
|||||||
def _tool_patches(pc, hits, recorder, cfg=None, retrieval_log=None):
|
def _tool_patches(pc, hits, recorder, cfg=None, retrieval_log=None):
|
||||||
return (
|
return (
|
||||||
patch.object(pc, "get_writepath_config",
|
patch.object(pc, "get_writepath_config",
|
||||||
AsyncMock(return_value=cfg or {
|
# Every key, derived from the surface registry (#4102).
|
||||||
"enabled": True, "threshold": 0.6,
|
# A stub missing one does not fail where a reader would
|
||||||
"top_k": 3, "rule_threshold": 0.6,
|
# see it: the arm fails open, so the KeyError becomes an
|
||||||
# The command arm reads its OWN bar since #3853, and
|
# empty hint and every case in _ARMS reports the arm went
|
||||||
# a stub missing this key does not fail where a
|
# silent instead.
|
||||||
# reader would see it: the arm fails open, so the
|
AsyncMock(return_value=cfg or writepath_cfg(
|
||||||
# KeyError becomes an empty hint and every case in
|
threshold=0.6, rule_threshold=0.6,
|
||||||
# _ARMS reports the arm went silent instead.
|
tool_rule_threshold=0.6,
|
||||||
"tool_rule_threshold": 0.6,
|
))),
|
||||||
})),
|
|
||||||
patch.object(pc, "semantic_search_rules", AsyncMock(return_value=hits)),
|
patch.object(pc, "semantic_search_rules", AsyncMock(return_value=hits)),
|
||||||
patch.object(pc, "record_retrieval", retrieval_log or MagicMock()),
|
patch.object(pc, "record_retrieval", retrieval_log or MagicMock()),
|
||||||
patch.object(pc, "record_rule_surfaced", recorder),
|
patch.object(pc, "record_rule_surfaced", recorder),
|
||||||
@@ -433,9 +431,10 @@ async def _run_tool_arm(hits, recorder, command="curl -s https://git.example/api
|
|||||||
|
|
||||||
def _prompt_patches(pc, hits, recorder, retrieval_log=None):
|
def _prompt_patches(pc, hits, recorder, retrieval_log=None):
|
||||||
return (
|
return (
|
||||||
# The arm reads its own threshold key rather than a shared config
|
# The arm reads its own floor and budget from the surface registry
|
||||||
# object — a third corpus with a bar nothing has yet tuned for it.
|
# rather than a shared config object (#4102) — a third corpus, with a
|
||||||
patch.object(pc, "get_setting", AsyncMock(return_value="0.6")),
|
# pair nothing has yet tuned for it.
|
||||||
|
patch.object(rs, "get_setting", AsyncMock(return_value="0.6")),
|
||||||
patch.object(pc, "semantic_search_rules", AsyncMock(return_value=hits)),
|
patch.object(pc, "semantic_search_rules", AsyncMock(return_value=hits)),
|
||||||
patch.object(pc, "record_retrieval", retrieval_log or MagicMock()),
|
patch.object(pc, "record_retrieval", retrieval_log or MagicMock()),
|
||||||
patch.object(pc, "record_rule_surfaced", recorder),
|
patch.object(pc, "record_rule_surfaced", recorder),
|
||||||
@@ -469,7 +468,7 @@ async def test_the_prompt_arm_retrieves_against_what_the_operator_SAID():
|
|||||||
))])
|
))])
|
||||||
from scribe.services import plugin_context as pc
|
from scribe.services import plugin_context as pc
|
||||||
with ExitStack() as stack:
|
with ExitStack() as stack:
|
||||||
stack.enter_context(patch.object(pc, "get_setting", AsyncMock(return_value="0.6")))
|
stack.enter_context(patch.object(rs, "get_setting", AsyncMock(return_value="0.6")))
|
||||||
stack.enter_context(patch.object(pc, "semantic_search_rules", search))
|
stack.enter_context(patch.object(pc, "semantic_search_rules", search))
|
||||||
stack.enter_context(patch.object(pc, "record_retrieval", MagicMock()))
|
stack.enter_context(patch.object(pc, "record_retrieval", MagicMock()))
|
||||||
stack.enter_context(patch.object(pc, "record_rule_surfaced", rec))
|
stack.enter_context(patch.object(pc, "record_rule_surfaced", rec))
|
||||||
@@ -503,7 +502,7 @@ async def test_the_prompt_arm_says_nothing_when_asked_nothing():
|
|||||||
log = MagicMock()
|
log = MagicMock()
|
||||||
from scribe.services import plugin_context as pc
|
from scribe.services import plugin_context as pc
|
||||||
with ExitStack() as stack:
|
with ExitStack() as stack:
|
||||||
stack.enter_context(patch.object(pc, "get_setting", AsyncMock(return_value="0.6")))
|
stack.enter_context(patch.object(rs, "get_setting", AsyncMock(return_value="0.6")))
|
||||||
stack.enter_context(patch.object(pc, "semantic_search_rules", search))
|
stack.enter_context(patch.object(pc, "semantic_search_rules", search))
|
||||||
stack.enter_context(patch.object(pc, "record_retrieval", log))
|
stack.enter_context(patch.object(pc, "record_retrieval", log))
|
||||||
stack.enter_context(patch.object(pc, "record_rule_surfaced", MagicMock()))
|
stack.enter_context(patch.object(pc, "record_rule_surfaced", MagicMock()))
|
||||||
@@ -1416,7 +1415,7 @@ async def _run_slot(general, preference, recorder=None, retrieval_log=None, **kw
|
|||||||
from scribe.services import plugin_context as pc
|
from scribe.services import plugin_context as pc
|
||||||
rec = recorder or MagicMock()
|
rec = recorder or MagicMock()
|
||||||
with ExitStack() as stack:
|
with ExitStack() as stack:
|
||||||
stack.enter_context(patch.object(pc, "get_setting", AsyncMock(return_value="0.6")))
|
stack.enter_context(patch.object(rs, "get_setting", AsyncMock(return_value="0.6")))
|
||||||
stack.enter_context(patch.object(
|
stack.enter_context(patch.object(
|
||||||
pc, "semantic_search_rules", _search_by_kind(general, preference)))
|
pc, "semantic_search_rules", _search_by_kind(general, preference)))
|
||||||
stack.enter_context(patch.object(
|
stack.enter_context(patch.object(
|
||||||
@@ -1480,7 +1479,7 @@ async def test_the_slot_query_can_only_answer_with_a_preference():
|
|||||||
from scribe.services import plugin_context as pc
|
from scribe.services import plugin_context as pc
|
||||||
search = _search_by_kind(_RULES_FILLING_THE_LIMIT, _PREF_HIT)
|
search = _search_by_kind(_RULES_FILLING_THE_LIMIT, _PREF_HIT)
|
||||||
with ExitStack() as stack:
|
with ExitStack() as stack:
|
||||||
stack.enter_context(patch.object(pc, "get_setting", AsyncMock(return_value="0.6")))
|
stack.enter_context(patch.object(rs, "get_setting", AsyncMock(return_value="0.6")))
|
||||||
stack.enter_context(patch.object(pc, "semantic_search_rules", search))
|
stack.enter_context(patch.object(pc, "semantic_search_rules", search))
|
||||||
stack.enter_context(patch.object(pc, "record_retrieval", MagicMock()))
|
stack.enter_context(patch.object(pc, "record_retrieval", MagicMock()))
|
||||||
stack.enter_context(patch.object(pc, "record_rule_surfaced", MagicMock()))
|
stack.enter_context(patch.object(pc, "record_rule_surfaced", MagicMock()))
|
||||||
@@ -1617,8 +1616,7 @@ async def test_each_act_arm_searches_at_its_own_bar():
|
|||||||
"""The split, where it actually takes effect."""
|
"""The split, where it actually takes effect."""
|
||||||
from scribe.services import plugin_context as pc
|
from scribe.services import plugin_context as pc
|
||||||
|
|
||||||
cfg = {"enabled": True, "threshold": 0.6, "top_k": 3,
|
cfg = writepath_cfg(threshold=0.6, top_k=3, rule_threshold=0.77, tool_rule_threshold=0.61)
|
||||||
"rule_threshold": 0.77, "tool_rule_threshold": 0.61}
|
|
||||||
|
|
||||||
search = AsyncMock(return_value=list(_THREE_HITS))
|
search = AsyncMock(return_value=list(_THREE_HITS))
|
||||||
with ExitStack() as stack:
|
with ExitStack() as stack:
|
||||||
@@ -1646,8 +1644,7 @@ async def test_an_act_arm_reports_the_bar_it_actually_searched_at():
|
|||||||
"""
|
"""
|
||||||
from scribe.services import plugin_context as pc
|
from scribe.services import plugin_context as pc
|
||||||
|
|
||||||
cfg = {"enabled": True, "threshold": 0.6, "top_k": 3,
|
cfg = writepath_cfg(threshold=0.6, top_k=3, rule_threshold=0.77, tool_rule_threshold=0.61)
|
||||||
"rule_threshold": 0.77, "tool_rule_threshold": 0.61}
|
|
||||||
|
|
||||||
search = AsyncMock(return_value=list(_THREE_HITS))
|
search = AsyncMock(return_value=list(_THREE_HITS))
|
||||||
log = MagicMock()
|
log = MagicMock()
|
||||||
@@ -1712,8 +1709,7 @@ async def test_the_act_arms_scope_their_search_to_the_bound_project(bound, scope
|
|||||||
global rules only, which the search spells as `project_id=None`."""
|
global rules only, which the search spells as `project_id=None`."""
|
||||||
from scribe.services import plugin_context as pc
|
from scribe.services import plugin_context as pc
|
||||||
|
|
||||||
cfg = {"enabled": True, "threshold": 0.6, "top_k": 3,
|
cfg = writepath_cfg(threshold=0.6, top_k=3, rule_threshold=0.6, tool_rule_threshold=0.6)
|
||||||
"rule_threshold": 0.6, "tool_rule_threshold": 0.6}
|
|
||||||
tool_search = AsyncMock(return_value=[])
|
tool_search = AsyncMock(return_value=[])
|
||||||
with ExitStack() as stack:
|
with ExitStack() as stack:
|
||||||
stack.enter_context(patch.object(
|
stack.enter_context(patch.object(
|
||||||
@@ -1726,7 +1722,7 @@ async def test_the_act_arms_scope_their_search_to_the_bound_project(bound, scope
|
|||||||
|
|
||||||
prompt_search = AsyncMock(return_value=[])
|
prompt_search = AsyncMock(return_value=[])
|
||||||
with ExitStack() as stack:
|
with ExitStack() as stack:
|
||||||
stack.enter_context(patch.object(pc, "get_setting", AsyncMock(return_value="0.6")))
|
stack.enter_context(patch.object(rs, "get_setting", AsyncMock(return_value="0.6")))
|
||||||
stack.enter_context(patch.object(pc, "semantic_search_rules", prompt_search))
|
stack.enter_context(patch.object(pc, "semantic_search_rules", prompt_search))
|
||||||
stack.enter_context(patch.object(pc, "record_retrieval", MagicMock()))
|
stack.enter_context(patch.object(pc, "record_retrieval", MagicMock()))
|
||||||
stack.enter_context(patch.object(pc, "record_rule_surfaced", MagicMock()))
|
stack.enter_context(patch.object(pc, "record_rule_surfaced", MagicMock()))
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ def test_backup_version_is_current():
|
|||||||
|
|
||||||
(Named for the number it asserted until v10, which is exactly the drift a
|
(Named for the number it asserted until v10, which is exactly the drift a
|
||||||
name-carrying-a-value invites; it now says what it checks.)"""
|
name-carrying-a-value invites; it now says what it checks.)"""
|
||||||
assert backup.BACKUP_VERSION == 15
|
assert backup.BACKUP_VERSION == 16
|
||||||
|
|
||||||
|
|
||||||
def _exportable_note(**over):
|
def _exportable_note(**over):
|
||||||
@@ -134,6 +134,7 @@ def _column_guard_targets():
|
|||||||
from scribe.models.note_supersession import NoteSupersession
|
from scribe.models.note_supersession import NoteSupersession
|
||||||
from scribe.models.note_usage import NoteUsageEvent
|
from scribe.models.note_usage import NoteUsageEvent
|
||||||
from scribe.models.rule_usage import RuleUsageEvent
|
from scribe.models.rule_usage import RuleUsageEvent
|
||||||
|
from scribe.models.retrieval_tuning import RetrievalTuningEvent
|
||||||
from scribe.models.note_version import NoteVersion
|
from scribe.models.note_version import NoteVersion
|
||||||
from scribe.models.rule_version import RuleVersion
|
from scribe.models.rule_version import RuleVersion
|
||||||
from scribe.models.project import Project
|
from scribe.models.project import Project
|
||||||
@@ -164,6 +165,9 @@ def _column_guard_targets():
|
|||||||
"rule_relations": (RuleRelation, backup._rule_relation_rows),
|
"rule_relations": (RuleRelation, backup._rule_relation_rows),
|
||||||
"note_usage_events": (NoteUsageEvent, backup._usage_event_rows),
|
"note_usage_events": (NoteUsageEvent, backup._usage_event_rows),
|
||||||
"rule_usage_events": (RuleUsageEvent, backup._rule_usage_event_rows),
|
"rule_usage_events": (RuleUsageEvent, backup._rule_usage_event_rows),
|
||||||
|
"retrieval_tuning_events": (
|
||||||
|
RetrievalTuningEvent, backup._retrieval_tuning_event_rows,
|
||||||
|
),
|
||||||
"design_systems": (DesignSystem, backup._design_system_rows),
|
"design_systems": (DesignSystem, backup._design_system_rows),
|
||||||
"design_tokens": (DesignToken, backup._design_token_rows),
|
"design_tokens": (DesignToken, backup._design_token_rows),
|
||||||
"repo_bindings": (RepoBinding, backup._repo_binding_rows),
|
"repo_bindings": (RepoBinding, backup._repo_binding_rows),
|
||||||
@@ -340,7 +344,9 @@ async def test_export_full_backup_contains_every_declared_section():
|
|||||||
"systems", "record_systems", "design_systems",
|
"systems", "record_systems", "design_systems",
|
||||||
"design_tokens", "note_usage_events", "repo_bindings",
|
"design_tokens", "note_usage_events", "repo_bindings",
|
||||||
"note_supersessions", "code_shapes", "code_shape_events",
|
"note_supersessions", "code_shapes", "code_shape_events",
|
||||||
"code_shape_uses"):
|
"code_shape_uses",
|
||||||
|
# v16: the reasons beside the settings they explain.
|
||||||
|
"retrieval_tuning_events"):
|
||||||
assert key in out, f"missing export section: {key}"
|
assert key in out, f"missing export section: {key}"
|
||||||
assert out[key] == []
|
assert out[key] == []
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from tests.helpers import fake_note
|
from scribe.services import retrieval_surfaces as rs
|
||||||
|
from tests.helpers import fake_note, writepath_cfg
|
||||||
|
|
||||||
|
|
||||||
pytestmark = pytest.mark.usefixtures("_no_supersession")
|
pytestmark = pytest.mark.usefixtures("_no_supersession")
|
||||||
@@ -17,7 +18,8 @@ async def test_get_autoinject_config_defaults_and_clamps():
|
|||||||
from scribe.services import plugin_context as pc
|
from scribe.services import plugin_context as pc
|
||||||
|
|
||||||
# No settings stored → defaults.
|
# No settings stored → defaults.
|
||||||
with patch.object(pc, "get_setting", AsyncMock(side_effect=lambda uid, k, d: d)):
|
with patch.object(pc, "get_setting", AsyncMock(side_effect=lambda uid, k, d: d)), \
|
||||||
|
patch.object(rs, "get_setting", AsyncMock(side_effect=lambda uid, k, d="": d)):
|
||||||
cfg = await pc.get_autoinject_config(1)
|
cfg = await pc.get_autoinject_config(1)
|
||||||
assert cfg == {
|
assert cfg == {
|
||||||
"enabled": pc.AUTOINJECT_DEFAULT_ENABLED,
|
"enabled": pc.AUTOINJECT_DEFAULT_ENABLED,
|
||||||
@@ -31,8 +33,12 @@ async def test_get_autoinject_config_defaults_and_clamps():
|
|||||||
pc.AUTOINJECT_THRESHOLD_KEY: "5",
|
pc.AUTOINJECT_THRESHOLD_KEY: "5",
|
||||||
pc.AUTOINJECT_TOP_K_KEY: "999",
|
pc.AUTOINJECT_TOP_K_KEY: "999",
|
||||||
}
|
}
|
||||||
with patch.object(pc, "get_setting",
|
# The switch comes from plugin_context, the two numbers from the registry.
|
||||||
AsyncMock(side_effect=lambda uid, k, d: stored.get(k, d))):
|
def _side(uid, k, d=""):
|
||||||
|
return stored.get(k, d)
|
||||||
|
|
||||||
|
with patch.object(pc, "get_setting", AsyncMock(side_effect=_side)), \
|
||||||
|
patch.object(rs, "get_setting", AsyncMock(side_effect=_side)):
|
||||||
cfg = await pc.get_autoinject_config(1)
|
cfg = await pc.get_autoinject_config(1)
|
||||||
assert cfg["enabled"] is False
|
assert cfg["enabled"] is False
|
||||||
assert cfg["threshold"] == 1.0
|
assert cfg["threshold"] == 1.0
|
||||||
@@ -437,9 +443,7 @@ async def test_write_path_semantic_arm_asks_for_experience_not_just_snippets():
|
|||||||
search = AsyncMock(return_value=hits)
|
search = AsyncMock(return_value=hits)
|
||||||
rec = MagicMock()
|
rec = MagicMock()
|
||||||
with patch.object(pc, "get_writepath_config",
|
with patch.object(pc, "get_writepath_config",
|
||||||
AsyncMock(return_value={"enabled": True, "threshold": 0.6,
|
AsyncMock(return_value=writepath_cfg(threshold=0.6))), \
|
||||||
"top_k": 3,
|
|
||||||
"rule_threshold": 0.72})), \
|
|
||||||
patch.object(pc.snippets_svc, "list_snippets",
|
patch.object(pc.snippets_svc, "list_snippets",
|
||||||
AsyncMock(return_value=([], 0))), \
|
AsyncMock(return_value=([], 0))), \
|
||||||
patch.object(pc, "semantic_search_notes", search), \
|
patch.object(pc, "semantic_search_notes", search), \
|
||||||
@@ -470,9 +474,7 @@ async def test_write_path_labels_a_non_snippet_hit_with_its_kind():
|
|||||||
hits = [(0.72, fake_note(id=9, title="debounce helper", user_id=1, note_type="snippet")),
|
hits = [(0.72, fake_note(id=9, title="debounce helper", user_id=1, note_type="snippet")),
|
||||||
(0.71, fake_note(id=7, title="Debounce dropped the trailing call", user_id=1, is_task=True, task_kind="issue"))]
|
(0.71, fake_note(id=7, title="Debounce dropped the trailing call", user_id=1, is_task=True, task_kind="issue"))]
|
||||||
with patch.object(pc, "get_writepath_config",
|
with patch.object(pc, "get_writepath_config",
|
||||||
AsyncMock(return_value={"enabled": True, "threshold": 0.6,
|
AsyncMock(return_value=writepath_cfg(threshold=0.6))), \
|
||||||
"top_k": 3,
|
|
||||||
"rule_threshold": 0.72})), \
|
|
||||||
patch.object(pc.snippets_svc, "list_snippets",
|
patch.object(pc.snippets_svc, "list_snippets",
|
||||||
AsyncMock(return_value=([], 0))), \
|
AsyncMock(return_value=([], 0))), \
|
||||||
patch.object(pc, "semantic_search_notes", AsyncMock(return_value=hits)), \
|
patch.object(pc, "semantic_search_notes", AsyncMock(return_value=hits)), \
|
||||||
|
|||||||
@@ -9,6 +9,9 @@ import re
|
|||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
M = "scribe.services.reply_preferences"
|
M = "scribe.services.reply_preferences"
|
||||||
|
# Its two numbers are resolved through the registry now (#4102), so the
|
||||||
|
# settings reader to patch lives there rather than in the arm's own module.
|
||||||
|
RS = "scribe.services.retrieval_surfaces"
|
||||||
|
|
||||||
|
|
||||||
def _rule(rid, kind="preference"):
|
def _rule(rid, kind="preference"):
|
||||||
@@ -26,7 +29,7 @@ async def _run(hits, *, searched=True, raises=None):
|
|||||||
|
|
||||||
search_mock = AsyncMock(side_effect=search)
|
search_mock = AsyncMock(side_effect=search)
|
||||||
with patch(f"{M}.semantic_search_rules", search_mock), \
|
with patch(f"{M}.semantic_search_rules", search_mock), \
|
||||||
patch(f"{M}.get_setting", AsyncMock(return_value="0.72")), \
|
patch(f"{RS}.get_setting", AsyncMock(return_value="0.72")), \
|
||||||
patch(f"{M}.record_retrieval") as logged, \
|
patch(f"{M}.record_retrieval") as logged, \
|
||||||
patch(f"{M}.record_rule_surfaced") as surfaced:
|
patch(f"{M}.record_rule_surfaced") as surfaced:
|
||||||
out = await completion_preferences(7, project_id=3)
|
out = await completion_preferences(7, project_id=3)
|
||||||
@@ -93,7 +96,7 @@ def test_its_bar_is_its_own_key_not_the_prompt_arm_s(monkeypatch):
|
|||||||
|
|
||||||
asked: list[str] = []
|
asked: list[str] = []
|
||||||
|
|
||||||
async def get_setting(user_id, key, default):
|
async def get_setting(user_id, key, default=""):
|
||||||
asked.append(key)
|
asked.append(key)
|
||||||
return default
|
return default
|
||||||
|
|
||||||
@@ -101,16 +104,26 @@ def test_its_bar_is_its_own_key_not_the_prompt_arm_s(monkeypatch):
|
|||||||
kw["report"].update({"searched": True})
|
kw["report"].update({"searched": True})
|
||||||
return []
|
return []
|
||||||
|
|
||||||
with patch(f"{M}.get_setting", AsyncMock(side_effect=get_setting)), \
|
with patch(f"{RS}.get_setting", AsyncMock(side_effect=get_setting)), \
|
||||||
patch(f"{M}.semantic_search_rules", AsyncMock(side_effect=search)), \
|
patch(f"{M}.semantic_search_rules", AsyncMock(side_effect=search)), \
|
||||||
patch(f"{M}.record_retrieval"):
|
patch(f"{M}.record_retrieval"):
|
||||||
asyncio.run(completion_preferences(7))
|
asyncio.run(completion_preferences(7))
|
||||||
|
|
||||||
assert asked == [plugin_context.REPORTPREF_THRESHOLD_KEY]
|
# Both numbers are its own since #4102 — a floor AND a budget — so the arm
|
||||||
assert plugin_context.REPORTPREF_THRESHOLD_KEY != plugin_context.PROMPTRULE_THRESHOLD_KEY, (
|
# asks for two keys, and neither may be the prompt arm's.
|
||||||
|
from scribe.services.retrieval_surfaces import get_surface
|
||||||
|
|
||||||
|
mine = get_surface("report_preference")
|
||||||
|
assert asked == [mine.floor_key, mine.budget_key]
|
||||||
|
theirs = get_surface("prompt_rule")
|
||||||
|
assert mine.floor_key != theirs.floor_key, (
|
||||||
"the two keys are the same string again, so the settings form has one "
|
"the two keys are the same string again, so the settings form has one "
|
||||||
"dial driving two arms — which is the defect, whatever the value is"
|
"dial driving two arms — which is the defect, whatever the value is"
|
||||||
)
|
)
|
||||||
|
assert mine.floor_key == plugin_context.REPORTPREF_THRESHOLD_KEY, (
|
||||||
|
"the registry and the module constant disagree about this arm's key, "
|
||||||
|
"so the Settings form would write one and the arm would read the other"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_the_query_assumes_no_particular_domain():
|
def test_the_query_assumes_no_particular_domain():
|
||||||
|
|||||||
@@ -108,14 +108,29 @@ def test_the_rules_ledger_survives_exactly_when_the_context_does(tmp_path):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_only_the_rule_ledger_is_cleared_and_the_note_ledgers_are_left(tmp_path):
|
def test_the_note_ledgers_are_cleared_too_now_that_a_repeat_is_shown(tmp_path):
|
||||||
"""Scope, asserted rather than described.
|
"""The decision this test used to pin, reversed — and why it could be.
|
||||||
|
|
||||||
The same directory holds `.ids`, `.sync.ids` and `.derive.ids` for the note
|
It read: the note arms were deliberately left out of #3749, whether a
|
||||||
arms. Whether a surfaced NOTE should come back after a compaction is a
|
surfaced NOTE should come back after a compaction is a different question,
|
||||||
different question with a different answer, and it is not being answered.
|
and a glob over `<sid>.*` would decide it by accident. That was right when
|
||||||
A `rm` glob over `<sid>.*` would pass every assertion in the test above
|
it was written, and the reason was the note arms' behaviour rather than the
|
||||||
while silently deciding it.
|
note arms' nature: a note repeat was WITHHELD, so clearing their ledger
|
||||||
|
meant re-injecting whole menu lines the session had already been given.
|
||||||
|
Against that, leaving them was the cheaper mistake.
|
||||||
|
|
||||||
|
#4101 removed the premise. A note repeat is now rendered again with a
|
||||||
|
`seen` marker instead of dropped from the search, so the ledger no longer
|
||||||
|
decides whether a record is reachable — only whether its line says the
|
||||||
|
session has met it before. Across a compaction that marker becomes a false
|
||||||
|
statement: it tells a freshly-summarised session it has already seen a
|
||||||
|
record that is nowhere in its context. Keeping the ledger buys nothing and
|
||||||
|
asserts something untrue.
|
||||||
|
|
||||||
|
Kept as a named test rather than deleted, because the roster it guards
|
||||||
|
moved to tests/test_session_ledger_clear.py and what is worth keeping HERE
|
||||||
|
is the record that the answer changed, with the reason attached — a
|
||||||
|
deleted test takes the old decision's reasoning with it.
|
||||||
"""
|
"""
|
||||||
env = _env(tmp_path)
|
env = _env(tmp_path)
|
||||||
sid = "sess-scope"
|
sid = "sess-scope"
|
||||||
@@ -127,10 +142,9 @@ def test_only_the_rule_ledger_is_cleared_and_the_note_ledgers_are_left(tmp_path)
|
|||||||
_fire("compact", sid, env)
|
_fire("compact", sid, env)
|
||||||
|
|
||||||
assert not rules.exists(), "the rule ledger should have been cleared"
|
assert not rules.exists(), "the rule ledger should have been cleared"
|
||||||
assert notes.exists() and sync.exists() and derive.exists(), (
|
assert not (notes.exists() or sync.exists() or derive.exists()), (
|
||||||
"a note ledger was cleared too. The note arms were deliberately left "
|
"a note ledger outlived the context it describes, so its records are "
|
||||||
"out of #3749 — clearing them is a decision about a different surface, "
|
"marked `seen` to a session that can no longer see them"
|
||||||
"and a glob that takes them along makes it by accident."
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,228 @@
|
|||||||
|
"""A compaction clears every session ledger, by convention not by list (#4101).
|
||||||
|
|
||||||
|
WHY THIS EXISTS
|
||||||
|
|
||||||
|
Milestone 386 established the claim: a ledger describes what a session HOLDS,
|
||||||
|
so the events that destroy context must destroy it too, or the records it names
|
||||||
|
become permanently unreachable mid-session. `scribe_session_context.sh`
|
||||||
|
implemented that — for the rules ledger, by name.
|
||||||
|
|
||||||
|
Five ledgers live in that directory and two were on the list. The note arms'
|
||||||
|
three (`.ids`, `.sync.ids`, `.derive.ids`) were left, under a comment asserting
|
||||||
|
this was "a decision rather than an oversight". It was not a decision, and the
|
||||||
|
arms left out are the ones where it costs most:
|
||||||
|
|
||||||
|
- their exclusions are HARD — `exclude_ids` is passed into
|
||||||
|
`semantic_search_notes` itself, so a surfaced note leaves the result set
|
||||||
|
entirely, with no weaker rendering to fall back to the way #3750 gave a
|
||||||
|
repeated rule one;
|
||||||
|
- and they never AGE — #3751's TTL was added to the rules ledger only.
|
||||||
|
|
||||||
|
Hard, permanent and never cleared: a note surfaced in a session's first minute
|
||||||
|
is unreachable for the rest of it, through any number of compactions.
|
||||||
|
|
||||||
|
WHAT THIS PINS
|
||||||
|
|
||||||
|
The list was the bug, so the fix cannot be a longer list and neither can the
|
||||||
|
test. Both sides are asserted:
|
||||||
|
|
||||||
|
1. BEHAVIOUR — the hook is run on a real `compact` event with all five
|
||||||
|
ledgers on disk, and all five are gone afterwards. Run rather than
|
||||||
|
grepped, because grepping for the names is the hand-maintained pattern
|
||||||
|
this step removes.
|
||||||
|
2. THE CONVENTION THE BEHAVIOUR RESTS ON — every per-session ledger any hook
|
||||||
|
builds is named `<sid>[.<kind>].ids`. That is what makes a sixth ledger
|
||||||
|
covered on the day it is written, and it is the assumption that would
|
||||||
|
rot silently, because a ledger named outside it simply never clears and
|
||||||
|
nothing says so.
|
||||||
|
|
||||||
|
And the negative: `<sid>.unreached` is not a ledger of held context but a
|
||||||
|
record that the instance could not be reached, and a glob that swept it away
|
||||||
|
would make a hook forget an outage it is meant to report (#2932).
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
HOOKS = Path(__file__).resolve().parents[1] / "plugin" / "hooks"
|
||||||
|
SESSION_START = HOOKS / "scribe_session_context.sh"
|
||||||
|
DEFS = HOOKS / "scribe_defs.sh"
|
||||||
|
|
||||||
|
# Every ledger the hooks write today, and where. Named here so a failure READS
|
||||||
|
# as "this one survived", but never used to build the hook's own delete list —
|
||||||
|
# the convention tests below are what keep this roster honest as it grows.
|
||||||
|
LEDGERS = {
|
||||||
|
"scribe-priorart": (".ids", ".rules.ids", ".opened.ids", ".sync.ids",
|
||||||
|
".derive.ids"),
|
||||||
|
"scribe-autoinject": (".ids",),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _swept_dirs() -> set[str]:
|
||||||
|
"""The directories the SHIPPED hook sweeps, read from the hook itself.
|
||||||
|
|
||||||
|
Read rather than restated: a test carrying its own copy of the roster would
|
||||||
|
agree with itself forever, which is precisely the failure that let the
|
||||||
|
auto-inject ledger sit in a directory nothing cleared.
|
||||||
|
"""
|
||||||
|
line = re.search(r'SCRIBE_LEDGER_DIRS="([^"]*)"', DEFS.read_text())
|
||||||
|
assert line, "SCRIBE_LEDGER_DIRS is gone; the clear has no roster"
|
||||||
|
return set(line.group(1).split())
|
||||||
|
|
||||||
|
|
||||||
|
def _run_session_start(source: str, tmp: Path) -> Path:
|
||||||
|
"""Run the SessionStart hook for real, with the ledger directories filled."""
|
||||||
|
for tool in ("bash", "jq"):
|
||||||
|
if shutil.which(tool) is None:
|
||||||
|
pytest.skip(f"hook runtime tool {tool!r} not installed")
|
||||||
|
|
||||||
|
for dirname, suffixes in LEDGERS.items():
|
||||||
|
state = tmp / dirname
|
||||||
|
state.mkdir(parents=True, exist_ok=True)
|
||||||
|
for suffix in suffixes:
|
||||||
|
(state / f"s1{suffix}").write_text("42\t1789600000\n")
|
||||||
|
# Not a ledger: an outage marker that must outlive the clear.
|
||||||
|
(tmp / "scribe-priorart" / "s1.unreached").write_text("1\n")
|
||||||
|
|
||||||
|
env = {"PATH": os.environ["PATH"], "HOME": str(tmp), "TMPDIR": str(tmp)}
|
||||||
|
out = subprocess.run(
|
||||||
|
["bash", str(SESSION_START)],
|
||||||
|
input=json.dumps({"session_id": "s1", "source": source}),
|
||||||
|
capture_output=True, text=True, env=env, timeout=60,
|
||||||
|
)
|
||||||
|
assert out.returncode == 0, out.stderr
|
||||||
|
return tmp
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("source", ["compact", "clear"])
|
||||||
|
def test_a_context_destroying_source_clears_every_ledger(source, tmp_path):
|
||||||
|
"""The step, stated as behaviour: all of them, in both directories."""
|
||||||
|
root = _run_session_start(source, tmp_path)
|
||||||
|
survived = [f"{d}/s1{s}" for d, suffixes in LEDGERS.items()
|
||||||
|
for s in suffixes if (root / d / f"s1{s}").exists()]
|
||||||
|
assert not survived, (
|
||||||
|
f"{survived} survived a {source!r} that destroyed what they describe"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("source", ["startup", "resume"])
|
||||||
|
def test_a_source_that_kept_the_context_keeps_the_ledgers(source, tmp_path):
|
||||||
|
"""The mirror error, and the more expensive one.
|
||||||
|
|
||||||
|
`resume` genuinely restores the context, so the ledger still describes what
|
||||||
|
the session holds; clearing there would re-surface every record after a
|
||||||
|
restore that lost nothing. A blanket glob makes over-clearing cheap to
|
||||||
|
write, which is exactly why this direction needs a test of its own.
|
||||||
|
"""
|
||||||
|
root = _run_session_start(source, tmp_path)
|
||||||
|
for dirname, suffixes in LEDGERS.items():
|
||||||
|
for suffix in suffixes:
|
||||||
|
assert (root / dirname / f"s1{suffix}").exists(), (
|
||||||
|
f"{dirname}/s1{suffix} was cleared on {source!r}, which lost "
|
||||||
|
"no context"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_outage_marker_is_not_swept_with_them(tmp_path):
|
||||||
|
"""`.unreached` records that the instance was down, not what was surfaced.
|
||||||
|
|
||||||
|
Different lifetime, different question. #2932's whole point is that "we
|
||||||
|
checked and found nothing" and "we never managed to check" must stay
|
||||||
|
distinguishable, and a clear that took this file out would quietly answer
|
||||||
|
the second with the first.
|
||||||
|
"""
|
||||||
|
root = _run_session_start("compact", tmp_path)
|
||||||
|
assert (root / "scribe-priorart" / "s1.unreached").exists()
|
||||||
|
|
||||||
|
|
||||||
|
# ── the two assumptions the sweep rests on ─────────────────────────────────
|
||||||
|
#
|
||||||
|
# Both are checked against the hooks rather than trusted, because neither fails
|
||||||
|
# loudly: a ledger outside them simply never clears, on the arm whose author had
|
||||||
|
# no reason to know a convention existed.
|
||||||
|
|
||||||
|
def _dir_vars(text: str) -> dict[str, str]:
|
||||||
|
"""`var="${TMPDIR:-/tmp}/<name>"` → {var: name}, per script."""
|
||||||
|
return dict(re.findall(
|
||||||
|
r'(\w+)="\$\{TMPDIR:-/tmp\}/([A-Za-z0-9_-]+)"', text))
|
||||||
|
|
||||||
|
|
||||||
|
def _session_paths(text: str):
|
||||||
|
"""Every `$<var>/${safe_sid}<suffix>` a script composes."""
|
||||||
|
return re.findall(r'\$(\w+)"?/\$\{safe_sid\}([A-Za-z0-9_.]*)', text)
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_directory_holding_a_ledger_is_one_the_clear_visits():
|
||||||
|
"""The failure that shipped once already, in the same step that fixed it.
|
||||||
|
|
||||||
|
The first cut swept `scribe-priorart` alone. Every ledger NAMED in the
|
||||||
|
hooks was there, so it read as complete — and `scribe_autoinject.sh` keeps
|
||||||
|
its note ledger in `scribe-autoinject`, which meant the arm that fires most
|
||||||
|
was the only one still carrying the bug. Nothing said so: the clear ran,
|
||||||
|
found nothing to remove, and exited 0.
|
||||||
|
"""
|
||||||
|
swept = _swept_dirs()
|
||||||
|
offenders = []
|
||||||
|
for script in sorted(HOOKS.glob("*.sh")):
|
||||||
|
text = script.read_text()
|
||||||
|
dirs = _dir_vars(text)
|
||||||
|
for var, suffix in _session_paths(text):
|
||||||
|
if not suffix.endswith(".ids"):
|
||||||
|
continue
|
||||||
|
where = dirs.get(var)
|
||||||
|
if where not in swept:
|
||||||
|
offenders.append(f"{script.name}: ${var} -> {where or '?'}")
|
||||||
|
|
||||||
|
assert not offenders, (
|
||||||
|
"these ledgers live in directories the compaction clear never visits, "
|
||||||
|
f"so they outlive the context they describe: {offenders}. Add the "
|
||||||
|
"directory to SCRIBE_LEDGER_DIRS in scribe_defs.sh."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_session_file_in_a_swept_directory_follows_the_convention():
|
||||||
|
"""The other half: the sweep matches `.ids`, so a ledger must be named it.
|
||||||
|
|
||||||
|
Stated as its own test because the two assumptions fail differently — this
|
||||||
|
one lets a ledger sit in the right directory and still never clear.
|
||||||
|
"""
|
||||||
|
# Session files in a swept directory that are NOT per-session ledgers.
|
||||||
|
NOT_LEDGERS = {".unreached"}
|
||||||
|
|
||||||
|
swept = _swept_dirs()
|
||||||
|
offenders = []
|
||||||
|
for script in sorted(HOOKS.glob("*.sh")):
|
||||||
|
text = script.read_text()
|
||||||
|
dirs = _dir_vars(text)
|
||||||
|
for var, suffix in _session_paths(text):
|
||||||
|
if dirs.get(var) not in swept:
|
||||||
|
continue # a different directory, a different question
|
||||||
|
if suffix in NOT_LEDGERS or suffix.endswith(".ids"):
|
||||||
|
continue
|
||||||
|
offenders.append(f"{script.name}: ${var}/${{safe_sid}}{suffix}")
|
||||||
|
|
||||||
|
assert not offenders, (
|
||||||
|
"these session files sit in a swept directory but clear on no "
|
||||||
|
f"compaction, because the sweep matches `.ids`: {offenders}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_clear_is_derived_and_not_a_list_of_names():
|
||||||
|
"""The regression that would look like a fix.
|
||||||
|
|
||||||
|
Appending an `rm -f` per ledger passes every behavioural test above while
|
||||||
|
rebuilding the trap for the next one. The property worth keeping is that
|
||||||
|
the hook names no ledger at all.
|
||||||
|
"""
|
||||||
|
sh = SESSION_START.read_text()
|
||||||
|
block = sh.split('case "$source" in')[1].split("esac")[0]
|
||||||
|
assert "scribe_clear_session_ledgers" in block
|
||||||
|
for suffix in {s for suffixes in LEDGERS.values() for s in suffixes}:
|
||||||
|
assert suffix not in block, (
|
||||||
|
f"the clear names {suffix} again — a list, not a convention"
|
||||||
|
)
|
||||||
@@ -20,9 +20,12 @@ string equals the Python default. Retuning either stays free as long as both
|
|||||||
move — which is the point, because these are tuning values and #3853 moved one
|
move — which is the point, because these are tuning values and #3853 moved one
|
||||||
of them the day this was written.
|
of them the day this was written.
|
||||||
|
|
||||||
Both sides are read from SOURCE rather than imported. The Vue file cannot be
|
The Vue side is read from SOURCE, because that file cannot be imported at all.
|
||||||
imported at all, and reading the Python constant through an import would tie
|
The Python side is read from source for a loose constant and IMPORTED for a
|
||||||
this to module-load side effects it has no interest in.
|
registry surface — the registry is a plain table of frozen dataclasses with
|
||||||
|
nothing to trigger on import, and importing it means this guard checks the value
|
||||||
|
the server will actually resolve rather than a literal that happens to look
|
||||||
|
right.
|
||||||
|
|
||||||
It cannot check that the form WRITES the right key — that is behaviour, and
|
It cannot check that the form WRITES the right key — that is behaviour, and
|
||||||
the keys are asserted where they are built. It catches the drift that has no
|
the keys are asserted where they are built. It catches the drift that has no
|
||||||
@@ -39,20 +42,29 @@ ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|||||||
_SERVICES = ROOT / "src" / "scribe" / "services"
|
_SERVICES = ROOT / "src" / "scribe" / "services"
|
||||||
_VUE = ROOT / "frontend" / "src" / "views" / "SettingsView.vue"
|
_VUE = ROOT / "frontend" / "src" / "views" / "SettingsView.vue"
|
||||||
|
|
||||||
# (services module, python constant, vue ref). Hand-written because the
|
# THE SIX RETRIEVAL FLOORS now live in one registry (#4102), so their side of
|
||||||
# pairing is an editorial fact — the names do not share a convention either
|
# the pairing is a SURFACE NAME rather than a module constant. That is strictly
|
||||||
# side could derive — but every entry is asserted to EXIST on both sides, so a
|
# better for this guard: the surface name is also the telemetry source, so a row
|
||||||
# rename fails loudly here rather than silently dropping that threshold from
|
# here names the same arm the readout does, and a floor that moved cannot be
|
||||||
# the check.
|
# checked against a stale constant that happened to keep its old value.
|
||||||
_PAIRS = (
|
_SURFACE_PAIRS = (
|
||||||
("plugin_context.py", "AUTOINJECT_DEFAULT_THRESHOLD", "kbInjectThreshold"),
|
("auto_inject", "kbInjectThreshold"),
|
||||||
("plugin_context.py", "WRITEPATH_DEFAULT_THRESHOLD", "kbWritePathThreshold"),
|
("write_path", "kbWritePathThreshold"),
|
||||||
("plugin_context.py", "RULEHINT_DEFAULT_THRESHOLD", "kbRuleHintThreshold"),
|
("write_path_rule", "kbRuleHintThreshold"),
|
||||||
("plugin_context.py", "TOOLRULE_DEFAULT_THRESHOLD", "kbToolRuleThreshold"),
|
("pre_tool_rule", "kbToolRuleThreshold"),
|
||||||
("plugin_context.py", "PROMPTRULE_DEFAULT_THRESHOLD", "kbPromptRuleThreshold"),
|
("prompt_rule", "kbPromptRuleThreshold"),
|
||||||
("plugin_context.py", "REPORTPREF_DEFAULT_THRESHOLD", "kbReportPrefThreshold"),
|
("report_preference", "kbReportPrefThreshold"),
|
||||||
|
)
|
||||||
|
|
||||||
|
# (services module, python constant, vue ref) for the defaults that are NOT
|
||||||
|
# retrieval surfaces. Hand-written because the pairing is an editorial fact —
|
||||||
|
# the names share no convention either side could derive — and asserted to
|
||||||
|
# exist on both sides, so a rename fails loudly rather than silently dropping
|
||||||
|
# that threshold from the check.
|
||||||
|
_CONSTANT_PAIRS = (
|
||||||
# The plan gate (milestone 415): it blocks a create, so a form showing a
|
# The plan gate (milestone 415): it blocks a create, so a form showing a
|
||||||
# looser bar than the one in force would be the more misleading drift.
|
# looser bar than the one in force would be the more misleading drift. Not
|
||||||
|
# a push surface, so it has no registry entry and keeps the older shape.
|
||||||
("dedup.py", "PLAN_MATCH_DEFAULT_THRESHOLD", "kbPlanMatchThreshold"),
|
("dedup.py", "PLAN_MATCH_DEFAULT_THRESHOLD", "kbPlanMatchThreshold"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -80,14 +92,42 @@ def _vue_default(ref_name: str) -> float:
|
|||||||
return float(m.group(1))
|
return float(m.group(1))
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(("module", "constant", "ref_name"), _PAIRS,
|
def _assert_agrees(server: float, form: float, ref_name: str, what: str) -> None:
|
||||||
ids=[p[1] for p in _PAIRS])
|
|
||||||
def test_the_form_shows_the_default_the_server_uses(module, constant, ref_name):
|
|
||||||
"""An untouched control must render the bar actually in force."""
|
|
||||||
server, form = _python_default(module, constant), _vue_default(ref_name)
|
|
||||||
assert form == server, (
|
assert form == server, (
|
||||||
f"SettingsView shows {form} for {ref_name} while the server defaults "
|
f"SettingsView shows {form} for {ref_name} while the server defaults "
|
||||||
f"to {server} ({constant}). An operator who has never set this reads "
|
f"to {server} ({what}). An operator who has never set this reads the "
|
||||||
f"the form as the value in force, so the two must move together — "
|
f"form as the value in force, so the two must move together — retune "
|
||||||
f"retune both, or neither."
|
f"both, or neither."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("surface", "ref_name"), _SURFACE_PAIRS,
|
||||||
|
ids=[p[0] for p in _SURFACE_PAIRS])
|
||||||
|
def test_the_form_shows_the_floor_the_surface_uses(surface, ref_name):
|
||||||
|
"""An untouched control must render the bar actually in force."""
|
||||||
|
from scribe.services.retrieval_surfaces import get_surface
|
||||||
|
|
||||||
|
_assert_agrees(get_surface(surface).floor_default, _vue_default(ref_name),
|
||||||
|
ref_name, f"{surface} floor")
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_tunable_surface_has_a_control():
|
||||||
|
"""Rule 25/27: a number an operator may need different has a UI or it is
|
||||||
|
not shipped. Derived from the registry, so a seventh surface arrives here
|
||||||
|
as a failure rather than as a setting only the model can reach."""
|
||||||
|
from scribe.services.retrieval_surfaces import surface_names
|
||||||
|
|
||||||
|
covered = {s for s, _ref in _SURFACE_PAIRS}
|
||||||
|
missing = [n for n in surface_names() if n not in covered]
|
||||||
|
assert not missing, (
|
||||||
|
f"these surfaces are tunable with no Settings control: {missing}. "
|
||||||
|
"Add the control and its row, or say in _SURFACE_PAIRS why it has none."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("module", "constant", "ref_name"), _CONSTANT_PAIRS,
|
||||||
|
ids=[p[1] for p in _CONSTANT_PAIRS])
|
||||||
|
def test_the_form_shows_the_default_the_server_uses(module, constant, ref_name):
|
||||||
|
"""Same claim, for the defaults that are not retrieval surfaces."""
|
||||||
|
_assert_agrees(_python_default(module, constant), _vue_default(ref_name),
|
||||||
|
ref_name, constant)
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ source, and the two ways this must stay silent. Plus the plugin hook contract
|
|||||||
a PreToolUse hook that returns a permission decision would be able to block the
|
a PreToolUse hook that returns a permission decision would be able to block the
|
||||||
operator's edit, which this feature must never do.
|
operator's edit, which this feature must never do.
|
||||||
"""
|
"""
|
||||||
|
import contextlib
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
@@ -13,7 +14,30 @@ from pathlib import Path
|
|||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from tests.helpers import fake_note, http_sink
|
from tests.helpers import fake_note, http_sink, writepath_cfg
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def _stored_settings(stored):
|
||||||
|
"""Patch BOTH readers, because one call now uses two (#4102).
|
||||||
|
|
||||||
|
`get_writepath_config` reads its `enabled` switch through
|
||||||
|
`plugin_context.get_setting` and all six tunable numbers through
|
||||||
|
`retrieval_surfaces.get_setting`. Patching only the first leaves the numbers
|
||||||
|
talking to a real database — which in a unit job is a connection error, and
|
||||||
|
in an integration job is worse: the test would pass or fail on whatever the
|
||||||
|
instance happened to have stored.
|
||||||
|
"""
|
||||||
|
from scribe.services import plugin_context as pc
|
||||||
|
from scribe.services import retrieval_surfaces as rs
|
||||||
|
|
||||||
|
def _side(uid, k, d=""):
|
||||||
|
return stored.get(k, d)
|
||||||
|
|
||||||
|
with patch.object(pc, "get_setting", AsyncMock(side_effect=_side)), \
|
||||||
|
patch.object(rs, "get_setting", AsyncMock(side_effect=_side)):
|
||||||
|
yield
|
||||||
|
|
||||||
|
|
||||||
PLUGIN = Path(__file__).resolve().parents[1] / "plugin"
|
PLUGIN = Path(__file__).resolve().parents[1] / "plugin"
|
||||||
HOOK = PLUGIN / "hooks" / "scribe_prior_art.sh"
|
HOOK = PLUGIN / "hooks" / "scribe_prior_art.sh"
|
||||||
@@ -31,9 +55,7 @@ def _cfg(**over):
|
|||||||
# missing key raises inside its fail-open except and turns the arm into a
|
# missing key raises inside its fail-open except and turns the arm into a
|
||||||
# silent no-op — which is indistinguishable from it working and finding
|
# silent no-op — which is indistinguishable from it working and finding
|
||||||
# nothing.
|
# nothing.
|
||||||
base = {"enabled": True, "threshold": 0.68, "top_k": 3, "rule_threshold": 0.72}
|
return writepath_cfg(**over)
|
||||||
base.update(over)
|
|
||||||
return base
|
|
||||||
|
|
||||||
|
|
||||||
# The semantic arm ignores payloads carrying less than WRITEPATH_MIN_CODE_CHARS
|
# The semantic arm ignores payloads carrying less than WRITEPATH_MIN_CODE_CHARS
|
||||||
@@ -310,9 +332,23 @@ async def test_semantic_arm_only_asks_for_the_budget_the_place_arm_left():
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_session_dedup_excludes_ids_from_the_reuse_arms():
|
async def test_session_dedup_marks_the_reuse_arms_rather_than_silencing_them():
|
||||||
"""exclude_ids governs the REUSE classes — nearby and semantic. (The sync
|
"""exclude_ids governs the REUSE classes — nearby and semantic. (The sync
|
||||||
class has its own channel; see the tests above.)"""
|
class has its own channel; see the tests above.)
|
||||||
|
|
||||||
|
WHAT CHANGED AND WHY (#4101). This used to assert the opposite: the nearby
|
||||||
|
hit was dropped and its id pushed into the search's `exclude_ids`. That was
|
||||||
|
#3750's defect on the note arms — the second time a record was the best
|
||||||
|
answer, the session got silence, which reads exactly like "nothing is
|
||||||
|
recorded here". Now the repeat is rendered with a `seen` marker, and the
|
||||||
|
ledger never reaches the query, so the score the search reports describes
|
||||||
|
the bar rather than a candidate set the caller had already edited.
|
||||||
|
|
||||||
|
Marked on BOTH reuse arms, which is what this asserts: one menu with two
|
||||||
|
rules — `seen` on the semantic hits, silence on the nearby ones — would
|
||||||
|
make the marker read as a complete account of what the session has met,
|
||||||
|
when it would only cover half the lines.
|
||||||
|
"""
|
||||||
from scribe.services import plugin_context as pc
|
from scribe.services import plugin_context as pc
|
||||||
|
|
||||||
async def _listing(uid, **kw):
|
async def _listing(uid, **kw):
|
||||||
@@ -327,8 +363,15 @@ async def test_session_dedup_excludes_ids_from_the_reuse_arms():
|
|||||||
patch.object(pc, "semantic_search_notes", search), \
|
patch.object(pc, "semantic_search_notes", search), \
|
||||||
patch.object(pc, "record_retrieval", MagicMock()):
|
patch.object(pc, "record_retrieval", MagicMock()):
|
||||||
out = await pc.build_write_path_hint(1, "src/x.py", code=REAL_CODE, exclude_ids=[12])
|
out = await pc.build_write_path_hint(1, "src/x.py", code=REAL_CODE, exclude_ids=[12])
|
||||||
# The nearby hit was already surfaced this session → dropped, not repeated.
|
assert out["note_ids"] == [12]
|
||||||
assert out["note_ids"] == []
|
assert "seen" in next(
|
||||||
|
line for line in out["context"].splitlines() if "#12" in line
|
||||||
|
)
|
||||||
|
# #12 IS in this call's `exclude_ids`, and that is the other rule working:
|
||||||
|
# the place arm has just listed it, so the semantic arm must not list it
|
||||||
|
# again. What must never reach the query is the LEDGER, which is asserted
|
||||||
|
# on its own in tests/test_ledger_references_not_silence.py against a
|
||||||
|
# record this call has not otherwise rendered.
|
||||||
assert 12 in search.await_args.kwargs["exclude_ids"]
|
assert 12 in search.await_args.kwargs["exclude_ids"]
|
||||||
|
|
||||||
|
|
||||||
@@ -402,11 +445,10 @@ async def test_sync_surfacing_is_measured_under_its_own_usage_source():
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_config_has_its_own_switch_and_threshold_but_shares_top_k():
|
async def test_config_has_its_own_switch_threshold_and_inherited_budget():
|
||||||
from scribe.services import plugin_context as pc
|
from scribe.services import plugin_context as pc
|
||||||
stored = {pc.WRITEPATH_ENABLED_KEY: "false"}
|
stored = {pc.WRITEPATH_ENABLED_KEY: "false"}
|
||||||
with patch.object(pc, "get_setting",
|
with _stored_settings(stored):
|
||||||
AsyncMock(side_effect=lambda uid, k, d: stored.get(k, d))):
|
|
||||||
cfg = await pc.get_writepath_config(1)
|
cfg = await pc.get_writepath_config(1)
|
||||||
# Its own switch is off while auto-inject stays on...
|
# Its own switch is off while auto-inject stays on...
|
||||||
assert cfg["enabled"] is False
|
assert cfg["enabled"] is False
|
||||||
@@ -415,8 +457,12 @@ async def test_config_has_its_own_switch_and_threshold_but_shares_top_k():
|
|||||||
# made unrelated code — including `x = 1` at 0.58 — clear the bar.
|
# made unrelated code — including `x = 1` at 0.58 — clear the bar.
|
||||||
assert cfg["threshold"] == pc.WRITEPATH_DEFAULT_THRESHOLD
|
assert cfg["threshold"] == pc.WRITEPATH_DEFAULT_THRESHOLD
|
||||||
assert cfg["threshold"] > pc.AUTOINJECT_DEFAULT_THRESHOLD
|
assert cfg["threshold"] > pc.AUTOINJECT_DEFAULT_THRESHOLD
|
||||||
# ...and top_k is still shared: "how many titles at once" means the same
|
# ...and the budget is INHERITED rather than shared (#4102). This arm has
|
||||||
# thing on both surfaces.
|
# its own key now, because it fires before every Write and Edit while
|
||||||
|
# auto-inject fires once a turn, so the same number buys very different
|
||||||
|
# amounts of attention. Unset, it still reads auto-inject's — which is what
|
||||||
|
# stops the split from silently resetting an install that had tuned the
|
||||||
|
# knob when it was shared.
|
||||||
assert cfg["top_k"] == pc.AUTOINJECT_DEFAULT_TOP_K
|
assert cfg["top_k"] == pc.AUTOINJECT_DEFAULT_TOP_K
|
||||||
|
|
||||||
|
|
||||||
@@ -428,8 +474,7 @@ async def test_writepath_threshold_is_operator_tunable_and_clamped():
|
|||||||
|
|
||||||
async def _cfg_with(raw):
|
async def _cfg_with(raw):
|
||||||
stored = {pc.WRITEPATH_THRESHOLD_KEY: raw}
|
stored = {pc.WRITEPATH_THRESHOLD_KEY: raw}
|
||||||
with patch.object(pc, "get_setting",
|
with _stored_settings(stored):
|
||||||
AsyncMock(side_effect=lambda uid, k, d: stored.get(k, d))):
|
|
||||||
return await pc.get_writepath_config(1)
|
return await pc.get_writepath_config(1)
|
||||||
|
|
||||||
assert (await _cfg_with("0.9"))["threshold"] == 0.9
|
assert (await _cfg_with("0.9"))["threshold"] == 0.9
|
||||||
@@ -453,8 +498,7 @@ async def test_the_rule_arm_has_its_own_tunable_bar():
|
|||||||
|
|
||||||
async def _cfg_with(raw):
|
async def _cfg_with(raw):
|
||||||
stored = {pc.RULEHINT_THRESHOLD_KEY: raw}
|
stored = {pc.RULEHINT_THRESHOLD_KEY: raw}
|
||||||
with patch.object(pc, "get_setting",
|
with _stored_settings(stored):
|
||||||
AsyncMock(side_effect=lambda uid, k, d: stored.get(k, d))):
|
|
||||||
return await pc.get_writepath_config(1)
|
return await pc.get_writepath_config(1)
|
||||||
|
|
||||||
assert (await _cfg_with("0.8"))["rule_threshold"] == 0.8
|
assert (await _cfg_with("0.8"))["rule_threshold"] == 0.8
|
||||||
@@ -473,8 +517,7 @@ async def test_the_two_write_path_bars_are_independent():
|
|||||||
from scribe.services import plugin_context as pc
|
from scribe.services import plugin_context as pc
|
||||||
|
|
||||||
stored = {pc.WRITEPATH_THRESHOLD_KEY: "0.90", pc.RULEHINT_THRESHOLD_KEY: "0.61"}
|
stored = {pc.WRITEPATH_THRESHOLD_KEY: "0.90", pc.RULEHINT_THRESHOLD_KEY: "0.61"}
|
||||||
with patch.object(pc, "get_setting",
|
with _stored_settings(stored):
|
||||||
AsyncMock(side_effect=lambda uid, k, d: stored.get(k, d))):
|
|
||||||
cfg = await pc.get_writepath_config(1)
|
cfg = await pc.get_writepath_config(1)
|
||||||
|
|
||||||
assert cfg["threshold"] == 0.90
|
assert cfg["threshold"] == 0.90
|
||||||
@@ -495,8 +538,7 @@ async def test_the_two_act_arms_read_independent_rule_bars():
|
|||||||
from scribe.services import plugin_context as pc
|
from scribe.services import plugin_context as pc
|
||||||
|
|
||||||
stored = {pc.RULEHINT_THRESHOLD_KEY: "0.75", pc.TOOLRULE_THRESHOLD_KEY: "0.61"}
|
stored = {pc.RULEHINT_THRESHOLD_KEY: "0.75", pc.TOOLRULE_THRESHOLD_KEY: "0.61"}
|
||||||
with patch.object(pc, "get_setting",
|
with _stored_settings(stored):
|
||||||
AsyncMock(side_effect=lambda uid, k, d: stored.get(k, d))):
|
|
||||||
cfg = await pc.get_writepath_config(1)
|
cfg = await pc.get_writepath_config(1)
|
||||||
|
|
||||||
assert cfg["rule_threshold"] == 0.75
|
assert cfg["rule_threshold"] == 0.75
|
||||||
@@ -515,8 +557,7 @@ async def test_a_garbage_command_bar_falls_back_to_its_own_default():
|
|||||||
from scribe.services import plugin_context as pc
|
from scribe.services import plugin_context as pc
|
||||||
|
|
||||||
stored = {pc.TOOLRULE_THRESHOLD_KEY: "banana"}
|
stored = {pc.TOOLRULE_THRESHOLD_KEY: "banana"}
|
||||||
with patch.object(pc, "get_setting",
|
with _stored_settings(stored):
|
||||||
AsyncMock(side_effect=lambda uid, k, d: stored.get(k, d))):
|
|
||||||
cfg = await pc.get_writepath_config(1)
|
cfg = await pc.get_writepath_config(1)
|
||||||
|
|
||||||
assert cfg["tool_rule_threshold"] == pc.TOOLRULE_DEFAULT_THRESHOLD
|
assert cfg["tool_rule_threshold"] == pc.TOOLRULE_DEFAULT_THRESHOLD
|
||||||
@@ -1264,8 +1305,16 @@ async def test_stamping_needs_named_shapes_and_a_recent_pull():
|
|||||||
async def test_a_pulled_snippet_already_seen_is_evidence_not_menu():
|
async def test_a_pulled_snippet_already_seen_is_evidence_not_menu():
|
||||||
"""The pulled-then-written flow IS the dedup-excluded flow: the hint offered
|
"""The pulled-then-written flow IS the dedup-excluded flow: the hint offered
|
||||||
#7 earlier (so it sits in exclude_ids), the session pulled it, and now
|
#7 earlier (so it sits in exclude_ids), the session pulled it, and now
|
||||||
writes code resembling it. #7 must be scored for this payload — and handed
|
writes code resembling it. #7 must be scored for this payload and handed to
|
||||||
to the stamp as resemblance — without being re-listed in the menu."""
|
the stamp as resemblance.
|
||||||
|
|
||||||
|
THE MENU HALF INVERTED AT #4101. This asserted that #7 was evidence and NOT
|
||||||
|
menu — scored for the stamp, kept out of the lines. That followed from the
|
||||||
|
ledger being a hard exclusion; it is now a marker, so #7 is both, and the
|
||||||
|
`limit` no longer needs widening for it because nothing is dropped after
|
||||||
|
the search. What survives unchanged is the part the stamp depends on: the
|
||||||
|
pulled id stays in the query and its score reaches `resembles`.
|
||||||
|
"""
|
||||||
from scribe.services import plugin_context as pc
|
from scribe.services import plugin_context as pc
|
||||||
search = AsyncMock(return_value=[(0.91, fake_note(id=7, title="pulled", user_id=1, note_type="snippet")), (0.80, fake_note(id=8, title="fresh", user_id=1, note_type="snippet"))])
|
search = AsyncMock(return_value=[(0.91, fake_note(id=7, title="pulled", user_id=1, note_type="snippet")), (0.80, fake_note(id=8, title="fresh", user_id=1, note_type="snippet"))])
|
||||||
stamp = AsyncMock(return_value=[{
|
stamp = AsyncMock(return_value=[{
|
||||||
@@ -1283,15 +1332,20 @@ async def test_a_pulled_snippet_already_seen_is_evidence_not_menu():
|
|||||||
1, "src/x.py", code=REAL_CODE, project_id=4, exclude_ids=[7],
|
1, "src/x.py", code=REAL_CODE, project_id=4, exclude_ids=[7],
|
||||||
stamp_shapes=[("sym", "debounce")], repo_key="git.example.com/a/b",
|
stamp_shapes=[("sym", "debounce")], repo_key="git.example.com/a/b",
|
||||||
)
|
)
|
||||||
# The query kept #7 eligible (and widened the budget by one for it)...
|
# The query kept #7 eligible, and needs no extra budget for it now that
|
||||||
|
# nothing is dropped between the search and the menu.
|
||||||
kw = search.call_args.kwargs
|
kw = search.call_args.kwargs
|
||||||
assert 7 not in kw["exclude_ids"]
|
assert 7 not in kw["exclude_ids"]
|
||||||
assert kw["limit"] == 4
|
assert kw["limit"] == 3
|
||||||
# ...but the menu still honours the session dedup.
|
# ...and the menu carries it, marked.
|
||||||
assert out["note_ids"] == [8]
|
seven = next(line for line in out["context"].splitlines() if "#7" in line)
|
||||||
assert "#7" not in "\n".join(
|
assert "seen" in seven
|
||||||
line for line in out["context"].splitlines() if "[similar" in line
|
# #8 (0.80) now falls OUTSIDE the margin band, because #7 (0.91) anchors it
|
||||||
)
|
# at 0.81 instead of being filtered out first. That is the point rather than
|
||||||
|
# a casualty: the band measures distance from the best answer, and the best
|
||||||
|
# answer here is #7 — suppressing it used to promote a materially weaker hit
|
||||||
|
# into a slot it had not earned, with nothing in the output saying so.
|
||||||
|
assert out["note_ids"] == [7]
|
||||||
# The stamp saw the pull and the resemblance score for this payload.
|
# The stamp saw the pull and the resemblance score for this payload.
|
||||||
skw = stamp.call_args.kwargs
|
skw = stamp.call_args.kwargs
|
||||||
assert skw["pulled"] == {7: skw["pulled"][7]}
|
assert skw["pulled"] == {7: skw["pulled"][7]}
|
||||||
@@ -1670,9 +1724,47 @@ async def _write_path_row(rec, **kwargs):
|
|||||||
async def test_a_record_this_arm_withheld_itself_is_not_a_near_miss():
|
async def test_a_record_this_arm_withheld_itself_is_not_a_near_miss():
|
||||||
"""The defect: the row's count is POST this arm's filter and the score was
|
"""The defect: the row's count is POST this arm's filter and the score was
|
||||||
captured PRE it, so a withheld record is indistinguishable from one the bar
|
captured PRE it, so a withheld record is indistinguishable from one the bar
|
||||||
rejected — while scoring higher than anything the bar ever let through."""
|
rejected — while scoring higher than anything the bar ever let through.
|
||||||
|
|
||||||
|
REACHED DIFFERENTLY SINCE #4101, and that is the news. The ledger used to
|
||||||
|
produce this case and no longer can: a repeat comes back from the search
|
||||||
|
and is rendered, so nothing is dropped and `best_available` describes the
|
||||||
|
bar alone. What still produces it is the one post-search filter left — a
|
||||||
|
PULLED record that this same call already listed by place. It stays in the
|
||||||
|
query because `resembles` needs its score, and is dropped from the menu
|
||||||
|
because it is already on it, which is exactly the pre/post split.
|
||||||
|
|
||||||
|
The property is unchanged and worth as much as it ever was; only the setup
|
||||||
|
that exhibits it moved. Written this way rather than deleted, because the
|
||||||
|
filter is still there and an arm that reported a score for it would be
|
||||||
|
making the same false claim about the bar.
|
||||||
|
"""
|
||||||
|
from scribe.services import plugin_context as pc
|
||||||
rec = MagicMock()
|
rec = MagicMock()
|
||||||
row = await _write_path_row(rec, exclude_ids=[7])
|
# Recorded at a sibling file → listed by PLACE in this same call, and
|
||||||
|
# pulled this session → kept in the query as resemblance evidence.
|
||||||
|
async def _listing(uid, **kw):
|
||||||
|
if kw["path"] == "src":
|
||||||
|
return ([_snippet_item(7, "already listed by place")], 1)
|
||||||
|
return ([], 0)
|
||||||
|
|
||||||
|
with patch.object(pc, "get_writepath_config", AsyncMock(return_value=_cfg())), \
|
||||||
|
patch.object(pc.snippets_svc, "list_snippets", _listing), \
|
||||||
|
patch.object(pc, "record_retrieval", rec), \
|
||||||
|
patch.object(pc, "owner_names_for", AsyncMock(return_value={})), \
|
||||||
|
patch.object(pc.shape_ledger_svc, "recent_pulls",
|
||||||
|
AsyncMock(return_value={7: _ts()})), \
|
||||||
|
patch.object(pc.shape_ledger_svc, "stamp_write_path_instances",
|
||||||
|
AsyncMock(return_value=[])), \
|
||||||
|
patch.object(pc, "semantic_search_notes",
|
||||||
|
_search_reporting(0.9, fake_note(
|
||||||
|
id=7, title="scored", user_id=1, note_type="snippet"))):
|
||||||
|
await pc.build_write_path_hint(
|
||||||
|
1, "src/x.py", code=REAL_CODE,
|
||||||
|
stamp_shapes=[("sym", "debounce")],
|
||||||
|
)
|
||||||
|
row = next(c for c in rec.call_args_list
|
||||||
|
if c.kwargs["source"] == "write_path")
|
||||||
|
|
||||||
assert row.kwargs["results"] == [], "the hit was withheld, so nothing shown"
|
assert row.kwargs["results"] == [], "the hit was withheld, so nothing shown"
|
||||||
assert row.kwargs["best_available"] is None, (
|
assert row.kwargs["best_available"] is None, (
|
||||||
@@ -1682,6 +1774,27 @@ async def test_a_record_this_arm_withheld_itself_is_not_a_near_miss():
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_ledger_repeat_no_longer_produces_that_case_at_all():
|
||||||
|
"""The other direction, and the reason the setup above had to move.
|
||||||
|
|
||||||
|
A record on the session ledger now clears the search, is rendered with a
|
||||||
|
marker, and is counted in `suppressed` — so nothing is withheld and the
|
||||||
|
reported score describes the bar. Pinned because the tempting way to keep
|
||||||
|
the old test passing would have been to null the score whenever the ledger
|
||||||
|
matched, which would delete the measurement #3670 was built for on exactly
|
||||||
|
the calls where the bar is most worth reading.
|
||||||
|
"""
|
||||||
|
rec = MagicMock()
|
||||||
|
row = await _write_path_row(rec, exclude_ids=[7])
|
||||||
|
|
||||||
|
assert row.kwargs["results"] == [], "a repeat is not a fresh result"
|
||||||
|
assert row.kwargs["suppressed"] == 1, "and it is counted rather than lost"
|
||||||
|
assert row.kwargs["best_available"] == 0.9, (
|
||||||
|
"nothing was withheld, so the reported score describes the bar"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_a_call_that_withheld_nothing_still_reports_what_the_bar_refused():
|
async def test_a_call_that_withheld_nothing_still_reports_what_the_bar_refused():
|
||||||
"""The other half, and what stops the fix being 'never report it'.
|
"""The other half, and what stops the fix being 'never report it'.
|
||||||
|
|||||||
Reference in New Issue
Block a user