diff --git a/src/scribe/services/embeddings.py b/src/scribe/services/embeddings.py index bf66e49..e5923c4 100644 --- a/src/scribe/services/embeddings.py +++ b/src/scribe/services/embeddings.py @@ -774,10 +774,17 @@ async def semantic_search_rules( is the surfacing question, and it has its own machinery (get_applicable_rules) rather than a second, subtly different copy here. - `tier` narrows to one tier. The write-path hint passes "conditional", - because an always-on rule is ALREADY in the session — surfacing it again as - a suggestion is pure noise, and noise on a hint that fires on every write - is how a hint gets ignored. + `tier` narrows to one tier, and NONE is the ordinary case. The write-path + and pre-tool hints deliberately pass nothing: an always-on rule is already + in the session, but being in a list from turn zero is not the same as being + in front of the reader when the action it governs is taken, and filtering + on tier made a whole class of rules permanently ineligible for the one + mechanism that surfaces a rule AT the moment. Relevance is the threshold's + job; see the block above RULEHINT_LIMIT in services/plugin_context.py for + the argument and for what the resulting scores are being read against. + + Pass a tier when a caller genuinely wants one class — a listing, an audit, + a UI that renders the tiers apart. Not to approximate relevance. Collapses to best-chunk-per-rule like the note search, so a long rule split across chunks competes once rather than crowding the results with itself. diff --git a/src/scribe/services/plugin_context.py b/src/scribe/services/plugin_context.py index 2b45615..3ce4dda 100644 --- a/src/scribe/services/plugin_context.py +++ b/src/scribe/services/plugin_context.py @@ -94,13 +94,17 @@ WRITEPATH_DEFAULT_THRESHOLD = 0.68 # THE STRUCTURAL ARGUMENT, which is the only kind admissible here (rule 115). # Two facts hold on any install, including one with six rules and no telemetry: # -# 1. The eligible corpus is TINY. The arm searches `tier="conditional"` -# rules only — a handful to a few dozen documents against thousands of -# notes. A top-k over forty candidates always returns something, so -# "the best match cleared the bar" stops meaning "a good match exists" -# and starts meaning "forty things were ranked". A bar calibrated for -# best-of-thousands is cleared by best-of-forty as arithmetic, not -# relevance. +# 1. The eligible corpus is SMALL — every rule an install owns, still only +# a few dozen documents against thousands of notes. A top-k over a small +# pool always returns something, so "the best match cleared the bar" +# drifts from "a good match exists" toward "N things were ranked". A bar +# calibrated for best-of-thousands is cleared by best-of-forty as +# arithmetic rather than relevance. +# This argument WEAKENED when the arms stopped filtering to one tier +# (see the note on that below): a larger pool makes clearing the bar +# mean more, not less. The threshold was deliberately left where it was +# anyway — moving two variables at once would make the resulting +# distribution unreadable, and this one errs toward silence on purpose. # 2. Rules are short imperative technical English — a far more HOMOGENEOUS # corpus than note prose. #2223 measured the floor for code against prose # at 0.55-0.63 and set 0.68 above it. A more homogeneous corpus has a @@ -128,9 +132,14 @@ RULEHINT_DEFAULT_THRESHOLD = 0.72 # ONE rule per write, not two — and this is deliberately NOT a knob. # # With a corpus this small, top-k does as much damage as the threshold: k=2 -# over forty candidates means the second line is almost always the second-best -# noise, arriving with the same confident framing as the first. Halving k -# halves that regardless of where the bar sits. +# over a few dozen candidates means the second line is almost always the +# second-best noise, arriving with the same confident framing as the first. +# Halving k halves that regardless of where the bar sits. +# +# It also BOUNDS the blast radius of widening the pool (below): with k=1 a +# wider corpus can change WHICH rule surfaces and how often one does, but it +# can never make a single hint longer. The loudness of one hint and the +# eligibility of a rule are separate controls, and only one of them moved. # # It stays a constant because it is a decision about how LOUD one hint may be, # not a per-install tuning question. The hint already carries prior art, shape @@ -140,6 +149,45 @@ RULEHINT_DEFAULT_THRESHOLD = 0.72 # adds a way to misconfigure the surface (rule 25 cuts both ways). RULEHINT_LIMIT = 1 +# WHY THE ARMS NO LONGER FILTER TO ONE TIER (#3702). +# +# Both arms used to pass `tier="conditional"`, on the reasoning that an +# always-on rule is already in the session, so surfacing it again is pure +# noise. That reasoning conflates two different things: +# +# PRESENT IN CONTEXT — the rule was delivered at session start. +# SALIENT AT THE MOMENT — the rule is in front of the reader when the +# action it governs is about to be taken. +# +# A rule handed over in a list at turn zero is present while a session writes +# a config value three hundred turns later. It is not surfaced. So the filter +# did not merely skip a redundant hint — it made a whole class of rules +# permanently ineligible for the only mechanism that puts a rule in front of +# an agent AT the moment, and the more important a rule is, the more likely +# it was in that class. +# +# The deeper defect is that the filter was doing the THRESHOLD's job. Whether +# a rule belongs in this hint is a relevance question, and a similarity bar is +# the control for relevance. A categorical exclusion standing in for a +# relevance judgment cannot be tuned, cannot be measured, and cannot be wrong +# in a way anybody notices. +# +# THIS IS A MEASURED CHANGE, NOT A SETTLED ONE. The old comment's fear is +# real — a hint that fires on every write and says obvious things teaches the +# reader to skip the block, and the surface is then lost along with its true +# positives. That fear had simply never been checked. `retrieval_logs` already +# records top_score, result_count and the query for every call, so the +# evidence now arrives on its own: +# +# - rules clear the bar often and at high scores -> the fear was justified, +# the filter was a crude proxy for a bar set too low, and the WORK IS THE +# BAR. Any reinstated filter should then carry a measured reason. +# - rules clear rarely, in a thin band near the bar -> the filter was never +# the right instrument and relevance was always sufficient. +# +# Only the eligibility moved. The bar and k=1 were both left exactly where +# they were, so the resulting distribution has one cause. + # How much of a command reaches the embedding (#3476). A shell call is not a # file: most are short, and the ones that are not are usually a heredoc or a # pasted script whose bulk says nothing about which rule applies. The VERB AND @@ -1205,7 +1253,7 @@ async def build_write_path_hint( rule_t0 = time.perf_counter() hits = await semantic_search_rules( user_id, code or path, limit=RULEHINT_LIMIT, - threshold=cfg["rule_threshold"], tier="conditional", + threshold=cfg["rule_threshold"], ) rule_ms = (time.perf_counter() - rule_t0) * 1000.0 fresh = [(score, rule) for score, rule in hits if rule.id not in already] @@ -1337,7 +1385,7 @@ async def build_tool_rule_hint( t0 = time.perf_counter() hits = await semantic_search_rules( user_id, query, limit=RULEHINT_LIMIT, - threshold=cfg["rule_threshold"], tier="conditional", + threshold=cfg["rule_threshold"], ) duration_ms = (time.perf_counter() - t0) * 1000.0 diff --git a/tests/test_rule_usage_wiring.py b/tests/test_rule_usage_wiring.py index bb76683..f985bae 100644 --- a/tests/test_rule_usage_wiring.py +++ b/tests/test_rule_usage_wiring.py @@ -160,7 +160,15 @@ async def test_the_arm_searches_on_its_OWN_bar_not_the_code_one(): kw = search.await_args.kwargs assert kw["threshold"] == 0.81, "the arm is still using the code threshold" assert kw["limit"] == pc.RULEHINT_LIMIT - assert kw["tier"] == "conditional" + # NO tier filter (#3702). The arms search every rule the caller owns, + # because "already in the session" is not the same as "in front of the + # reader at the moment it applies" — and relevance is the threshold's + # job, not a category's. If this assertion is failing because a tier + # argument came back, read the block above RULEHINT_LIMIT first: the + # filter may legitimately return, but only carrying a measured reason. + assert "tier" not in kw or kw["tier"] is None, ( + "the arm is filtering the rule corpus by tier again" + ) @pytest.mark.asyncio