Merge pull request 'feat(rules): the rule arms stop filtering the corpus to one tier (#3702)' (#142) from dev into main

src/scribe/services/embeddings.py

@@ -774,10 +774,17 @@ async def semantic_search_rules(
     is the surfacing question, and it has its own machinery
     (get_applicable_rules) rather than a second, subtly different copy here.
 
-    `tier` narrows to one tier. The write-path hint passes "conditional",
+    `tier` narrows to one tier, and NONE is the ordinary case. The write-path
-    because an always-on rule is ALREADY in the session — surfacing it again as
+    and pre-tool hints deliberately pass nothing: an always-on rule is already
-    a suggestion is pure noise, and noise on a hint that fires on every write
+    in the session, but being in a list from turn zero is not the same as being
-    is how a hint gets ignored.
+    in front of the reader when the action it governs is taken, and filtering
+    on tier made a whole class of rules permanently ineligible for the one
+    mechanism that surfaces a rule AT the moment. Relevance is the threshold's
+    job; see the block above RULEHINT_LIMIT in services/plugin_context.py for
+    the argument and for what the resulting scores are being read against.
+
+    Pass a tier when a caller genuinely wants one class — a listing, an audit,
+    a UI that renders the tiers apart. Not to approximate relevance.
 
     Collapses to best-chunk-per-rule like the note search, so a long rule split
     across chunks competes once rather than crowding the results with itself.

src/scribe/services/plugin_context.py

@@ -94,13 +94,17 @@ WRITEPATH_DEFAULT_THRESHOLD = 0.68
 # THE STRUCTURAL ARGUMENT, which is the only kind admissible here (rule 115).
 # Two facts hold on any install, including one with six rules and no telemetry:
 #
-#   1. The eligible corpus is TINY. The arm searches `tier="conditional"`
+#   1. The eligible corpus is SMALL — every rule an install owns, still only
-#      rules only — a handful to a few dozen documents against thousands of
+#      a few dozen documents against thousands of notes. A top-k over a small
-#      notes. A top-k over forty candidates always returns something, so
+#      pool always returns something, so "the best match cleared the bar"
-#      "the best match cleared the bar" stops meaning "a good match exists"
+#      drifts from "a good match exists" toward "N things were ranked". A bar
-#      and starts meaning "forty things were ranked". A bar calibrated for
+#      calibrated for best-of-thousands is cleared by best-of-forty as
-#      best-of-thousands is cleared by best-of-forty as arithmetic, not
+#      arithmetic rather than relevance.
-#      relevance.
+#      This argument WEAKENED when the arms stopped filtering to one tier
+#      (see the note on that below): a larger pool makes clearing the bar
+#      mean more, not less. The threshold was deliberately left where it was
+#      anyway — moving two variables at once would make the resulting
+#      distribution unreadable, and this one errs toward silence on purpose.
 #   2. Rules are short imperative technical English — a far more HOMOGENEOUS
 #      corpus than note prose. #2223 measured the floor for code against prose
 #      at 0.55-0.63 and set 0.68 above it. A more homogeneous corpus has a
@@ -128,9 +132,14 @@ RULEHINT_DEFAULT_THRESHOLD = 0.72
 # ONE rule per write, not two — and this is deliberately NOT a knob.
 #
 # With a corpus this small, top-k does as much damage as the threshold: k=2
-# over forty candidates means the second line is almost always the second-best
+# over a few dozen candidates means the second line is almost always the
-# noise, arriving with the same confident framing as the first. Halving k
+# second-best noise, arriving with the same confident framing as the first.
-# halves that regardless of where the bar sits.
+# Halving k halves that regardless of where the bar sits.
+#
+# It also BOUNDS the blast radius of widening the pool (below): with k=1 a
+# wider corpus can change WHICH rule surfaces and how often one does, but it
+# can never make a single hint longer. The loudness of one hint and the
+# eligibility of a rule are separate controls, and only one of them moved.
 #
 # It stays a constant because it is a decision about how LOUD one hint may be,
 # not a per-install tuning question. The hint already carries prior art, shape
@@ -140,6 +149,45 @@ RULEHINT_DEFAULT_THRESHOLD = 0.72
 # adds a way to misconfigure the surface (rule 25 cuts both ways).
 RULEHINT_LIMIT = 1
 
+# WHY THE ARMS NO LONGER FILTER TO ONE TIER (#3702).
+#
+# Both arms used to pass `tier="conditional"`, on the reasoning that an
+# always-on rule is already in the session, so surfacing it again is pure
+# noise. That reasoning conflates two different things:
+#
+#   PRESENT IN CONTEXT  — the rule was delivered at session start.
+#   SALIENT AT THE MOMENT  — the rule is in front of the reader when the
+#                            action it governs is about to be taken.
+#
+# A rule handed over in a list at turn zero is present while a session writes
+# a config value three hundred turns later. It is not surfaced. So the filter
+# did not merely skip a redundant hint — it made a whole class of rules
+# permanently ineligible for the only mechanism that puts a rule in front of
+# an agent AT the moment, and the more important a rule is, the more likely
+# it was in that class.
+#
+# The deeper defect is that the filter was doing the THRESHOLD's job. Whether
+# a rule belongs in this hint is a relevance question, and a similarity bar is
+# the control for relevance. A categorical exclusion standing in for a
+# relevance judgment cannot be tuned, cannot be measured, and cannot be wrong
+# in a way anybody notices.
+#
+# THIS IS A MEASURED CHANGE, NOT A SETTLED ONE. The old comment's fear is
+# real — a hint that fires on every write and says obvious things teaches the
+# reader to skip the block, and the surface is then lost along with its true
+# positives. That fear had simply never been checked. `retrieval_logs` already
+# records top_score, result_count and the query for every call, so the
+# evidence now arrives on its own:
+#
+#   - rules clear the bar often and at high scores  -> the fear was justified,
+#     the filter was a crude proxy for a bar set too low, and the WORK IS THE
+#     BAR. Any reinstated filter should then carry a measured reason.
+#   - rules clear rarely, in a thin band near the bar -> the filter was never
+#     the right instrument and relevance was always sufficient.
+#
+# Only the eligibility moved. The bar and k=1 were both left exactly where
+# they were, so the resulting distribution has one cause.
+
 # How much of a command reaches the embedding (#3476). A shell call is not a
 # file: most are short, and the ones that are not are usually a heredoc or a
 # pasted script whose bulk says nothing about which rule applies. The VERB AND
@@ -1205,7 +1253,7 @@ async def build_write_path_hint(
         rule_t0 = time.perf_counter()
         hits = await semantic_search_rules(
             user_id, code or path, limit=RULEHINT_LIMIT,
-            threshold=cfg["rule_threshold"], tier="conditional",
+            threshold=cfg["rule_threshold"],
         )
         rule_ms = (time.perf_counter() - rule_t0) * 1000.0
         fresh = [(score, rule) for score, rule in hits if rule.id not in already]
@@ -1337,7 +1385,7 @@ async def build_tool_rule_hint(
         t0 = time.perf_counter()
         hits = await semantic_search_rules(
             user_id, query, limit=RULEHINT_LIMIT,
-            threshold=cfg["rule_threshold"], tier="conditional",
+            threshold=cfg["rule_threshold"],
         )
         duration_ms = (time.perf_counter() - t0) * 1000.0
 

tests/test_rule_usage_wiring.py

@@ -160,7 +160,15 @@ async def test_the_arm_searches_on_its_OWN_bar_not_the_code_one():
     kw = search.await_args.kwargs
     assert kw["threshold"] == 0.81, "the arm is still using the code threshold"
     assert kw["limit"] == pc.RULEHINT_LIMIT
-    assert kw["tier"] == "conditional"
+    # NO tier filter (#3702). The arms search every rule the caller owns,
+    # because "already in the session" is not the same as "in front of the
+    # reader at the moment it applies" — and relevance is the threshold's
+    # job, not a category's. If this assertion is failing because a tier
+    # argument came back, read the block above RULEHINT_LIMIT first: the
+    # filter may legitimately return, but only carrying a measured reason.
+    assert "tier" not in kw or kw["tier"] is None, (
+        "the arm is filtering the rule corpus by tier again"
+    )
 
 
 @pytest.mark.asyncio