align(4909d): every rule arm that annotates reads one bar, kb_rule_threshold at 0.70
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 13s
CI & Build / TypeScript typecheck (push) Successful in 55s
CI & Build / integration (push) Successful in 1m7s
CI & Build / Python tests (push) Failing after 1m30s
CI & Build / Build & push image (push) Skipped
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 13s
CI & Build / TypeScript typecheck (push) Successful in 55s
CI & Build / integration (push) Successful in 1m7s
CI & Build / Python tests (push) Failing after 1m30s
CI & Build / Build & push image (push) Skipped
prompt_rule, pre_tool_rule and write_path_rule now share one settings key, RULE_FLOOR_KEY = "kb_rule_threshold", default 0.70. The reply backstop keeps its own 0.80, because there the floor is a stop bar. Moving the shared floor through tune_retrieval or the Settings form records a tuning event on every surface that reads it (set_dial, via retrieval_surfaces.sharing_floor), and the result names the surfaces it carried as `moved_with`. Settings shows one "Standing-rule confidence threshold" control where it showed three. Operator ruling (2026-10-05): one shared bar for the rule arms, "especially since we're only taking the first x number of them" -- the budget limits what is shown; the bar only decides what is worth ranking. Kept a dial. Why 0.70, from records read rather than percentiles: - write_path_rule near-misses at 0.7199-0.72 were all false positives (rule 149 against a Lidarr comment, 157 against a retention Svelte file, 121 against an Archives restore, 164 against a config test, 131 against a Go package), so the old 0.72 was not hiding real hits -- but the band does noise control on this arm, so dropping 0.02 costs little. - prompt_rule: the operator set 0.70 by hand on 2026-09-18, after a 0.60 experiment admitted junk and was reverted. 0.70 is the number with a human decision behind it. - pre_tool_rule lines near 0.68 this session were mostly noise; rising to 0.70 trims them, and the band still narrows what clears. Retired keys: kb_promptrule_threshold, kb_toolrule_threshold and kb_rulehint_threshold are no longer read. Rows an install stored under them stay in the settings table, inert. On this instance the effective bars move pre_tool_rule 0.68 -> 0.70, write_path_rule 0.72 -> 0.70, prompt_rule stays 0.70; record_release_defaults writes those per-surface release rows on boot, so the telemetry shift has an event to explain it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -71,13 +71,29 @@ def test_every_surface_name_is_a_real_telemetry_source():
|
||||
)
|
||||
|
||||
|
||||
def test_no_two_surfaces_share_a_settings_key():
|
||||
"""One dial, one label."""
|
||||
floors = [s.floor_key for s in rs.SURFACES.values()]
|
||||
def test_a_shared_settings_key_is_one_bar_with_one_default():
|
||||
"""One dial, one label. Surfaces may share a floor key, as the rule arms
|
||||
do, but then they must ship the same default: otherwise the one number
|
||||
means something different on each surface until somebody sets it."""
|
||||
by_key: dict[str, set[float]] = {}
|
||||
for s in rs.SURFACES.values():
|
||||
by_key.setdefault(s.floor_key, set()).add(s.floor_default)
|
||||
split = {k: v for k, v in by_key.items() if len(v) > 1}
|
||||
assert not split, f"a shared floor key ships two defaults: {split}"
|
||||
|
||||
budgets = [s.budget_key for s in rs.SURFACES.values()]
|
||||
assert len(set(floors)) == len(floors), f"duplicate floor key in {floors}"
|
||||
assert len(set(budgets)) == len(budgets), f"duplicate budget key in {budgets}"
|
||||
assert not (set(floors) & set(budgets)), "a floor key doubles as a budget key"
|
||||
assert not (set(by_key) & set(budgets)), "a floor key doubles as a budget key"
|
||||
|
||||
|
||||
def test_the_rule_arms_that_annotate_share_one_bar_and_the_reply_stop_does_not():
|
||||
from grimoire.services.retrieval_pipeline import RULE_FLOOR_KEY
|
||||
|
||||
assert sorted(rs.sharing_floor("prompt_rule")) == [
|
||||
"pre_tool_rule", "prompt_rule", "write_path_rule",
|
||||
]
|
||||
assert rs.SURFACES["prompt_rule"].floor_key == RULE_FLOOR_KEY
|
||||
assert rs.sharing_floor("reply_rule") == ["reply_rule"]
|
||||
|
||||
|
||||
def test_every_surface_says_what_it_asks_over_what_and_how_often():
|
||||
|
||||
@@ -86,9 +86,9 @@ async def test_a_good_change_writes_the_setting_and_the_event():
|
||||
|
||||
setter.assert_awaited_once()
|
||||
_uid, key, stored = setter.await_args.args
|
||||
assert key == "kb_promptrule_threshold" and stored == "0.66"
|
||||
assert key == "kb_rule_threshold" and stored == "0.66"
|
||||
|
||||
event = session.add.call_args.args[0]
|
||||
event = session.add.call_args_list[0].args[0]
|
||||
assert event.surface == "prompt_rule" and event.dial == "floor"
|
||||
# The before-value is what makes the history answerable.
|
||||
assert event.old_value == 0.72 and event.new_value == 0.66
|
||||
@@ -97,6 +97,37 @@ async def test_a_good_change_writes_the_setting_and_the_event():
|
||||
assert out["clamped"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_shared_floor_records_the_move_on_every_surface_that_reads_it():
|
||||
"""One key, one write, an event per surface. A rule arm whose bar moved
|
||||
with no event of its own would show a telemetry shift nothing explains."""
|
||||
session, ctx = _patches(floor=0.70)
|
||||
with ctx[0], patch.object(rt, "set_setting", AsyncMock()) as setter, \
|
||||
ctx[2], ctx[3]:
|
||||
out = await rt.set_dial(
|
||||
1, "pre_tool_rule", "floor", 0.74,
|
||||
reason="read the 10 highest declines: every one was a real miss",
|
||||
)
|
||||
|
||||
setter.assert_awaited_once()
|
||||
recorded = {c.args[0].surface for c in session.add.call_args_list}
|
||||
assert recorded == {"pre_tool_rule", "prompt_rule", "write_path_rule"}
|
||||
assert sorted(out["moved_with"]) == ["prompt_rule", "write_path_rule"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_budget_moves_only_its_own_surface():
|
||||
session, ctx = _patches(budget=3)
|
||||
with ctx[0], patch.object(rt, "set_setting", AsyncMock()), ctx[2], ctx[3]:
|
||||
out = await rt.set_dial(
|
||||
1, "prompt_rule", "budget", 4,
|
||||
reason="on-point hits sit at ranks 4 and 5 as often as rank 1",
|
||||
)
|
||||
|
||||
assert [c.args[0].surface for c in session.add.call_args_list] == ["prompt_rule"]
|
||||
assert out["moved_with"] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("dial, sent, applied", [
|
||||
("floor", 1.4, 1.0),
|
||||
|
||||
@@ -76,10 +76,15 @@ def test_the_route_does_not_take_the_actor_from_the_caller():
|
||||
def test_every_registry_key_is_recognised_as_a_dial():
|
||||
"""Derived from the registry, both directions. A seventh surface added
|
||||
without a row here is a floor that can be written silently again."""
|
||||
from grimoire.services.retrieval_surfaces import SURFACES, dial_for_key
|
||||
from grimoire.services.retrieval_surfaces import (
|
||||
SURFACES, dial_for_key, sharing_floor,
|
||||
)
|
||||
|
||||
for name, surface in SURFACES.items():
|
||||
assert dial_for_key(surface.floor_key) == (name, "floor")
|
||||
# A shared floor resolves to one of the surfaces that read it, and
|
||||
# `set_dial` records the move on all of them.
|
||||
owner, dial = dial_for_key(surface.floor_key)
|
||||
assert dial == "floor" and name in sharing_floor(owner)
|
||||
assert dial_for_key(surface.budget_key) == (name, "budget")
|
||||
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
_SERVICES = ROOT / "src" / "grimoire" / "services"
|
||||
_VUE = ROOT / "frontend" / "src" / "views" / "SettingsView.vue"
|
||||
|
||||
# THE SIX RETRIEVAL FLOORS live in one registry, so their side of
|
||||
# THE RETRIEVAL FLOORS live in one registry, so their side of
|
||||
# the pairing is a SURFACE NAME rather than a module constant. That is strictly
|
||||
# better for this guard: the surface name is also the telemetry source, so a row
|
||||
# here names the same arm the readout does, and a floor that moved cannot be
|
||||
@@ -49,9 +49,10 @@ _VUE = ROOT / "frontend" / "src" / "views" / "SettingsView.vue"
|
||||
_SURFACE_PAIRS = (
|
||||
("auto_inject", "kbInjectThreshold"),
|
||||
("write_path", "kbWritePathThreshold"),
|
||||
("write_path_rule", "kbRuleHintThreshold"),
|
||||
("pre_tool_rule", "kbToolRuleThreshold"),
|
||||
("prompt_rule", "kbPromptRuleThreshold"),
|
||||
# The three rule hints read one bar, so one control serves them.
|
||||
("write_path_rule", "kbRuleThreshold"),
|
||||
("pre_tool_rule", "kbRuleThreshold"),
|
||||
("prompt_rule", "kbRuleThreshold"),
|
||||
("reply_rule", "kbReplyRuleThreshold"),
|
||||
)
|
||||
|
||||
|
||||
@@ -485,13 +485,13 @@ async def test_writepath_threshold_is_operator_tunable_and_clamped():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_rule_arm_has_its_own_tunable_bar():
|
||||
async def test_the_rule_arms_have_their_own_tunable_bar():
|
||||
"""Settable and clamped again, for the THIRD corpus.
|
||||
|
||||
Separate from the code threshold above and separately settable, because the
|
||||
two are measured against different things: 0.68 was derived from code
|
||||
against note PROSE, and rules are short imperative technical
|
||||
English — a more homogeneous corpus whose noise floor sits higher.
|
||||
two are measured against different things: the code bar against note
|
||||
PROSE, the rule bar against short imperative technical English — a more
|
||||
homogeneous corpus whose noise floor sits higher.
|
||||
"""
|
||||
from grimoire.services import plugin_context as pc
|
||||
|
||||
@@ -524,33 +524,28 @@ async def test_the_two_write_path_bars_are_independent():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_two_act_arms_read_independent_rule_bars():
|
||||
"""The command arm's bar moves without the write path's.
|
||||
async def test_both_act_arms_read_the_one_rule_bar():
|
||||
"""One stored value moves the command arm and the write arm together.
|
||||
|
||||
The two act arms behave like different subsystems at the same number,
|
||||
because a code payload is long and rich where a shell command is short. A config assembler that
|
||||
reads one key into both fields would silently undo that, and the symptom
|
||||
would be invisible: both arms would simply agree again.
|
||||
They search the same rules, and what limits each is its budget and band.
|
||||
A config assembler that read a second key into either field would split
|
||||
the bar again with nothing in the settings to show it.
|
||||
"""
|
||||
from grimoire.services import plugin_context as pc
|
||||
|
||||
stored = {pc.RULEHINT_THRESHOLD_KEY: "0.75", pc.TOOLRULE_THRESHOLD_KEY: "0.61"}
|
||||
assert pc.TOOLRULE_THRESHOLD_KEY == pc.RULEHINT_THRESHOLD_KEY
|
||||
stored = {pc.RULEHINT_THRESHOLD_KEY: "0.61"}
|
||||
with _stored_settings(stored):
|
||||
cfg = await pc.get_writepath_config(1)
|
||||
|
||||
assert cfg["rule_threshold"] == 0.75
|
||||
assert cfg["rule_threshold"] == 0.61
|
||||
assert cfg["tool_rule_threshold"] == 0.61
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_garbage_command_bar_falls_back_to_its_own_default():
|
||||
"""Not to 0.0, and not to the write path's default.
|
||||
|
||||
Falling back to 0.0 would attach a rule to every Bash call in the session;
|
||||
falling back to the sibling's default would quietly re-merge the two bars
|
||||
that are kept separate, which is the harder failure to see because the arm
|
||||
keeps working.
|
||||
"""
|
||||
async def test_a_garbage_rule_bar_falls_back_to_its_default():
|
||||
"""Not to 0.0, which would attach a rule to every Bash call and every
|
||||
write in the session."""
|
||||
from grimoire.services import plugin_context as pc
|
||||
|
||||
stored = {pc.TOOLRULE_THRESHOLD_KEY: "banana"}
|
||||
@@ -558,20 +553,7 @@ async def test_a_garbage_command_bar_falls_back_to_its_own_default():
|
||||
cfg = await pc.get_writepath_config(1)
|
||||
|
||||
assert cfg["tool_rule_threshold"] == pc.TOOLRULE_DEFAULT_THRESHOLD
|
||||
|
||||
|
||||
def test_the_command_bar_defaults_below_the_write_path_bar():
|
||||
"""A DIRECTION check, like its sibling above, and for the same reason: the
|
||||
value is measured against one corpus, the relationship is not.
|
||||
|
||||
A shell command carries less text than a code payload and therefore scores
|
||||
lower for the same relevance; at the write path's bar, consequential
|
||||
commands retrieve nothing. Tuning either value stays free; inverting the
|
||||
relationship would mute the command arm.
|
||||
"""
|
||||
from grimoire.services import plugin_context as pc
|
||||
|
||||
assert pc.TOOLRULE_DEFAULT_THRESHOLD < pc.RULEHINT_DEFAULT_THRESHOLD
|
||||
assert cfg["rule_threshold"] == pc.RULEHINT_DEFAULT_THRESHOLD
|
||||
|
||||
|
||||
def test_the_rule_bar_defaults_above_the_code_bar():
|
||||
|
||||
Reference in New Issue
Block a user