Rule outcomes, the contract hint, and four extractor/backup fixes #174
@@ -374,7 +374,18 @@ def writepath_cfg(**over):
|
|||||||
So the keys come from `retrieval_surfaces.SURFACES`. A seventh surface, or a
|
So the keys come from `retrieval_surfaces.SURFACES`. A seventh surface, or a
|
||||||
rename, changes this helper for free and cannot quietly disable an arm in
|
rename, changes this helper for free and cannot quietly disable an arm in
|
||||||
ten hand-written dicts that each looked complete on the day they were typed.
|
ten hand-written dicts that each looked complete on the day they were typed.
|
||||||
|
|
||||||
|
NOT EVERY KEY IS A SURFACE, and that gap already bit once (#4214). The
|
||||||
|
checkpoint bar has no entry in SURFACES on purpose — everything in that
|
||||||
|
table is a floor/budget pair belonging to one QUERY, and the checkpoint
|
||||||
|
runs none, it re-reads hits the rule arms already produced. Adding it there
|
||||||
|
would give it a phantom budget. So it is taken from the module constant
|
||||||
|
instead, which keeps the "one literal, in the product" property even though
|
||||||
|
the derivation differs. `test_the_config_stand_in_carries_every_key_the_
|
||||||
|
real_one_does` is what makes the next addition fail loudly here rather than
|
||||||
|
silently no-op an arm, which is the whole claim this docstring makes.
|
||||||
"""
|
"""
|
||||||
|
from scribe.services.plugin_context import _CHECKPOINT_DEFAULT
|
||||||
from scribe.services.retrieval_surfaces import SURFACES
|
from scribe.services.retrieval_surfaces import SURFACES
|
||||||
|
|
||||||
cfg = {
|
cfg = {
|
||||||
@@ -385,6 +396,7 @@ def writepath_cfg(**over):
|
|||||||
"rule_top_k": SURFACES["write_path_rule"].budget_default,
|
"rule_top_k": SURFACES["write_path_rule"].budget_default,
|
||||||
"tool_rule_threshold": SURFACES["pre_tool_rule"].floor_default,
|
"tool_rule_threshold": SURFACES["pre_tool_rule"].floor_default,
|
||||||
"tool_rule_top_k": SURFACES["pre_tool_rule"].budget_default,
|
"tool_rule_top_k": SURFACES["pre_tool_rule"].budget_default,
|
||||||
|
"checkpoint_threshold": _CHECKPOINT_DEFAULT,
|
||||||
}
|
}
|
||||||
cfg.update(over)
|
cfg.update(over)
|
||||||
return cfg
|
return cfg
|
||||||
|
|||||||
@@ -183,5 +183,9 @@ async def test_the_write_path_config_carries_every_arm_it_drives():
|
|||||||
cfg = await pc.get_writepath_config(1)
|
cfg = await pc.get_writepath_config(1)
|
||||||
|
|
||||||
for key in ("threshold", "top_k", "rule_threshold", "rule_top_k",
|
for key in ("threshold", "top_k", "rule_threshold", "rule_top_k",
|
||||||
"tool_rule_threshold", "tool_rule_top_k"):
|
"tool_rule_threshold", "tool_rule_top_k",
|
||||||
|
# Not a surface, and that is why it is spelled out here: it
|
||||||
|
# has no floor/budget pair to derive from, so nothing else in
|
||||||
|
# this file would notice it going missing (#4214).
|
||||||
|
"checkpoint_threshold"):
|
||||||
assert key in cfg, f"{key} missing — its arm will silently no-op"
|
assert key in cfg, f"{key} missing — its arm will silently no-op"
|
||||||
|
|||||||
@@ -508,7 +508,7 @@ async def test_the_prompt_arm_says_nothing_when_asked_nothing():
|
|||||||
stack.enter_context(patch.object(pc, "record_rule_surfaced", MagicMock()))
|
stack.enter_context(patch.object(pc, "record_rule_surfaced", MagicMock()))
|
||||||
out = await pc.build_prompt_rule_hint(1, " ")
|
out = await pc.build_prompt_rule_hint(1, " ")
|
||||||
|
|
||||||
assert out == {"context": "", "rule_ids": []}
|
assert out == {"context": "", "rule_ids": [], "checkpoint": {}}
|
||||||
search.assert_not_called()
|
search.assert_not_called()
|
||||||
log.assert_not_called()
|
log.assert_not_called()
|
||||||
|
|
||||||
@@ -653,7 +653,7 @@ async def test_an_empty_command_asks_the_ranker_nothing():
|
|||||||
stack.enter_context(patch.object(pc, "record_rule_surfaced", rec))
|
stack.enter_context(patch.object(pc, "record_rule_surfaced", rec))
|
||||||
out = await pc.build_tool_rule_hint(1, "Bash", " ")
|
out = await pc.build_tool_rule_hint(1, "Bash", " ")
|
||||||
|
|
||||||
assert out == {"context": "", "rule_ids": []}
|
assert out == {"context": "", "rule_ids": [], "checkpoint": {}}
|
||||||
search.assert_not_called()
|
search.assert_not_called()
|
||||||
rec.assert_not_called()
|
rec.assert_not_called()
|
||||||
|
|
||||||
@@ -669,7 +669,7 @@ async def test_the_tool_arm_fails_open():
|
|||||||
AsyncMock(side_effect=RuntimeError("boom"))))
|
AsyncMock(side_effect=RuntimeError("boom"))))
|
||||||
out = await pc.build_tool_rule_hint(1, "Bash", "docker compose up -d")
|
out = await pc.build_tool_rule_hint(1, "Bash", "docker compose up -d")
|
||||||
|
|
||||||
assert out == {"context": "", "rule_ids": []}
|
assert out == {"context": "", "rule_ids": [], "checkpoint": {}}
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@@ -833,7 +833,7 @@ async def test_the_tool_arm_logs_the_call_that_found_nothing():
|
|||||||
log, rec = MagicMock(), MagicMock()
|
log, rec = MagicMock(), MagicMock()
|
||||||
out = await _run_tool_arm([], rec, retrieval_log=log)
|
out = await _run_tool_arm([], rec, retrieval_log=log)
|
||||||
|
|
||||||
assert out == {"context": "", "rule_ids": []}
|
assert out == {"context": "", "rule_ids": [], "checkpoint": {}}
|
||||||
assert log.call_count == 1
|
assert log.call_count == 1
|
||||||
assert log.call_args.kwargs["source"] == "pre_tool_rule"
|
assert log.call_args.kwargs["source"] == "pre_tool_rule"
|
||||||
assert log.call_args.kwargs["results"] == []
|
assert log.call_args.kwargs["results"] == []
|
||||||
|
|||||||
@@ -518,3 +518,37 @@ async def test_write_path_labels_a_non_snippet_hit_with_its_kind():
|
|||||||
assert '[similar 0.72] "debounce helper"' in ctx # the snippet does not
|
assert '[similar 0.72] "debounce helper"' in ctx # the snippet does not
|
||||||
# The header now names the right opener for each kind.
|
# The header now names the right opener for each kind.
|
||||||
assert "get_task(id)" in ctx and "get_snippet(id)" in ctx
|
assert "get_task(id)" in ctx and "get_snippet(id)" in ctx
|
||||||
|
|
||||||
|
|
||||||
|
# ── The config stand-in cannot fall behind the real one (#4214) ───────────
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_the_config_stand_in_carries_every_key_the_real_one_does():
|
||||||
|
"""THE GUARD `writepath_cfg`'s DOCSTRING ALREADY CLAIMED AND DID NOT HAVE.
|
||||||
|
|
||||||
|
Three arms read their numbers out of that dict inside a fail-open
|
||||||
|
`except`, so a missing key does not raise where anyone can see it — the
|
||||||
|
arm silently becomes a no-op, which is indistinguishable from the arm
|
||||||
|
working and finding nothing. The helper derives its SURFACE keys from the
|
||||||
|
registry to prevent exactly that, and then #4214 added a key that is
|
||||||
|
deliberately not a surface: the whole derivation missed it, ten tests went
|
||||||
|
red at once, and the diagnosis cost a CI round.
|
||||||
|
|
||||||
|
Asserting the KEY SETS match, not the values: the stand-in exists to let a
|
||||||
|
test set different numbers.
|
||||||
|
"""
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
from scribe.services import plugin_context as pc
|
||||||
|
from tests.helpers import writepath_cfg
|
||||||
|
|
||||||
|
with patch.object(pc, "get_setting", AsyncMock(return_value="0.6")), \
|
||||||
|
patch.object(pc, "floor_for", AsyncMock(return_value=0.6)), \
|
||||||
|
patch.object(pc, "budget_for", AsyncMock(return_value=3)):
|
||||||
|
real = await pc.get_writepath_config(1)
|
||||||
|
|
||||||
|
assert set(writepath_cfg()) == set(real), (
|
||||||
|
"tests/helpers.writepath_cfg has fallen behind get_writepath_config; "
|
||||||
|
"a key the real config has and the stand-in does not turns an arm "
|
||||||
|
"into a silent no-op under test"
|
||||||
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user