feat(6251): the rule arms can be reviewed — menus_to_review re-runs prompt_rule, pre_tool_rule and write_path_rule
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 12s
CI & Build / TypeScript typecheck (push) Successful in 53s
CI & Build / integration (push) Successful in 1m2s
CI & Build / Python tests (push) Failing after 1m26s
CI & Build / Build & push image (push) Skipped

A rule arm logs its query, floor, budget and project, which is everything
semantic_search_rules takes, so a logged call is re-run to a depth past its
budget and judged like an auto_inject menu. within_budget is the arm's own
cut: inside the budget and, for the banded command and write arms, inside the
band of the top hit. "Opened after" reads rule pulls, and rules created
after the call are dropped as notes are.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-22 19:56:17 -04:00
co-authored by Claude Opus 5.5
parent 4297c88cd2
commit 7c1d98dd2a
3 changed files with 179 additions and 16 deletions
+100 -3
View File
@@ -81,7 +81,7 @@ async def test_the_re_run_searches_the_way_the_arm_does():
rerun_kw.update(kw)
return []
row = SimpleNamespace(query="q", threshold=0.5, limit_n=3, project_id=None,
row = SimpleNamespace(source="auto_inject", query="q", threshold=0.5, limit_n=3, project_id=None,
created_at=CALL)
with patch("grimoire.services.embeddings.semantic_search_notes", side_effect=rerun_search):
await review._rerun(1, row, 3)
@@ -99,7 +99,7 @@ BEFORE = CALL - timedelta(days=1)
@pytest.mark.asyncio
async def test_the_re_run_ranks_from_one_and_marks_the_budget_cut():
row = SimpleNamespace(query="q", threshold=0.55, limit_n=2, project_id=None,
row = SimpleNamespace(source="auto_inject", query="q", threshold=0.55, limit_n=2, project_id=None,
created_at=CALL)
hits = [
(0.81, fake_note(id=11, title="first", created_at=BEFORE, updated_at=BEFORE)),
@@ -127,7 +127,7 @@ async def test_a_record_written_after_the_call_is_dropped_before_ranking():
"""The session that made a call goes on to write about it, and that record
outranks everything in a re-run. The call could never have been offered
it, so it must not take a rank — or a verdict — from what was."""
row = SimpleNamespace(query="q", threshold=0.55, limit_n=1, project_id=None,
row = SimpleNamespace(source="auto_inject", query="q", threshold=0.55, limit_n=1, project_id=None,
created_at=CALL)
after = CALL + timedelta(hours=1)
hits = [
@@ -147,6 +147,103 @@ async def test_a_record_written_after_the_call_is_dropped_before_ranking():
for ln in lines] == [(1, 22, True, False), (2, 23, False, True)]
# ── the rule arms ────────────────────────────────────────────────────────
def _rule(rid, title, created=BEFORE, updated=BEFORE):
return SimpleNamespace(
id=rid, title=title, kind="rule", when_to_apply=f"when {title}",
statement="s", created_at=created, updated_at=updated,
)
def test_the_reviewable_rule_arms_are_the_ones_that_annotate():
"""Every rule arm that shows lines can be reviewed; the reply backstop
shows none, so there is no menu to judge."""
from grimoire.services import retrieval_pipeline as rp
assert set(review.RULE_REVIEWABLE) == {
a.source for a in rp.RULE_ARMS if not a.stop_only
}
@pytest.mark.asyncio
async def test_the_rule_re_run_searches_the_way_the_arm_does():
"""The floor and the project the arm searched with, run and compared."""
from grimoire.services import retrieval_pipeline as rp
arm_kw: dict = {}
async def arm_search(_uid, _q, **kw):
if "kind" not in kw:
arm_kw.update(kw)
return []
await rp.run_rule_arm(
rp.PROMPT_RULE, rp.RuleMoment(user_id=1, query="q", project_id=7),
floor=0.7, budget=3,
io=rp.RuleIO(search=arm_search, record_retrieval=MagicMock(),
record_rule_surfaced=MagicMock()),
)
rerun_kw: dict = {}
async def rerun_search(_uid, _q, **kw):
rerun_kw.update(kw)
return []
row = SimpleNamespace(source="prompt_rule", query="q", threshold=0.7, limit_n=3,
project_id=7, created_at=CALL)
with patch("grimoire.services.embeddings.semantic_search_rules", side_effect=rerun_search):
await review._rerun(1, row, 3)
for key in ("threshold", "project_id"):
assert rerun_kw[key] == arm_kw[key], key
assert "kind" not in rerun_kw
@pytest.mark.asyncio
@pytest.mark.parametrize(("source", "within"), [
# No band: the budget alone is the cut.
("prompt_rule", [True, True, True, False]),
# Banded: the third sits outside the band of the top hit, so it was
# not shown though the budget had room for it.
("pre_tool_rule", [True, True, False, False]),
])
async def test_a_rule_line_is_inside_the_cut_only_where_the_arm_showed_it(source, within):
row = SimpleNamespace(source=source, query="git push", threshold=0.7, limit_n=3,
project_id=None, created_at=CALL)
hits = [(0.86, _rule(1, "one")), (0.84, _rule(2, "two")),
(0.76, _rule(3, "three")), (0.74, _rule(4, "four"))]
async def search(_uid, _q, **kw):
assert kw["threshold"] == 0.7 and kw["project_id"] is None
return hits
with patch("grimoire.services.embeddings.semantic_search_rules", side_effect=search):
lines = await review._rerun(1, row, 4)
assert [ln["within_budget"] for ln in lines] == within
assert lines[0]["passage"] == "when one"
assert lines[0]["kind"] == "rule"
@pytest.mark.asyncio
async def test_a_rule_written_after_the_call_is_dropped_before_ranking():
row = SimpleNamespace(source="write_path_rule", query="q", threshold=0.7, limit_n=2,
project_id=None, created_at=CALL)
after = CALL + timedelta(hours=1)
hits = [(0.9, _rule(31, "written after", created=after, updated=after)),
(0.8, _rule(32, "held then"))]
async def search(_uid, _q, **kw):
return hits
rep: dict = {}
with patch("grimoire.services.embeddings.semantic_search_rules", side_effect=search):
lines = await review._rerun(1, row, 2, report=rep)
assert rep["postdated"] == [31]
assert [(ln["rank"], ln["record_id"]) for ln in lines] == [(1, 32)]
# ── against Postgres ─────────────────────────────────────────────────────