feat(4636): rule pull-through per surface, and per-rule usage with confirmed lessons on list_rules
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 13s
CI & Build / TypeScript typecheck (push) Successful in 55s
CI & Build / integration (push) Successful in 1m3s
CI & Build / Python tests (push) Successful in 1m56s
CI & Build / Build & push image (push) Successful in 22s

retrieval_telemetry gains rule_usage.by_source, the twin of usage.by_source:
distinct rules each surface showed, how many an agent opened, and a ratio
that is null on a bulk delivery. list_rules(with_usage=True, days=N) adds
each rule's usage, windowed, and how many lessons are confirmed instances
of it, so linked and unlinked rules can be compared through MCP.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-22 19:20:32 -04:00
co-authored by Claude Opus 5.5
parent 71260a28e2
commit d05d4f3632
9 changed files with 287 additions and 9 deletions
@@ -618,6 +618,54 @@ async def test_the_preload_lands_in_ambient_and_never_in_the_ratio(_dispose_engi
await cleanup()
@pytest.mark.integration
@pytest.mark.asyncio
async def test_rule_by_source_says_which_arm_earns_its_lines(_dispose_engine):
"""The rule ratio blends every arm. Per surface it can say which one is
working: distinct rules per arm, the agent opens among them, and no ratio
on a bulk delivery."""
from grimoire.services.retrieval_telemetry import retrieval_summary
cleanup = await _rule_events(990016, [
# The write arm chose rule 5301 twice — one rule — and 5302 once.
(5301, "surfaced", "write_path_rule"),
(5301, "surfaced", "write_path_rule"),
(5302, "surfaced", "write_path_rule"),
# The tool arm chose 5303 and nothing came of it.
(5303, "surfaced", "pre_tool_rule"),
# The preload carried 5301 as well.
(5301, "surfaced", "session_start"),
# An agent opened 5301; a person opened 5303, which is not a hint
# landing.
(5301, "pulled", "mcp_get_rule"),
(5303, "pulled", "rest_rule"),
])
try:
ru = (await retrieval_summary(990016, days=30))["rule_usage"]
assert "by_source_failed" not in ru, "the join did not execute"
by_source = ru["by_source"]
wp = by_source["write_path_rule"]
assert (wp["rules_surfaced"], wp["rules_pulled"]) == (2, 1)
assert wp["pull_through"] == 0.5
assert wp["ambient"] is False
tool = by_source["pre_tool_rule"]
assert (tool["rules_surfaced"], tool["rules_pulled"]) == (1, 0)
assert tool["pull_through"] == 0.0
preload = by_source["session_start"]
# The pull is counted against every surface that showed the rule,
# the upper bound the readout states.
assert (preload["rules_surfaced"], preload["rules_pulled"]) == (1, 1)
assert preload["pull_through"] is None
assert preload["ambient"] is True
assert "mcp_get_rule" not in by_source, "a pull is not a surface"
finally:
await cleanup()
@pytest.mark.integration
@pytest.mark.asyncio
async def test_ambient_alone_reports_no_ratio(_dispose_engine):