feat(4636): rule pull-through per surface, and per-rule usage with confirmed lessons on list_rules
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 13s
CI & Build / TypeScript typecheck (push) Successful in 55s
CI & Build / integration (push) Successful in 1m3s
CI & Build / Python tests (push) Successful in 1m56s
CI & Build / Build & push image (push) Successful in 22s
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 13s
CI & Build / TypeScript typecheck (push) Successful in 55s
CI & Build / integration (push) Successful in 1m3s
CI & Build / Python tests (push) Successful in 1m56s
CI & Build / Build & push image (push) Successful in 22s
retrieval_telemetry gains rule_usage.by_source, the twin of usage.by_source: distinct rules each surface showed, how many an agent opened, and a ratio that is null on a bulk delivery. list_rules(with_usage=True, days=N) adds each rule's usage, windowed, and how many lessons are confirmed instances of it, so linked and unlinked rules can be compared through MCP. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -618,6 +618,54 @@ async def test_the_preload_lands_in_ambient_and_never_in_the_ratio(_dispose_engi
|
||||
await cleanup()
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_rule_by_source_says_which_arm_earns_its_lines(_dispose_engine):
|
||||
"""The rule ratio blends every arm. Per surface it can say which one is
|
||||
working: distinct rules per arm, the agent opens among them, and no ratio
|
||||
on a bulk delivery."""
|
||||
from grimoire.services.retrieval_telemetry import retrieval_summary
|
||||
|
||||
cleanup = await _rule_events(990016, [
|
||||
# The write arm chose rule 5301 twice — one rule — and 5302 once.
|
||||
(5301, "surfaced", "write_path_rule"),
|
||||
(5301, "surfaced", "write_path_rule"),
|
||||
(5302, "surfaced", "write_path_rule"),
|
||||
# The tool arm chose 5303 and nothing came of it.
|
||||
(5303, "surfaced", "pre_tool_rule"),
|
||||
# The preload carried 5301 as well.
|
||||
(5301, "surfaced", "session_start"),
|
||||
# An agent opened 5301; a person opened 5303, which is not a hint
|
||||
# landing.
|
||||
(5301, "pulled", "mcp_get_rule"),
|
||||
(5303, "pulled", "rest_rule"),
|
||||
])
|
||||
try:
|
||||
ru = (await retrieval_summary(990016, days=30))["rule_usage"]
|
||||
assert "by_source_failed" not in ru, "the join did not execute"
|
||||
by_source = ru["by_source"]
|
||||
|
||||
wp = by_source["write_path_rule"]
|
||||
assert (wp["rules_surfaced"], wp["rules_pulled"]) == (2, 1)
|
||||
assert wp["pull_through"] == 0.5
|
||||
assert wp["ambient"] is False
|
||||
|
||||
tool = by_source["pre_tool_rule"]
|
||||
assert (tool["rules_surfaced"], tool["rules_pulled"]) == (1, 0)
|
||||
assert tool["pull_through"] == 0.0
|
||||
|
||||
preload = by_source["session_start"]
|
||||
# The pull is counted against every surface that showed the rule,
|
||||
# the upper bound the readout states.
|
||||
assert (preload["rules_surfaced"], preload["rules_pulled"]) == (1, 1)
|
||||
assert preload["pull_through"] is None
|
||||
assert preload["ambient"] is True
|
||||
|
||||
assert "mcp_get_rule" not in by_source, "a pull is not a surface"
|
||||
finally:
|
||||
await cleanup()
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_ambient_alone_reports_no_ratio(_dispose_engine):
|
||||
|
||||
Reference in New Issue
Block a user