feat(telemetry): the preload emits, and the always-on set stops being unfalsifiable (#3473)
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 11s
CI & Build / TypeScript typecheck (push) Successful in 23s
CI & Build / integration (push) Successful in 31s
CI & Build / Python tests (push) Successful in 1m6s
CI & Build / Build & push image (push) Successful in 27s
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 11s
CI & Build / TypeScript typecheck (push) Successful in 23s
CI & Build / integration (push) Successful in 31s
CI & Build / Python tests (push) Successful in 1m6s
CI & Build / Build & push image (push) Successful in 27s
The ranked rule arm became measurable in M333. The preload did not — and that is the surface whose value is actually in question. `list_always_on_rules`, the SessionStart block and every `rules_payload` caller handed rules over wholesale and emitted nothing, so the resident set's token cost was certain and its usefulness could not be tested even in principle. Bulk deliveries now record as AMBIENT, beside the ranked count and never inside pull-through. Folding them in would mean growing the always-on set depressed the arm's measured precision and trimming it flattered the arm, neither for any reason to do with the arm. `RANKED_SOURCES` inverts the note twin's `AMBIENT_SOURCES` deliberately: there is one ranked rule source and this change adds seven bulk ones, so naming the rare half makes a forgotten surface default to ambient — under-counting it — rather than padding the denominator with surfacings nobody chose. Two lookalike call sites are deliberately left silent, with a test to keep them that way: the write-path etag arm and `rules_etag_for` read the rules to build or compare a MARKER and show nobody anything. No migration — `event` and `source` are plain Text with no CHECK (rule 36 does not apply). Snippet #2858 updated to the new `rules_payload` contract. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011cPyzNnegXHr5iRMzzy5KJ
This commit is contained in:
@@ -274,3 +274,130 @@ def test_the_bulk_loaders_are_not_counted_as_pulls():
|
||||
"applicable rule at once — so counting it would drown the "
|
||||
"surfaced:pulled ratio in ambient delivery."
|
||||
)
|
||||
|
||||
|
||||
# ── The AMBIENT end: bulk deliveries (#3473) ───────────────────────────
|
||||
#
|
||||
# The preload was the largest rule surface in the product and emitted nothing,
|
||||
# so its cost was certain and its usefulness unfalsifiable. These assert the
|
||||
# three delivery shapes now emit — and, just as importantly, that the two
|
||||
# lookalike call sites which show nobody anything do NOT.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_session_start_preload_records_what_it_delivered():
|
||||
"""The block every session opens with. Chosen by nobody, paid for every
|
||||
turn — and until it emitted, invisible to the scoreboard that judges every
|
||||
other surface."""
|
||||
from scribe.services import plugin_context as pc
|
||||
|
||||
rec = MagicMock()
|
||||
rules = [fake_rule(id=1, title="`dev` is home"),
|
||||
fake_rule(id=2, title="`main` — never without explicit request")]
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(
|
||||
patch.object(pc.rulebooks_svc, "list_always_on_rules",
|
||||
AsyncMock(return_value=rules))
|
||||
)
|
||||
stack.enter_context(
|
||||
patch.object(pc.rulebooks_svc, "excluded_always_on_rulebooks",
|
||||
AsyncMock(return_value=[]))
|
||||
)
|
||||
stack.enter_context(patch.object(pc, "record_rule_surfaced", rec))
|
||||
stack.enter_context(
|
||||
patch.object(pc, "_topic_titles", AsyncMock(return_value={}))
|
||||
)
|
||||
await pc.build_session_context(1, project_id=0)
|
||||
|
||||
assert rec.call_count == 1, "the preload recorded nothing"
|
||||
kw = rec.call_args.kwargs
|
||||
assert kw["rule_ids"] == [1, 2]
|
||||
assert kw["source"] == "session_start"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_always_on_tool_records_what_it_handed_over():
|
||||
from scribe.mcp.tools import rulebooks as tools
|
||||
|
||||
rec = MagicMock()
|
||||
rules = [fake_rule(id=3, title="No GitHub — Fabled-Git only")]
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(
|
||||
patch.object(tools.rulebooks_svc, "list_always_on_rules",
|
||||
AsyncMock(return_value=rules))
|
||||
)
|
||||
stack.enter_context(
|
||||
patch.object(tools.rulebooks_svc, "rules_etag",
|
||||
MagicMock(return_value="etag"))
|
||||
)
|
||||
stack.enter_context(patch.object(tools, "record_rule_surfaced", rec))
|
||||
await tools.list_always_on_rules()
|
||||
|
||||
assert rec.call_args.kwargs["rule_ids"] == [3]
|
||||
assert rec.call_args.kwargs["source"] == "list_always_on_rules"
|
||||
|
||||
|
||||
def test_rules_payload_records_both_the_family_and_project_halves():
|
||||
"""One emit site for all five `rules_payload` surfaces.
|
||||
|
||||
Per-caller emission would be five sites to remember, and #3430 gap 2 is
|
||||
what that costs: the process→skill sync went un-emitted through an entire
|
||||
dedicated telemetry survey because nothing forced its surface to be
|
||||
accounted for.
|
||||
"""
|
||||
from scribe.services import rulebooks as svc
|
||||
|
||||
rec = MagicMock()
|
||||
with patch.object(svc, "record_rule_surfaced", rec):
|
||||
svc.rules_payload(
|
||||
{
|
||||
"rules": [{"id": 10}, {"id": 11}],
|
||||
"project_rules": [{"id": 12}],
|
||||
"truncated": False,
|
||||
"subscribed_rulebooks": [],
|
||||
},
|
||||
user_id=1,
|
||||
source="enter_project",
|
||||
)
|
||||
|
||||
kw = rec.call_args.kwargs
|
||||
assert kw["rule_ids"] == [10, 11, 12], "project-scoped rules were delivered too"
|
||||
assert kw["source"] == "enter_project"
|
||||
|
||||
|
||||
def test_every_rules_payload_caller_names_itself():
|
||||
"""`source` is the CALLER's name, so the readout can still separate the
|
||||
session handshake from a mid-session milestone read. A shared constant here
|
||||
would collapse five distinguishable surfaces into one."""
|
||||
import re
|
||||
|
||||
seen = set()
|
||||
for path in Path("src/scribe").rglob("*.py"):
|
||||
for m in re.finditer(r"rules_payload\([^)]*source=\"([a-z_]+)\"", path.read_text()):
|
||||
seen.add(m.group(1))
|
||||
assert seen == {
|
||||
"enter_project", "get_project", "get_milestone",
|
||||
"start_planning", "get_task",
|
||||
}, f"a rules_payload caller is missing or misnamed: {sorted(seen)}"
|
||||
|
||||
|
||||
def test_the_marker_paths_stay_silent():
|
||||
"""The two call sites that read the rules and show NOBODY anything.
|
||||
|
||||
`rules_etag_for` and the write-path staleness arm both call
|
||||
`list_always_on_rules` to build or compare a marker. Emitting there would
|
||||
put rules in the denominator that no agent ever saw — the exact inflation
|
||||
`record_rule_surfaced`'s docstring forbids, arriving from the one direction
|
||||
nothing else guards.
|
||||
"""
|
||||
svc_src = Path("src/scribe/services/rulebooks.py").read_text()
|
||||
etag_fn = svc_src.split("async def rules_etag_for")[1].split("\ndef ")[0]
|
||||
assert "record_rule_surfaced" not in etag_fn, (
|
||||
"rules_etag_for emits a surfacing — it builds a marker, it shows nothing"
|
||||
)
|
||||
|
||||
pc_src = Path("src/scribe/services/plugin_context.py").read_text()
|
||||
staleness = pc_src.split("if rules_etag:")[1].split("# The guard sits BELOW")[0]
|
||||
assert "record_rule_surfaced" not in staleness, (
|
||||
"the staleness arm emits a surfacing — it compares a marker, it shows nothing"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user