feat(telemetry): the preload emits, and the always-on set stops being unfalsifiable (#3473)

The ranked rule arm became measurable in M333. The preload did not — and
that is the surface whose value is actually in question. `list_always_on_rules`,
the SessionStart block and every `rules_payload` caller handed rules over
wholesale and emitted nothing, so the resident set's token cost was certain
and its usefulness could not be tested even in principle.

Bulk deliveries now record as AMBIENT, beside the ranked count and never
inside pull-through. Folding them in would mean growing the always-on set
depressed the arm's measured precision and trimming it flattered the arm,
neither for any reason to do with the arm.

`RANKED_SOURCES` inverts the note twin's `AMBIENT_SOURCES` deliberately: there
is one ranked rule source and this change adds seven bulk ones, so naming the
rare half makes a forgotten surface default to ambient — under-counting it —
rather than padding the denominator with surfacings nobody chose.

Two lookalike call sites are deliberately left silent, with a test to keep
them that way: the write-path etag arm and `rules_etag_for` read the rules to
build or compare a MARKER and show nobody anything.

No migration — `event` and `source` are plain Text with no CHECK (rule 36
does not apply). Snippet #2858 updated to the new `rules_payload` contract.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011cPyzNnegXHr5iRMzzy5KJ

tests/test_inception_rules.py

@@ -15,14 +15,17 @@ def test_rules_payload_carries_excluded_always_on_as_the_seventh_key():
     out = rules_payload({
         "rules": [], "truncated": False, "subscribed_rulebooks": [],
         "excluded_always_on": [{"id": 1, "title": "Family"}],
-    })
+    }, user_id=1, source="enter_project")
     assert set(out) == {
         "applicable_rules", "applicable_rules_truncated", "subscribed_rulebooks",
         "project_rules", "suppressed_rules", "suppressed_topics", "excluded_always_on",
     }
     assert out["excluded_always_on"] == [{"id": 1, "title": "Family"}]
     # An older applicable dict without the key still renders (empty list).
-    assert rules_payload({"rules": [], "truncated": False, "subscribed_rulebooks": []})["excluded_always_on"] == []
+    assert rules_payload(
+        {"rules": [], "truncated": False, "subscribed_rulebooks": []},
+        user_id=1, source="enter_project",
+    )["excluded_always_on"] == []
 
 
 def test_list_always_on_rules_service_and_tool_take_a_project_id():

tests/test_rule_usage_wiring.py

@@ -274,3 +274,130 @@ def test_the_bulk_loaders_are_not_counted_as_pulls():
                 "applicable rule at once — so counting it would drown the "
                 "surfaced:pulled ratio in ambient delivery."
             )
+
+
+# ── The AMBIENT end: bulk deliveries (#3473) ───────────────────────────
+#
+# The preload was the largest rule surface in the product and emitted nothing,
+# so its cost was certain and its usefulness unfalsifiable. These assert the
+# three delivery shapes now emit — and, just as importantly, that the two
+# lookalike call sites which show nobody anything do NOT.
+
+
+@pytest.mark.asyncio
+async def test_the_session_start_preload_records_what_it_delivered():
+    """The block every session opens with. Chosen by nobody, paid for every
+    turn — and until it emitted, invisible to the scoreboard that judges every
+    other surface."""
+    from scribe.services import plugin_context as pc
+
+    rec = MagicMock()
+    rules = [fake_rule(id=1, title="`dev` is home"),
+             fake_rule(id=2, title="`main` — never without explicit request")]
+    with ExitStack() as stack:
+        stack.enter_context(
+            patch.object(pc.rulebooks_svc, "list_always_on_rules",
+                         AsyncMock(return_value=rules))
+        )
+        stack.enter_context(
+            patch.object(pc.rulebooks_svc, "excluded_always_on_rulebooks",
+                         AsyncMock(return_value=[]))
+        )
+        stack.enter_context(patch.object(pc, "record_rule_surfaced", rec))
+        stack.enter_context(
+            patch.object(pc, "_topic_titles", AsyncMock(return_value={}))
+        )
+        await pc.build_session_context(1, project_id=0)
+
+    assert rec.call_count == 1, "the preload recorded nothing"
+    kw = rec.call_args.kwargs
+    assert kw["rule_ids"] == [1, 2]
+    assert kw["source"] == "session_start"
+
+
+@pytest.mark.asyncio
+async def test_the_always_on_tool_records_what_it_handed_over():
+    from scribe.mcp.tools import rulebooks as tools
+
+    rec = MagicMock()
+    rules = [fake_rule(id=3, title="No GitHub — Fabled-Git only")]
+    with ExitStack() as stack:
+        stack.enter_context(
+            patch.object(tools.rulebooks_svc, "list_always_on_rules",
+                         AsyncMock(return_value=rules))
+        )
+        stack.enter_context(
+            patch.object(tools.rulebooks_svc, "rules_etag",
+                         MagicMock(return_value="etag"))
+        )
+        stack.enter_context(patch.object(tools, "record_rule_surfaced", rec))
+        await tools.list_always_on_rules()
+
+    assert rec.call_args.kwargs["rule_ids"] == [3]
+    assert rec.call_args.kwargs["source"] == "list_always_on_rules"
+
+
+def test_rules_payload_records_both_the_family_and_project_halves():
+    """One emit site for all five `rules_payload` surfaces.
+
+    Per-caller emission would be five sites to remember, and #3430 gap 2 is
+    what that costs: the process→skill sync went un-emitted through an entire
+    dedicated telemetry survey because nothing forced its surface to be
+    accounted for.
+    """
+    from scribe.services import rulebooks as svc
+
+    rec = MagicMock()
+    with patch.object(svc, "record_rule_surfaced", rec):
+        svc.rules_payload(
+            {
+                "rules": [{"id": 10}, {"id": 11}],
+                "project_rules": [{"id": 12}],
+                "truncated": False,
+                "subscribed_rulebooks": [],
+            },
+            user_id=1,
+            source="enter_project",
+        )
+
+    kw = rec.call_args.kwargs
+    assert kw["rule_ids"] == [10, 11, 12], "project-scoped rules were delivered too"
+    assert kw["source"] == "enter_project"
+
+
+def test_every_rules_payload_caller_names_itself():
+    """`source` is the CALLER's name, so the readout can still separate the
+    session handshake from a mid-session milestone read. A shared constant here
+    would collapse five distinguishable surfaces into one."""
+    import re
+
+    seen = set()
+    for path in Path("src/scribe").rglob("*.py"):
+        for m in re.finditer(r"rules_payload\([^)]*source=\"([a-z_]+)\"", path.read_text()):
+            seen.add(m.group(1))
+    assert seen == {
+        "enter_project", "get_project", "get_milestone",
+        "start_planning", "get_task",
+    }, f"a rules_payload caller is missing or misnamed: {sorted(seen)}"
+
+
+def test_the_marker_paths_stay_silent():
+    """The two call sites that read the rules and show NOBODY anything.
+
+    `rules_etag_for` and the write-path staleness arm both call
+    `list_always_on_rules` to build or compare a marker. Emitting there would
+    put rules in the denominator that no agent ever saw — the exact inflation
+    `record_rule_surfaced`'s docstring forbids, arriving from the one direction
+    nothing else guards.
+    """
+    svc_src = Path("src/scribe/services/rulebooks.py").read_text()
+    etag_fn = svc_src.split("async def rules_etag_for")[1].split("\ndef ")[0]
+    assert "record_rule_surfaced" not in etag_fn, (
+        "rules_etag_for emits a surfacing — it builds a marker, it shows nothing"
+    )
+
+    pc_src = Path("src/scribe/services/plugin_context.py").read_text()
+    staleness = pc_src.split("if rules_etag:")[1].split("# The guard sits BELOW")[0]
+    assert "record_rule_surfaced" not in staleness, (
+        "the staleness arm emits a surfacing — it compares a marker, it shows nothing"
+    )

tests/test_services_retrieval_telemetry.py

@@ -511,3 +511,68 @@ async def test_rule_usage_sees_only_its_own_users_events(_dispose_engine):
         assert (await retrieval_summary(990014, days=30))["rule_usage"]["surfaced"] == 1
     finally:
         await cleanup()
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+async def test_the_preload_lands_in_ambient_and_never_in_the_ratio(_dispose_engine):
+    """The split that makes the always-on set judgeable (#3473).
+
+    Pull-through asks "was that hint any use", and only a surface that CHOSE
+    what it showed can be judged by it. If the preload counted toward the
+    denominator, growing the always-on set would DEPRESS the arm's measured
+    precision and trimming it would flatter it — neither for any reason to do
+    with the arm. So the resident deliveries are counted, reported, and kept
+    out of the ratio.
+    """
+    from scribe.services.retrieval_telemetry import retrieval_summary
+
+    cleanup = await _rule_events(990012, [
+        # One rule the arm actually chose, and opened.
+        (5101, "surfaced", "write_path_rule"),
+        (5101, "pulled", "mcp_get_rule"),
+        # Four bulk deliveries across every shape of preload. Nobody chose any
+        # of them, and none may touch the denominator.
+        (5102, "surfaced", "session_start"),
+        (5103, "surfaced", "list_always_on_rules"),
+        (5104, "surfaced", "enter_project"),
+        (5105, "surfaced", "get_milestone"),
+    ])
+    try:
+        ru = (await retrieval_summary(990012, days=30))["rule_usage"]
+
+        assert ru["surfaced"] == 1, "only the arm chose a rule"
+        assert ru["ambient"] == 4, "the four bulk deliveries are reported, not dropped"
+
+        # 1 agent pull over 1 RANKED surfacing. Were the ambient four folded in
+        # the ratio would read 0.2 — the arm looking four times worse for
+        # having a large resident set beside it.
+        assert ru["pull_through"] == 1.0
+
+        # Dead-weight detection needs both classes: a rule delivered by the
+        # preload and never opened is the case that reading matters most for.
+        assert ru["distinct_rules_surfaced"] == 5
+        assert ru["distinct_rules_pulled"] == 1
+    finally:
+        await cleanup()
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+async def test_ambient_alone_reports_no_ratio(_dispose_engine):
+    """A brand-new install loads rules every session and may never trigger the
+    arm. That must read as "no ranked surfacings yet", not as a precision of
+    zero — the reading that would make a working install look broken."""
+    from scribe.services.retrieval_telemetry import retrieval_summary
+
+    cleanup = await _rule_events(990013, [
+        (5201, "surfaced", "session_start"),
+        (5202, "surfaced", "session_start"),
+    ])
+    try:
+        ru = (await retrieval_summary(990013, days=30))["rule_usage"]
+        assert ru["ambient"] == 2
+        assert ru["surfaced"] == 0
+        assert ru["pull_through"] is None
+    finally:
+        await cleanup()

tests/test_services_rule_usage.py

@@ -99,12 +99,32 @@ def test_the_zero_readout_names_every_key():
     — a missing key here would read as a broken readout on almost every row."""
     assert rule_usage.empty_rule_usage() == {
         "surfaced_count": 0,
+        "ambient_count": 0,
         "pull_count": 0,
         "last_surfaced_at": None,
         "last_pulled_at": None,
     }
 
 
+def test_only_a_ranker_counts_as_ranked():
+    """The bulk surfaces are ambient; the write-path arm is the only chooser.
+
+    Inverted against the note twin on purpose (see the module docstring): the
+    RARE half is the one that gets named, so a bulk surface added later and
+    forgotten defaults to ambient — under-counting it — instead of defaulting
+    to ranked and padding the pull-through denominator with surfacings nobody
+    chose.
+    """
+    assert not rule_usage.is_ambient("write_path_rule")
+    for bulk in (
+        "session_start", "list_always_on_rules", "enter_project",
+        "get_project", "get_milestone", "start_planning", "get_task",
+    ):
+        assert rule_usage.is_ambient(bulk), bulk
+    # The safe default is the whole point of the inversion.
+    assert rule_usage.is_ambient("some_surface_invented_next_year")
+
+
 def test_the_model_serialises_the_fields_the_ratio_needs():
     ev = RuleUsageEvent(
         user_id=7, rule_id=156, event=SURFACED, source="write_path_rule"
@@ -161,6 +181,10 @@ async def test_usage_for_rules_aggregates_per_rule(_dispose_engine):
         # never have to tell "no events" from "not in the result" — and on any
         # existing install that is nearly every rule.
         assert out[6003] == rule_usage.empty_rule_usage()
+
+        # Nothing ambient in this fixture, so the ambient bucket stays empty
+        # rather than absorbing the ranked hits.
+        assert out[6001]["ambient_count"] == 0
     finally:
         async with async_session() as s:
             await s.execute(
@@ -178,3 +202,56 @@ async def test_usage_for_rules_on_an_empty_id_list_asks_the_database_nothing(
     empty topic is nothing. An unguarded `IN ()` is both a pointless round trip
     and, on some drivers, a syntax error."""
     assert await rule_usage.usage_for_rules([]) == {}
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+async def test_a_preloaded_rule_does_not_read_as_a_ranked_surfacing(_dispose_engine):
+    """The split that makes the always-on set judgeable (#3473).
+
+    A resident rule is delivered every session by a surface that chose
+    nothing. Counting those as `surfaced_count` would rank the always-on set
+    as the most-surfaced rules in the install purely for being resident — and
+    the badge's "shown often, opened never → dead weight" reading, which is
+    the whole reason the counter exists, would then be exactly backwards.
+    """
+    from sqlalchemy import delete
+
+    from scribe.models import async_session
+    from scribe.models.rule_usage import RuleUsageEvent
+
+    async with async_session() as s:
+        s.add_all([
+            # Delivered by the preload three times over: ambient, all of it.
+            RuleUsageEvent(user_id=990021, rule_id=6101,
+                           event=SURFACED, source="session_start"),
+            RuleUsageEvent(user_id=990021, rule_id=6101,
+                           event=SURFACED, source="list_always_on_rules"),
+            RuleUsageEvent(user_id=990021, rule_id=6101,
+                           event=SURFACED, source="enter_project"),
+            # ...and once by the arm, which DID choose it.
+            RuleUsageEvent(user_id=990021, rule_id=6101,
+                           event=SURFACED, source="write_path_rule"),
+            # Opened once after a hint and once from the list: pulls are pulls
+            # however the rule was found, so both land in the one counter.
+            RuleUsageEvent(user_id=990021, rule_id=6101,
+                           event=PULLED, source="mcp_get_rule"),
+            RuleUsageEvent(user_id=990021, rule_id=6101,
+                           event=PULLED, source="rest_rule"),
+        ])
+        await s.commit()
+    try:
+        out = await rule_usage.usage_for_rules([6101])
+
+        assert out[6101]["surfaced_count"] == 1, "only the arm chose this rule"
+        assert out[6101]["ambient_count"] == 3, "three bulk deliveries"
+        # Both PULLED rows accumulate — the loop ADDS rather than assigns, so a
+        # rule opened after a hint and again from the list reports two, not one.
+        assert out[6101]["pull_count"] == 2
+        assert out[6101]["last_pulled_at"] is not None
+    finally:
+        async with async_session() as s:
+            await s.execute(
+                delete(RuleUsageEvent).where(RuleUsageEvent.user_id == 990021)
+            )
+            await s.commit()