feat(telemetry): the readout names rules that were opened and changed nothing (#4213)

Milestone 419 step 2. Step 1 made an outcome recordable; this makes it
readable. `retrieval_summary`'s rule block gains `applied`, `departed` and
`distinct_rules_acted`, and `_compute_warnings` gains two codes.

TWO CODES, NOT ONE WITH A ZERO IN IT. `read_and_unacted` reports rules that
were opened and left no outcome, against the ones that did. It only fires once
outcomes exist anywhere in the window, because a window with none cannot tell
"every rule was ignored" from "nothing calls `rule_outcome` yet" — and on
every install the day this ships, the truth is the second. Claiming the first
there would be #3311's failure exactly: a statistic that could not vary being
read as a fact about the corpus. The cold case gets its own code,
`outcomes_never_recorded`, whose prose says in as many words that it does NOT
mean the rules were ignored.

`applied` AND `departed` ARE NOT SUMMED. A departure carries the reason the
agent gave and is evidence about the RULE; an application is evidence about
the agent. Folded together they would say only "an outcome exists", which is
true of both and useful about neither. `distinct_rules_acted` counts either,
because for the unacted arithmetic the distinction does not matter.

An outcome is not a pull. The fold branches on OUTCOMES first and never routes
an outcome through the surfaced/ambient split: `source` on an outcome row
names the door the outcome came through, not a ranker, so the ambient
distinction has nothing to say about it. An integration test holds that line —
if an outcome leaked into the pull counters the silently-unchanged rule would
vanish into a compliant-looking total, which is the confusion #4212 was opened
to end.

Verified by lifting the shipped `_compute_warnings` out of source with `ast`
and exercising it against the six populations the new tests assert: cold
instrument, warm instrument, departures-only, full compliance, nothing opened,
and a failed read. The integration tests for the new counts run against real
Postgres in CI — count(distinct) with an IN over an unconstrained column is a
SQL shape a mock would agree with whatever it did, which is what #2663 was.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01821k5B3Ysecp9fNYs92Kuy

tests/test_retrieval_warnings.py

@@ -266,3 +266,86 @@ def test_a_point_seen_only_in_usage_counts_as_having_emitted() -> None:
 ])
 def test_a_setting_falls_back_rather_than_raising(raw, fallback, want) -> None:
     assert _num(raw, fallback) == want
+
+
+# ── read_and_unacted / outcomes_never_recorded (#4213, milestone 419) ─────
+#
+# The pair exists because ZERO OUTCOMES IS AMBIGUOUS, and getting that wrong
+# would have been this milestone's own failure mode in miniature: a window
+# with no outcome rows cannot tell "every rule was ignored" from "nothing
+# reports outcomes yet". Reporting the first when the truth is the second
+# manufactures a finding out of an unwired feature — #3311, where a statistic
+# that could not vary was read as a fact about the corpus.
+
+def ru(**kw) -> dict:
+    base = {
+        "distinct_rules_surfaced": 0, "distinct_rules_pulled": 0,
+        "distinct_rules_acted": 0, "applied": 0, "departed": 0,
+    }
+    base.update(kw)
+    return base
+
+
+def test_rules_opened_with_no_outcome_machinery_running_says_so() -> None:
+    """The cold-instrument case, which is what an install looks like the day
+    this ships. It must NOT read as "47 rules ignored"."""
+    ws = warn({}, rule_usage=ru(distinct_rules_pulled=47))
+    assert "outcomes_never_recorded" in codes(ws)
+    assert "read_and_unacted" not in codes(ws)
+    [w] = [w for w in ws if w["code"] == "outcomes_never_recorded"]
+    assert w["numbers"]["opened"] == 47
+    # The distinction is in the prose, because the prose is what gets read.
+    assert "does NOT mean they were ignored" in w["detail"]
+
+
+def test_once_outcomes_exist_the_unacted_rules_are_named() -> None:
+    """The instrument is live — some rules recorded an outcome — so the ones
+    that did not are a real finding rather than an artefact."""
+    ws = warn({}, rule_usage=ru(
+        distinct_rules_pulled=20, distinct_rules_acted=6, applied=5, departed=2,
+    ))
+    assert "read_and_unacted" in codes(ws)
+    assert "outcomes_never_recorded" not in codes(ws)
+    [w] = [w for w in ws if w["code"] == "read_and_unacted"]
+    assert w["numbers"]["unacted"] == 14
+    assert w["numbers"]["opened"] == 20 and w["numbers"]["acted"] == 6
+    assert w["numbers"]["applied"] == 5 and w["numbers"]["departed"] == 2
+
+
+def test_a_departure_alone_is_enough_to_warm_the_instrument() -> None:
+    """Departures count as outcomes. An install whose every recorded outcome
+    is a departure is saying something loudly, and must not be mistaken for
+    one that records nothing."""
+    ws = warn({}, rule_usage=ru(
+        distinct_rules_pulled=9, distinct_rules_acted=2, departed=3,
+    ))
+    assert "read_and_unacted" in codes(ws)
+    assert "outcomes_never_recorded" not in codes(ws)
+
+
+def test_every_opened_rule_acted_on_reports_nothing() -> None:
+    ws = warn({}, rule_usage=ru(
+        distinct_rules_pulled=4, distinct_rules_acted=4, applied=4,
+    ))
+    assert "read_and_unacted" not in codes(ws)
+    assert "outcomes_never_recorded" not in codes(ws)
+
+
+def test_no_rules_opened_at_all_reports_neither() -> None:
+    """Silence is not a finding. A window where nothing was opened has nothing
+    to say about outcomes, and saying it anyway would put a warning on every
+    fresh install (rule 115)."""
+    ws = warn({}, rule_usage=ru(distinct_rules_surfaced=12))
+    assert "read_and_unacted" not in codes(ws)
+    assert "outcomes_never_recorded" not in codes(ws)
+
+
+def test_an_absent_rule_usage_block_is_not_a_finding() -> None:
+    """A failed rule-usage read leaves the keys missing or zero. Neither may
+    become a warning, because a warning computed over rows that could not be
+    loaded describes the outage, not the corpus (#2663)."""
+    assert "read_and_unacted" not in codes(warn({}, rule_usage={}))
+    assert "outcomes_never_recorded" not in codes(warn({}, rule_usage={}))
+    assert "outcomes_never_recorded" not in codes(
+        warn({}, rule_usage={"rule_usage_failed": True})
+    )

tests/test_services_retrieval_telemetry.py

@@ -643,6 +643,72 @@ async def test_ambient_alone_reports_no_ratio(_dispose_engine):
         await cleanup()
 
 
+@pytest.mark.integration
+@pytest.mark.asyncio
+async def test_the_outcome_counts_come_back_split(_dispose_engine):
+    """A pull says the rule was read; an outcome says it changed something
+    (#4213). Integration rather than a mock because `distinct_rules_acted` is
+    a count(distinct) with an IN over a column that carries no CHECK — the
+    kind of SQL shape #2663 was, where a mock agrees with whatever the code
+    does including nothing.
+
+    `applied` and `departed` stay APART. Summed they would say "an outcome was
+    recorded", which is true of both and useful about neither: a departure is
+    evidence about the rule, an application is evidence about the agent.
+    """
+    from scribe.services.retrieval_telemetry import retrieval_summary
+
+    cleanup = await _rule_events(990016, [
+        # Opened and followed.
+        (5301, "surfaced", "write_path_rule"),
+        (5301, "pulled", "mcp_get_rule"),
+        (5301, "applied", "mcp_rule_outcome"),
+        # Opened and deliberately departed from.
+        (5302, "surfaced", "write_path_rule"),
+        (5302, "pulled", "mcp_get_rule"),
+        (5302, "departed", "mcp_rule_outcome"),
+        # Opened, and nothing after it. The state the milestone exists for,
+        # and the one that is counted by its ABSENCE.
+        (5303, "surfaced", "write_path_rule"),
+        (5303, "pulled", "mcp_get_rule"),
+    ])
+    try:
+        ru = (await retrieval_summary(990016, days=30))["rule_usage"]
+        assert ru["applied"] == 1
+        assert ru["departed"] == 1
+        assert ru["distinct_rules_acted"] == 2
+        assert ru["distinct_rules_pulled"] == 3
+
+        # An outcome is NOT a pull. If `applied` leaked into the pull counters
+        # the silently-unchanged rule would vanish into a compliant-looking
+        # total, which is the exact confusion #4212 was opened to end.
+        assert ru["pulled"] == 3
+    finally:
+        await cleanup()
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+async def test_an_outcome_is_never_split_by_ambient(_dispose_engine):
+    """`source` on an outcome row names the door the outcome came through, not
+    a ranker, so the surfaced/ambient split has nothing to say about it. An
+    outcome recorded from an unranked source must still count."""
+    from scribe.services.retrieval_telemetry import retrieval_summary
+
+    cleanup = await _rule_events(990017, [
+        (5401, "surfaced", "session_start"),
+        (5401, "pulled", "mcp_get_rule"),
+        (5401, "applied", "some_door_nobody_has_ranked"),
+    ])
+    try:
+        ru = (await retrieval_summary(990017, days=30))["rule_usage"]
+        assert ru["applied"] == 1
+        assert ru["distinct_rules_acted"] == 1
+        assert ru["ambient"] == 1, "the surfacing was ambient; the outcome is not"
+    finally:
+        await cleanup()
+
+
 # ── Window coverage (#3712) ────────────────────────────────────────────
 #
 # A counter added last week, read over a 30-day window, reports a real count