feat(retrieval): a review pass judges whether injected lines related — menus_to_review, judge_menu and a judged readout (#4772)

An open rate cannot say whether a menu line related: every line carries its
matched passage (#4364), so "not opened" covers unrelated, enough as shown,
and already in context. #4772 "Injected notes are never judged".

- retrieval_judgments (0115): a reviewer verdict per line of a logged call,
  on_point / adjacent / unrelated, with its reason, rank, budget side and
  whether the agent opened it within the hour.
- menus_to_review re-runs a random sample of unjudged auto_inject calls with
  the arm's own parameters, past its budget, passage on every line.
  judge_menu records verdicts, re-deriving rank from a fresh re-run.
- retrieval_telemetry gains a judged block (by rank, within/beyond budget,
  on_point_unopened). surfaced_never_pulled stops blaming titles.
- missed-retrieval guidance names the review before a budget move.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_retrieval_review.py

@@ -0,0 +1,250 @@
+"""The review pass: judging whether an injected line RELATED (#4772).
+
+Unit tests pin the refusals and the re-run's fidelity to the arm it replays.
+Integration tests pin the parts a mock would make true by construction: which
+logged calls are offered, that a verdict lands and replaces, that "opened
+after" reads the usage table, and that the readout counts by rank.
+"""
+import ast
+import inspect
+import textwrap
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+
+import pytest
+import pytest_asyncio
+
+from scribe.models import async_session
+from scribe.services import retrieval_review as review
+from tests.helpers import ensure_user, fake_note
+
+
+# ── refusals ─────────────────────────────────────────────────────────────
+
+
+@pytest.mark.asyncio
+async def test_only_an_arm_that_can_be_re_run_exactly_is_offered():
+    with pytest.raises(ValueError, match="source must be one of"):
+        await review.menus_to_review(1, source="write_path")
+
+
+@pytest.mark.asyncio
+async def test_a_verdict_outside_the_vocabulary_is_refused():
+    with pytest.raises(ValueError, match="verdict must be one of"):
+        await review.judge_menu(1, log_id=1, verdicts=[
+            {"record_id": 5, "verdict": "relevant", "reason": "x"},
+        ])
+
+
+@pytest.mark.asyncio
+async def test_a_verdict_without_its_reason_is_refused():
+    with pytest.raises(ValueError, match="needs its reason"):
+        await review.judge_menu(1, log_id=1, verdicts=[
+            {"record_id": 5, "verdict": "unrelated", "reason": "  "},
+        ])
+
+
+@pytest.mark.asyncio
+async def test_an_empty_verdict_list_is_refused():
+    with pytest.raises(ValueError, match="at least one"):
+        await review.judge_menu(1, log_id=1, verdicts=[])
+
+
+# ── the re-run replays the arm ───────────────────────────────────────────
+
+
+def _search_kwargs(fn) -> dict:
+    """The constant keywords `fn` passes to semantic_search_notes."""
+    tree = ast.parse(textwrap.dedent(inspect.getsource(fn)))
+    for node in ast.walk(tree):
+        if (isinstance(node, ast.Call) and getattr(node.func, "id", "")
+                == "semantic_search_notes"):
+            return {
+                kw.arg: kw.value.value for kw in node.keywords
+                if isinstance(kw.value, ast.Constant)
+            }
+    return {}
+
+
+def test_the_re_run_searches_the_way_the_arm_does():
+    """A re-run with different visibility or kinds from the arm's would judge
+    a menu nobody was shown. Read from both call sites, so a change to the
+    arm's search fails here until the review follows it."""
+    from scribe.services.plugin_context import build_autoinject_hint
+
+    arm = _search_kwargs(build_autoinject_hint)
+    rerun = _search_kwargs(review._rerun)
+    assert arm, "found no semantic_search_notes call in the arm"
+    for key in ("include_global_kinds", "scope"):
+        assert key in arm, f"the arm no longer passes {key} as a constant"
+        assert rerun.get(key) == arm[key], key
+
+
+@pytest.mark.asyncio
+async def test_the_re_run_ranks_from_one_and_marks_the_budget_cut():
+    row = SimpleNamespace(query="q", threshold=0.55, limit_n=2, project_id=None)
+    hits = [
+        (0.81, fake_note(id=11, title="first")),
+        (0.74, fake_note(id=12, title="second")),
+        (0.66, fake_note(id=13, title="third")),
+    ]
+
+    async def search(user_id, query, **kw):
+        assert kw["limit"] == 3 and kw["threshold"] == 0.55
+        assert kw["project_id"] is None
+        kw["report"]["best_chunk"] = {12: {"text": "second\nthe passage that matched"}}
+        return hits
+
+    with patch("scribe.services.embeddings.semantic_search_notes", side_effect=search):
+        lines = await review._rerun(1, row, 3)
+    assert [(ln["rank"], ln["record_id"], ln["within_budget"]) for ln in lines] == [
+        (1, 11, True), (2, 12, True), (3, 13, False),
+    ]
+    assert lines[1]["passage"] == "the passage that matched"
+    assert lines[0]["passage"] is None
+
+
+# ── against Postgres ─────────────────────────────────────────────────────
+
+
+@pytest_asyncio.fixture
+async def logged_calls():
+    """A reviewer's logged calls: two reviewable, one that offered nothing,
+    one of another arm — and another user's reviewable call."""
+    from scribe.models.retrieval_log import RetrievalLog
+
+    async with async_session() as s:
+        me = await ensure_user(s, "review_owner")
+        other = await ensure_user(s, "review_other")
+        now = datetime.now(timezone.utc)
+
+        def log(uid, source="auto_inject", count=2, query="how is the cache invalidated"):
+            return RetrievalLog(
+                user_id=uid, source=source, query=query, threshold=0.55,
+                limit_n=3, result_count=count, created_at=now - timedelta(hours=2),
+                result_ids=[{"id": 501, "score": 0.8, "rank": 0}],
+            )
+
+        rows = {
+            "a": log(me.id), "b": log(me.id),
+            "empty": log(me.id, count=0),
+            "other_arm": log(me.id, source="write_path"),
+            "theirs": log(other.id),
+        }
+        s.add_all(rows.values())
+        await s.commit()
+        ids = {k: int(v.id) for k, v in rows.items()}
+        ids |= {"me": me.id, "other": other.id}
+    yield ids
+
+    from sqlalchemy import delete
+    from scribe.models.note_usage import NoteUsageEvent
+    from scribe.models.retrieval_judgment import RetrievalJudgment
+    async with async_session() as s:
+        logs = [ids[k] for k in ("a", "b", "empty", "other_arm", "theirs")]
+        await s.execute(delete(RetrievalJudgment).where(
+            RetrievalJudgment.retrieval_log_id.in_(logs)))
+        await s.execute(delete(RetrievalLog).where(RetrievalLog.id.in_(logs)))
+        await s.execute(delete(NoteUsageEvent).where(
+            NoteUsageEvent.user_id == ids["me"], NoteUsageEvent.note_id.in_([501, 502])))
+        await s.commit()
+
+
+def _lines(*specs):
+    """A stand-in re-run: (record_id, rank, within_budget)."""
+    return AsyncMock(return_value=[
+        {"record_id": rid, "rank": rank, "score": 0.9 - rank / 10,
+         "within_budget": within, "kind": "note", "name": f"n{rid}", "passage": "p"}
+        for rid, rank, within in specs
+    ])
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("_dispose_engine")
+async def test_the_sample_offers_only_the_reviewers_unjudged_reviewable_calls(logged_calls):
+    ids = logged_calls
+    with patch.object(review, "_rerun", _lines((501, 1, True))):
+        out = await review.menus_to_review(ids["me"], n=20, days=1)
+        assert {m["log_id"] for m in out["menus"]} == {ids["a"], ids["b"]}
+        assert out["remaining"] == 2
+
+        await review.judge_menu(ids["me"], log_id=ids["a"], verdicts=[
+            {"record_id": 501, "verdict": "on_point", "reason": "names the cache"},
+        ])
+        out = await review.menus_to_review(ids["me"], n=20, days=1)
+    assert {m["log_id"] for m in out["menus"]} == {ids["b"]}
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("_dispose_engine")
+async def test_another_users_call_cannot_be_judged(logged_calls):
+    ids = logged_calls
+    with patch.object(review, "_rerun", _lines((501, 1, True))), \
+            pytest.raises(ValueError, match="not found"):
+        await review.judge_menu(ids["me"], log_id=ids["theirs"], verdicts=[
+            {"record_id": 501, "verdict": "unrelated", "reason": "r"},
+        ])
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("_dispose_engine")
+async def test_a_record_the_re_run_does_not_hold_is_refused(logged_calls):
+    ids = logged_calls
+    with patch.object(review, "_rerun", _lines((501, 1, True))), \
+            pytest.raises(ValueError, match=r"\[999\]"):
+        await review.judge_menu(ids["me"], log_id=ids["a"], verdicts=[
+            {"record_id": 999, "verdict": "unrelated", "reason": "r"},
+        ])
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("_dispose_engine")
+async def test_verdicts_land_replace_and_read_back_by_rank(logged_calls):
+    from scribe.models.note_usage import NoteUsageEvent
+    from scribe.services.retrieval_telemetry import retrieval_summary
+
+    ids = logged_calls
+    # The reviewer's agent opened 501 an hour inside the call; nothing opened 502.
+    async with async_session() as s:
+        s.add(NoteUsageEvent(
+            user_id=ids["me"], note_id=501, event="pulled", source="mcp_get_note",
+            created_at=datetime.now(timezone.utc) - timedelta(hours=1, minutes=30),
+        ))
+        await s.commit()
+
+    rerun = _lines((501, 1, True), (502, 4, False))
+    with patch.object(review, "_rerun", rerun):
+        await review.judge_menu(ids["me"], log_id=ids["a"], verdicts=[
+            {"record_id": 501, "verdict": "adjacent", "reason": "first look"},
+            {"record_id": 502, "verdict": "on_point", "reason": "the exact fix"},
+        ])
+        again = await review.judge_menu(ids["me"], log_id=ids["a"], verdicts=[
+            {"record_id": 501, "verdict": "on_point", "reason": "on reflection"},
+        ])
+    assert again == {"log_id": ids["a"], "recorded": 1, "replaced": 1}
+
+    block = (await retrieval_summary(ids["me"], days=1))["judged"]
+    assert "judged_failed" not in block, "the readout did not execute"
+    ai = block["auto_inject"]
+    assert ai["judged_calls"] == 1 and ai["judged_lines"] == 2
+    assert ai["within_budget"] == {
+        "on_point": 1, "adjacent": 0, "unrelated": 0, "opened": 1}
+    assert ai["beyond_budget"] == {
+        "on_point": 1, "adjacent": 0, "unrelated": 0, "opened": 0}
+    assert [r["rank"] for r in ai["by_rank"]] == [1, 4]
+    # Related and left closed: the number an open rate could never show.
+    assert ai["on_point_unopened"] == 1
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("_dispose_engine")
+async def test_a_fresh_install_reads_an_empty_judged_block_not_a_failure():
+    from scribe.services.retrieval_telemetry import retrieval_summary
+
+    assert (await retrieval_summary(990041, days=30))["judged"] == {}