"""Carrying a floor across a calibration change (#4104). WHAT THIS PINS 1. **The percentile is what transfers, not the number.** A floor's content is a decision about selectivity; the cosine expressing it is units. So a migration that admitted 30% before admits 30% after, whatever the new scores look like. 2. **Nothing is written unless asked twice.** `apply` defaults to False. A model change makes every number uncertain at once, which is the worst moment to let a statistic move six dials unattended. 3. **Missing evidence is a refusal, not a guess.** No logged calls, or a corpus that re-scores to nothing, returns `migrated: False` with a reason — never a floor computed off an empty distribution, which is how an install mid-backfill would end up with every bar at zero. 4. **Every registry surface can be migrated.** A seventh arm that nobody adds a re-scorer for is one whose floor silently cannot survive a model change. """ from unittest.mock import AsyncMock, MagicMock, patch import pytest from scribe.services import retrieval_migration as rm from scribe.services.retrieval_surfaces import SURFACES from tests.helpers import make_mock_session, tool_doc def _logs(pairs): """Rows as `migrate_floor` reads them: (query, project_id, old score).""" return [MagicMock(query=q, project_id=p, best_available_score=s) for q, p, s in pairs] def _session_with(rows): """`.all()` is SYNCHRONOUS on a Result, so it needs a MagicMock. `make_mock_session` is an AsyncMock, and every child of an AsyncMock is one too — leaving `.all` as it comes hands the service a coroutine where it expects a list, the same trap the helper's docstring flags for `add`. """ session = make_mock_session() session.execute.return_value = MagicMock(all=MagicMock(return_value=rows)) return session def test_every_surface_has_a_rescorer(): """Otherwise a surface's floor cannot cross a model change at all.""" assert set(rm._RESCORERS) == set(SURFACES) def test_the_floor_that_admits_a_fraction_is_an_observed_score(): scores = [0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1, 0.05] # 30% of ten is three; the third-best score is the bar that admits exactly # those three. Exact rather than interpolated, so the answer can be checked # against the sample it came from. assert rm._floor_admitting(scores, 0.3) == 0.7 assert sum(1 for s in scores if s >= 0.7) == 3 def test_a_surface_that_admitted_nothing_keeps_admitting_nothing(): """A migration must not quietly reopen an arm the operator had shut.""" scores = [0.5, 0.4, 0.3] assert rm._floor_admitting(scores, 0.0) > max(scores) @pytest.mark.asyncio async def test_selectivity_is_preserved_across_a_scale_change(): """The whole idea, on numbers that move a long way. Old scores cluster near 0.7 with a bar at 0.72 admitting two of five. New scores sit far lower — a different geometry — and the proposal is the value that admits two of five there, not anything resembling 0.72. """ rows = _logs([ ("q1", 2, 0.80), ("q2", 2, 0.75), ("q3", 2, 0.70), ("q4", 2, 0.60), ("q5", 2, 0.50), ]) new = {"q1": 0.42, "q2": 0.38, "q3": 0.31, "q4": 0.22, "q5": 0.10} rescore = AsyncMock(side_effect=lambda u, q, p: new[q]) with patch.object(rm, "async_session", MagicMock(return_value=_session_with(rows))), \ patch.object(rm, "floor_for", AsyncMock(return_value=0.72)), \ patch.dict(rm._RESCORERS, {"prompt_rule": rescore}): out = await rm.migrate_floor(1, "prompt_rule") assert out["old_admit_rate"] == 0.4 # 0.80 and 0.75 cleared 0.72 assert out["proposed_floor"] == 0.38 # admits 0.42 and 0.38 — also two assert out["migrated"] is False # dry run by default @pytest.mark.asyncio async def test_a_dry_run_writes_nothing(): rows = _logs([("q1", None, 0.9), ("q2", None, 0.1)]) with patch.object(rm, "async_session", MagicMock(return_value=_session_with(rows))), \ patch.object(rm, "floor_for", AsyncMock(return_value=0.5)), \ patch.object(rm, "set_dial", AsyncMock()) as set_dial, \ patch.dict(rm._RESCORERS, {"prompt_rule": AsyncMock(return_value=0.4)}): out = await rm.migrate_floor(1, "prompt_rule") set_dial.assert_not_called() assert "proposed_floor" in out @pytest.mark.asyncio async def test_applying_writes_an_ordinary_tuning_event_with_the_arithmetic(): """A migrated floor is reviewable and revertible like any other change.""" rows = _logs([("q1", None, 0.9), ("q2", None, 0.1)]) with patch.object(rm, "async_session", MagicMock(return_value=_session_with(rows))), \ patch.object(rm, "floor_for", AsyncMock(return_value=0.5)), \ patch.object(rm, "set_dial", AsyncMock(return_value={})) as set_dial, \ patch.dict(rm._RESCORERS, {"prompt_rule": AsyncMock(return_value=0.4)}): out = await rm.migrate_floor(1, "prompt_rule", apply=True) assert out["migrated"] is True kwargs = set_dial.call_args.kwargs reason = kwargs["reason"] # The reason has to carry the working, not just the verdict — it is what the # operator reads to decide whether to keep the number. assert "0.5" in reason and "sampled" in reason assert kwargs["actor"] == "model" @pytest.mark.asyncio async def test_no_logged_calls_refuses_rather_than_inventing_a_distribution(): with patch.object(rm, "async_session", MagicMock(return_value=_session_with([]))), \ patch.object(rm, "floor_for", AsyncMock(return_value=0.5)), \ patch.object(rm, "set_dial", AsyncMock()) as set_dial: out = await rm.migrate_floor(1, "prompt_rule", apply=True) assert out["migrated"] is False assert "no logged calls" in out["why"] set_dial.assert_not_called() @pytest.mark.asyncio async def test_a_corpus_that_rescores_to_nothing_refuses(): """The mid-backfill case: every bar would otherwise be set off no data.""" rows = _logs([("q1", None, 0.9), ("q2", None, 0.8)]) with patch.object(rm, "async_session", MagicMock(return_value=_session_with(rows))), \ patch.object(rm, "floor_for", AsyncMock(return_value=0.5)), \ patch.object(rm, "set_dial", AsyncMock()) as set_dial, \ patch.dict(rm._RESCORERS, {"prompt_rule": AsyncMock(return_value=None)}): out = await rm.migrate_floor(1, "prompt_rule", apply=True) assert out["migrated"] is False assert "not embedded" in out["why"] set_dial.assert_not_called() @pytest.mark.asyncio async def test_an_unknown_surface_is_refused(): with pytest.raises(ValueError): await rm.migrate_floor(1, "promptrule") def test_the_tool_says_it_is_a_starting_point_and_defaults_to_a_dry_run(): doc = tool_doc("scribe.mcp.tools.retrieval_tuning", "migrate_retrieval_floor") assert "DRY RUN BY DEFAULT" in doc # Percentile-preserving carries the old floor's wrongness forward faithfully. # A reader who misses that will treat a migrated number as a measured one. assert "STARTING POINT" in doc.upper() assert "stale" in doc